def requestUrl(fileUrl):
try:
userAgent = {'User-agent': baseConfig.userAgent}
request = requests.get(fileUrl, headers=userAgent, proxies=proxies, timeout=(20,20))
if request.status_code == 200:
response = request.content
tmpName = randomString(32)
tmpFilePath = os.path.join(baseConfig.outputFolder, tmpName)
fileName = urlparse.urlparse(fileUrl).path
open(tmpFilePath,'wb').write(response)
processed = processDownload(tmpFilePath, fileName, fileUrl)
return processed
else:
logging.warning('Problem connecting to {0}. Status code: {1}. Aborting task.'.format(fileUrl, request.status_code))
return False
except requests.exceptions.ConnectionError as e:
logging.warning('Problem connecting to {0}. Error: {1}'.format(fileUrl, e))
return False
except Exception as e:
logging.warning('Problem connecting to {0}. Aborting task.'.format(fileUrl))
logging.exception(sys.exc_info())
logging.exception(type(e))
logging.exception(e.args)
logging.exception(e)
return False
def getWildFile(url, urlMD5):
try:
userAgent = {'User-agent': baseConfig.userAgent}
if baseConfig.useTor == 'yes':
torProxy = 'socks5://localhost:{0}'.format(baseConfig.torPort)
proxies = {'http': torProxy, 'https': torProxy}
request = requests.get(url, headers=userAgent, proxies=proxies)
else:
request = requests.get(url, headers=userAgent)
if request.status_code == 200:
response = request.content
tmpName = randomString(32)
tmpFilePath = os.path.join(baseConfig.outputFolder, tmpName)
open(tmpFilePath, "wb").write(response)
logging.info(
"Saved as temporary file: {0}. Calculating MD5.".format(
tmpFilePath))
fileMD5 = md5SumFile(tmpFilePath)
filePath = os.path.join(baseConfig.outputFolder, fileMD5)
os.rename(tmpFilePath, filePath)
logging.info(
"Renamed as file: {0}. Checking Viper again.".format(filePath))
if isNewEntry(fileHash=fileMD5):
fileName = url.split('/')[-1]
tags = getTags(fileMD5, url, "wild-spider", urlHash=urlMD5)
uploadToViper(filePath, fileName, tags)
if baseConfig.deleteOutput.lower() == "yes":
logging.info("Removing file: {0}".format(filePath))
os.remove(filePath)
return True
else:
logging.info("Removing file: {0}".format(filePath))
os.remove(filePath)
return False
else:
logging.warning(
"Problem connecting to {0}. Status code: {1}. Continuing.".
format(url, request.status_code))
return False
except requests.exceptions.ConnectionError as e:
logging.warning("Problem connecting to {0}. Error: {1}".format(url, e))
return False
except Exception as e:
logging.warning("Problem connecting to {0}. Continuing.".format(url))
logging.exception(sys.exc_info())
logging.exception(type(e))
logging.exception(e.args)
logging.exception(e)
return False
def requestUrl(fileUrl):
try:
userAgent = {'User-agent': baseConfig.userAgent}
request = requests.get(fileUrl, headers=userAgent, proxies=proxies, timeout=(20,20))
if request.status_code == 200:
response = request.content
tmpName = randomString(32)
tmpFilePath = os.path.join(baseConfig.outputFolder, tmpName)
fileName = urlparse.urlparse(fileUrl).path.strip('/')
open(tmpFilePath,'wb').write(response)
processed = processDownload(tmpFilePath, fileName, fileUrl)
return processed
else:
logging.warning('Problem connecting to {0}. Status code: {1}. Aborting task.'.format(fileUrl, request.status_code))
return False
except requests.exceptions.ConnectionError as e:
logging.warning('Problem connecting to {0}. Error: {1}'.format(fileUrl, e))
return False
except Exception as e:
logging.warning('Problem connecting to {0}. Aborting task.'.format(fileUrl))
logging.exception(sys.exc_info())
logging.exception(type(e))
logging.exception(e.args)
logging.exception(e)
return False
def getMalShareFile(fileHash):
try:
payload = {
'action': 'getfile',
'api_key': baseConfig.malShareApiKey,
'hash': fileHash
}
userAgent = {'User-agent': baseConfig.userAgent}
request = requests.get(baseConfig.malShareApi,
params=payload,
headers=userAgent)
if request.status_code == 200:
response = request.content
if "Sample not found" in response:
logging.warning("Sample not found.")
return None
if "Account not activated" in response:
logging.error("Bad API key.")
sys.exit(1)
if "Over Request Limit" in response:
logging.error(
"Exceeded MalShare request quota. Please temporarily disable MalShare."
)
sys.exit(1)
tmpName = randomString(32)
tmpFilePath = os.path.join(baseConfig.outputFolder, tmpName)
open(tmpFilePath, "wb").write(response)
logging.info(
"Saved as temporary file: {0}. Calculating MD5.".format(
tmpFilePath))
fileMD5 = md5SumFile(tmpFilePath)
filePath = os.path.join(baseConfig.outputFolder, fileMD5)
os.rename(tmpFilePath, filePath)
logging.info(
"Renamed as file: {0}. Checking Viper again.".format(filePath))
if isNewEntry(fileHash=fileMD5):
url = getMalShareSource(fileHash)
fileName = url.split('/')[-1]
tags = getTags(fileMD5, url, "malshare-spider")
uploadToViper(filePath, fileName, tags)
if baseConfig.deleteOutput.lower() == "yes":
logging.info("Removing file: {0}".format(filePath))
os.remove(filePath)
return True
else:
logging.info("Removing file: {0}".format(filePath))
os.remove(filePath)
return False
else:
logging.error(
"Problem connecting to MalShare. Status code: {0}. Please try again later."
.format(request.status_code))
sys.exit(1)
except Exception as e:
logging.error(
"Problem connecting to MalShare. Please try again later.")
logging.exception(sys.exc_info())
logging.exception(type(e))
logging.exception(e.args)
logging.exception(e)
sys.exit(1)