def getWildFile(url, urlMD5):
try:
userAgent = {'User-agent': baseConfig.userAgent}
if baseConfig.useTor == 'yes':
torProxy = 'socks5://localhost:{0}'.format(baseConfig.torPort)
proxies = {'http': torProxy, 'https': torProxy}
request = requests.get(url, headers=userAgent, proxies=proxies)
else:
request = requests.get(url, headers=userAgent)
if request.status_code == 200:
response = request.content
tmpName = randomString(32)
tmpFilePath = os.path.join(baseConfig.outputFolder, tmpName)
open(tmpFilePath, "wb").write(response)
logging.info(
"Saved as temporary file: {0}. Calculating MD5.".format(
tmpFilePath))
fileMD5 = md5SumFile(tmpFilePath)
filePath = os.path.join(baseConfig.outputFolder, fileMD5)
os.rename(tmpFilePath, filePath)
logging.info(
"Renamed as file: {0}. Checking Viper again.".format(filePath))
if isNewEntry(fileHash=fileMD5):
fileName = url.split('/')[-1]
tags = getTags(fileMD5, url, "wild-spider", urlHash=urlMD5)
uploadToViper(filePath, fileName, tags)
if baseConfig.deleteOutput.lower() == "yes":
logging.info("Removing file: {0}".format(filePath))
os.remove(filePath)
return True
else:
logging.info("Removing file: {0}".format(filePath))
os.remove(filePath)
return False
else:
logging.warning(
"Problem connecting to {0}. Status code: {1}. Continuing.".
format(url, request.status_code))
return False
except requests.exceptions.ConnectionError as e:
logging.warning("Problem connecting to {0}. Error: {1}".format(url, e))
return False
except Exception as e:
logging.warning("Problem connecting to {0}. Continuing.".format(url))
logging.exception(sys.exc_info())
logging.exception(type(e))
logging.exception(e.args)
logging.exception(e)
return False
def processDownload(tmpFilePath, fileName, fileUrl):
logging.info('Downloaded as temporary file: {0}. Beginning processing...'.format(tmpFilePath))
fileSize = os.path.getsize(tmpFilePath) >> 20
if (fileSize > 10):
logging.error('File is {0}MB. Too large to process.'.format(fileSize))
cleanUp(tmpFilePath)
return False
fileHash = sha256SumFile(tmpFilePath)
if not isAcceptedHash(fileHash):
cleanUp(tmpFilePath)
return False
filePath = os.path.join(baseConfig.outputFolder, fileHash)
os.rename(tmpFilePath, filePath)
# Trust only the content type of the downloaded file.
mimeType = magic.from_file(filePath, mime=True)
if mimeType not in ['application/octet-stream', 'application/x-dosexec', 'application/x-msdownload', 'application/x-ms-installer', 'application/pdf', 'application/x-pdf', 'application/msword', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'application/vnd.openxmlformats-officedocument.wordprocessingml.template', 'application/vnd.ms-word.document.macroEnabled', 'application/vnd.ms-excel', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'application/vnd.openxmlformats-officedocument.spreadsheetml.template', 'application/vnd.ms-excel.sheet.macroEnabled', 'application/vnd.ms-excel.template.macroEnabled', 'application/vnd.ms-excel.addin.macroEnabled', 'application/vnd.ms-excel.sheet.binary.macroEnabled', 'application/x-shockwave-flash']:
logging.error('Detected non-binary or executable file type ({0}). Skipping: {1}'.format(mimeType, filePath))
cleanUp(filePath)
return False
logging.info('File with hash: {0} identified as type: {1}'.format(fileHash, mimeType))
uploaded = uploadToViper(filePath, fileName, fileHash, fileUrl)
addToHashCache(fileHash)
cleanUp(filePath)
return uploaded
def processDownload(tmpFilePath, fileName, fileUrl):
logging.info('Downloaded as temporary file: {0}. Beginning processing...'.format(tmpFilePath))
fileSize = os.path.getsize(tmpFilePath) >> 20
if (fileSize > 10):
logging.error('File is {0}MB. Too large to process.'.format(fileSize))
cleanUp(tmpFilePath)
return False
fileHash = sha256SumFile(tmpFilePath)
if not isAcceptedHash(fileHash):
cleanUp(tmpFilePath)
return False
filePath = os.path.join(baseConfig.outputFolder, fileHash)
os.rename(tmpFilePath, filePath)
# Trust only the content type of the downloaded file.
mimeType = magic.from_file(filePath, mime=True)
if mimeType not in ['application/octet-stream', 'application/x-dosexec', 'application/x-msdownload', 'application/x-ms-installer', 'application/pdf', 'application/x-pdf', 'application/msword', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'application/vnd.openxmlformats-officedocument.wordprocessingml.template', 'application/vnd.ms-word.document.macroEnabled', 'application/vnd.ms-excel', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'application/vnd.openxmlformats-officedocument.spreadsheetml.template', 'application/vnd.ms-excel.sheet.macroEnabled', 'application/vnd.ms-excel.template.macroEnabled', 'application/vnd.ms-excel.addin.macroEnabled', 'application/vnd.ms-excel.sheet.binary.macroEnabled', 'application/x-shockwave-flash']:
logging.error('Detected non-binary or executable file type ({0}). Skipping: {1}'.format(mimeType, filePath))
cleanUp(filePath)
return False
logging.info('File with hash: {0} identified as type: {1}'.format(fileHash, mimeType))
uploaded = uploadToViper(filePath, fileName, fileUrl)
addToHashCache(fileHash)
cleanUp(filePath)
return uploaded
def getMalShareFile(fileHash):
try:
payload = {
'action': 'getfile',
'api_key': baseConfig.malShareApiKey,
'hash': fileHash
}
userAgent = {'User-agent': baseConfig.userAgent}
request = requests.get(baseConfig.malShareApi,
params=payload,
headers=userAgent)
if request.status_code == 200:
response = request.content
if "Sample not found" in response:
logging.warning("Sample not found.")
return None
if "Account not activated" in response:
logging.error("Bad API key.")
sys.exit(1)
if "Over Request Limit" in response:
logging.error(
"Exceeded MalShare request quota. Please temporarily disable MalShare."
)
sys.exit(1)
tmpName = randomString(32)
tmpFilePath = os.path.join(baseConfig.outputFolder, tmpName)
open(tmpFilePath, "wb").write(response)
logging.info(
"Saved as temporary file: {0}. Calculating MD5.".format(
tmpFilePath))
fileMD5 = md5SumFile(tmpFilePath)
filePath = os.path.join(baseConfig.outputFolder, fileMD5)
os.rename(tmpFilePath, filePath)
logging.info(
"Renamed as file: {0}. Checking Viper again.".format(filePath))
if isNewEntry(fileHash=fileMD5):
url = getMalShareSource(fileHash)
fileName = url.split('/')[-1]
tags = getTags(fileMD5, url, "malshare-spider")
uploadToViper(filePath, fileName, tags)
if baseConfig.deleteOutput.lower() == "yes":
logging.info("Removing file: {0}".format(filePath))
os.remove(filePath)
return True
else:
logging.info("Removing file: {0}".format(filePath))
os.remove(filePath)
return False
else:
logging.error(
"Problem connecting to MalShare. Status code: {0}. Please try again later."
.format(request.status_code))
sys.exit(1)
except Exception as e:
logging.error(
"Problem connecting to MalShare. Please try again later.")
logging.exception(sys.exc_info())
logging.exception(type(e))
logging.exception(e.args)
logging.exception(e)
sys.exit(1)