def requestUrl(fileUrl): try: userAgent = {'User-agent': baseConfig.userAgent} request = requests.get(fileUrl, headers=userAgent, proxies=proxies, timeout=(20,20)) if request.status_code == 200: response = request.content tmpName = randomString(32) tmpFilePath = os.path.join(baseConfig.outputFolder, tmpName) fileName = urlparse.urlparse(fileUrl).path open(tmpFilePath,'wb').write(response) processed = processDownload(tmpFilePath, fileName, fileUrl) return processed else: logging.warning('Problem connecting to {0}. Status code: {1}. Aborting task.'.format(fileUrl, request.status_code)) return False except requests.exceptions.ConnectionError as e: logging.warning('Problem connecting to {0}. Error: {1}'.format(fileUrl, e)) return False except Exception as e: logging.warning('Problem connecting to {0}. Aborting task.'.format(fileUrl)) logging.exception(sys.exc_info()) logging.exception(type(e)) logging.exception(e.args) logging.exception(e) return False
def getWildFile(url, urlMD5): try: userAgent = {'User-agent': baseConfig.userAgent} if baseConfig.useTor == 'yes': torProxy = 'socks5://localhost:{0}'.format(baseConfig.torPort) proxies = {'http': torProxy, 'https': torProxy} request = requests.get(url, headers=userAgent, proxies=proxies) else: request = requests.get(url, headers=userAgent) if request.status_code == 200: response = request.content tmpName = randomString(32) tmpFilePath = os.path.join(baseConfig.outputFolder, tmpName) open(tmpFilePath, "wb").write(response) logging.info( "Saved as temporary file: {0}. Calculating MD5.".format( tmpFilePath)) fileMD5 = md5SumFile(tmpFilePath) filePath = os.path.join(baseConfig.outputFolder, fileMD5) os.rename(tmpFilePath, filePath) logging.info( "Renamed as file: {0}. Checking Viper again.".format(filePath)) if isNewEntry(fileHash=fileMD5): fileName = url.split('/')[-1] tags = getTags(fileMD5, url, "wild-spider", urlHash=urlMD5) uploadToViper(filePath, fileName, tags) if baseConfig.deleteOutput.lower() == "yes": logging.info("Removing file: {0}".format(filePath)) os.remove(filePath) return True else: logging.info("Removing file: {0}".format(filePath)) os.remove(filePath) return False else: logging.warning( "Problem connecting to {0}. Status code: {1}. Continuing.". format(url, request.status_code)) return False except requests.exceptions.ConnectionError as e: logging.warning("Problem connecting to {0}. Error: {1}".format(url, e)) return False except Exception as e: logging.warning("Problem connecting to {0}. Continuing.".format(url)) logging.exception(sys.exc_info()) logging.exception(type(e)) logging.exception(e.args) logging.exception(e) return False
def requestUrl(fileUrl): try: userAgent = {'User-agent': baseConfig.userAgent} request = requests.get(fileUrl, headers=userAgent, proxies=proxies, timeout=(20,20)) if request.status_code == 200: response = request.content tmpName = randomString(32) tmpFilePath = os.path.join(baseConfig.outputFolder, tmpName) fileName = urlparse.urlparse(fileUrl).path.strip('/') open(tmpFilePath,'wb').write(response) processed = processDownload(tmpFilePath, fileName, fileUrl) return processed else: logging.warning('Problem connecting to {0}. Status code: {1}. Aborting task.'.format(fileUrl, request.status_code)) return False except requests.exceptions.ConnectionError as e: logging.warning('Problem connecting to {0}. Error: {1}'.format(fileUrl, e)) return False except Exception as e: logging.warning('Problem connecting to {0}. Aborting task.'.format(fileUrl)) logging.exception(sys.exc_info()) logging.exception(type(e)) logging.exception(e.args) logging.exception(e) return False
def getMalShareFile(fileHash): try: payload = { 'action': 'getfile', 'api_key': baseConfig.malShareApiKey, 'hash': fileHash } userAgent = {'User-agent': baseConfig.userAgent} request = requests.get(baseConfig.malShareApi, params=payload, headers=userAgent) if request.status_code == 200: response = request.content if "Sample not found" in response: logging.warning("Sample not found.") return None if "Account not activated" in response: logging.error("Bad API key.") sys.exit(1) if "Over Request Limit" in response: logging.error( "Exceeded MalShare request quota. Please temporarily disable MalShare." ) sys.exit(1) tmpName = randomString(32) tmpFilePath = os.path.join(baseConfig.outputFolder, tmpName) open(tmpFilePath, "wb").write(response) logging.info( "Saved as temporary file: {0}. Calculating MD5.".format( tmpFilePath)) fileMD5 = md5SumFile(tmpFilePath) filePath = os.path.join(baseConfig.outputFolder, fileMD5) os.rename(tmpFilePath, filePath) logging.info( "Renamed as file: {0}. Checking Viper again.".format(filePath)) if isNewEntry(fileHash=fileMD5): url = getMalShareSource(fileHash) fileName = url.split('/')[-1] tags = getTags(fileMD5, url, "malshare-spider") uploadToViper(filePath, fileName, tags) if baseConfig.deleteOutput.lower() == "yes": logging.info("Removing file: {0}".format(filePath)) os.remove(filePath) return True else: logging.info("Removing file: {0}".format(filePath)) os.remove(filePath) return False else: logging.error( "Problem connecting to MalShare. Status code: {0}. Please try again later." .format(request.status_code)) sys.exit(1) except Exception as e: logging.error( "Problem connecting to MalShare. Please try again later.") logging.exception(sys.exc_info()) logging.exception(type(e)) logging.exception(e.args) logging.exception(e) sys.exit(1)