def db_inconsistency(file_hash):
if(not valid_hash(file_hash)):
raise ValueError("db_inconsistency invalid hash")
pc = PackageController()
v = VersionController()
file_id = get_file_id(file_hash)
if file_id is not None: # meta exists
file_bin = pc.getFile(file_id)
if file_bin is not None: # sample exists
version = v.searchVersion(file_id)
if version is not None:
return 0 # ok
else: # version does not exist
logging.info(
"inconsistency: meta and sample exists. Version does not")
return 3
else: # has meta but not sample
logging.info("inconsistency: meta exists, sample does not")
return 2
else: # does not have meta
if len(file_hash) == 64:
return 0 # cant search in grid by sha256
if len(file_hash) == 40:
file_bin = pc.getFile(file_hash)
else: # md5
sha1 = pc.md5_to_sha1(file_hash)
if sha1 is None:
return 0 # does not have meta or sample
file_bin = pc.getFile(file_hash)
if file_bin is None:
return 0
else:
logging.info("inconsistency: does not have meta. has sample")
return 1
def generic_process_hash(hash_str):
hash_str = clean_hash(hash_str)
if(not valid_hash(hash_str)):
return None
if(len(hash_str)==32):
hash_str=get_file_id(hash_str)
if(hash_str is not None):
return process_file(hash_str)
else :
return None
def get_result_from_av():
hash_id = request.query.file_hash
if len(hash_id) == 0:
response.status = 400
return jsonize({
'error': 4,
'error_message': 'file_hash parameter is missing.'
})
hash_id = clean_hash(hash_id)
if not valid_hash(hash_id):
return jsonize({'error': 5, 'error_message': 'Invalid hash format.'})
if (len(hash_id) != 40):
data = "1=" + str(hash_id)
res = SearchModule.search_by_id(data, 1, [], True)
if (len(res) == 0):
response.status = 400
return jsonize({'error': 6, 'error_message': 'File not found'})
else:
sha1 = res[0]["sha1"]
else:
sha1 = hash_id
key_manager = KeyManager()
if (key_manager.check_keys_in_secrets()):
av_result = get_av_result(sha1, 'high')
else:
return jsonize({
'error':
7,
"error_message":
"Error: VirusTotal API key missing from secrets.py file"
})
if (av_result.get('status') == "added"):
return jsonize({"message": "AV scans downloaded."})
elif (av_result.get('status') == "already_had_it"):
return jsonize({"message": "File already have AV scans."})
elif (av_result.get('status') == "not_found"):
return jsonize({"error": 10, "error_message": "Not found on VT."})
elif (av_result.get('status') == "no_key_available"):
return jsonize({
"error":
11,
"error_message":
"No key available right now. Please try again later."
})
else:
logging.error("av_result for hash=" + str(sha1))
logging.error("av_result=" + str(av_result))
return jsonize({"error": 9, "error_message": "Cannot get analysis."})
def generic_process_hash(hash_str):
if hash_str is None:
return None
hash_str = clean_hash(hash_str)
if(not valid_hash(hash_str)):
return None
if(len(hash_str) == 64):
hash_str = get_file_id(hash_str)
elif(len(hash_str) == 32):
pc = PackageController()
hash_str = pc.md5_to_sha1(hash_str)
logging.debug("generic_process_hash-->sha1: " + str(hash_str))
if(hash_str is not None):
return process_file(hash_str)
else:
return None
def generic_process_hash(hash_str):
if hash_str is None:
return None
hash_str = clean_hash(hash_str)
if (not valid_hash(hash_str)):
return None
if (len(hash_str) == 64):
hash_str = get_file_id(hash_str)
elif (len(hash_str) == 32):
pc = PackageController()
hash_str = pc.md5_to_sha1(hash_str)
logging.debug("generic_process_hash-->sha1: " + str(hash_str))
if (hash_str is not None):
return process_file(hash_str)
else:
return None
def get_metadata():
if request.query.file_hash == '':
response.status = 400
return jsonize({'message': 'file_hash parameter is missing'})
file_hash = clean_hash(request.query.file_hash)
if not valid_hash(file_hash):
response.status = 400
return jsonize({'message': 'Invalid hash format (use MD5, SHA1 or SHA2)'})
file_hash = get_file_id(file_hash)
if file_hash is None:
response.status = 404
return jsonize({'message': 'Metadata not found in the database'})
mdc = MetaController()
res = mdc.read(file_hash)
if res is None:
log_event("metadata", file_hash)
return dumps(change_date_to_str(res))
def get_av_result(file_id,priority="low"):
if not valid_hash(file_id):
raise ValueError("Invalid hash")
mdc=MetaController()
analysis_result=mdc.search_av_analysis(file_id)
added=False
status = None
if analysis_result==None:
logging.info("Searching analysis of %s in VT" % file_id)
vt_av_result = get_vt_av_result(file_id,priority)
status = vt_av_result.get('status')
if vt_av_result.get('status') == "ok":
vt_av_result_response = vt_av_result.get('response')
analysis_result=parse_vt_response(vt_av_result_response)
# Save in mongo
if(analysis_result is not None):
logging.info( "saving vt av from "+str(file_id)+ " in mongo")
mdc.save_av_analysis(file_id,analysis_result)
status = "added"
elif vt_av_result.get('status') == "error":
return {"scans": None, "hash": file_id, "status": "error", "error_message": vt_av_result.get('error_message')}
else:
status = "already_had_it"
if analysis_result is not None:
scans=analysis_result.get("scans")
positives = analysis_result.get('positives')
total = analysis_result.get('total')
else:
positives = 0
total = 0
scans = None
response = {"scans": scans, "positives": positives,
"total": total, "hash": file_id, "status": status}
return response
def download_from_virus_total(file_id):
logging.debug("download_from_virus_total(): " + str(file_id))
if not valid_hash(file_id):
raise ValueError("download_from_virus_total recieved an invalid hash")
key_manager = KeyManager()
has_key = False
while not has_key:
apikey = key_manager.get_key('download_sample')
if(apikey.get('key') is None and apikey.get('timeleft') is None):
return None
elif apikey.get('key') is not None:
has_key = True
elif((isinstance(apikey.get('timeleft'), int) or
isinstance(apikey.get('timeleft'), float)) and
apikey.get('timeleft') > 0):
logging.debug("download_from_virus_total(): timeleft=" +
str(apikey.get('timeleft')))
time.sleep(apikey.get('timeleft'))
params = {'apikey': apikey.get('key'), 'hash': file_id}
try_again = True
fail_count = 0
response = None
while(try_again):
try:
response = requests.get(
'https://www.virustotal.com/vtapi/v2/file/download', params=params, timeout=30)
try_again = False
except Exception, e:
logging.exception("requests to virustotal / download")
# print(str(e))
# print(traceback.format_exc())
try_again = True
fail_count += 1
if(fail_count >= 3):
break
def get_av_result(file_id, priority="low"):
if not valid_hash(file_id):
raise ValueError("Invalid hash")
mdc = MetaController()
analysis_result = mdc.search_av_analysis(file_id)
added = False
status = None
if analysis_result is None:
logging.info("Searching analysis of %s in VT" % file_id)
vt_av_result = get_vt_av_result(file_id, priority)
status = vt_av_result.get('status')
if vt_av_result.get('status') == "ok":
vt_av_result_response = vt_av_result.get('response')
analysis_result = parse_vt_response(vt_av_result_response)
# Save in mongo
if(analysis_result is not None):
logging.info("saving vt av from " + str(file_id) + " in mongo")
mdc.save_av_analysis(file_id, analysis_result)
status = "added"
elif vt_av_result.get('status') == "error":
return {"scans": None, "hash": file_id, "status": "error", "error_message": vt_av_result.get('error_message')}
else:
status = "already_had_it"
if analysis_result is not None:
scans = analysis_result.get("scans")
positives = analysis_result.get('positives')
total = analysis_result.get('total')
else:
positives = 0
total = 0
scans = None
response = {"scans": scans, "positives": positives,
"total": total, "hash": file_id, "status": status}
return response
def get_result_from_av():
hash_id = request.query.file_hash
if len(hash_id) == 0:
response.status = 400
return jsonize({'error': 4, 'error_message': 'file_hash parameter is missing.'})
hash_id = clean_hash(hash_id)
if not valid_hash(hash_id):
return jsonize({'error': 5, 'error_message': 'Invalid hash format.'})
if(len(hash_id) != 40):
data = "1=" + str(hash_id)
res = SearchModule.search_by_id(data, 1, [], True)
if(len(res) == 0):
response.status = 400
return jsonize({'error': 6, 'error_message': 'File not found'})
else:
sha1 = res[0]["sha1"]
else:
sha1 = hash_id
key_manager = KeyManager()
if(key_manager.check_keys_in_secrets()):
av_result = get_av_result(sha1, 'high')
else:
return jsonize({'error': 7, "error_message": "Error: VirusTotal API key missing from secrets.py file"})
if(av_result.get('status') == "added"):
return jsonize({"message": "AV scans downloaded."})
elif(av_result.get('status') == "already_had_it"):
return jsonize({"message": "File already have AV scans."})
elif(av_result.get('status') == "not_found"):
return jsonize({"error": 10, "error_message": "Not found on VT."})
elif(av_result.get('status') == "no_key_available"):
return jsonize({"error": 11, "error_message": "No key available right now. Please try again later."})
else:
logging.error("av_result for hash=" + str(sha1))
logging.error("av_result=" + str(av_result))
return jsonize({"error": 9, "error_message": "Cannot get analysis."})