def db_inconsistency(file_hash):
if(not valid_hash(file_hash)):
raise ValueError("db_inconsistency invalid hash")
pc = PackageController()
v = VersionController()
file_id = get_file_id(file_hash)
if file_id is not None: # meta exists
file_bin = pc.getFile(file_id)
if file_bin is not None: # sample exists
version = v.searchVersion(file_id)
if version is not None:
return 0 # ok
else: # version does not exist
logging.info(
"inconsistency: meta and sample exists. Version does not")
return 3
else: # has meta but not sample
logging.info("inconsistency: meta exists, sample does not")
return 2
else: # does not have meta
if len(file_hash) == 64:
return 0 # cant search in grid by sha256
if len(file_hash) == 40:
file_bin = pc.getFile(file_hash)
else: # md5
sha1 = pc.md5_to_sha1(file_hash)
if sha1 is None:
return 0 # does not have meta or sample
file_bin = pc.getFile(file_hash)
if file_bin is None:
return 0
else:
logging.info("inconsistency: does not have meta. has sample")
return 1
def db_inconsistency(file_hash):
if (not valid_hash(file_hash)):
raise ValueError("db_inconsistency invalid hash")
pc = PackageController()
v = VersionController()
file_id = get_file_id(file_hash)
if file_id is not None: #meta exists
file_bin = pc.getFile(file_id)
if file_bin is not None: #sample exists
version = v.searchVersion(file_id)
if version is not None:
return 0 # ok
else: #version does not exist
logging.info(
"inconsistency: meta and sample exists. Version does not")
return 3
else: # has meta but not sample
logging.info("inconsistency: meta exists, sample does not")
return 2
else: # does not have meta
if len(file_hash) == 64:
return 0 # cant search in grid by sha256
if len(file_hash) == 40:
file_bin = pc.getFile(file_hash)
else: # md5
sha1 = pc.md5_to_sha1(file_hash)
if sha1 is None:
return 0 # does not have meta or sample
file_bin = pc.getFile(file_hash)
if file_bin is None:
return 0
else:
logging.info("inconsistency: does not have meta. has sample")
return 1
def fix_inconsistency(file_hash):
status = db_inconsistency(file_hash)
if status == 1 or status == 3:
generic_process_hash(file_hash)
return 1
elif status == 2:
file_id = get_file_id(file_hash)
save_file_from_vt(file_id)
return 1
else:
return 0
def fix_inconsistency(file_hash):
status = db_inconsistency(file_hash)
if status == 1 or status == 3:
generic_process_hash(file_hash)
return {"inconsistency": True, "fixed": True, "credit_spent": False}
elif status == 2 and envget('spend_credit_to_fix_inconsistency'):
file_id = get_file_id(file_hash)
save_file_from_vt(file_id)
return {"inconsistency": True, "fixed": True, "credit_spent": True}
elif status == 2 and not envget('spend_credit_to_fix_inconsistency'):
return {"inconsistency": True, "fixed": False}
elif status == 0:
return {"inconsistency": False, "fixed": False}
def fix_inconsistency(file_hash):
status = db_inconsistency(file_hash)
if status == 1 or status == 3:
generic_process_hash(file_hash)
return {"inconsistency": True, "fixed": True, "credit_spent": False}
elif status == 2 and envget('spend_credit_to_fix_inconsistency'):
file_id = get_file_id(file_hash)
save_file_from_vt(file_id)
return {"inconsistency": True, "fixed": True, "credit_spent": True}
elif status == 2 and not envget('spend_credit_to_fix_inconsistency'):
return {"inconsistency": True, "fixed": False}
elif status == 0:
return {"inconsistency": False, "fixed": False}
def generic_process_hash(hash_str):
if hash_str is None:
return None
hash_str = clean_hash(hash_str)
if(not valid_hash(hash_str)):
return None
if(len(hash_str) == 64):
hash_str = get_file_id(hash_str)
elif(len(hash_str) == 32):
pc = PackageController()
hash_str = pc.md5_to_sha1(hash_str)
logging.debug("generic_process_hash-->sha1: " + str(hash_str))
if(hash_str is not None):
return process_file(hash_str)
else:
return None
def generic_process_hash(hash_str):
if hash_str is None:
return None
hash_str = clean_hash(hash_str)
if (not valid_hash(hash_str)):
return None
if (len(hash_str) == 64):
hash_str = get_file_id(hash_str)
elif (len(hash_str) == 32):
pc = PackageController()
hash_str = pc.md5_to_sha1(hash_str)
logging.debug("generic_process_hash-->sha1: " + str(hash_str))
if (hash_str is not None):
return process_file(hash_str)
else:
return None
def get_metadata():
if request.query.file_hash == '':
response.status = 400
return jsonize({'message': 'file_hash parameter is missing'})
file_hash = clean_hash(request.query.file_hash)
if not valid_hash(file_hash):
response.status = 400
return jsonize({'message': 'Invalid hash format (use MD5, SHA1 or SHA2)'})
file_hash = get_file_id(file_hash)
if file_hash is None:
response.status = 404
return jsonize({'message': 'Metadata not found in the database'})
mdc = MetaController()
res = mdc.read(file_hash)
if res is None:
log_event("metadata", file_hash)
return dumps(change_date_to_str(res))
def generic_task(process,
file_hash,
vt_av,
vt_samples,
email,
task_id,
document_name=""):
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
logging.info("task_id=" + str(task_id))
generic_count = 0
response = {}
response["date_start"] = datetime.datetime.now()
response["document_name"] = document_name
response["task_id"] = task_id
check_hashes_output = check_hashes(file_hash)
errors = check_hashes_output.get('errors')
for error in errors:
key = error.get('error')
value = error.get('error_message')
logging.error("errors (key=" + str(key) + ", value=" + str(value) +
")")
response = add_error(response, key, value)
hashes = check_hashes_output.get('hashes')
remove_dups_output = remove_dups(hashes)
# remove duplicated hashes
hashes = remove_dups_output.get('list')
response["duplicated_hashes"] = remove_dups_output.get('dups')
response["hashes"] = hashes
hash_dicts = []
mc = MetaController()
for x in hashes:
x_dict = {}
x_dict["original"] = x
x_dict["sha1"] = get_file_id(x)
if (x_dict["sha1"] is not None):
doc = mc.read(x_dict["sha1"])
if doc is not None and doc.get('hash') is not None:
if doc.get('hash').get('md5') is not None:
x_dict["md5"] = doc.get('hash').get('md5')
if doc.get('hash').get('sha2') is not None:
x_dict["sha2"] = doc.get('hash').get('sha2')
hash_dicts.append(x_dict)
response["duplicated_samples"] = []
for x in hash_dicts:
for y in hash_dicts:
if x.get('original') != y.get('original') and (
x.get('original') == y.get('sha1') or x.get('original')
== y.get('md5') or x.get('original') == y.get('sha2')):
response["duplicated_samples"].append(y.get('original'))
hash_dicts.remove(y)
hashes = []
for x in hash_dicts:
hashes.append(x.get('original'))
response["hashes"] = hashes
if (len(hashes) == 0):
response = add_error(response, 6, "No valid hashes provided.")
response["date_end"] = datetime.datetime.now()
save(response)
return change_date_to_str(response)
save(response)
response["inconsistencies"] = []
if (vt_samples or process):
for hash_id in hashes:
if fix_inconsistency(hash_id) == 1:
response["inconsistencies"].append(hash_id)
save(response)
response["not_found_on_vt"] = []
response["private_credits_spent"] = 0
if vt_samples:
response["downloaded"] = []
for hash_id in hashes:
if (get_file_id(hash_id) is None or db_inconsistency(hash_id)):
logging.debug("task(): " + hash_id +
" was not found (get_file_id returned None). ")
generic_count += 1
if (generic_count % 20 == 0):
save(response)
output = save_file_from_vt(hash_id)
sha1 = output.get('hash')
if (output.get('status') == 'out_of_credits'):
request_successful = False
while not request_successful:
output = save_file_from_vt(hash_id)
if output.get('status') != 'out_of_credits':
request_successful = True
if (output.get('status') == 'added'):
response["downloaded"].append(hash_id)
# we need to process the sha1, and not the sha2 because
# the grid does not save the sha2.
generic_process_hash(sha1)
response["private_credits_spent"] += 1
elif (output.get('status') == 'inconsistency_found'):
response["private_credits_spent"] += 1
generic_process_hash(sha1)
elif (output.get('status') == 'not_found'):
response["not_found_on_vt"].append(hash_id)
else:
logging.error("task_id=" + str(task_id))
logging.error(str(output))
response = add_error(
response, 11,
"Unknown error when downloading sample from VT.")
save(response)
save(response)
response["processed"] = []
response["not_found_for_processing"] = []
if process:
logging.debug("process=true")
for hash_id in hashes:
logging.debug("task: hash_id=" + str(hash_id))
process_start_time = datetime.datetime.now()
generic_count += 1
if (generic_count % 20 == 0):
save(response)
if (generic_process_hash(hash_id) == 0):
process_end_time = datetime.datetime.now()
response["processed"].append({
"hash":
hash_id,
"seconds": (process_end_time - process_start_time).seconds
})
else:
response["not_found_for_processing"].append(hash_id)
save(response)
if vt_av:
response["vt_av_added"] = []
response["vt_av_out_of_credits"] = []
response["not_found_on_vt_av"] = []
response["vt_av_already_downloaded"] = []
response["public_credits_spent"] = 0
for hash_id in hashes:
sha1 = get_file_id(hash_id)
if (sha1 is not None):
av_result_output = get_av_result(sha1)
if (av_result_output.get('status') == 'out_of_credits'):
request_successful = False
count = 0
while not request_successful:
av_result_output = get_av_result(sha1)
count += 1
if av_result_output.get('status') != 'out_of_credits':
response["vt_av_out_of_credits"].append(hash_id)
response = add_error(
response, 10, "Had to retried " + str(count) +
" times in av_result(out_of_credits) for hash= "
+ str(hash_id) +
". Is someone else using the same public key?")
request_successful = True
if (av_result_output.get('status') == "added"):
response["vt_av_added"].append(hash_id)
response["public_credits_spent"] += 1
elif (av_result_output.get('status') == "already_had_it"):
response["vt_av_already_downloaded"].append(hash_id)
elif (av_result_output.get('status') == 'error'):
response = add_error(
response, 9, "Error in av_result: " +
str(av_result_output.get('error_message')))
elif (av_result_output.get('status') == 'not_found'):
response["not_found_on_vt_av"].append(hash_id)
else:
logging.error("task_id=" + str(task_id))
logging.error("unknown error in av_result: " +
str(hash_id) + " ; " + str(av_result_output))
response = add_error(response, 12,
"Unknown error in av_result()")
save(response)
if (bool(email)):
send_mail(email, "task done", str(response))
response["date_end"] = datetime.datetime.now()
save(response)
return response
def generic_task(task_id):
response = load_task(task_id)
if response.get('date_end') is not None:
logging.error(
"Task already done. why was this on the queue? task_id=" + str(task_id))
return response
process = response['requested']['process']
file_hash = response['requested']['file_hash']
vt_av = response['requested']['vt_av']
vt_samples = response['requested']['vt_samples']
email = response['requested']['email']
document_name = response['requested'].get('document_name', '')
ip = response['requested']['ip']
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
logging.info("task_id=" + str(task_id))
logging.info("response['requested']=" + str(response['requested']))
generic_count = 0
response = {}
response["date_start"] = datetime.datetime.now()
response["document_name"] = document_name
response["task_id"] = task_id
response["ip"] = ip
check_hashes_output = check_hashes(file_hash)
errors = check_hashes_output.get('errors')
for error in errors:
key = error.get('error')
value = error.get('error_message')
logging.error("errors (key=" + str(key) +
", value=" + str(value) + ")")
response = add_error(response, key, value)
hashes = check_hashes_output.get('hashes')
remove_dups_output = remove_dups(hashes)
# remove duplicated hashes
hashes = remove_dups_output.get('list')
response["duplicated_hashes"] = remove_dups_output.get('dups')
response["hashes"] = hashes
hash_dicts = []
mc = MetaController()
for x in hashes:
x_dict = {}
x_dict["original"] = x
x_dict["sha1"] = get_file_id(x)
if(x_dict["sha1"] is not None):
doc = mc.read(x_dict["sha1"])
if doc is not None and doc.get('hash') is not None:
if doc.get('hash').get('md5') is not None:
x_dict["md5"] = doc.get('hash').get('md5')
if doc.get('hash').get('sha2') is not None:
x_dict["sha2"] = doc.get('hash').get('sha2')
hash_dicts.append(x_dict)
response["duplicated_samples"] = []
for x in hash_dicts:
for y in hash_dicts:
if x.get('original') != y.get('original') and (
x.get('original') == y.get('sha1') or
x.get('original') == y.get('md5') or
x.get('original') == y.get('sha2')):
response["duplicated_samples"].append(y.get('original'))
hash_dicts.remove(y)
hashes = []
for x in hash_dicts:
hashes.append(x.get('original'))
response["hashes"] = hashes
if(len(hashes) == 0):
response = add_error(response, 6, "No valid hashes provided.")
response["date_end"] = datetime.datetime.now()
save(response)
return change_date_to_str(response)
save(response)
response["private_credits_spent"] = 0
response["inconsistencies"] = []
if(vt_samples or process):
for hash_id in hashes:
inconsistency_output = fix_inconsistency(hash_id)
if inconsistency_output.get('inconsistency'):
response["inconsistencies"].append(hash_id)
if inconsistency_output.get('credit_spent'):
response["private_credits_spent"] += 1
save(response)
response["not_found_on_vt"] = []
if vt_samples:
response["downloaded"] = []
for hash_id in hashes:
if(get_file_id(hash_id) is None or db_inconsistency(hash_id)):
logging.debug("task(): " + hash_id +
" was not found (get_file_id returned None). ")
generic_count += 1
if (generic_count % 20 == 0):
save(response)
output = save_file_from_vt(hash_id)
sha1 = output.get('hash')
if(output.get('status') == 'out_of_credits'):
request_successful = False
while not request_successful:
output = save_file_from_vt(hash_id)
if output.get('status') != 'out_of_credits':
request_successful = True
if(output.get('status') == 'added'):
response["downloaded"].append(hash_id)
# we need to process the sha1, and not the sha2 because
# the grid does not save the sha2.
generic_process_hash(sha1)
response["private_credits_spent"] += 1
elif(output.get('status') == 'inconsistency_found'):
response["private_credits_spent"] += 1
generic_process_hash(sha1)
elif(output.get('status') == 'not_found'):
response["not_found_on_vt"].append(hash_id)
else:
logging.error("task_id=" + str(task_id))
logging.error(str(output))
response = add_error(
response, 11, "Unknown error when downloading sample from VT.")
save(response)
save(response)
response["processed"] = []
response["not_found_for_processing"] = []
if process:
logging.debug("process=true")
for hash_id in hashes:
logging.debug("task: hash_id=" + str(hash_id))
process_start_time = datetime.datetime.now()
generic_count += 1
if (generic_count % 20 == 0):
save(response)
if(generic_process_hash(hash_id) == 0):
process_end_time = datetime.datetime.now()
response["processed"].append({"hash": hash_id,
"seconds": (process_end_time - process_start_time).seconds})
else:
response["not_found_for_processing"].append(hash_id)
save(response)
if vt_av:
response["vt_av_added"] = []
response["vt_av_out_of_credits"] = []
response["not_found_on_vt_av"] = []
response["vt_av_already_downloaded"] = []
response["public_credits_spent"] = 0
for hash_id in hashes:
sha1 = get_file_id(hash_id)
if(sha1 is not None):
av_result_output = get_av_result(sha1)
if (av_result_output.get('status') == 'out_of_credits'):
request_successful = False
count = 0
while not request_successful:
av_result_output = get_av_result(sha1)
count += 1
if av_result_output.get('status') != 'out_of_credits':
response["vt_av_out_of_credits"].append(hash_id)
response = add_error(response, 10, "Had to retried " + str(count) + " times in av_result(out_of_credits) for hash= " + str(
hash_id) + ". Is someone else using the same public key?")
request_successful = True
if(av_result_output.get('status') == "added"):
response["vt_av_added"].append(hash_id)
response["public_credits_spent"] += 1
elif(av_result_output.get('status') == "already_had_it"):
response["vt_av_already_downloaded"].append(hash_id)
elif(av_result_output.get('status') == 'error'):
response = add_error(
response, 9, "Error in av_result: " + str(av_result_output.get('error_message')))
elif(av_result_output.get('status') == 'not_found'):
response["not_found_on_vt_av"].append(hash_id)
else:
logging.error("task_id=" + str(task_id))
logging.error("unknown error in av_result: " +
str(hash_id) + " ; " + str(av_result_output))
response = add_error(
response, 12, "Unknown error in av_result()")
save(response)
if(bool(email)):
send_mail(email, "task done", str(response))
response["date_end"] = datetime.datetime.now()
save(response)
return response