def export_metadata():
mdc=MetaController()
hashes = request.forms.dict.get("file_hash[]")
dump_to_save=""
for hash in hashes:
hash = clean_hash(hash.replace('\r',''))
res=mdc.read(hash)
dump=dumps(res,indent=4)
line="\n\n#### File:%s\n"%hash
dump_to_save=dump_to_save+line+dump
id_random=id_generator()
tmp_folder="/tmp/meta_export"
subprocess.call(["mkdir","-p",tmp_folder])
file_name=os.path.join(tmp_folder,str(id_random)+'.txt')
fd=open(file_name,"w")
fd.write(dump_to_save)
fd.close()
resp = static_file(str(id_random)+'.txt',root=tmp_folder,download=True)
resp.set_cookie('fileDownload','true');
return resp
def export_metadata():
mdc = MetaController()
hashes = request.forms.dict.get("file_hash[]")
dump_to_save = ""
for hash in hashes:
hash = clean_hash(hash.replace('\r', ''))
res = mdc.read(hash)
dump = dumps(res, indent=4)
line = "\n\n#### File:%s\n" % hash
dump_to_save = dump_to_save + line + dump
id_random = id_generator()
tmp_folder = "/tmp/meta_export"
subprocess.call(["mkdir", "-p", tmp_folder])
file_name = os.path.join(tmp_folder, str(id_random) + '.txt')
fd = open(file_name, "w")
fd.write(dump_to_save)
fd.close()
resp = static_file(str(id_random) + '.txt', root=tmp_folder, download=True)
resp.set_cookie('fileDownload', 'true')
return resp
def get_metadata():
if request.query.file_hash == '':
response.status = 400
return jsonize({'message': 'file_hash parameter is missing'})
file_hash = clean_hash(request.query.file_hash)
if not valid_hash(file_hash):
response.status = 400
return jsonize({'message': 'Invalid hash format (use MD5, SHA1 or SHA2)'})
file_hash = get_file_id(file_hash)
if file_hash is None:
response.status = 404
return jsonize({'message': 'Metadata not found in the database'})
mdc = MetaController()
res = mdc.read(file_hash)
if res is None:
log_event("metadata", file_hash)
return dumps(change_date_to_str(res))
def get_metadata():
file_hash=clean_hash(request.query.file_hash)
if file_hash is None:
return
if len(file_hash) == 32: #ToDo: validate hash
key = 'md5'
elif len(file_hash) == 40:
key = 'sha1'
else:
response.code = 400
return jsonize({'message':'Invalid hash format (use MD5, SHA1 or SHA2)'})
mdc=MetaController()
res=mdc.read(file_hash)
if res==None:
response.code = 404
return jsonize({'message':'Metadata not found in the database'})
log_event("metadata",file_hash)
return dumps(change_date_to_str(res))
def get_metadata():
file_hash=clean_hash(request.query.file_hash)
if file_hash is None:
return
if len(file_hash) == 32: #ToDo: validate hash
key = 'md5'
elif len(file_hash) == 40:
key = 'sha1'
#elif len(file_hash) == 64:
# key = 'sha256'
else:
response.code = 400
return jsonize({'message':'Invalid hash format (use MD5, SHA1 or SHA2)'})
mdc=MetaController()
res=mdc.read(file_hash)
if res==None:
response.code = 404
return jsonize({'message':'Metadata not found in the database'})
log_event("metadata",file_hash)
return dumps(res)
def export_metadata():
mdc = MetaController()
hashes = request.forms.dict.get("file_hash[]")
dump_to_save = ""
random_id = id_generator()
tmp_path = "/tmp/meta_export"
tmp_folder = os.path.join(tmp_path, random_id)
call_with_output(["mkdir", "-p", tmp_folder])
for hash in hashes:
hash = clean_hash(hash.replace('\r', ''))
res = mdc.read(hash)
dump = dumps(res, indent=4)
file_name = os.path.join(tmp_folder, str(hash) + '.txt')
fd = open(file_name, "w")
fd.write(dump)
fd.close()
zip_path = os.path.join(tmp_path, random_id + '.zip')
call_with_output(["zip", "-jr", zip_path, tmp_folder])
resp = static_file(str(random_id) + '.zip', root=tmp_path, download=True)
resp.set_cookie('fileDownload', 'true')
shutil.rmtree(tmp_folder)
os.remove(zip_path)
return resp
def export_metadata():
mdc = MetaController()
hashes = request.forms.dict.get("file_hash[]")
dump_to_save = ""
random_id = id_generator()
tmp_path = "/tmp/meta_export"
tmp_folder = os.path.join(tmp_path, random_id)
call_with_output(["mkdir", "-p", tmp_folder])
for hash in hashes:
hash = clean_hash(hash.replace('\r', ''))
res = mdc.read(hash)
dump = dumps(res, indent=4)
file_name = os.path.join(tmp_folder, str(hash) + '.txt')
fd = open(file_name, "w")
fd.write(dump)
fd.close()
zip_path = os.path.join(tmp_path, random_id + '.zip')
call_with_output(["zip", "-jr", zip_path, tmp_folder])
resp = static_file(str(random_id) + '.zip', root=tmp_path, download=True)
resp.set_cookie('fileDownload', 'true')
shutil.rmtree(tmp_folder)
os.remove(zip_path)
return resp
def generic_task(process,
file_hash,
vt_av,
vt_samples,
email,
task_id,
document_name=""):
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
logging.info("task_id=" + str(task_id))
generic_count = 0
response = {}
response["date_start"] = datetime.datetime.now()
response["document_name"] = document_name
response["task_id"] = task_id
check_hashes_output = check_hashes(file_hash)
errors = check_hashes_output.get('errors')
for error in errors:
key = error.get('error')
value = error.get('error_message')
logging.error("errors (key=" + str(key) + ", value=" + str(value) +
")")
response = add_error(response, key, value)
hashes = check_hashes_output.get('hashes')
remove_dups_output = remove_dups(hashes)
# remove duplicated hashes
hashes = remove_dups_output.get('list')
response["duplicated_hashes"] = remove_dups_output.get('dups')
response["hashes"] = hashes
hash_dicts = []
mc = MetaController()
for x in hashes:
x_dict = {}
x_dict["original"] = x
x_dict["sha1"] = get_file_id(x)
if (x_dict["sha1"] is not None):
doc = mc.read(x_dict["sha1"])
if doc is not None and doc.get('hash') is not None:
if doc.get('hash').get('md5') is not None:
x_dict["md5"] = doc.get('hash').get('md5')
if doc.get('hash').get('sha2') is not None:
x_dict["sha2"] = doc.get('hash').get('sha2')
hash_dicts.append(x_dict)
response["duplicated_samples"] = []
for x in hash_dicts:
for y in hash_dicts:
if x.get('original') != y.get('original') and (
x.get('original') == y.get('sha1') or x.get('original')
== y.get('md5') or x.get('original') == y.get('sha2')):
response["duplicated_samples"].append(y.get('original'))
hash_dicts.remove(y)
hashes = []
for x in hash_dicts:
hashes.append(x.get('original'))
response["hashes"] = hashes
if (len(hashes) == 0):
response = add_error(response, 6, "No valid hashes provided.")
response["date_end"] = datetime.datetime.now()
save(response)
return change_date_to_str(response)
save(response)
response["inconsistencies"] = []
if (vt_samples or process):
for hash_id in hashes:
if fix_inconsistency(hash_id) == 1:
response["inconsistencies"].append(hash_id)
save(response)
response["not_found_on_vt"] = []
response["private_credits_spent"] = 0
if vt_samples:
response["downloaded"] = []
for hash_id in hashes:
if (get_file_id(hash_id) is None or db_inconsistency(hash_id)):
logging.debug("task(): " + hash_id +
" was not found (get_file_id returned None). ")
generic_count += 1
if (generic_count % 20 == 0):
save(response)
output = save_file_from_vt(hash_id)
sha1 = output.get('hash')
if (output.get('status') == 'out_of_credits'):
request_successful = False
while not request_successful:
output = save_file_from_vt(hash_id)
if output.get('status') != 'out_of_credits':
request_successful = True
if (output.get('status') == 'added'):
response["downloaded"].append(hash_id)
# we need to process the sha1, and not the sha2 because
# the grid does not save the sha2.
generic_process_hash(sha1)
response["private_credits_spent"] += 1
elif (output.get('status') == 'inconsistency_found'):
response["private_credits_spent"] += 1
generic_process_hash(sha1)
elif (output.get('status') == 'not_found'):
response["not_found_on_vt"].append(hash_id)
else:
logging.error("task_id=" + str(task_id))
logging.error(str(output))
response = add_error(
response, 11,
"Unknown error when downloading sample from VT.")
save(response)
save(response)
response["processed"] = []
response["not_found_for_processing"] = []
if process:
logging.debug("process=true")
for hash_id in hashes:
logging.debug("task: hash_id=" + str(hash_id))
process_start_time = datetime.datetime.now()
generic_count += 1
if (generic_count % 20 == 0):
save(response)
if (generic_process_hash(hash_id) == 0):
process_end_time = datetime.datetime.now()
response["processed"].append({
"hash":
hash_id,
"seconds": (process_end_time - process_start_time).seconds
})
else:
response["not_found_for_processing"].append(hash_id)
save(response)
if vt_av:
response["vt_av_added"] = []
response["vt_av_out_of_credits"] = []
response["not_found_on_vt_av"] = []
response["vt_av_already_downloaded"] = []
response["public_credits_spent"] = 0
for hash_id in hashes:
sha1 = get_file_id(hash_id)
if (sha1 is not None):
av_result_output = get_av_result(sha1)
if (av_result_output.get('status') == 'out_of_credits'):
request_successful = False
count = 0
while not request_successful:
av_result_output = get_av_result(sha1)
count += 1
if av_result_output.get('status') != 'out_of_credits':
response["vt_av_out_of_credits"].append(hash_id)
response = add_error(
response, 10, "Had to retried " + str(count) +
" times in av_result(out_of_credits) for hash= "
+ str(hash_id) +
". Is someone else using the same public key?")
request_successful = True
if (av_result_output.get('status') == "added"):
response["vt_av_added"].append(hash_id)
response["public_credits_spent"] += 1
elif (av_result_output.get('status') == "already_had_it"):
response["vt_av_already_downloaded"].append(hash_id)
elif (av_result_output.get('status') == 'error'):
response = add_error(
response, 9, "Error in av_result: " +
str(av_result_output.get('error_message')))
elif (av_result_output.get('status') == 'not_found'):
response["not_found_on_vt_av"].append(hash_id)
else:
logging.error("task_id=" + str(task_id))
logging.error("unknown error in av_result: " +
str(hash_id) + " ; " + str(av_result_output))
response = add_error(response, 12,
"Unknown error in av_result()")
save(response)
if (bool(email)):
send_mail(email, "task done", str(response))
response["date_end"] = datetime.datetime.now()
save(response)
return response
def generic_task(task_id):
response = load_task(task_id)
if response.get('date_end') is not None:
logging.error(
"Task already done. why was this on the queue? task_id=" + str(task_id))
return response
process = response['requested']['process']
file_hash = response['requested']['file_hash']
vt_av = response['requested']['vt_av']
vt_samples = response['requested']['vt_samples']
email = response['requested']['email']
document_name = response['requested'].get('document_name', '')
ip = response['requested']['ip']
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
logging.info("task_id=" + str(task_id))
logging.info("response['requested']=" + str(response['requested']))
generic_count = 0
response = {}
response["date_start"] = datetime.datetime.now()
response["document_name"] = document_name
response["task_id"] = task_id
response["ip"] = ip
check_hashes_output = check_hashes(file_hash)
errors = check_hashes_output.get('errors')
for error in errors:
key = error.get('error')
value = error.get('error_message')
logging.error("errors (key=" + str(key) +
", value=" + str(value) + ")")
response = add_error(response, key, value)
hashes = check_hashes_output.get('hashes')
remove_dups_output = remove_dups(hashes)
# remove duplicated hashes
hashes = remove_dups_output.get('list')
response["duplicated_hashes"] = remove_dups_output.get('dups')
response["hashes"] = hashes
hash_dicts = []
mc = MetaController()
for x in hashes:
x_dict = {}
x_dict["original"] = x
x_dict["sha1"] = get_file_id(x)
if(x_dict["sha1"] is not None):
doc = mc.read(x_dict["sha1"])
if doc is not None and doc.get('hash') is not None:
if doc.get('hash').get('md5') is not None:
x_dict["md5"] = doc.get('hash').get('md5')
if doc.get('hash').get('sha2') is not None:
x_dict["sha2"] = doc.get('hash').get('sha2')
hash_dicts.append(x_dict)
response["duplicated_samples"] = []
for x in hash_dicts:
for y in hash_dicts:
if x.get('original') != y.get('original') and (
x.get('original') == y.get('sha1') or
x.get('original') == y.get('md5') or
x.get('original') == y.get('sha2')):
response["duplicated_samples"].append(y.get('original'))
hash_dicts.remove(y)
hashes = []
for x in hash_dicts:
hashes.append(x.get('original'))
response["hashes"] = hashes
if(len(hashes) == 0):
response = add_error(response, 6, "No valid hashes provided.")
response["date_end"] = datetime.datetime.now()
save(response)
return change_date_to_str(response)
save(response)
response["private_credits_spent"] = 0
response["inconsistencies"] = []
if(vt_samples or process):
for hash_id in hashes:
inconsistency_output = fix_inconsistency(hash_id)
if inconsistency_output.get('inconsistency'):
response["inconsistencies"].append(hash_id)
if inconsistency_output.get('credit_spent'):
response["private_credits_spent"] += 1
save(response)
response["not_found_on_vt"] = []
if vt_samples:
response["downloaded"] = []
for hash_id in hashes:
if(get_file_id(hash_id) is None or db_inconsistency(hash_id)):
logging.debug("task(): " + hash_id +
" was not found (get_file_id returned None). ")
generic_count += 1
if (generic_count % 20 == 0):
save(response)
output = save_file_from_vt(hash_id)
sha1 = output.get('hash')
if(output.get('status') == 'out_of_credits'):
request_successful = False
while not request_successful:
output = save_file_from_vt(hash_id)
if output.get('status') != 'out_of_credits':
request_successful = True
if(output.get('status') == 'added'):
response["downloaded"].append(hash_id)
# we need to process the sha1, and not the sha2 because
# the grid does not save the sha2.
generic_process_hash(sha1)
response["private_credits_spent"] += 1
elif(output.get('status') == 'inconsistency_found'):
response["private_credits_spent"] += 1
generic_process_hash(sha1)
elif(output.get('status') == 'not_found'):
response["not_found_on_vt"].append(hash_id)
else:
logging.error("task_id=" + str(task_id))
logging.error(str(output))
response = add_error(
response, 11, "Unknown error when downloading sample from VT.")
save(response)
save(response)
response["processed"] = []
response["not_found_for_processing"] = []
if process:
logging.debug("process=true")
for hash_id in hashes:
logging.debug("task: hash_id=" + str(hash_id))
process_start_time = datetime.datetime.now()
generic_count += 1
if (generic_count % 20 == 0):
save(response)
if(generic_process_hash(hash_id) == 0):
process_end_time = datetime.datetime.now()
response["processed"].append({"hash": hash_id,
"seconds": (process_end_time - process_start_time).seconds})
else:
response["not_found_for_processing"].append(hash_id)
save(response)
if vt_av:
response["vt_av_added"] = []
response["vt_av_out_of_credits"] = []
response["not_found_on_vt_av"] = []
response["vt_av_already_downloaded"] = []
response["public_credits_spent"] = 0
for hash_id in hashes:
sha1 = get_file_id(hash_id)
if(sha1 is not None):
av_result_output = get_av_result(sha1)
if (av_result_output.get('status') == 'out_of_credits'):
request_successful = False
count = 0
while not request_successful:
av_result_output = get_av_result(sha1)
count += 1
if av_result_output.get('status') != 'out_of_credits':
response["vt_av_out_of_credits"].append(hash_id)
response = add_error(response, 10, "Had to retried " + str(count) + " times in av_result(out_of_credits) for hash= " + str(
hash_id) + ". Is someone else using the same public key?")
request_successful = True
if(av_result_output.get('status') == "added"):
response["vt_av_added"].append(hash_id)
response["public_credits_spent"] += 1
elif(av_result_output.get('status') == "already_had_it"):
response["vt_av_already_downloaded"].append(hash_id)
elif(av_result_output.get('status') == 'error'):
response = add_error(
response, 9, "Error in av_result: " + str(av_result_output.get('error_message')))
elif(av_result_output.get('status') == 'not_found'):
response["not_found_on_vt_av"].append(hash_id)
else:
logging.error("task_id=" + str(task_id))
logging.error("unknown error in av_result: " +
str(hash_id) + " ; " + str(av_result_output))
response = add_error(
response, 12, "Unknown error in av_result()")
save(response)
if(bool(email)):
send_mail(email, "task done", str(response))
response["date_end"] = datetime.datetime.now()
save(response)
return response