def db_inconsistency(file_hash):
if (not valid_hash(file_hash)):
raise ValueError("db_inconsistency invalid hash")
pc = PackageController()
v = VersionController()
file_id = get_file_id(file_hash)
if file_id is not None: #meta exists
file_bin = pc.getFile(file_id)
if file_bin is not None: #sample exists
version = v.searchVersion(file_id)
if version is not None:
return 0 # ok
else: #version does not exist
logging.info(
"inconsistency: meta and sample exists. Version does not")
return 3
else: # has meta but not sample
logging.info("inconsistency: meta exists, sample does not")
return 2
else: # does not have meta
if len(file_hash) == 64:
return 0 # cant search in grid by sha256
if len(file_hash) == 40:
file_bin = pc.getFile(file_hash)
else: # md5
sha1 = pc.md5_to_sha1(file_hash)
if sha1 is None:
return 0 # does not have meta or sample
file_bin = pc.getFile(file_hash)
if file_bin is None:
return 0
else:
logging.info("inconsistency: does not have meta. has sample")
return 1
def db_inconsistency(file_hash):
if(not valid_hash(file_hash)):
raise ValueError("db_inconsistency invalid hash")
pc = PackageController()
v = VersionController()
file_id = get_file_id(file_hash)
if file_id is not None: # meta exists
file_bin = pc.getFile(file_id)
if file_bin is not None: # sample exists
version = v.searchVersion(file_id)
if version is not None:
return 0 # ok
else: # version does not exist
logging.info(
"inconsistency: meta and sample exists. Version does not")
return 3
else: # has meta but not sample
logging.info("inconsistency: meta exists, sample does not")
return 2
else: # does not have meta
if len(file_hash) == 64:
return 0 # cant search in grid by sha256
if len(file_hash) == 40:
file_bin = pc.getFile(file_hash)
else: # md5
sha1 = pc.md5_to_sha1(file_hash)
if sha1 is None:
return 0 # does not have meta or sample
file_bin = pc.getFile(file_hash)
if file_bin is None:
return 0
else:
logging.info("inconsistency: does not have meta. has sample")
return 1
def get_file():
tmp_folder = "/tmp/mass_download"
subprocess.call(["mkdir", "-p", tmp_folder])
file_hash = clean_hash(request.query.file_hash)
key = ''
if len(file_hash) == 40:
key = 'sha1'
else:
response.code = 400
return jsonize({'message': 'Invalid hash format (use sha1)'})
pc = PackageController()
res = pc.searchFile(file_hash)
if res == None:
response.code = 404
return jsonize({'message': 'File not found in the database'})
if res == 1:
response.code = 400
return jsonize({'message': 'File not available for downloading'})
res = pc.getFile(file_hash)
zip_name = os.path.join(tmp_folder, str(file_hash) + '.zip')
file_name = os.path.join(tmp_folder, str(file_hash) + '.codex')
fd = open(file_name, "wb")
fd.write(res)
fd.close()
subprocess.call(["zip", "-ju", "-P", "codex", zip_name, file_name])
return static_file(str(file_hash) + ".zip", root=tmp_folder, download=True)
def get_file():
tmp_folder="/tmp/mass_download"
subprocess.call(["mkdir","-p",tmp_folder])
file_hash = clean_hash(request.query.file_hash)
key = ''
if len(file_hash) == 40:
key = 'sha1'
else:
response.code = 400
return jsonize({'message':'Invalid hash format (use sha1)'})
pc=PackageController()
res=pc.searchFile(file_hash)
if res==None:
response.code = 404
return jsonize({'message':'File not found in the database'})
if res==1:
response.code = 400
return jsonize({'message':'File not available for downloading'})
res=pc.getFile(file_hash)
zip_name=os.path.join(tmp_folder,str(file_hash)+'.zip')
file_name=os.path.join(tmp_folder,str(file_hash)+'.codex')
fd=open(file_name,"wb")
fd.write(res)
fd.close()
subprocess.call(["zip","-ju","-P","codex",zip_name,file_name])
return static_file(str(file_hash)+".zip",root=tmp_folder,download=True)
def yara():
tmp_folder = "/tmp/yara_working_dir"
subprocess.call(["mkdir", "-p", tmp_folder])
hashes = request.forms.dict.get("file_hash[]")
if hashes is not None:
if len(hashes) == 1:
random_id = hashes[0]
else:
random_id = id_generator()
else:
return jsonize({'message': 'Error. no file selected'})
folder_path = os.path.join(tmp_folder, random_id)
subprocess.call(["mkdir", "-p", folder_path])
yara_output_file = os.path.join(tmp_folder, random_id + ".txt")
for file_hash in hashes:
key = ''
if len(file_hash) == 40:
key = 'sha1'
else:
response.code = 400
return jsonize({'message': 'Invalid hash format (use sha1)'})
pc = PackageController()
res = pc.searchFile(file_hash)
if res == None:
response.code = 404
return jsonize({'message': 'File not found in the database'
}) #needs a better fix
res = pc.getFile(file_hash)
file_name = os.path.join(folder_path, str(file_hash) + ".codex")
if not os.path.isfile(file_name):
fd = open(file_name, "wb")
fd.write(res)
fd.close()
yara_cli_output = call_with_output([
"python", env['yara-script2'], "--opcodes", "--excludegood",
"--nosimple", "-z", "5", "-m", folder_path, "-o", yara_output_file
])
#yara_cli_output = call_with_output(["python",env['yara-script1'],"-f","exe","-a","Codex Gigas","-r",yara_output_file, folder_path+"/"])
# yara_output_file += ".yar" # because the script yara-script2 is ugly and saves the file to x.yar.yar
if os.path.isfile(yara_output_file) is False:
fp = open(yara_output_file, 'w+')
fp.write(yara_cli_output)
fp.close()
yara_output_fp = open(yara_output_file, 'r')
output_cleaned = yara_output_fp.read().replace(
"[!] Rule Name Can Not Contain Spaces or Begin With A Non Alpha Character",
"")
output_cleaned = re.sub(
r"\[\+\] Generating Yara Rule \/tmp\/yara_working_dir\/[A-Z0-9]+\.txt from files located in: /tmp/yara_working_dir/[A-Z0-9]+/",
"", output_cleaned)
output_cleaned = re.sub(r"rule /tmp/yara_working_dir/([a-zA-Z0-9]+).txt",
r"rule \1", output_cleaned)
# lines = [line for line in output_with_credits_removed if line.strip()]
return jsonize({"message": output_cleaned})
def yara():
tmp_folder = "/tmp/yara_working_dir"
subprocess.call(["mkdir", "-p", tmp_folder])
hashes = request.forms.dict.get("file_hash[]")
if hashes is not None:
if len(hashes) == 1:
random_id = hashes[0]
else:
random_id = id_generator()
else:
return jsonize({'message': 'Error. no file selected'})
folder_path = os.path.join(tmp_folder, random_id)
subprocess.call(["mkdir", "-p", folder_path])
yara_output_file = os.path.join(tmp_folder, random_id + ".txt")
for file_hash in hashes:
key = ''
if len(file_hash) == 40:
key = 'sha1'
else:
response.status = 400
return jsonize({'message': 'Invalid hash format (use sha1)'})
pc = PackageController()
res = pc.searchFile(file_hash)
if res is None:
response.status = 404
# needs a better fix
return jsonize({'message': 'File not found in the database'})
res = pc.getFile(file_hash)
file_name = os.path.join(folder_path, str(file_hash) + ".codex")
if not os.path.isfile(file_name):
fd = open(file_name, "wb")
fd.write(res)
fd.close()
yara_cli_output = call_with_output(["python", envget(
'yara-script2'), "--opcodes", "--excludegood", "--nosimple", "-z", "5", "-m", folder_path, "-o", yara_output_file])
# yara_cli_output = call_with_output(["python",envget('yara-script1'),"-f","exe","-a","Codex Gigas","-r",yara_output_file, folder_path+"/"])
# yara_output_file += ".yar" # because the script yara-script2 is ugly and
# saves the file to x.yar.yar
if os.path.isfile(yara_output_file) is False:
fp = open(yara_output_file, 'w+')
fp.write(yara_cli_output)
fp.close()
yara_output_fp = open(yara_output_file, 'r')
output_cleaned = yara_output_fp.read().replace(
"[!] Rule Name Can Not Contain Spaces or Begin With A Non Alpha Character", "")
output_cleaned = re.sub(
r"\[\+\] Generating Yara Rule \/tmp\/yara_working_dir\/[A-Z0-9]+\.txt from files located in: /tmp/yara_working_dir/[A-Z0-9]+/", "", output_cleaned)
output_cleaned = re.sub(
r"rule /tmp/yara_working_dir/([a-zA-Z0-9]+).txt", r"rule \1", output_cleaned)
# lines = [line for line in output_with_credits_removed if line.strip()]
return jsonize({"message": output_cleaned})
def get_package_file():
tmp_folder = "/tmp/mass_download"
subprocess.call(["mkdir", "-p", tmp_folder])
hashes = request.forms.dict.get("file_hash[]")
if hashes is None:
hashes = request.forms.get("file_hash").split("\n")
if hashes is not None:
if len(hashes) == 1:
random_id = hashes[0]
else:
random_id = id_generator()
else:
return jsonize({'message': 'Error. no file selected'})
folder_path = os.path.join(tmp_folder, random_id)
subprocess.call(["mkdir", "-p", folder_path])
zip_name = os.path.join(tmp_folder, random_id + ".zip")
pc = PackageController()
for file_hash in hashes:
file_hash = clean_hash(file_hash.replace('\r', ''))
data = "1=" + file_hash
res = SearchModule.search_by_id(data, 1)
if (len(res) == 0):
pass
else:
file_hash = res[0]["sha1"]
res = pc.searchFile(file_hash)
if res != 1 and res is not None:
res = pc.getFile(file_hash)
file_name = os.path.join(folder_path, str(file_hash) + ".codex")
fd = open(file_name, "wb")
fd.write(res)
fd.close()
elif res == 1:
fd = open(os.path.join(folder_path, 'readme.txt'), 'a+')
fd.write(str(file_hash) + " is not available to download.\n")
fd.close()
elif res is None:
fd = open(os.path.join(folder_path, 'readme.txt'), 'a+')
fd.write(str(file_hash) + " not found.")
fd.close()
else:
print "Unknown res:" + str(res)
subprocess.call(["zip", "-P", "codex", "-jr", zip_name, folder_path])
resp = static_file(str(random_id) + ".zip", root=tmp_folder, download=True)
resp.set_cookie('fileDownload', 'true')
# http://johnculviner.com/jquery-file-download-plugin-for-ajax-like-feature-rich-file-downloads/
return resp
def get_package_file():
tmp_folder="/tmp/mass_download"
subprocess.call(["mkdir","-p",tmp_folder])
hashes = request.forms.dict.get("file_hash[]")
if hashes is None:
hashes = request.forms.get("file_hash").split("\n")
if hashes is not None:
if len(hashes) == 1:
random_id=hashes[0]
else:
random_id = id_generator()
else:
return jsonize({'message':'Error. no file selected'})
folder_path=os.path.join(tmp_folder,random_id)
subprocess.call(["mkdir","-p",folder_path])
zip_name=os.path.join(tmp_folder,random_id+".zip")
pc=PackageController()
for file_hash in hashes:
file_hash = clean_hash(file_hash.replace('\r',''))
data="1="+file_hash
res=SearchModule.search_by_id(data,1)
if(len(res)==0):
pass
else:
file_hash=res[0]["sha1"]
res=pc.searchFile(file_hash)
if res != 1 and res is not None:
res=pc.getFile(file_hash)
file_name=os.path.join(folder_path,str(file_hash)+".codex")
fd=open(file_name,"wb")
fd.write(res)
fd.close()
elif res == 1:
fd=open(os.path.join(folder_path,'readme.txt'),'a+')
fd.write(str(file_hash)+" is not available to download.\n")
fd.close()
elif res is None:
fd=open(os.path.join(folder_path,'readme.txt'),'a+')
fd.write(str(file_hash)+" not found.")
fd.close()
else:
print "Unknown res:"+str(res)
subprocess.call(["zip","-P","codex","-jr", zip_name,folder_path])
resp = static_file(str(random_id)+".zip",root=tmp_folder,download=True)
resp.set_cookie('fileDownload','true');
# http://johnculviner.com/jquery-file-download-plugin-for-ajax-like-feature-rich-file-downloads/
return resp
def getBinary(self):
if(self.binary != None):
return self.binary
if(not self.binary_try_to_load):
return None
self.binary_try_to_load=False
if(self.pc==None):
tmp_pc=PackageController() #we use a temporary PackageController so we don't leave a mongo cursor open.
self.binary=tmp_pc.getFile(self.sample_id)
return self.binary
self.binary=self.pc.getFile(self.sample_id)
return self.binary
def process_file(file_hash):
#print "process_file("+str(file_hash)+")"
pc = PackageController()
res = pc.getFile(file_hash)
if res == None: return None
sam_id = file_hash
sample = Sample()
sample.setID(sam_id)
sample.setBinary(res)
sample.setStorageVersion({})
lc = Launcher()
lc.launchAnalysisByID(sample)
return 0
def getBinary(self):
if (self.binary != None):
return self.binary
if (not self.binary_try_to_load):
return None
self.binary_try_to_load = False
if (self.pc == None):
tmp_pc = PackageController(
) #we use a temporary PackageController so we don't leave a mongo cursor open.
self.binary = tmp_pc.getFile(self.sample_id)
return self.binary
self.binary = self.pc.getFile(self.sample_id)
return self.binary
def process_file(file_hash):
#print "process_file("+str(file_hash)+")"
pc=PackageController()
res=pc.getFile(file_hash)
if res==None:return None
sam_id=file_hash
sample=Sample()
sample.setID(sam_id)
sample.setBinary(res)
sample.setStorageVersion({})
lc=Launcher()
lc.launchAnalysisByID(sample)
return 0
def process_file(file_hash, force=False):
if file_hash is None:
return None
print "process_file(" + str(file_hash) + ")"
pc = PackageController()
res = pc.getFile(file_hash)
if res == None:
return None
sam_id = file_hash
sample = Sample()
sample.setID(sam_id)
sample.setBinary(res)
if force:
sample.setStorageVersion({})
lc = Launcher()
lc.launchAnalysisByID(sample)
log_event("process", str(file_hash))
return 0
def process_file(file_hash, force=False):
if not is_sha1(file_hash):
raise ValueError("process_file only accepts sha1")
logging.debug("process_file(" + str(file_hash) + ")")
pc = PackageController()
res = pc.getFile(file_hash)
if res is None:
logging.warning(
"Error: process_file(" + str(file_hash) + "): pc.getFile returned None")
return None
sam_id = file_hash
sample = Sample()
sample.setID(sam_id)
sample.setBinary(res)
if force:
sample.setStorageVersion({})
lc = Launcher()
lc.launchAnalysisByID(sample)
log_event("process", str(file_hash))
return 0
def process_file(file_hash, force=False):
if not is_sha1(file_hash):
raise ValueError("process_file only accepts sha1")
logging.debug("process_file(" + str(file_hash) + ")")
pc = PackageController()
res = pc.getFile(file_hash)
if res is None:
logging.warning("Error: process_file(" + str(file_hash) +
"): pc.getFile returned None")
return None
sam_id = file_hash
sample = Sample()
sample.setID(sam_id)
sample.setBinary(res)
if force:
sample.setStorageVersion({})
lc = Launcher()
lc.launchAnalysisByID(sample)
log_event("process", str(file_hash))
return 0
def api_batch_process_debug_file():
yield "<html><body><pre>"
yield "Running Batch process\n"
file_hashes = request.forms.get('file_hash')
if file_hashes is None:
response.status = 422
logging.debug("api_batch_process_debug_file(): file_hash is missing")
yield "file_hash parameter is missing"
# transform file_hashes in a list of hashes.
not_found = []
added_to_queue = 0
downloaded_from_vt = 0
for hash_id in file_hashes.split("\n"):
hash_id = clean_hash(hash_id)
if hash_id is None:
continue
data = "1=" + hash_id
if(len(hash_id) == 40 or len(hash_id) == 32):
pc = PackageController()
res = pc.getFile(hash_id)
if res is not None and len(SearchModule.search_by_id(data, 1, [], False)) == 0:
logging.debug("Processing right now: " + str(hash_id))
process_file(hash_id)
if(envget('auto_get_av_result')):
add_task_to_download_av_result(hash_id)
continue
res = SearchModule.search_by_id(data, 1, [], False)
if(len(res) == 0):
legging.debug("process_debug(): metadata of " + str(hash_id) +
" was not found. We will look in Pc. hash length: " + str(len(hash_id)))
if(len(hash_id) == 40 or len(hash_id) == 32):
pc = PackageController()
res = pc.getFile(hash_id)
if res is not None:
logging.debug(
"process_debug(): hash was found (" + str(hash_id) + ")")
else:
logging.debug(
"process_debug(): hash was not found(" + str(hash_id) + ")")
logging.debug("process_debug():")
logging.debug("process_debug(): going to search " +
str(hash_id) + " in vt")
add_response = SearchModule.add_file_from_vt(hash_id)
sha1 = add_response.get('hash')
if(sha1 is None):
logging.debug("process_debug(): sha1 is None: " + str(hash_id))
not_found.append(hash_id)
continue
else:
downloaded_from_vt += 1
else:
sha1 = res[0]["sha1"]
added_to_queue += 1
add_hash_to_process_queue(sha1)
if(envget('auto_get_av_result')):
add_task_to_download_av_result(sha1)
yield str(sha1) + "\n"
responsex = str(added_to_queue) + " files added to the process queue.\n"
if(downloaded_from_vt > 0):
responsex += str(downloaded_from_vt) + " new hashes.\n"
if(len(not_found) != 0):
responsex += str(len(not_found)) + " hashes not found.\n"
responsex += "Not Found:\n"
for aux in not_found:
responsex = responsex + str(aux) + "\n"
yield responsex
yield "END"
def api_batch_process_debug_file():
yield "<html><body><pre>"
yield "Running Batch process\n"
file_hashes = request.forms.get('file_hash')
if file_hashes is None:
response.status = 422
logging.debug("api_batch_process_debug_file(): file_hash is missing")
yield "file_hash parameter is missing"
# transform file_hashes in a list of hashes.
not_found = []
added_to_queue = 0
downloaded_from_vt = 0
for hash_id in file_hashes.split("\n"):
hash_id = clean_hash(hash_id)
if hash_id is None:
continue
data = "1=" + hash_id
if (len(hash_id) == 40 or len(hash_id) == 32):
pc = PackageController()
res = pc.getFile(hash_id)
if res is not None and len(
SearchModule.search_by_id(data, 1, [], False)) == 0:
logging.debug("Processing right now: " + str(hash_id))
process_file(hash_id)
if (env['auto_get_av_result']):
add_task_to_download_av_result(hash_id)
continue
res = SearchModule.search_by_id(data, 1, [], False)
if (len(res) == 0):
legging.debug("process_debug(): metadata of " + str(hash_id) +
" was not found. We will look in Pc. hash length: " +
str(len(hash_id)))
if (len(hash_id) == 40 or len(hash_id) == 32):
pc = PackageController()
res = pc.getFile(hash_id)
if res is not None:
logging.debug("process_debug(): hash was found (" +
str(hash_id) + ")")
else:
logging.debug("process_debug(): hash was not found(" +
str(hash_id) + ")")
logging.debug("process_debug():")
logging.debug("process_debug(): going to search " + str(hash_id) +
" in vt")
add_response = SearchModule.add_file_from_vt(hash_id)
sha1 = add_response.get('hash')
if (sha1 == None):
logging.debug("process_debug(): sha1 is None: " + str(hash_id))
not_found.append(hash_id)
continue
else:
downloaded_from_vt += 1
else:
sha1 = res[0]["sha1"]
added_to_queue += 1
add_hash_to_process_queue(sha1)
if (env['auto_get_av_result']):
add_task_to_download_av_result(sha1)
yield str(sha1) + "\n"
responsex = str(added_to_queue) + " files added to the process queue.\n"
if (downloaded_from_vt > 0):
responsex += str(downloaded_from_vt) + " new hashes.\n"
if (len(not_found) != 0):
responsex += str(len(not_found)) + " hashes not found.\n"
responsex += "Not Found:\n"
for aux in not_found:
responsex = responsex + str(aux) + "\n"
yield responsex
yield "END"
