def add_file(): # tags = request.forms.get('name') upload = request.files.get('file') form_date = request.forms.get('file_date') try: # validate process_date(form_date) except ValueError: # response.status = 422 #status can't be added because angular will not # show the message. return jsonize({'message': 'Invalid date format'}) logging.debug("add_file(). date=" + str(form_date)) if form_date is None: form_date = datetime.datetime.now() name = upload.filename data_bin = upload.file.read() file_id = hashlib.sha1(data_bin).hexdigest() logging.debug("add_file(): file_id=" + str(file_id)) status = upload_file(data_bin) process_file(file_id) # ToDo: add a redis job update_date(file_id, form_date) if(status == "ok"): return jsonize({'message': 'Added with ' + str(file_id)}) elif(status == "already exists"): return jsonize({'message': 'Already exists ' + str(file_id)}) elif(status == "virustotal"): return jsonize({'message': 'Already exists ' + str(file_id)}) else: return jsonize({'message': 'Error'})
def get_file(): tmp_folder = "/tmp/mass_download" subprocess.call(["mkdir", "-p", tmp_folder]) file_hash = clean_hash(request.query.file_hash) key = '' if len(file_hash) == 40: key = 'sha1' else: response.status = 400 return jsonize({'message': 'Invalid hash format (use sha1)'}) pc = PackageController() res = pc.searchFile(file_hash) if res == None: response.status = 404 return jsonize({'message': 'File not found in the database'}) if res == 1: response.status = 400 return jsonize({'message': 'File not available for downloading'}) res = pc.getFile(file_hash) zip_name = os.path.join(tmp_folder, str(file_hash) + '.zip') file_name = os.path.join(tmp_folder, str(file_hash) + '.codex') fd = open(file_name, "wb") fd.write(res) fd.close() subprocess.call(["zip", "-ju", "-P", "codex", zip_name, file_name]) return static_file(str(file_hash) + ".zip", root=tmp_folder, download=True)
def add_file(): #tags = request.forms.get('name') upload = request.files.get('file') form_date = request.forms.get('file_date') try: # validate process_date(form_date) except ValueError: #response.status = 422 #status can't be added because angular will not show the message. return jsonize({'message': 'Invalid date format'}) logging.debug("add_file(). date=" + str(form_date)) if form_date is None: form_date = datetime.datetime.now() name = upload.filename data_bin = upload.file.read() file_id = hashlib.sha1(data_bin).hexdigest() logging.debug("add_file(): file_id=" + str(file_id)) status = upload_file(data_bin) process_file(file_id) #ToDo: add a redis job update_date(file_id, form_date) if (status == "ok"): return jsonize({'message': 'Added with ' + str(file_id)}) elif (status == "already exists"): return jsonize({'message': 'Already exists ' + str(file_id)}) elif (status == "virustotal"): return jsonize({'message': 'Already exists ' + str(file_id)}) else: return jsonize({'message': 'Error'})
def api_batch_process_file(): logging.debug("api_batch_process_file(): Running Batch process") file_hashes = request.forms.get('file_hash') # transform file_hashes in a list of hashes if file_hashes is None: return jsonize({"Error: file_hash parameter is missing."}) not_found = [] added_to_queue = 0 downloaded_from_vt = 0 for hash_id in file_hashes.split("\n"): hash_id = clean_hash(hash_id) if hash_id is None: continue data = "1=" + str(hash_id) res = SearchModule.search_by_id(data, 1, [], True) if (len(res) == 0): not_found.append(hash_id) continue else: sha1 = res[0]["sha1"] added_to_queue += 1 logging.debug(str(hash_id) + " added to queue") add_hash_to_process_queue(sha1) responsex = str(added_to_queue) + " files added to the process queue.\n" if (downloaded_from_vt > 0): responsex += str(downloaded_from_vt) + " new hashes.\n" if (len(not_found) != 0): responsex += str(len(not_found)) + " hashes not found.\n" responsex += "Not Found:\n" for aux in not_found: responsex = responsex + str(aux) + "\n" return jsonize({"message": responsex})
def api_batch_process_file(): logging.debug("api_batch_process_file(): Running Batch process") file_hashes = request.forms.get('file_hash') # transform file_hashes in a list of hashes if file_hashes is None: return jsonize({"Error: file_hash parameter is missing."}) not_found = [] added_to_queue = 0 downloaded_from_vt = 0 for hash_id in file_hashes.split("\n"): hash_id = clean_hash(hash_id) if hash_id is None: continue data = "1=" + str(hash_id) res = SearchModule.search_by_id(data, 1, [], True) if(len(res) == 0): not_found.append(hash_id) continue else: sha1 = res[0]["sha1"] added_to_queue += 1 logging.debug(str(hash_id) + " added to queue") add_hash_to_process_queue(sha1) responsex = str(added_to_queue) + " files added to the process queue.\n" if(downloaded_from_vt > 0): responsex += str(downloaded_from_vt) + " new hashes.\n" if(len(not_found) != 0): responsex += str(len(not_found)) + " hashes not found.\n" responsex += "Not Found:\n" for aux in not_found: responsex = responsex + str(aux) + "\n" return jsonize({"message": responsex})
def get_file(): tmp_folder = "/tmp/mass_download" subprocess.call(["mkdir", "-p", tmp_folder]) file_hash = clean_hash(request.query.file_hash) key = '' if len(file_hash) == 40: key = 'sha1' else: response.status = 400 return jsonize({'message': 'Invalid hash format (use sha1)'}) pc = PackageController() res = pc.searchFile(file_hash) if res is None: response.status = 404 return jsonize({'message': 'File not found in the database'}) if res == 1: response.status = 400 return jsonize({'message': 'File not available for downloading'}) res = pc.getFile(file_hash) zip_name = os.path.join(tmp_folder, str(file_hash) + '.zip') file_name = os.path.join(tmp_folder, str(file_hash) + '.codex') fd = open(file_name, "wb") fd.write(res) fd.close() subprocess.call(["zip", "-ju", "-P", "codex", zip_name, file_name]) return static_file(str(file_hash) + ".zip", root=tmp_folder, download=True)
def check_lib(): lib = str(request.query.q) mdc = MetaController() res = mdc.searchDllByName("'" + lib.lower() + "'") if (res != None): return jsonize({"valid": True}) else: return jsonize({"valid": False})
def check_imp(): imp = str(request.query.q) mdc = MetaController() res = mdc.searchImportByName("'" + imp.lower() + "'") if (res != None): return jsonize({"valid": True}) else: return jsonize({"valid": False})
def check_lib(): lib = str(request.query.q) mdc = MetaController() res = mdc.searchDllByName("'" + lib.lower() + "'") if(res is not None): return jsonize({"valid": True}) else: return jsonize({"valid": False})
def check_imp(): imp = str(request.query.q) mdc = MetaController() res = mdc.searchImportByName("'" + imp.lower() + "'") if(res is not None): return jsonize({"valid": True}) else: return jsonize({"valid": False})
def yara(): tmp_folder = "/tmp/yara_working_dir" subprocess.call(["mkdir", "-p", tmp_folder]) hashes = request.forms.dict.get("file_hash[]") if hashes is not None: if len(hashes) == 1: random_id = hashes[0] else: random_id = id_generator() else: return jsonize({'message': 'Error. no file selected'}) folder_path = os.path.join(tmp_folder, random_id) subprocess.call(["mkdir", "-p", folder_path]) yara_output_file = os.path.join(tmp_folder, random_id + ".txt") for file_hash in hashes: key = '' if len(file_hash) == 40: key = 'sha1' else: response.status = 400 return jsonize({'message': 'Invalid hash format (use sha1)'}) pc = PackageController() res = pc.searchFile(file_hash) if res == None: response.status = 404 return jsonize({'message': 'File not found in the database' }) #needs a better fix res = pc.getFile(file_hash) file_name = os.path.join(folder_path, str(file_hash) + ".codex") if not os.path.isfile(file_name): fd = open(file_name, "wb") fd.write(res) fd.close() yara_cli_output = call_with_output([ "python", env['yara-script2'], "--opcodes", "--excludegood", "--nosimple", "-z", "5", "-m", folder_path, "-o", yara_output_file ]) #yara_cli_output = call_with_output(["python",env['yara-script1'],"-f","exe","-a","Codex Gigas","-r",yara_output_file, folder_path+"/"]) # yara_output_file += ".yar" # because the script yara-script2 is ugly and saves the file to x.yar.yar if os.path.isfile(yara_output_file) is False: fp = open(yara_output_file, 'w+') fp.write(yara_cli_output) fp.close() yara_output_fp = open(yara_output_file, 'r') output_cleaned = yara_output_fp.read().replace( "[!] Rule Name Can Not Contain Spaces or Begin With A Non Alpha Character", "") output_cleaned = re.sub( r"\[\+\] Generating Yara Rule \/tmp\/yara_working_dir\/[A-Z0-9]+\.txt from files located in: /tmp/yara_working_dir/[A-Z0-9]+/", "", output_cleaned) output_cleaned = re.sub(r"rule /tmp/yara_working_dir/([a-zA-Z0-9]+).txt", r"rule \1", output_cleaned) # lines = [line for line in output_with_credits_removed if line.strip()] return jsonize({"message": output_cleaned})
def get_result_from_av(): file_hash = clean_hash(request.query.file_hash) if len(file_hash) != 40: response.code = 400 return jsonize({'message': 'Invalid hash format (use sha1)'}) av_result = get_av_result(file_hash) if (av_result == None): return jsonize("Can not get analysis") return jsonize("File processed")
def api_process_file(): file_hash = clean_hash(request.query.file_hash) if len(file_hash) != 40: response.status = 400 return jsonize({'message': 'Invalid hash format (use sha1)'}) res = process_file(file_hash, True) if res is None: response.status = 404 return jsonize("File not found in the database") return jsonize("File processed")
def api_process_file(): file_hash = clean_hash(request.query.file_hash) if len(file_hash) != 40: response.status = 400 return jsonize({'message': 'Invalid hash format (use sha1)'}) res = process_file(file_hash, True) if res == None: response.status = 404 return jsonize("File not found in the database") return jsonize("File processed")
def yara(): tmp_folder = "/tmp/yara_working_dir" subprocess.call(["mkdir", "-p", tmp_folder]) hashes = request.forms.dict.get("file_hash[]") if hashes is not None: if len(hashes) == 1: random_id = hashes[0] else: random_id = id_generator() else: return jsonize({'message': 'Error. no file selected'}) folder_path = os.path.join(tmp_folder, random_id) subprocess.call(["mkdir", "-p", folder_path]) yara_output_file = os.path.join(tmp_folder, random_id + ".txt") for file_hash in hashes: key = '' if len(file_hash) == 40: key = 'sha1' else: response.status = 400 return jsonize({'message': 'Invalid hash format (use sha1)'}) pc = PackageController() res = pc.searchFile(file_hash) if res is None: response.status = 404 # needs a better fix return jsonize({'message': 'File not found in the database'}) res = pc.getFile(file_hash) file_name = os.path.join(folder_path, str(file_hash) + ".codex") if not os.path.isfile(file_name): fd = open(file_name, "wb") fd.write(res) fd.close() yara_cli_output = call_with_output(["python", envget( 'yara-script2'), "--opcodes", "--excludegood", "--nosimple", "-z", "5", "-m", folder_path, "-o", yara_output_file]) # yara_cli_output = call_with_output(["python",envget('yara-script1'),"-f","exe","-a","Codex Gigas","-r",yara_output_file, folder_path+"/"]) # yara_output_file += ".yar" # because the script yara-script2 is ugly and # saves the file to x.yar.yar if os.path.isfile(yara_output_file) is False: fp = open(yara_output_file, 'w+') fp.write(yara_cli_output) fp.close() yara_output_fp = open(yara_output_file, 'r') output_cleaned = yara_output_fp.read().replace( "[!] Rule Name Can Not Contain Spaces or Begin With A Non Alpha Character", "") output_cleaned = re.sub( r"\[\+\] Generating Yara Rule \/tmp\/yara_working_dir\/[A-Z0-9]+\.txt from files located in: /tmp/yara_working_dir/[A-Z0-9]+/", "", output_cleaned) output_cleaned = re.sub( r"rule /tmp/yara_working_dir/([a-zA-Z0-9]+).txt", r"rule \1", output_cleaned) # lines = [line for line in output_with_credits_removed if line.strip()] return jsonize({"message": output_cleaned})
def last_uploaded(): number = request.query.get("n") if number is None: response.status = 400 return jsonize({"error": 1, "error_message": "Parameter n is missing"}) if number.isdigit() is False: response.status = 400 return jsonize({"error": 2, "error_message": "Parameter n must be a number"}) if int(number) == 0: return jsonize({"error": 3, "error_message": "Parameter n must be greater than zero."}) pc = PackageController() lasts = pc.last_updated(int(number)) for i in range(0, len(lasts)): # Convert datetime objects lasts[i] = change_date_to_str(lasts[i]) return jsonize(lasts)
def add_file(): #tags = request.forms.get('name') upload = request.files.get('file') name = upload.filename data_bin = upload.file.read() file_id = hashlib.sha1(data_bin).hexdigest() print "file_id=" + str(file_id) status = upload_file(data_bin) process_file(file_id) #ToDo: add a redis job if (status == "ok"): return jsonize({'message': 'Added with ' + str(file_id)}) elif (status == "already exists"): return jsonize({'message': 'Already exists ' + str(file_id)}) elif (status == "virustotal"): return jsonize({'message': 'Already exists ' + str(file_id)}) else: return jsonize({'message': 'Error'})
def get_metadata(): if request.query.file_hash == '': response.status = 400 return jsonize({'message': 'file_hash parameter is missing'}) file_hash = clean_hash(request.query.file_hash) if not valid_hash(file_hash): response.status = 400 return jsonize({'message': 'Invalid hash format (use MD5, SHA1 or SHA2)'}) file_hash = get_file_id(file_hash) if file_hash is None: response.status = 404 return jsonize({'message': 'Metadata not found in the database'}) mdc = MetaController() res = mdc.read(file_hash) if res is None: log_event("metadata", file_hash) return dumps(change_date_to_str(res))
def get_result_from_av(): hash_id = request.query.file_hash if len(hash_id) == 0: response.status = 400 return jsonize({ 'error': 4, 'error_message': 'file_hash parameter is missing.' }) hash_id = clean_hash(hash_id) if not valid_hash(hash_id): return jsonize({'error': 5, 'error_message': 'Invalid hash format.'}) if (len(hash_id) != 40): data = "1=" + str(hash_id) res = SearchModule.search_by_id(data, 1, [], True) if (len(res) == 0): response.status = 400 return jsonize({'error': 6, 'error_message': 'File not found'}) else: sha1 = res[0]["sha1"] else: sha1 = hash_id key_manager = KeyManager() if (key_manager.check_keys_in_secrets()): av_result = get_av_result(sha1, 'high') else: return jsonize({ 'error': 7, "error_message": "Error: VirusTotal API key missing from secrets.py file" }) if (av_result.get('status') == "added"): return jsonize({"message": "AV scans downloaded."}) elif (av_result.get('status') == "already_had_it"): return jsonize({"message": "File already have AV scans."}) elif (av_result.get('status') == "not_found"): return jsonize({"error": 10, "error_message": "Not found on VT."}) elif (av_result.get('status') == "no_key_available"): return jsonize({ "error": 11, "error_message": "No key available right now. Please try again later." }) else: logging.error("av_result for hash=" + str(sha1)) logging.error("av_result=" + str(av_result)) return jsonize({"error": 9, "error_message": "Cannot get analysis."})
def get_metadata(): file_hash=clean_hash(request.query.file_hash) if file_hash is None: return if len(file_hash) == 32: #ToDo: validate hash key = 'md5' elif len(file_hash) == 40: key = 'sha1' else: response.code = 400 return jsonize({'message':'Invalid hash format (use MD5, SHA1 or SHA2)'}) mdc=MetaController() res=mdc.read(file_hash) if res==None: response.code = 404 return jsonize({'message':'Metadata not found in the database'}) log_event("metadata",file_hash) return dumps(change_date_to_str(res))
def get_result_from_av(): hash_id=request.query.file_hash if len(hash_id) == 0: response.code = 400 return jsonize({'error': 4, 'error_message':'file_hash parameter is missing.'}) hash_id=clean_hash(hash_id) if not valid_hash(hash_id): return jsonize({'error': 5, 'error_message':'Invalid hash format.'}) if(len(hash_id)!=40): data="1="+str(hash_id) res=SearchModule.search_by_id(data,1,[],True) if(len(res)==0): response.code = 400 return jsonize({'error': 6, 'error_message':'File not found'}) else: sha1=res[0]["sha1"] else: sha1=hash_id if(vt_key()): av_result=get_av_result(sha1) else: return jsonize({'error': 7, "error_message": "Error: VirusTotal API key missing from secrets.py file"}) if(av_result==None): return jsonize({"error": 8, "error_message": "Cannot get analysis (hash not found in VT? out of credits?)"}) return jsonize({"message": "AV scans downloaded."})
def get_package_file(): tmp_folder = "/tmp/mass_download" subprocess.call(["mkdir", "-p", tmp_folder]) hashes = request.forms.dict.get("file_hash[]") if hashes is None: hashes = request.forms.get("file_hash").split("\n") if hashes is not None: if len(hashes) == 1: random_id = hashes[0] else: random_id = id_generator() else: return jsonize({'message': 'Error. no file selected'}) folder_path = os.path.join(tmp_folder, random_id) subprocess.call(["mkdir", "-p", folder_path]) zip_name = os.path.join(tmp_folder, random_id + ".zip") pc = PackageController() for file_hash in hashes: file_hash = clean_hash(file_hash.replace('\r', '')) data = "1=" + file_hash res = SearchModule.search_by_id(data, 1) if (len(res) == 0): pass else: file_hash = res[0]["sha1"] res = pc.searchFile(file_hash) if res != 1 and res is not None: res = pc.getFile(file_hash) file_name = os.path.join(folder_path, str(file_hash) + ".codex") fd = open(file_name, "wb") fd.write(res) fd.close() elif res == 1: fd = open(os.path.join(folder_path, 'readme.txt'), 'a+') fd.write(str(file_hash) + " is not available to download.\n") fd.close() elif res is None: fd = open(os.path.join(folder_path, 'readme.txt'), 'a+') fd.write(str(file_hash) + " not found.") fd.close() else: logging.error("get_package_file(). Unknown res:" + str(res)) subprocess.call(["zip", "-P", "codex", "-jr", zip_name, folder_path]) resp = static_file(str(random_id) + ".zip", root=tmp_folder, download=True) resp.set_cookie('fileDownload', 'true') # http://johnculviner.com/jquery-file-download-plugin-for-ajax-like-feature-rich-file-downloads/ return resp
def get_package_file(): tmp_folder = "/tmp/mass_download" subprocess.call(["mkdir", "-p", tmp_folder]) hashes = request.forms.dict.get("file_hash[]") if hashes is None: hashes = request.forms.get("file_hash").split("\n") if hashes is not None: if len(hashes) == 1: random_id = hashes[0] else: random_id = id_generator() else: return jsonize({'message': 'Error. no file selected'}) folder_path = os.path.join(tmp_folder, random_id) subprocess.call(["mkdir", "-p", folder_path]) zip_name = os.path.join(tmp_folder, random_id + ".zip") pc = PackageController() for file_hash in hashes: file_hash = clean_hash(file_hash.replace('\r', '')) data = "1=" + file_hash res = SearchModule.search_by_id(data, 1) if(len(res) == 0): pass else: file_hash = res[0]["sha1"] res = pc.searchFile(file_hash) if res != 1 and res is not None: res = pc.getFile(file_hash) file_name = os.path.join(folder_path, str(file_hash) + ".codex") fd = open(file_name, "wb") fd.write(res) fd.close() elif res == 1: fd = open(os.path.join(folder_path, 'readme.txt'), 'a+') fd.write(str(file_hash) + " is not available to download.\n") fd.close() elif res is None: fd = open(os.path.join(folder_path, 'readme.txt'), 'a+') fd.write(str(file_hash) + " not found.") fd.close() else: logging.error("get_package_file(). Unknown res:" + str(res)) subprocess.call(["zip", "-P", "codex", "-jr", zip_name, folder_path]) resp = static_file(str(random_id) + ".zip", root=tmp_folder, download=True) resp.set_cookie('fileDownload', 'true') # http://johnculviner.com/jquery-file-download-plugin-for-ajax-like-feature-rich-file-downloads/ return resp
def api_batch_process_file(): print("Running Batch process") file_hashes = request.forms.get('file_hash') #print(dir(request.forms)) #print(request.forms.keys()) #transformar file_hashes a una lista de hashes not_found = [] added_to_queue = 0 downloaded_from_vt = 0 for hash_id in file_hashes.split("\n"): hash_id = clean_hash(hash_id) if hash_id is None: continue data = "1=" + str(hash_id) res = SearchModule.search_by_id(data, 1, [], True) if (len(res) == 0): not_found.append(hash_id) continue """ print "downloading "+str(hash_id)+" from vt" sha1=SearchModule.add_file_from_vt(hash_id) if(sha1==None): print "not found on vt: "+str(hash_id) not_found.append(hash_id) continue else: downloaded_from_vt+=1 """ else: sha1 = res[0]["sha1"] added_to_queue += 1 print str(hash_id) + " added to queue" add_hash_to_process_queue(sha1) if (env['auto_get_av_result']): get_av_result(sha1) responsex = str(added_to_queue) + " files added to the process queue.\n" if (downloaded_from_vt > 0): responsex += str(downloaded_from_vt) + " new hashes.\n" if (len(not_found) != 0): responsex += str(len(not_found)) + " hashes not found.\n" responsex += "Not Found:\n" for aux in not_found: responsex = responsex + str(aux) + "\n" return jsonize({"message": responsex})
def get_result_from_av(): hash_id = request.query.file_hash if len(hash_id) == 0: response.status = 400 return jsonize({'error': 4, 'error_message': 'file_hash parameter is missing.'}) hash_id = clean_hash(hash_id) if not valid_hash(hash_id): return jsonize({'error': 5, 'error_message': 'Invalid hash format.'}) if(len(hash_id) != 40): data = "1=" + str(hash_id) res = SearchModule.search_by_id(data, 1, [], True) if(len(res) == 0): response.status = 400 return jsonize({'error': 6, 'error_message': 'File not found'}) else: sha1 = res[0]["sha1"] else: sha1 = hash_id key_manager = KeyManager() if(key_manager.check_keys_in_secrets()): av_result = get_av_result(sha1, 'high') else: return jsonize({'error': 7, "error_message": "Error: VirusTotal API key missing from secrets.py file"}) if(av_result.get('status') == "added"): return jsonize({"message": "AV scans downloaded."}) elif(av_result.get('status') == "already_had_it"): return jsonize({"message": "File already have AV scans."}) elif(av_result.get('status') == "not_found"): return jsonize({"error": 10, "error_message": "Not found on VT."}) elif(av_result.get('status') == "no_key_available"): return jsonize({"error": 11, "error_message": "No key available right now. Please try again later."}) else: logging.error("av_result for hash=" + str(sha1)) logging.error("av_result=" + str(av_result)) return jsonize({"error": 9, "error_message": "Cannot get analysis."})
def logs(): try: csvfile = open('logs.csv', 'r') except Exception, e: print str(e) return jsonize([])
def test(): enable_cors() return jsonize({'message': 'Server Runing'})
def search_tree(): return jsonize(tree_menu.tree)
def get_sample_count(): count = SearchModule.count_documents() res = {"count": count} return jsonize(res)
def logs(): try: csvfile = open('logs.csv', 'r') except Exception, e: logging.exception("logs() exception") return jsonize([])
def jsonp(data, callback): reply = {"status": "OK", "data": data} return callback + "([" + jsonize(reply) + "]);"
def cron(): key_manager = KeyManager() key_manager.reset_daily_counter() return jsonize({"status": "ok"})