def get_file(): tmp_folder = "/tmp/mass_download" subprocess.call(["mkdir", "-p", tmp_folder]) file_hash = clean_hash(request.query.file_hash) key = '' if len(file_hash) == 40: key = 'sha1' else: response.code = 400 return jsonize({'message': 'Invalid hash format (use sha1)'}) pc = PackageController() res = pc.searchFile(file_hash) if res == None: response.code = 404 return jsonize({'message': 'File not found in the database'}) if res == 1: response.code = 400 return jsonize({'message': 'File not available for downloading'}) res = pc.getFile(file_hash) zip_name = os.path.join(tmp_folder, str(file_hash) + '.zip') file_name = os.path.join(tmp_folder, str(file_hash) + '.codex') fd = open(file_name, "wb") fd.write(res) fd.close() subprocess.call(["zip", "-ju", "-P", "codex", zip_name, file_name]) return static_file(str(file_hash) + ".zip", root=tmp_folder, download=True)
def get_file(): tmp_folder="/tmp/mass_download" subprocess.call(["mkdir","-p",tmp_folder]) file_hash = clean_hash(request.query.file_hash) key = '' if len(file_hash) == 40: key = 'sha1' else: response.code = 400 return jsonize({'message':'Invalid hash format (use sha1)'}) pc=PackageController() res=pc.searchFile(file_hash) if res==None: response.code = 404 return jsonize({'message':'File not found in the database'}) if res==1: response.code = 400 return jsonize({'message':'File not available for downloading'}) res=pc.getFile(file_hash) zip_name=os.path.join(tmp_folder,str(file_hash)+'.zip') file_name=os.path.join(tmp_folder,str(file_hash)+'.codex') fd=open(file_name,"wb") fd.write(res) fd.close() subprocess.call(["zip","-ju","-P","codex",zip_name,file_name]) return static_file(str(file_hash)+".zip",root=tmp_folder,download=True)
def yara(): tmp_folder = "/tmp/yara_working_dir" subprocess.call(["mkdir", "-p", tmp_folder]) hashes = request.forms.dict.get("file_hash[]") if hashes is not None: if len(hashes) == 1: random_id = hashes[0] else: random_id = id_generator() else: return jsonize({'message': 'Error. no file selected'}) folder_path = os.path.join(tmp_folder, random_id) subprocess.call(["mkdir", "-p", folder_path]) yara_output_file = os.path.join(tmp_folder, random_id + ".txt") for file_hash in hashes: key = '' if len(file_hash) == 40: key = 'sha1' else: response.code = 400 return jsonize({'message': 'Invalid hash format (use sha1)'}) pc = PackageController() res = pc.searchFile(file_hash) if res == None: response.code = 404 return jsonize({'message': 'File not found in the database' }) #needs a better fix res = pc.getFile(file_hash) file_name = os.path.join(folder_path, str(file_hash) + ".codex") if not os.path.isfile(file_name): fd = open(file_name, "wb") fd.write(res) fd.close() yara_cli_output = call_with_output([ "python", env['yara-script2'], "--opcodes", "--excludegood", "--nosimple", "-z", "5", "-m", folder_path, "-o", yara_output_file ]) #yara_cli_output = call_with_output(["python",env['yara-script1'],"-f","exe","-a","Codex Gigas","-r",yara_output_file, folder_path+"/"]) # yara_output_file += ".yar" # because the script yara-script2 is ugly and saves the file to x.yar.yar if os.path.isfile(yara_output_file) is False: fp = open(yara_output_file, 'w+') fp.write(yara_cli_output) fp.close() yara_output_fp = open(yara_output_file, 'r') output_cleaned = yara_output_fp.read().replace( "[!] Rule Name Can Not Contain Spaces or Begin With A Non Alpha Character", "") output_cleaned = re.sub( r"\[\+\] Generating Yara Rule \/tmp\/yara_working_dir\/[A-Z0-9]+\.txt from files located in: /tmp/yara_working_dir/[A-Z0-9]+/", "", output_cleaned) output_cleaned = re.sub(r"rule /tmp/yara_working_dir/([a-zA-Z0-9]+).txt", r"rule \1", output_cleaned) # lines = [line for line in output_with_credits_removed if line.strip()] return jsonize({"message": output_cleaned})
def yara(): tmp_folder = "/tmp/yara_working_dir" subprocess.call(["mkdir", "-p", tmp_folder]) hashes = request.forms.dict.get("file_hash[]") if hashes is not None: if len(hashes) == 1: random_id = hashes[0] else: random_id = id_generator() else: return jsonize({'message': 'Error. no file selected'}) folder_path = os.path.join(tmp_folder, random_id) subprocess.call(["mkdir", "-p", folder_path]) yara_output_file = os.path.join(tmp_folder, random_id + ".txt") for file_hash in hashes: key = '' if len(file_hash) == 40: key = 'sha1' else: response.status = 400 return jsonize({'message': 'Invalid hash format (use sha1)'}) pc = PackageController() res = pc.searchFile(file_hash) if res is None: response.status = 404 # needs a better fix return jsonize({'message': 'File not found in the database'}) res = pc.getFile(file_hash) file_name = os.path.join(folder_path, str(file_hash) + ".codex") if not os.path.isfile(file_name): fd = open(file_name, "wb") fd.write(res) fd.close() yara_cli_output = call_with_output(["python", envget( 'yara-script2'), "--opcodes", "--excludegood", "--nosimple", "-z", "5", "-m", folder_path, "-o", yara_output_file]) # yara_cli_output = call_with_output(["python",envget('yara-script1'),"-f","exe","-a","Codex Gigas","-r",yara_output_file, folder_path+"/"]) # yara_output_file += ".yar" # because the script yara-script2 is ugly and # saves the file to x.yar.yar if os.path.isfile(yara_output_file) is False: fp = open(yara_output_file, 'w+') fp.write(yara_cli_output) fp.close() yara_output_fp = open(yara_output_file, 'r') output_cleaned = yara_output_fp.read().replace( "[!] Rule Name Can Not Contain Spaces or Begin With A Non Alpha Character", "") output_cleaned = re.sub( r"\[\+\] Generating Yara Rule \/tmp\/yara_working_dir\/[A-Z0-9]+\.txt from files located in: /tmp/yara_working_dir/[A-Z0-9]+/", "", output_cleaned) output_cleaned = re.sub( r"rule /tmp/yara_working_dir/([a-zA-Z0-9]+).txt", r"rule \1", output_cleaned) # lines = [line for line in output_with_credits_removed if line.strip()] return jsonize({"message": output_cleaned})
def get_package_file(): tmp_folder="/tmp/mass_download" subprocess.call(["mkdir","-p",tmp_folder]) hashes = request.forms.dict.get("file_hash[]") if hashes is None: hashes = request.forms.get("file_hash").split("\n") if hashes is not None: if len(hashes) == 1: random_id=hashes[0] else: random_id = id_generator() else: return jsonize({'message':'Error. no file selected'}) folder_path=os.path.join(tmp_folder,random_id) subprocess.call(["mkdir","-p",folder_path]) zip_name=os.path.join(tmp_folder,random_id+".zip") pc=PackageController() for file_hash in hashes: file_hash = clean_hash(file_hash.replace('\r','')) data="1="+file_hash res=SearchModule.search_by_id(data,1) if(len(res)==0): pass else: file_hash=res[0]["sha1"] res=pc.searchFile(file_hash) if res != 1 and res is not None: res=pc.getFile(file_hash) file_name=os.path.join(folder_path,str(file_hash)+".codex") fd=open(file_name,"wb") fd.write(res) fd.close() elif res == 1: fd=open(os.path.join(folder_path,'readme.txt'),'a+') fd.write(str(file_hash)+" is not available to download.\n") fd.close() elif res is None: fd=open(os.path.join(folder_path,'readme.txt'),'a+') fd.write(str(file_hash)+" not found.") fd.close() else: print "Unknown res:"+str(res) subprocess.call(["zip","-P","codex","-jr", zip_name,folder_path]) resp = static_file(str(random_id)+".zip",root=tmp_folder,download=True) resp.set_cookie('fileDownload','true'); # http://johnculviner.com/jquery-file-download-plugin-for-ajax-like-feature-rich-file-downloads/ return resp
def get_package_file(): tmp_folder = "/tmp/mass_download" subprocess.call(["mkdir", "-p", tmp_folder]) hashes = request.forms.dict.get("file_hash[]") if hashes is None: hashes = request.forms.get("file_hash").split("\n") if hashes is not None: if len(hashes) == 1: random_id = hashes[0] else: random_id = id_generator() else: return jsonize({'message': 'Error. no file selected'}) folder_path = os.path.join(tmp_folder, random_id) subprocess.call(["mkdir", "-p", folder_path]) zip_name = os.path.join(tmp_folder, random_id + ".zip") pc = PackageController() for file_hash in hashes: file_hash = clean_hash(file_hash.replace('\r', '')) data = "1=" + file_hash res = SearchModule.search_by_id(data, 1) if (len(res) == 0): pass else: file_hash = res[0]["sha1"] res = pc.searchFile(file_hash) if res != 1 and res is not None: res = pc.getFile(file_hash) file_name = os.path.join(folder_path, str(file_hash) + ".codex") fd = open(file_name, "wb") fd.write(res) fd.close() elif res == 1: fd = open(os.path.join(folder_path, 'readme.txt'), 'a+') fd.write(str(file_hash) + " is not available to download.\n") fd.close() elif res is None: fd = open(os.path.join(folder_path, 'readme.txt'), 'a+') fd.write(str(file_hash) + " not found.") fd.close() else: print "Unknown res:" + str(res) subprocess.call(["zip", "-P", "codex", "-jr", zip_name, folder_path]) resp = static_file(str(random_id) + ".zip", root=tmp_folder, download=True) resp.set_cookie('fileDownload', 'true') # http://johnculviner.com/jquery-file-download-plugin-for-ajax-like-feature-rich-file-downloads/ return resp
def save_file_from_vt(hash_id): downloaded_file = download_from_virus_total(hash_id) if (downloaded_file == None): return None data_bin = downloaded_file file_id = hashlib.sha1(data_bin).hexdigest() # print "file_id="+str(file_id) pc = PackageController() res = pc.searchFile(file_id) if (res == None): # File not found. Add it to the package. pc.append(file_id, data_bin, True) print("Added: %s" % (file_id, )) return file_id
def save_file_from_vt(hash_id): downloaded_file=download_from_virus_total(hash_id) if(downloaded_file==None): return None data_bin=downloaded_file file_id=hashlib.sha1(data_bin).hexdigest() # print "file_id="+str(file_id) pc=PackageController() res=pc.searchFile(file_id) if(res==None): # File not found. Add it to the package. pc.append(file_id,data_bin,True) print("Added: %s" % (file_id,)) return file_id
def add_file_from_vt(hash_id): #return None # FUNCION DESABILITADA - SACAR LA LINEA PARA PONER downloaded_file = download_from_virus_total(hash_id) if (downloaded_file == None): return None data_bin = downloaded_file file_id = hashlib.sha1(data_bin).hexdigest() #print "file_id="+str(file_id) pc = PackageController() res = pc.searchFile(file_id) if (res == None): # File not found. Add it to the package. pc.append(file_id, data_bin, True) #print("Added: %s" % (file_id,)) return file_id
def add_file_from_vt(hash_id): #return None # FUNCION DESABILITADA - SACAR LA LINEA PARA PONER downloaded_file=download_from_virus_total(hash_id) if(downloaded_file==None): return None data_bin=downloaded_file file_id=hashlib.sha1(data_bin).hexdigest() #print "file_id="+str(file_id) pc=PackageController() res=pc.searchFile(file_id) if(res==None): # File not found. Add it to the package. pc.append(file_id,data_bin,True) #print("Added: %s" % (file_id,)) return file_id
def load_to_mongo2(folder_path): pc=PackageController() ram = Ram() files=recursive_read(folder_path) count=0 reset=0 already_loaded=0 time_start = datetime.datetime.now() uploaded=0 in_mem=0 loaded_ram_counter=0 lc=Launcher() if(files is None): return "No files where found." while (uploaded < len(files)): loaded_ram_counter=0 data_vector=[] print "loading files to memory" while (in_mem < len(files)): f=files[in_mem] file_cursor=open(f,"r") data_vector.append(file_cursor.read()) in_mem=in_mem+1 loaded_ram_counter=loaded_ram_counter+1 if(loaded_ram_counter > 100): if(ram.free_percent() < 0.3): print "Ram full" break for data in data_vector: file_id=hashlib.sha1(data).hexdigest() print "loading to db: "+str(file_id) res=pc.searchFile(file_id) if(res==None): pc.append(file_id,data) sample=Sample() sample.setID(file_id) sample.setBinary(data) sample.setStorageVersion({}) count+=1 lc.launchAnalysisByID(sample) else: already_loaded+=1 uploaded=uploaded+1 result=str(already_loaded)+" were already loaded to mongo.\n" result+=thetime(time_start,datetime.datetime.now(),count) print result return result
def upload_file(data_bin): pc=PackageController() file_id=hashlib.sha1(data_bin).hexdigest() res=pc.searchFile(file_id) if(res==None): # File not found. Add it to the package. pc.append(file_id,data_bin) print("Added: %s" % (file_id,)) log_event("file added",str(file_id)) return "ok" else: if(res==0):#file already exists log_event("file already exists",str(file_id)) return "already exists" else:#existe y esta bloqueado por vt log_event("file already exists",str(file_id)) return "virustotal"
def upload_file(data_bin): pc = PackageController() file_id = hashlib.sha1(data_bin).hexdigest() res = pc.searchFile(file_id) if (res == None): # File not found. Add it to the package. pc.append(file_id, data_bin) print("Added: %s" % (file_id, )) log_event("file added", str(file_id)) return "ok" else: if (res == 0): #file already exists log_event("file already exists", str(file_id)) return "already exists" else: #existe y esta bloqueado por vt log_event("file already exists", str(file_id)) return "virustotal"
def save_file_from_vt(hash_id): downloaded_file=download_from_virus_total(hash_id) if(downloaded_file==None): return {"status": "unknown", "hash": None} if downloaded_file.get('status') == "out_of_credits": return {"status": "out_of_credits", "hash": None} if downloaded_file.get('status') == "not_found": return {"status": "not_found", "hash": None} if downloaded_file.get('status') == 'ok': data_bin=downloaded_file.get('file') file_id=hashlib.sha1(data_bin).hexdigest() pc=PackageController() res=pc.searchFile(file_id) if(res==None): # File not found. Add it to the package. pc.append(file_id,data_bin,True) return {"status": "added", "hash": file_id} else: process_file(file_id) return {"status": "inconsistency_found", "hash": file_id}
def save_file_from_vt(hash_id): downloaded_file = download_from_virus_total(hash_id) if(downloaded_file is None): return {"status": "unknown", "hash": None} if downloaded_file.get('status') == "out_of_credits": return {"status": "out_of_credits", "hash": None} if downloaded_file.get('status') == "not_found": return {"status": "not_found", "hash": None} if downloaded_file.get('status') == 'ok': data_bin = downloaded_file.get('file') file_id = hashlib.sha1(data_bin).hexdigest() pc = PackageController() res = pc.searchFile(file_id) if(res is None): # File not found. Add it to the package. pc.append(file_id, data_bin, True) return {"status": "added", "hash": file_id} else: process_file(file_id) return {"status": "inconsistency_found", "hash": file_id}
def add_file_from_vt(hash_id): downloaded_file=download_from_virus_total(hash_id) if(downloaded_file==None): print "add_file_from_vt(): "+str(hash_id)+" not found in VT." return None print "add_file_from_vt(): downloaded_file is not None."+str(hash_id) data_bin=downloaded_file file_id=hashlib.sha1(data_bin).hexdigest() #print "file_id="+str(file_id) pc=PackageController() res=pc.searchFile(file_id) if(res==None): # File not found. Add it to the package. pc.append(file_id,data_bin,True) print str(hash_id)+" added to DB from VT." #print("Added: %s" % (file_id,)) else: print "add_file_from_vt(): "+str(hash_id)+" was found in the DB and asked in VT: BUG. Going to process right now." process_file(file_id) return file_id