def api_status_files_to_load_folder(): path=os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)),'..','files_to_load')) files=recursive_read(path) if(files is not None): flen=len(files) else: flen=0 return str(flen)+" files in files_to_load folder."
def testCode(): from Utils.Functions import recursive_read object = "./Test_files/" files = recursive_read(object) if (files == None): sys.exit() lc = Launcher() for fp in files: fd = open(fp, 'r') data = fd.read() file_id = hashlib.sha1(data).hexdigest() print("%s %s" % (fp, file_id)) lc.launchFileAnalitics((fp, data)) print("") print("")
def testCode(): from Utils.Functions import recursive_read object="./Test_files/" files=recursive_read(object) if(files==None): sys.exit() lc=Launcher() for fp in files: fd=open(fp,'r') data=fd.read() file_id=hashlib.sha1(data).hexdigest() print("%s %s"%(fp,file_id)) lc.launchFileAnalitics((fp,data)) print("") print("")
def load_to_mongo2(folder_path): pc=PackageController() ram = Ram() files=recursive_read(folder_path) count=0 reset=0 already_loaded=0 time_start = datetime.datetime.now() uploaded=0 in_mem=0 loaded_ram_counter=0 lc=Launcher() if(files is None): return "No files where found." while (uploaded < len(files)): loaded_ram_counter=0 data_vector=[] print "loading files to memory" while (in_mem < len(files)): f=files[in_mem] file_cursor=open(f,"r") data_vector.append(file_cursor.read()) in_mem=in_mem+1 loaded_ram_counter=loaded_ram_counter+1 if(loaded_ram_counter > 100): if(ram.free_percent() < 0.3): print "Ram full" break for data in data_vector: file_id=hashlib.sha1(data).hexdigest() print "loading to db: "+str(file_id) res=pc.searchFile(file_id) if(res==None): pc.append(file_id,data) sample=Sample() sample.setID(file_id) sample.setBinary(data) sample.setStorageVersion({}) count+=1 lc.launchAnalysisByID(sample) else: already_loaded+=1 uploaded=uploaded+1 result=str(already_loaded)+" were already loaded to mongo.\n" result+=thetime(time_start,datetime.datetime.now(),count) print result return result