return f
def write_task(self, task_id, data_dic):
command = {"$set": data_dic}
logging.debug("write_task(): data_dic=")
logging.debug(str(data_dic))
return self.tasks.update_one({"task_id": task_id}, command, upsert=True)
def save_av_analysis(self, file_id, analysis_result):
command = {"$set": analysis_result}
try:
self.av_coll.update_one({"sha1": file_id}, command, upsert=True)
except WriteError:
logging.exception("**** Error File: %s ****" % (file_id,))
# print(command)
# err=str(traceback.format_exc())
# print(err)
return -1
self.save_first_seen(file_id, analysis_result.get('date'))
return 0
# ****************TEST_CODE******************
def testCode():
pass
# ****************TEST_EXECUTE******************
test("-test_MetaController", testCode)
def __delete__(self):
pass
def updateVersion(self,file_id,ver_dic):
command={"$set":ver_dic}
#print(command)
self.collection.update_one({"file_id":file_id},command,upsert=True)
def searchVersion(self,file_id):
f=self.collection.find_one({"file_id":file_id})
#print(f)
return f
#****************TEST_CODE******************
def testCode():
db=DBVersion()
ver={}
for i in range(0,10):
ver[str(i)]=i+10
#db.updateVersion("0000",ver)
lver=db.loadVersion("0000")
n=lver["3"]
print(type(n))
#****************TEST_EXECUTE******************
from Utils.test import test
test("-test_VersionController",testCode)
class ProcessorFactory():
def __init__(self):
pass
def __delete__(self):
pass
def getHashProcessor(self, processor, sample):
return HashProcessor(sample)
def createProcessor(self, processor, sample):
obj = None
if (processor == "application/x-dosexec"):
obj = PEProcessor(sample)
else:
obj = DummyProcessor(sample)
return obj
# ****************TEST_CODE******************
def testCode():
pass
# ****************TEST_EXECUTE******************
test("-test_ProcessorFactory", testCode)
self.data = str(data)
def __delete__(self):
pass
def catalogData(self):
mime = MIME_TYPE(self.data, True)
return mime
def catalog(self, data):
mime = MIME_TYPE(data, True)
return mime
#****************TEST_CODE******************
import os
def testCode():
dir = os.getcwd()
#file=dir+"/Test_files/test.exe"
file = dir + "/Test_files/t.lz"
cat = Cataloger(open(file, 'rb').read())
res = cat.catalog()
print(res)
#****************TEST_EXECUTE******************
from Utils.test import test
test("-test_Cataloger", testCode)
#----------------------------------------------
def testCode6():
inicio = 0
client = MongoClient(env["files"]["host"], env["files"]["port"])
db = client[env["db_files_name"]]
fs = gridfs.GridFS(db)
res = fs.find(timeout=False).skip(inicio)
lc = Launcher()
count = inicio
reset = 0
start = time.time()
first = True
for f in res:
sam_id = f.filename
sample = Sample()
sample.setID(sam_id)
sample.setStorageVersion({})
lc.launchAnalysisByID(sample)
reset += 1
count += 1
if (reset >= 1000):
print(
time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
+ " processed:" + str(count / 1000) + "K")
reset = 0
print(str(count) + " processed")
#****************TEST_EXECUTE******************
test("-test_Launcher", testCode6)
#self.version[name]=code_version
self.result_version[name] = code_version
self.metadata_to_store[info_string] = res
return 0
# returns up to date versions.
def getVersion(self):
return self.result_version
#redefine str()
#def __str__(self):
# string=""
# for word in self.result:
# #tabs='\t'
# tabs=" "
# #for i in range(6-int((len(word)+1)/8)):
# # tabs+=" "
# string+=(str(word)+":"+tabs+str(self.result[word])+'\n')
#
# return string
#****************TEST_CODE******************
def testCode():
pass
#****************TEST_EXECUTE******************
from Utils.test import test
test("-test_Processor", testCode)
command = {"$set": data_dic}
logging.debug("write_task(): data_dic=")
logging.debug(str(data_dic))
return self.tasks.update_one({"task_id": task_id},
command,
upsert=True)
def save_av_analysis(self, file_id, analysis_result):
command = {"$set": analysis_result}
try:
self.av_coll.update_one({"sha1": file_id}, command, upsert=True)
except WriteError:
logging.exception("**** Error File: %s ****" % (file_id, ))
# print(command)
# err=str(traceback.format_exc())
# print(err)
return -1
self.save_first_seen(file_id, analysis_result.get('date'))
return 0
# ****************TEST_CODE******************
def testCode():
pass
# ****************TEST_EXECUTE******************
test("-test_MetaController", testCode)
# self.version[name]=code_version
self.result_version[name] = code_version
self.metadata_to_store[info_string] = res
return 0
# returns up to date versions.
def getVersion(self):
return self.result_version
# redefine str()
# def __str__(self):
# string=""
# for word in self.result:
# #tabs='\t'
# tabs=" "
# #for i in range(6-int((len(word)+1)/8)):
# # tabs+=" "
# string+=(str(word)+":"+tabs+str(self.result[word])+'\n')
#
# return string
# ****************TEST_CODE******************
def testCode():
pass
# ****************TEST_EXECUTE******************
test("-test_Processor", testCode)
self.data = str(data)
def __delete__(self):
pass
def catalogData(self):
mime = MIME_TYPE(self.data, True)
return mime
def catalog(self, data):
mime = MIME_TYPE(data, True)
return mime
# ****************TEST_CODE******************
def testCode():
dir = os.getcwd()
# file=dir+"/Test_files/test.exe"
file = dir + "/Test_files/t.lz"
cat = Cataloger(open(file, 'rb').read())
res = cat.catalog()
print(res)
# ****************TEST_EXECUTE******************
test("-test_Cataloger", testCode)
def testCode6():
inicio = 0
client = MongoClient(envget('files.host'), envget('files.port'))
db = client[envget('db_files_name')]
fs = gridfs.GridFS(db)
res = fs.find(timeout=False).skip(inicio)
lc = Launcher()
count = inicio
reset = 0
start = time.time()
first = True
for f in res:
sam_id = f.filename
sample = Sample()
sample.setID(sam_id)
sample.setStorageVersion({})
lc.launchAnalysisByID(sample)
reset += 1
count += 1
if(reset >= 1000):
print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())
) + " processed:" + str(count / 1000) + "K")
reset = 0
print(str(count) + " processed")
# ****************TEST_EXECUTE******************
test("-test_Launcher", testCode6)
from Processors.DummyProcessor import *
class ProcessorFactory():
def __init__(self):
pass
def __delete__(self):
pass
def getHashProcessor(self,processor,sample):
return HashProcessor(sample)
def createProcessor(self,processor,sample):
obj=None
if(processor=="application/x-dosexec"):
obj=PEProcessor(sample)
else:
obj=DummyProcessor(sample)
return obj
#****************TEST_CODE******************
def testCode():
pass
#****************TEST_EXECUTE******************
from Utils.test import test
test("-test_ProcessorFactory",testCode)
