def build_obj(file, dir=''):
if dir != '':
file = dir + file
vt_hash = hash_maker.get_hash_data(file, "md5")
#get the json decoded data
fhashes = json.loads(get_hash_obj(file))
fstructure = json.loads(get_structure(file))
fscore = json.loads(get_scores(file))
fvt = json.loads(get_vt_obj(vt_hash))
fcontents = json.loads(get_contents_obj(file))
# frelated = json.loads(get_related_files(file))
frelated = "null"
#build the object and then re-encode
fobj = {
"hash_data": fhashes,
"structure": fstructure,
"scores": fscore,
"scans": {
"virustotal": fvt,
"wepawet": "null"
},
"contents": fcontents,
'related': frelated
}
return json.dumps(fobj)
def build_obj(malpdf, vt=False, wepawet=False, hashes=False, exhaustive=False,
hexa=False, allobjects=False):
# get the json decoded data
fstructure = json.loads(get_structure(malpdf, exhaustive))
if not allobjects:
(fcontents, raw_fcontents) = filter_suspect_objects(
get_indirect_objects(malpdf), SUSPECT_KEYWORDS)
else:
fcontents = get_contents_obj(malpdf, hexa)
# TODO scoring
# fscore = json.loads(get_scores(malpdf))
fscore = "NotImplemented"
# TODO related
# frelated = json.loads(get_related_files(malpdf))
frelated = "NotImplemented"
# build the object and then re-encode
fobj = {"structure": fstructure, "scores": fscore, "scans":
{}, "contents": fcontents, "related": frelated}
if vt:
vt_hash = hash_maker.get_hash_data(malpdf, "md5")
fobj["scans"]["virustotal"] = get_vt_obj(vt_hash)
if wepawet:
fobj["scans"]["wepawet"] = get_wepawet_obj()
if hashes:
fobj["hash"] = get_hash_obj(malpdf)
return fobj
def build_obj(file, dir=''):
if dir != '':
file = dir + file
#vt_hash = hash_maker.get_hash_data(file, "md5")
#fhashes = json.loads(get_hash_obj(file))
#fstructure = json.loads(get_structure(file))
#fvt = json.loads(get_vt_obj(vt_hash))
#fversion = json.loads(get_version_details(file))
#fcontents = json.loads(get_contents_obj(file))
frelated = "null"
try:
vt_hash = hash_maker.get_hash_data(file, "md5")
except:
print str(traceback.print_exc())
print "VT Hash error"
try:
fhashes = json.loads(get_hash_obj(file))
except:
print str(traceback.print_exc())
print "Hash error"
try:
fstructure = json.loads(get_structure(file))
except:
print str(traceback.print_exc())
print "Structure error"
try:
fvt = json.loads(get_vt_obj(vt_hash))
except:
print str(traceback.print_exc())
print "VT error"
try:
fversion = json.loads(get_version_details(file))
except:
print str(traceback.print_exc())
print "Versions error"
try:
fcontents = json.loads(get_contents_obj(file))
except:
print str(traceback.print_exc())
print "Content error"
#build the object and then re-encode
try:
fobj = { "hash_data": fhashes, "structure": fstructure, "scans": { "virustotal": fvt, "wepawet": "null" }, "contents" : fcontents, 'related' : frelated, 'versions': fversion, 'tags': ['malware'] }
except:
print "Obj error"
print str(traceback.print_exc())
return json.dumps(fobj)
def main():
oParser = optparse.OptionParser(usage='usage: %prog [options]\n' + __description__, version='%prog ' + __version__)
oParser.add_option('-f', '--file', default='', type='string', help='file to build an object from')
oParser.add_option('-d', '--dir', default='', type='string', help='dir to build an object from')
oParser.add_option('-m', '--mongo', action='store_true', default=False, help='dump to a mongodb database')
oParser.add_option('-v', '--verbose', action='store_true', default=False, help='verbose outpout')
oParser.add_option('-l', '--log', action='store_true', default=False, help='log errors to file')
(options, args) = oParser.parse_args()
if options.log:
log = open("error_log",'w')
if options.mongo:
con = connect_to_mongo("localhost", 27017, "pdfs", "malware")
#file assumes the following: absolute path, filename is "hash.pdf.vir"
if options.file:
output = build_obj(options.file)
if options.mongo:
con.insert(json.loads(output))
if options.verbose:
print output
elif options.dir:
files = []
dirlist = os.listdir(options.dir)
for fname in dirlist:
files.append(fname)
files.sort()
count = 0
for file in files:
if count == 20:
if options.verbose:
print "Sleeping for 5 minutes"
time.sleep(300)
count = 0
else:
hash = hash_maker.get_hash_data(options.dir + file, "md5")
pres = con.find({"hash_data.file.md5":hash}).count()
if pres != 1:
output = build_obj(file, options.dir)
if options.mongo:
try:
con.insert(json.loads(output))
if options.verbose:
print file + " inserted"
except:
print "Something went wrong with" + file
traceback.print_exc()
if options.log:
log.write("ERROR: " + file + "\n")
count += 1
if options.log:
log.close()
else:
oParser.print_help()
return
def build_obj(file, dir=''):
if dir != '':
file = dir + file
frelated = "null"
try:
vt_hash = hash_maker.get_hash_data(file, "md5")
except:
print str(traceback.print_exc())
print "VT Hash error"
try:
fhashes = json.loads(get_hash_obj(file))
except:
print str(traceback.print_exc())
print "Hash error"
try:
fstructure = json.loads(get_structure(file))
except:
print str(traceback.print_exc())
print "Structure error"
try:
fvt = json.loads(get_vt_obj(vt_hash))
except:
print str(traceback.print_exc())
print "VT error"
try:
fversion = json.loads(get_version_details(file))
except:
print str(traceback.print_exc())
print "Versions error"
try:
fcontents = json.loads(get_contents_obj(file))
except:
print str(traceback.print_exc())
print "Content error"
#build the object and then re-encode
try:
fobj = { "hash_data": fhashes, "structure": fstructure, "scans": { "virustotal": fvt, "wepawet": "null" }, "contents" : fcontents, 'versions': fversion, 'tags': ['public'] }
except:
print "Obj error"
print str(traceback.print_exc())
return json.dumps(fobj)
def build_obj(file, dir=''):
if dir != '':
file = dir + file
vt_hash = hash_maker.get_hash_data(file, "md5")
#get the json decoded data
fhashes = json.loads(get_hash_obj(file))
fstructure = json.loads(get_structure(file))
fscore = json.loads(get_scores(file))
fvt = json.loads(get_vt_obj(vt_hash))
fcontents = json.loads(get_contents_obj(file))
# frelated = json.loads(get_related_files(file))
frelated = "null"
#build the object and then re-encode
fobj = { "hash_data": fhashes, "structure": fstructure, "scores" : fscore, "scans": { "virustotal": fvt, "wepawet": "null" }, "contents" : fcontents, 'related' : frelated }
return json.dumps(fobj)
def get_data(file, collection):
rhashes = []
hash_look = hash_maker.get_hash_data(file, "md5")
for md5 in collection.find({"value.hashes":hash_look},{"value.hashes":1,"_id":1}):
rjson = json.dumps(md5)
ruse = json.loads(rjson)
value = ruse.get("value")
hashes = value.get("hashes")
sig = ruse.get("_id")
for hash in hashes:
if hash in rhashes:
break
else:
if hash != hash_look:
rhashes.append(hash)
data = { 'hashes' : rhashes }
return data
def main():
oParser = optparse.OptionParser(usage='usage: %prog [options]\n' +
__description__,
version='%prog ' + __version__)
oParser.add_option('-f',
'--file',
default='',
type='string',
help='file to build an object from')
oParser.add_option('-d',
'--dir',
default='',
type='string',
help='dir to build an object from')
oParser.add_option('-m',
'--mongo',
action='store_true',
default=False,
help='dump to a mongodb database')
oParser.add_option('-v',
'--verbose',
action='store_true',
default=False,
help='verbose outpout')
oParser.add_option('-a',
'--auto',
action='store_true',
default=False,
help='auto run for web portal')
oParser.add_option('-l',
'--log',
action='store_true',
default=False,
help='log errors to file')
(options, args) = oParser.parse_args()
if options.log:
log = open("error_log", 'w')
if options.mongo:
con = connect_to_mongo("localhost", 27017, "pdfs", "pdf_repo")
if options.file:
output = build_obj(options.file)
if options.mongo:
con.insert(json.loads(output))
if options.verbose:
print output
elif options.dir:
files = []
dirlist = os.listdir(options.dir)
for fname in dirlist:
files.append(fname)
files.sort()
count = 0
for file in files:
if count == 20:
if options.verbose:
print "Sleeping for 5 minutes"
time.sleep(300)
count = 0
else:
try:
hash = hash_maker.get_hash_data(options.dir + file, "md5")
pres = con.find({"hash_data.file.md5": hash}).count()
except:
print "Hash error"
pres = 1
if pres != 1:
try:
output = build_obj(file, options.dir)
if options.mongo:
try:
con.insert(json.loads(output))
if options.verbose:
print file + " inserted"
except:
print "Something went wrong with" + file
traceback.print_exc()
if options.log:
log.write("ERROR: " + file + "\n")
count += 1
except:
print "Complete build failed"
if options.log:
log.close()
elif options.auto:
while True:
queue = connect_to_mongo("localhost", 27017, "pdfs", "file_queue")
malware = connect_to_mongo("localhost", 27017, "pdfs", "malware")
core = connect_to_mongo("localhost", 27017, "pdfs", "tests")
for row in queue.find({"processed": "false"}, {
"hash": 1,
"filename": 1,
"_id": 0
}):
row = json.dumps(row)
ruse = json.loads(row)
hash = ruse.get("hash")
filename = ruse.get("filename")
print "proccessing " + filename
path = "/var/www/mop_rest/uploads/" + filename
hash = hash_maker.get_hash_data(path, "md5")
pres = core.find({"hash_data.file.md5": hash}).count()
if pres != 1:
output = build_obj(path)
try:
core.insert(json.loads(output))
if options.verbose:
print file + " inserted"
except:
print "Something went wrong with" + filename
traceback.print_exc()
if options.log:
log.write("ERROR: " + file + "\n")
queue.update({"hash": hash},
{"$set": {
"processed": "true"
}})
print "processed " + filename
time.sleep(20)
else:
oParser.print_help()
return
def main():
oParser = optparse.OptionParser(usage='usage: %prog [options]\n' +
__description__,
version='%prog ' + __version__)
oParser.add_option('-f',
'--file',
default='',
type='string',
help='file to build an object from')
oParser.add_option('-d',
'--dir',
default='',
type='string',
help='dir to build an object from')
oParser.add_option('-m',
'--mongo',
action='store_true',
default=False,
help='dump to a mongodb database')
oParser.add_option('-v',
'--verbose',
action='store_true',
default=False,
help='verbose outpout')
oParser.add_option('-l',
'--log',
action='store_true',
default=False,
help='log errors to file')
(options, args) = oParser.parse_args()
if options.log:
log = open("error_log", 'w')
if options.mongo:
con = connect_to_mongo("localhost", 27017, "pdfs", "malware")
#file assumes the following: absolute path, filename is "hash.pdf.vir"
if options.file:
output = build_obj(options.file)
if options.mongo:
con.insert(json.loads(output))
if options.verbose:
print output
elif options.dir:
files = []
dirlist = os.listdir(options.dir)
for fname in dirlist:
files.append(fname)
files.sort()
count = 0
for file in files:
if count == 20:
if options.verbose:
print "Sleeping for 5 minutes"
time.sleep(300)
count = 0
else:
hash = hash_maker.get_hash_data(options.dir + file, "md5")
pres = con.find({"hash_data.file.md5": hash}).count()
if pres != 1:
output = build_obj(file, options.dir)
if options.mongo:
try:
con.insert(json.loads(output))
if options.verbose:
print file + " inserted"
except:
print "Something went wrong with" + file
traceback.print_exc()
if options.log:
log.write("ERROR: " + file + "\n")
count += 1
if options.log:
log.close()
else:
oParser.print_help()
return
def build_obj(file, dir=""):
if dir != "":
file = dir + file
# vt_hash = hash_maker.get_hash_data(file, "md5")
# fhashes = json.loads(get_hash_obj(file))
# fstructure = json.loads(get_structure(file))
# fvt = json.loads(get_vt_obj(vt_hash))
# fversion = json.loads(get_version_details(file))
# fcontents = json.loads(get_contents_obj(file))
frelated = "null"
try:
vt_hash = hash_maker.get_hash_data(file, "md5")
except:
# print str(traceback.print_exc())
# print "VT Hash error"
vt_hash = "error"
try:
fhashes = json.loads(get_hash_obj(file))
except:
# print str(traceback.print_exc())
# print "Hash error"
fhashes = "error"
try:
fstructure = json.loads(get_structure(file))
except:
# print str(traceback.print_exc())
# print "Structure error"
fstructure = "error"
try:
fvt = json.loads(get_vt_obj(vt_hash))
except:
# print str(traceback.print_exc())
# print "VT error"
fvt = "error"
try:
fversion = json.loads(get_version_details(file))
except:
# print str(traceback.print_exc())
# print "Versions error"
fversion = "error"
try:
fcontents = json.loads(get_contents_obj(file))
except:
# print str(traceback.print_exc())
# print "Content error"
fail = "add"
# build the object and then re-encode
try:
fobj = {
"hash_data": fhashes,
"structure": fstructure,
"scans": {"virustotal": fvt, "wepawet": "null"},
"contents": fcontents,
"related": frelated,
"versions": fversion,
"tags": ["public"],
}
except:
# print "Obj error"
# print str(traceback.print_exc())
fail = "fail"
return json.dumps(fobj)
def main():
oParser = optparse.OptionParser(usage="usage: %prog [options]\n" + __description__, version="%prog " + __version__)
oParser.add_option("-f", "--file", default="", type="string", help="file to build an object from")
oParser.add_option("-d", "--dir", default="", type="string", help="dir to build an object from")
oParser.add_option("-m", "--mongo", action="store_true", default=False, help="dump to a mongodb database")
oParser.add_option("-v", "--verbose", action="store_true", default=False, help="verbose outpout")
oParser.add_option("-a", "--auto", action="store_true", default=False, help="auto run for web portal")
oParser.add_option("-l", "--log", action="store_true", default=False, help="log errors to file")
(options, args) = oParser.parse_args()
if options.log:
log = open("error_log", "w")
if options.mongo:
con = connect_to_mongo("localhost", 27017, "pdfs", "pdf_repo")
if options.file:
output = build_obj(options.file)
if options.mongo:
con.insert(json.loads(output))
if options.verbose:
print output
elif options.dir:
files = []
dirlist = os.listdir(options.dir)
for fname in dirlist:
files.append(fname)
files.sort()
count = 0
for file in files:
if count == 20:
if options.verbose:
print "Sleeping for 5 minutes"
time.sleep(300)
count = 0
else:
try:
hash = hash_maker.get_hash_data(options.dir + file, "md5")
pres = con.find({"hash_data.file.md5": hash}).count()
except:
print "Hash error"
pres = 1
if pres != 1:
try:
output = build_obj(file, options.dir)
if options.mongo:
try:
con.insert(json.loads(output))
if options.verbose:
print file + " inserted"
except:
print "Something went wrong with" + file
traceback.print_exc()
if options.log:
log.write("ERROR: " + file + "\n")
count += 1
except:
print "Complete build failed"
if options.log:
log.close()
elif options.auto:
while True:
queue = connect_to_mongo("localhost", 27017, "pdfs", "file_queue")
malware = connect_to_mongo("localhost", 27017, "pdfs", "malware")
core = connect_to_mongo("localhost", 27017, "pdfs", "tests")
for row in queue.find({"processed": "false"}, {"hash": 1, "filename": 1, "_id": 0}):
row = json.dumps(row)
ruse = json.loads(row)
hash = ruse.get("hash")
filename = ruse.get("filename")
print "proccessing " + filename
path = "/var/www/mop_rest/uploads/" + filename
hash = hash_maker.get_hash_data(path, "md5")
pres = core.find({"hash_data.file.md5": hash}).count()
if pres != 1:
output = build_obj(path)
try:
core.insert(json.loads(output))
if options.verbose:
print file + " inserted"
except:
print "Something went wrong with" + filename
traceback.print_exc()
if options.log:
log.write("ERROR: " + file + "\n")
queue.update({"hash": hash}, {"$set": {"processed": "true"}})
print "processed " + filename
time.sleep(20)
else:
oParser.print_help()
return
def main():
oParser = optparse.OptionParser(usage='usage: %prog [options]\n' + __description__, version='%prog ' + __version__)
oParser.add_option('-f', '--file', default='', type='string', help='file to build an object from')
oParser.add_option('-d', '--dir', default='', type='string', help='dir to build an object from')
oParser.add_option('-m', '--mongo', action='store_true', default=False, help='dump to a mongodb database')
oParser.add_option('-v', '--verbose', action='store_true', default=False, help='verbose outpout')
oParser.add_option('-a', '--auto', action='store_true', default=False, help='auto run for web portal')
oParser.add_option('-l', '--log', action='store_true', default=False, help='log errors to file')
(options, args) = oParser.parse_args()
if options.log:
log = open("error_log",'w')
if options.mongo:
con = connect_to_mongo("localhost", 27017, "pdfs", "pdf_repo")
file_stat = connect_to_mongo("localhost", 27017, "pdfs", "file_statistics")
if options.file:
output = build_obj(options.file)
if options.mongo:
con.insert(json.loads(output))
if options.verbose:
print output
elif options.dir:
files = []
dirlist = os.listdir(options.dir)
for fname in dirlist:
files.append(fname)
files.sort()
count = 0
for file in files:
if count == 20:
if options.verbose:
print "Sleeping for 5 minutes"
other_time.sleep(300)
count = 0
else:
try:
hash = hash_maker.get_hash_data(options.dir + file, "md5")
pres = con.find({"hash_data.file.md5":hash}).count()
except:
print "Hash error"
pres = 1
if pres != 1:
try:
t = time()
output = build_obj(file, options.dir)
process_time = time() - t
if options.mongo:
try:
con.insert(json.loads(output))
file_stat.insert({'stored':True,'date_time':time(),'hash':hash,'user':'******','process_time':process_time})
if options.verbose:
print file + " inserted"
except:
print "Something went wrong with" + file
traceback.print_exc()
if options.log:
log.write("ERROR: " + file + "\n")
count += 1
except:
print "Complete build failed"
if options.log:
log.close()
elif options.auto:
while True:
queue = connect_to_mongo("localhost", 27017, "pdfs", "file_queue")
malware = connect_to_mongo("localhost",27017,"pdfs","malware")
core = connect_to_mongo("localhost", 27017, "pdfs", "tests")
for row in queue.find({"processed":"false"},{"hash":1,"filename":1,"_id":0}):
row = json.dumps(row)
ruse = json.loads(row)
hash = ruse.get("hash")
filename = ruse.get("filename")
print "proccessing " + filename
path = "/var/www/mop_rest/uploads/" + filename
hash = hash_maker.get_hash_data(path, "md5")
pres = core.find({"hash_data.file.md5":hash}).count()
if pres != 1:
output = build_obj(path)
try:
core.insert(json.loads(output))
if options.verbose:
print file + " inserted"
except:
print "Something went wrong with" + filename
traceback.print_exc()
if options.log:
log.write("ERROR: " + file + "\n")
queue.update({"hash":hash},{"$set":{"processed":"true"}})
print "processed " + filename
other_time.sleep(20)
else:
oParser.print_help()
return
def build_obj(file, dir='', output='text', vt=True):
if dir != '':
file = dir + file
#vt_hash = hash_maker.get_hash_data(file, "md5")
#fhashes = json.loads(get_hash_obj(file))
#fstructure = json.loads(get_structure(file))
#fvt = json.loads(get_vt_obj(vt_hash))
#fversion = json.loads(get_version_details(file))
#fcontents = json.loads(get_contents_obj(file))
frelated = "null"
try:
vt_hash = hash_maker.get_hash_data(file, "md5")
except:
#print str(traceback.print_exc())
#print "VT Hash error"
vt_hash = "error"
try:
fhashes = json.loads(get_hash_obj(file))
except:
#print str(traceback.print_exc())
#print "Hash error"
fhashes = "error"
try:
fstructure = json.loads(get_structure(file))
except:
#print str(traceback.print_exc())
#print "Structure error"
fstructure = "error"
if (vt):
try:
fvt = json.loads(get_vt_obj(vt_hash))
except:
#print str(traceback.print_exc())
#print "VT error"
fvt = "error"
else:
fvt = "disabled"
try:
fversion = json.loads(get_version_details(file))
except:
#print str(traceback.print_exc())
#print "Versions error"
fversion = "error"
try:
fcontents = json.loads(get_contents_obj(file))
except:
#print str(traceback.print_exc())
#print "Content error"
fail = "add"
#build the object and then re-encode
try:
fobj = { "hash_data": fhashes, "structure": fstructure, "scans": { "virustotal": fvt, "wepawet": "null" }, "contents" : fcontents, 'related' : frelated, 'versions': fversion, 'tags': ['public'] }
except:
#print str(traceback.print_exc())
fobj = {}
fail = "fail"
if output == 'obj':
return fobj
return json.dumps(fobj)