def msbox_insert(srcfile,
jobuuid,
md5hash,
sha256hash,
fileext,
target,
runtime=120,
priority=beanstalkc.DEFAULT_PRIORITY):
#all paths in lowercase
md5hash = md5hash.lower()
# copy file into malware storage directory
destfilepath = MALWARESTORAGEBASE + '/' + srcfile
destfile = destfilepath + '/' + sha256hash
#if not os.path.exists(destfile):
# os.umask(0)
# os.makedirs(destfilepath, mode = 0777)
# shutil.copyfile(srcfile, destfile)
#Extra check to make sure the file is there (ie. the copy worked) before we proceed
#if not os.path.exists(destfile):
# return -1
basefilename = os.path.basename(srcfile)
# add a .seen file (timestamp on file is first seen)
# sampletimestamp = ".seen-ms-%s-%s-%s" % (time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime()), jobuuid, basefilename)
# seenfile = destfilepath + '/' + sampletimestamp
# Possible race condition here...I'll take my chances
# open(seenfile, "a").close()
makefeeder(srcfile, sha256hash + ".xml")
# Insert job into beanstalk so a vm can service it
beanstalk = beanstalkc.Connection(host=BSHOST, port=BSPORT)
beanstalk.use(BSMSBOXQ)
print "target is" + target
jobstring = "{'jobuuid':'%s','md5':'%s','sha256':'%s','basename':'%s','ext':'%s', 'runtime':'%d', 'target':'%s', 'fullpath':'%s'}" % (
jobuuid, md5hash, sha256hash, basefilename, fileext, runtime, target,
srcfile)
jobid = beanstalk.put(jobstring, priority=priority)
beanstalk.close()
return jobid
def msbox_insert(srcfile, jobuuid, md5hash, sha256hash, fileext, target, runtime = 120, priority = beanstalkc.DEFAULT_PRIORITY):
#all paths in lowercase
md5hash = md5hash.lower()
# copy file into malware storage directory
destfilepath = MALWARESTORAGEBASE + '/' + srcfile
destfile = destfilepath + '/' + sha256hash
#if not os.path.exists(destfile):
# os.umask(0)
# os.makedirs(destfilepath, mode = 0777)
# shutil.copyfile(srcfile, destfile)
#Extra check to make sure the file is there (ie. the copy worked) before we proceed
#if not os.path.exists(destfile):
# return -1
basefilename = os.path.basename(srcfile)
# add a .seen file (timestamp on file is first seen)
# sampletimestamp = ".seen-ms-%s-%s-%s" % (time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime()), jobuuid, basefilename)
# seenfile = destfilepath + '/' + sampletimestamp
# Possible race condition here...I'll take my chances
# open(seenfile, "a").close()
makefeeder(srcfile,sha256hash + ".xml")
# Insert job into beanstalk so a vm can service it
beanstalk = beanstalkc.Connection(host=BSHOST, port=BSPORT)
beanstalk.use(BSMSBOXQ)
print "target is" + target
jobstring = "{'jobuuid':'%s','md5':'%s','sha256':'%s','basename':'%s','ext':'%s', 'runtime':'%d', 'target':'%s', 'fullpath':'%s'}" % (jobuuid, md5hash, sha256hash, basefilename, fileext, runtime, target,srcfile)
jobid = beanstalk.put(jobstring, priority = priority)
beanstalk.close()
return jobid
def msbox_insert(srcfile,
jobuuid,
md5hash,
sha1hash,
sha256hash,
fileext,
target,
msversion,
size,
runtime=180,
priority=beanstalkc.DEFAULT_PRIORITY):
#all paths in lowercase
md5hash = md5hash.lower()
sha1hash = sha1hash.lower()
sha256hash = sha256hash.lower()
existingSample = msdb.findSample(sha256hash)
logger.info("existing sample? %s" % (existingSample))
basefilename = os.path.basename(srcfile)
destfile = None
#we only save each unique sample once
if (existingSample):
logger.info("sample exists: %s" % existingSample)
destfile = msdb.getExistingPath(existingSample)
else:
# copy file into malware storage directory; we save the original name in the db, but the stored file is by hash
destfilepath = MALWARESTORAGEBASE + '/' + sha256hash
destfile = destfilepath + '/' + sha256hash + ".apk"
if not os.path.exists(destfile):
os.umask(0)
os.makedirs(destfilepath, mode=0733)
shutil.copyfile(srcfile, destfile)
#Extra check to make sure the file is there (ie. the copy worked) before we proceed
if not os.path.exists(destfile):
logger.info("copy failed!")
return -1
os.chmod(destfile, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
#3.3+ only :-( shutil.chown(destfile, "root","apache")
existingSample = msdb.insertSample(destfile, md5hash, sha1hash,
sha256hash, size)
if destfile == None:
logger.info("path to malware is empty, exiting")
return -1
if not existingSample:
logger.info("malware id is empty, exiting")
return -1
jobpath = MALWAREJOBSBASE + "/" + jobuuid
if not os.path.exists(jobpath):
os.umask(0)
os.makedirs(jobpath, mode=0733)
msdb.insertSubmission(basefilename, jobuuid, "not done", "NOT ANALYZED",
time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime()),
"127.0.0.2", existingSample, target, runtime)
makefeeder(destfile, jobpath + "/" + sha256hash + ".xml")
# Insert job into beanstalk so a vm can service it
beanstalk = beanstalkc.Connection(host=BSHOST, port=BSPORT)
beanstalk.use(BSMSBOXQ)
logger.info("sdk target is " + target)
jobstring = "{'jobuuid':'%s','md5':'%s','sha256':'%s','basename':'%s','ext':'%s', 'runtime':'%d', 'target':'%s', 'fullpath':'%s'}" % (
jobuuid, md5hash, sha256hash, basefilename, fileext, runtime, target,
destfile)
jobid = beanstalk.put(jobstring, priority=priority)
beanstalk.close()
logger.info("dir" + os.path.dirname(destfile))
#for DB testing, not for real ingest
#time.sleep(3)
#permissions_filename = sha256hash + ".xml"
#image_used = "some vm"
#start_time = time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime())
#msdb.updateStartRun(sha256hash,image_used,start_time,permissions_filename)
#time.sleep(3)
#complete_time = time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime())
#results_file = "some cool file"
#msdb.updateFinishRun(sha256hash,complete_time,results_file)
return jobid
def msbox_update(srcfile,
jobuuid,
md5hash,
sha256hash,
fileext,
target,
msversion,
runtime=120,
priority=beanstalkc.DEFAULT_PRIORITY):
#all paths in lowercase
existingSubmissionId = msdb.findSubmissionToUpdate(sha256hash)
if not existingSubmissionId:
logger.info("job doesn't exist!: %s" % sha256hash)
return
basefilename = os.path.basename(srcfile)
# copy file into malware storage directory
destfilepath = MALWARESTORAGEBASE + '/' + sha256hash
destfile = destfilepath + '/' + sha256hash + ".apk"
if not os.path.exists(destfile):
os.umask(0)
os.makedirs(destfilepath, mode=0733)
shutil.copyfile(srcfile, destfile)
#Extra check to make sure the file is there (ie. the copy worked) before we proceed
if not os.path.exists(destfile):
return -1
#apache can't thos this, but it doesn't need to because it's already apache
# os.chmod(destfile,stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH)
#3.3+ only :-( shutil.chown(destfile, "root","apache")
else:
msdb.updateSample(destfile, sha256hash)
if destfile == None:
logger.info("path to malware is empty, exiting")
return -1
if not existingSubmissionId:
logger.info("malware id is empty, exiting")
return -1
jobpath = MALWAREJOBSBASE + "/" + jobuuid
if not os.path.exists(jobpath):
os.umask(0)
os.makedirs(jobpath, mode=0733)
makefeeder(destfile, jobpath + "/" + sha256hash + ".xml")
# Insert job into beanstalk so a vm can service it
beanstalk = beanstalkc.Connection(host=BSHOST, port=BSPORT)
beanstalk.use(BSMSBOXQ)
logger.info("target is" + target)
jobstring = "{'jobuuid':'%s','md5':'%s','sha256':'%s','basename':'%s','ext':'%s', 'runtime':'%d', 'target':'%s', 'fullpath':'%s'}" % (
jobuuid, md5hash, sha256hash, basefilename, fileext, runtime, target,
destfile)
jobid = beanstalk.put(jobstring, priority=priority)
beanstalk.close()
msdb.updateSubmission(jobuuid, "not done", "NOT ANALYZED",
time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime()),
"127.0.0.2", existingSubmissionId, target, runtime)
logger.info("dir" + os.path.dirname(destfile))
#for DB testing, not for real ingest
#time.sleep(3)
#permissions_filename = sha256hash + ".xml"
#image_used = "some vm"
#start_time = time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime())
#msdb.updateStartRun(sha256hash,image_used,start_time,permissions_filename)
#time.sleep(3)
#complete_time = time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime())
#results_file = "some cool file"
#msdb.updateFinishRun(sha256hash,complete_time,results_file)
return jobid
def msbox_update(srcfile, jobuuid, md5hash, sha256hash, fileext, target, msversion, runtime = 120, priority = beanstalkc.DEFAULT_PRIORITY):
#all paths in lowercase
existingSubmissionId = msdb.findSubmissionToUpdate(sha256hash);
if not existingSubmissionId:
logger.info( "job doesn't exist!: %s" % sha256hash)
return
basefilename = os.path.basename(srcfile)
# copy file into malware storage directory
destfilepath = MALWARESTORAGEBASE + '/' + sha256hash
destfile = destfilepath + '/' + sha256hash + ".apk"
if not os.path.exists(destfile):
os.umask(0)
os.makedirs(destfilepath, mode = 0733)
shutil.copyfile(srcfile, destfile)
#Extra check to make sure the file is there (ie. the copy worked) before we proceed
if not os.path.exists(destfile):
return -1
#apache can't thos this, but it doesn't need to because it's already apache
# os.chmod(destfile,stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH)
#3.3+ only :-( shutil.chown(destfile, "root","apache")
else:
msdb.updateSample(destfile,sha256hash)
if destfile == None:
logger.info( "path to malware is empty, exiting")
return -1
if not existingSubmissionId:
logger.info( "malware id is empty, exiting")
return -1
jobpath = MALWAREJOBSBASE +"/" + jobuuid
if not os.path.exists(jobpath):
os.umask(0)
os.makedirs(jobpath, mode = 0733)
makefeeder(destfile,jobpath +"/" + sha256hash + ".xml")
# Insert job into beanstalk so a vm can service it
beanstalk = beanstalkc.Connection(host=BSHOST, port=BSPORT)
beanstalk.use(BSMSBOXQ)
logger.info( "target is" + target)
jobstring = "{'jobuuid':'%s','md5':'%s','sha256':'%s','basename':'%s','ext':'%s', 'runtime':'%d', 'target':'%s', 'fullpath':'%s'}" % (jobuuid, md5hash, sha256hash, basefilename, fileext, runtime, target,destfile)
jobid = beanstalk.put(jobstring, priority = priority)
beanstalk.close()
msdb.updateSubmission(jobuuid,"not done","NOT ANALYZED",time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime()),"127.0.0.2",existingSubmissionId,target,runtime)
logger.info( "dir" + os.path.dirname(destfile))
#for DB testing, not for real ingest
#time.sleep(3)
#permissions_filename = sha256hash + ".xml"
#image_used = "some vm"
#start_time = time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime())
#msdb.updateStartRun(sha256hash,image_used,start_time,permissions_filename)
#time.sleep(3)
#complete_time = time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime())
#results_file = "some cool file"
#msdb.updateFinishRun(sha256hash,complete_time,results_file)
return jobid
def msbox_insert(
srcfile,
jobuuid,
md5hash,
sha1hash,
sha256hash,
fileext,
target,
msversion,
size,
runtime=180,
priority=beanstalkc.DEFAULT_PRIORITY,
):
# all paths in lowercase
md5hash = md5hash.lower()
sha1hash = sha1hash.lower()
sha256hash = sha256hash.lower()
existingSample = msdb.findSample(sha256hash)
logger.info("existing sample? %s" % (existingSample))
basefilename = os.path.basename(srcfile)
destfile = None
# we only save each unique sample once
if existingSample:
logger.info("sample exists: %s" % existingSample)
destfile = msdb.getExistingPath(existingSample)
else:
# copy file into malware storage directory; we save the original name in the db, but the stored file is by hash
destfilepath = MALWARESTORAGEBASE + "/" + sha256hash
destfile = destfilepath + "/" + sha256hash + ".apk"
if not os.path.exists(destfile):
os.umask(0)
os.makedirs(destfilepath, mode=0733)
shutil.copyfile(srcfile, destfile)
# Extra check to make sure the file is there (ie. the copy worked) before we proceed
if not os.path.exists(destfile):
logger.info("copy failed!")
return -1
os.chmod(destfile, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
# 3.3+ only :-( shutil.chown(destfile, "root","apache")
existingSample = msdb.insertSample(destfile, md5hash, sha1hash, sha256hash, size)
if destfile == None:
logger.info("path to malware is empty, exiting")
return -1
if not existingSample:
logger.info("malware id is empty, exiting")
return -1
jobpath = MALWAREJOBSBASE + "/" + jobuuid
if not os.path.exists(jobpath):
os.umask(0)
os.makedirs(jobpath, mode=0733)
msdb.insertSubmission(
basefilename,
jobuuid,
"not done",
"NOT ANALYZED",
time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime()),
"127.0.0.2",
existingSample,
target,
runtime,
)
makefeeder(destfile, jobpath + "/" + sha256hash + ".xml")
# Insert job into beanstalk so a vm can service it
beanstalk = beanstalkc.Connection(host=BSHOST, port=BSPORT)
beanstalk.use(BSMSBOXQ)
logger.info("sdk target is " + target)
jobstring = (
"{'jobuuid':'%s','md5':'%s','sha256':'%s','basename':'%s','ext':'%s', 'runtime':'%d', 'target':'%s', 'fullpath':'%s'}"
% (jobuuid, md5hash, sha256hash, basefilename, fileext, runtime, target, destfile)
)
jobid = beanstalk.put(jobstring, priority=priority)
beanstalk.close()
logger.info("dir" + os.path.dirname(destfile))
# for DB testing, not for real ingest
# time.sleep(3)
# permissions_filename = sha256hash + ".xml"
# image_used = "some vm"
# start_time = time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime())
# msdb.updateStartRun(sha256hash,image_used,start_time,permissions_filename)
# time.sleep(3)
# complete_time = time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime())
# results_file = "some cool file"
# msdb.updateFinishRun(sha256hash,complete_time,results_file)
return jobid
