def msbox_insert(srcfile, jobuuid, md5hash, sha256hash, fileext, target, runtime=120, priority=beanstalkc.DEFAULT_PRIORITY): #all paths in lowercase md5hash = md5hash.lower() # copy file into malware storage directory destfilepath = MALWARESTORAGEBASE + '/' + srcfile destfile = destfilepath + '/' + sha256hash #if not os.path.exists(destfile): # os.umask(0) # os.makedirs(destfilepath, mode = 0777) # shutil.copyfile(srcfile, destfile) #Extra check to make sure the file is there (ie. the copy worked) before we proceed #if not os.path.exists(destfile): # return -1 basefilename = os.path.basename(srcfile) # add a .seen file (timestamp on file is first seen) # sampletimestamp = ".seen-ms-%s-%s-%s" % (time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime()), jobuuid, basefilename) # seenfile = destfilepath + '/' + sampletimestamp # Possible race condition here...I'll take my chances # open(seenfile, "a").close() makefeeder(srcfile, sha256hash + ".xml") # Insert job into beanstalk so a vm can service it beanstalk = beanstalkc.Connection(host=BSHOST, port=BSPORT) beanstalk.use(BSMSBOXQ) print "target is" + target jobstring = "{'jobuuid':'%s','md5':'%s','sha256':'%s','basename':'%s','ext':'%s', 'runtime':'%d', 'target':'%s', 'fullpath':'%s'}" % ( jobuuid, md5hash, sha256hash, basefilename, fileext, runtime, target, srcfile) jobid = beanstalk.put(jobstring, priority=priority) beanstalk.close() return jobid
def msbox_insert(srcfile, jobuuid, md5hash, sha256hash, fileext, target, runtime = 120, priority = beanstalkc.DEFAULT_PRIORITY): #all paths in lowercase md5hash = md5hash.lower() # copy file into malware storage directory destfilepath = MALWARESTORAGEBASE + '/' + srcfile destfile = destfilepath + '/' + sha256hash #if not os.path.exists(destfile): # os.umask(0) # os.makedirs(destfilepath, mode = 0777) # shutil.copyfile(srcfile, destfile) #Extra check to make sure the file is there (ie. the copy worked) before we proceed #if not os.path.exists(destfile): # return -1 basefilename = os.path.basename(srcfile) # add a .seen file (timestamp on file is first seen) # sampletimestamp = ".seen-ms-%s-%s-%s" % (time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime()), jobuuid, basefilename) # seenfile = destfilepath + '/' + sampletimestamp # Possible race condition here...I'll take my chances # open(seenfile, "a").close() makefeeder(srcfile,sha256hash + ".xml") # Insert job into beanstalk so a vm can service it beanstalk = beanstalkc.Connection(host=BSHOST, port=BSPORT) beanstalk.use(BSMSBOXQ) print "target is" + target jobstring = "{'jobuuid':'%s','md5':'%s','sha256':'%s','basename':'%s','ext':'%s', 'runtime':'%d', 'target':'%s', 'fullpath':'%s'}" % (jobuuid, md5hash, sha256hash, basefilename, fileext, runtime, target,srcfile) jobid = beanstalk.put(jobstring, priority = priority) beanstalk.close() return jobid
def msbox_insert(srcfile, jobuuid, md5hash, sha1hash, sha256hash, fileext, target, msversion, size, runtime=180, priority=beanstalkc.DEFAULT_PRIORITY): #all paths in lowercase md5hash = md5hash.lower() sha1hash = sha1hash.lower() sha256hash = sha256hash.lower() existingSample = msdb.findSample(sha256hash) logger.info("existing sample? %s" % (existingSample)) basefilename = os.path.basename(srcfile) destfile = None #we only save each unique sample once if (existingSample): logger.info("sample exists: %s" % existingSample) destfile = msdb.getExistingPath(existingSample) else: # copy file into malware storage directory; we save the original name in the db, but the stored file is by hash destfilepath = MALWARESTORAGEBASE + '/' + sha256hash destfile = destfilepath + '/' + sha256hash + ".apk" if not os.path.exists(destfile): os.umask(0) os.makedirs(destfilepath, mode=0733) shutil.copyfile(srcfile, destfile) #Extra check to make sure the file is there (ie. the copy worked) before we proceed if not os.path.exists(destfile): logger.info("copy failed!") return -1 os.chmod(destfile, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) #3.3+ only :-( shutil.chown(destfile, "root","apache") existingSample = msdb.insertSample(destfile, md5hash, sha1hash, sha256hash, size) if destfile == None: logger.info("path to malware is empty, exiting") return -1 if not existingSample: logger.info("malware id is empty, exiting") return -1 jobpath = MALWAREJOBSBASE + "/" + jobuuid if not os.path.exists(jobpath): os.umask(0) os.makedirs(jobpath, mode=0733) msdb.insertSubmission(basefilename, jobuuid, "not done", "NOT ANALYZED", time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime()), "127.0.0.2", existingSample, target, runtime) makefeeder(destfile, jobpath + "/" + sha256hash + ".xml") # Insert job into beanstalk so a vm can service it beanstalk = beanstalkc.Connection(host=BSHOST, port=BSPORT) beanstalk.use(BSMSBOXQ) logger.info("sdk target is " + target) jobstring = "{'jobuuid':'%s','md5':'%s','sha256':'%s','basename':'%s','ext':'%s', 'runtime':'%d', 'target':'%s', 'fullpath':'%s'}" % ( jobuuid, md5hash, sha256hash, basefilename, fileext, runtime, target, destfile) jobid = beanstalk.put(jobstring, priority=priority) beanstalk.close() logger.info("dir" + os.path.dirname(destfile)) #for DB testing, not for real ingest #time.sleep(3) #permissions_filename = sha256hash + ".xml" #image_used = "some vm" #start_time = time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime()) #msdb.updateStartRun(sha256hash,image_used,start_time,permissions_filename) #time.sleep(3) #complete_time = time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime()) #results_file = "some cool file" #msdb.updateFinishRun(sha256hash,complete_time,results_file) return jobid
def msbox_update(srcfile, jobuuid, md5hash, sha256hash, fileext, target, msversion, runtime=120, priority=beanstalkc.DEFAULT_PRIORITY): #all paths in lowercase existingSubmissionId = msdb.findSubmissionToUpdate(sha256hash) if not existingSubmissionId: logger.info("job doesn't exist!: %s" % sha256hash) return basefilename = os.path.basename(srcfile) # copy file into malware storage directory destfilepath = MALWARESTORAGEBASE + '/' + sha256hash destfile = destfilepath + '/' + sha256hash + ".apk" if not os.path.exists(destfile): os.umask(0) os.makedirs(destfilepath, mode=0733) shutil.copyfile(srcfile, destfile) #Extra check to make sure the file is there (ie. the copy worked) before we proceed if not os.path.exists(destfile): return -1 #apache can't thos this, but it doesn't need to because it's already apache # os.chmod(destfile,stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH) #3.3+ only :-( shutil.chown(destfile, "root","apache") else: msdb.updateSample(destfile, sha256hash) if destfile == None: logger.info("path to malware is empty, exiting") return -1 if not existingSubmissionId: logger.info("malware id is empty, exiting") return -1 jobpath = MALWAREJOBSBASE + "/" + jobuuid if not os.path.exists(jobpath): os.umask(0) os.makedirs(jobpath, mode=0733) makefeeder(destfile, jobpath + "/" + sha256hash + ".xml") # Insert job into beanstalk so a vm can service it beanstalk = beanstalkc.Connection(host=BSHOST, port=BSPORT) beanstalk.use(BSMSBOXQ) logger.info("target is" + target) jobstring = "{'jobuuid':'%s','md5':'%s','sha256':'%s','basename':'%s','ext':'%s', 'runtime':'%d', 'target':'%s', 'fullpath':'%s'}" % ( jobuuid, md5hash, sha256hash, basefilename, fileext, runtime, target, destfile) jobid = beanstalk.put(jobstring, priority=priority) beanstalk.close() msdb.updateSubmission(jobuuid, "not done", "NOT ANALYZED", time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime()), "127.0.0.2", existingSubmissionId, target, runtime) logger.info("dir" + os.path.dirname(destfile)) #for DB testing, not for real ingest #time.sleep(3) #permissions_filename = sha256hash + ".xml" #image_used = "some vm" #start_time = time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime()) #msdb.updateStartRun(sha256hash,image_used,start_time,permissions_filename) #time.sleep(3) #complete_time = time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime()) #results_file = "some cool file" #msdb.updateFinishRun(sha256hash,complete_time,results_file) return jobid
def msbox_update(srcfile, jobuuid, md5hash, sha256hash, fileext, target, msversion, runtime = 120, priority = beanstalkc.DEFAULT_PRIORITY): #all paths in lowercase existingSubmissionId = msdb.findSubmissionToUpdate(sha256hash); if not existingSubmissionId: logger.info( "job doesn't exist!: %s" % sha256hash) return basefilename = os.path.basename(srcfile) # copy file into malware storage directory destfilepath = MALWARESTORAGEBASE + '/' + sha256hash destfile = destfilepath + '/' + sha256hash + ".apk" if not os.path.exists(destfile): os.umask(0) os.makedirs(destfilepath, mode = 0733) shutil.copyfile(srcfile, destfile) #Extra check to make sure the file is there (ie. the copy worked) before we proceed if not os.path.exists(destfile): return -1 #apache can't thos this, but it doesn't need to because it's already apache # os.chmod(destfile,stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH) #3.3+ only :-( shutil.chown(destfile, "root","apache") else: msdb.updateSample(destfile,sha256hash) if destfile == None: logger.info( "path to malware is empty, exiting") return -1 if not existingSubmissionId: logger.info( "malware id is empty, exiting") return -1 jobpath = MALWAREJOBSBASE +"/" + jobuuid if not os.path.exists(jobpath): os.umask(0) os.makedirs(jobpath, mode = 0733) makefeeder(destfile,jobpath +"/" + sha256hash + ".xml") # Insert job into beanstalk so a vm can service it beanstalk = beanstalkc.Connection(host=BSHOST, port=BSPORT) beanstalk.use(BSMSBOXQ) logger.info( "target is" + target) jobstring = "{'jobuuid':'%s','md5':'%s','sha256':'%s','basename':'%s','ext':'%s', 'runtime':'%d', 'target':'%s', 'fullpath':'%s'}" % (jobuuid, md5hash, sha256hash, basefilename, fileext, runtime, target,destfile) jobid = beanstalk.put(jobstring, priority = priority) beanstalk.close() msdb.updateSubmission(jobuuid,"not done","NOT ANALYZED",time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime()),"127.0.0.2",existingSubmissionId,target,runtime) logger.info( "dir" + os.path.dirname(destfile)) #for DB testing, not for real ingest #time.sleep(3) #permissions_filename = sha256hash + ".xml" #image_used = "some vm" #start_time = time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime()) #msdb.updateStartRun(sha256hash,image_used,start_time,permissions_filename) #time.sleep(3) #complete_time = time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime()) #results_file = "some cool file" #msdb.updateFinishRun(sha256hash,complete_time,results_file) return jobid
def msbox_insert( srcfile, jobuuid, md5hash, sha1hash, sha256hash, fileext, target, msversion, size, runtime=180, priority=beanstalkc.DEFAULT_PRIORITY, ): # all paths in lowercase md5hash = md5hash.lower() sha1hash = sha1hash.lower() sha256hash = sha256hash.lower() existingSample = msdb.findSample(sha256hash) logger.info("existing sample? %s" % (existingSample)) basefilename = os.path.basename(srcfile) destfile = None # we only save each unique sample once if existingSample: logger.info("sample exists: %s" % existingSample) destfile = msdb.getExistingPath(existingSample) else: # copy file into malware storage directory; we save the original name in the db, but the stored file is by hash destfilepath = MALWARESTORAGEBASE + "/" + sha256hash destfile = destfilepath + "/" + sha256hash + ".apk" if not os.path.exists(destfile): os.umask(0) os.makedirs(destfilepath, mode=0733) shutil.copyfile(srcfile, destfile) # Extra check to make sure the file is there (ie. the copy worked) before we proceed if not os.path.exists(destfile): logger.info("copy failed!") return -1 os.chmod(destfile, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) # 3.3+ only :-( shutil.chown(destfile, "root","apache") existingSample = msdb.insertSample(destfile, md5hash, sha1hash, sha256hash, size) if destfile == None: logger.info("path to malware is empty, exiting") return -1 if not existingSample: logger.info("malware id is empty, exiting") return -1 jobpath = MALWAREJOBSBASE + "/" + jobuuid if not os.path.exists(jobpath): os.umask(0) os.makedirs(jobpath, mode=0733) msdb.insertSubmission( basefilename, jobuuid, "not done", "NOT ANALYZED", time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime()), "127.0.0.2", existingSample, target, runtime, ) makefeeder(destfile, jobpath + "/" + sha256hash + ".xml") # Insert job into beanstalk so a vm can service it beanstalk = beanstalkc.Connection(host=BSHOST, port=BSPORT) beanstalk.use(BSMSBOXQ) logger.info("sdk target is " + target) jobstring = ( "{'jobuuid':'%s','md5':'%s','sha256':'%s','basename':'%s','ext':'%s', 'runtime':'%d', 'target':'%s', 'fullpath':'%s'}" % (jobuuid, md5hash, sha256hash, basefilename, fileext, runtime, target, destfile) ) jobid = beanstalk.put(jobstring, priority=priority) beanstalk.close() logger.info("dir" + os.path.dirname(destfile)) # for DB testing, not for real ingest # time.sleep(3) # permissions_filename = sha256hash + ".xml" # image_used = "some vm" # start_time = time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime()) # msdb.updateStartRun(sha256hash,image_used,start_time,permissions_filename) # time.sleep(3) # complete_time = time.strftime("%Y-%m-%d-%H:%M:%S", time.gmtime()) # results_file = "some cool file" # msdb.updateFinishRun(sha256hash,complete_time,results_file) return jobid