def get_malware(q, dumpdir): while True: url = q.get() logging.info("Fetched URL %s from queue", url) mal = get_URL(url) if mal: malfile = mal.read() md5 = hashlib.md5(malfile).hexdigest() # Is this a big race condition problem? if md5 not in hashes: logging.info("Found file %s at URL %s", md5, url) logging.debug("Going to put file in directory %s", dumpdir) # store the file and log the data with open(os.path.join(dumpdir, md5), 'wb') as f: f.write(malfile) logging.info("Stored %s in %s", md5, dumpdir) hashes.add(md5) pasturls.add(url) if args.cuckoo: f = open(os.path.join(dumpdir, md5), 'rb') form = MultiPartForm() form.add_file('file', md5, fileHandle=f) request = urllib2.Request( 'http://localhost:8090/tasks/create/file') request.add_header('User-agent', 'Maltrieve') body = str(form) request.add_header('Content-type', form.get_content_type()) request.add_header('Content-length', len(body)) request.add_data(body) response = urllib2.urlopen(request).read() responsedata = json.loads(response) logging.info("Submitted %s to cuckoo, task ID %s", md5, responsedata["task_id"]) q.task_done()
def get_malware(q, dumpdir): while True: url = q.get() logging.info("Fetched URL %s from queue", url) logging.info("%s items remaining in queue", q.qsize()) mal = get_URL(url) if mal: malfile = mal.read() md5 = hashlib.md5(malfile).hexdigest() # Is this a big race condition problem? if md5 not in hashes: logging.info("Found file %s at URL %s", md5, url) logging.debug("Going to put file in directory %s", dumpdir) # see http://stackoverflow.com/a/5032238 # may resolve issue #21 if not os.path.isdir(dumpdir): try: logging.info("Creating dumpdir %s", dumpdir) os.makedirs(dumpdir) except OSError as exception: if exception.errno != errno.EEXIST: raise # store the file and log the data with open(os.path.join(dumpdir, md5), 'wb') as f: f.write(malfile) logging.info("Stored %s in %s", md5, dumpdir) if args.vxcage: if os.path.exists(os.path.join(dumpdir, md5)): f = open(os.path.join(dumpdir, md5), 'rb') form = MultiPartForm() form.add_file('file', md5, fileHandle=f) form.add_field('tags', 'maltrieve') request = urllib2.Request( 'http://localhost:8080/malware/add') request.add_header('User-agent', 'Maltrieve') body = str(form) request.add_header('Content-type', form.get_content_type()) request.add_header('Content-length', len(body)) request.add_data(body) try: response = urllib2.urlopen(request).read() except: logging.info("Exception caught from VxCage") responsedata = json.loads(response) logging.info("Submitted %s to VxCage, response was %s", md5, responsedata["message"]) logging.info( "Deleting file as it has been uploaded to VxCage") try: os.remove(os.path.join(dumpdir, md5)) except: logging.info( "Exception when attempting to delete file: %s", os.path.join(dumpdir, md5)) if args.cuckoo: f = open(os.path.join(dumpdir, md5), 'rb') form = MultiPartForm() form.add_file('file', md5, fileHandle=f) request = urllib2.Request( 'http://localhost:8090/tasks/create/file') request.add_header('User-agent', 'Maltrieve') body = str(form) request.add_header('Content-type', form.get_content_type()) request.add_header('Content-length', len(body)) request.add_data(body) response = urllib2.urlopen(request).read() responsedata = json.loads(response) logging.info("Submitted %s to cuckoo, task ID %s", md5, responsedata["task_id"]) hashes.add(md5) q.task_done()