def file_hashes(filename, algo="sha256", blocksize=65536): if filename and filename != "": file_handle = open(filename, "rb") data = file_handle.read(blocksize) if algo == "crc32": return int("%d" % (zlib.crc32(open(filename, "rb").read()) & 0xffffffff)) elif algo == "adler32": return "%d" % (zlib.adler32(open(filename, "rb").read()) & 0xffffffff) elif algo == "md5": hasher = hashlib.md5() elif algo == "sha1": hasher = hashlib.sha1() elif algo == "sha224": hasher = hashlib.sha224() elif algo == "sha256": hasher = hashlib.sha256() elif algo == "sha384": hasher = hashlib.sha384() elif algo == "sha512": hasher = hashlib.sha512() elif algo == "ssdeep": return pydeep.hash_file(filename) else: return None while len(data) > 0: hasher.update(data) data = file_handle.read(blocksize) return hasher.hexdigest()
def run(self): results = {} results["magic"] = magic.from_file(self.filepath) results["mimetype"] = magic.from_file(self.filepath, mime=True) binary = get_binary(self.job_id) results["md5"] = hashlib.md5(binary).hexdigest() results["sha1"] = hashlib.sha1(binary).hexdigest() results["sha256"] = hashlib.sha256(binary).hexdigest() results["ssdeep"] = pydeep.hash_file(self.filepath).decode() try: with ExifTool(self.exiftool_path) as et: exif_report = et.execute_json(self.filepath) if exif_report: exif_single_report = exif_report[0] exif_report_cleaned = { key: value for key, value in exif_single_report.items() if not (key.startswith("File") or key.startswith("SourceFile")) } # compatibility with the previous version of this analyzer results["filetype"] = exif_single_report.get("File:FileType", "") results["exiftool"] = exif_report_cleaned except Exception as e: logger.exception(e) return results
def file_hashes(filename, algo='sha256', blocksize=65536): file_handle = open(filename, 'rb') buf = file_handle.read(blocksize) if algo == 'crc32': return "%X" % (zlib.crc32(open(filename,"rb").read()) & 0xffffffff) elif algo == 'adler32': return "%X" % (zlib.adler32(open(filename,"rb").read()) & 0xffffffff) elif algo == 'md5': hasher = hashlib.md5() elif algo == 'sha1': hasher = hashlib.sha1() elif algo == 'sha224': hasher = hashlib.sha224() elif algo == 'sha256': hasher = hashlib.sha256() elif algo == 'sha384': hasher = hashlib.sha384() elif algo == 'sha512': hasher = hashlib.sha512() elif algo == 'ssdeep': return pydeep.hash_file(filename) else: return None while len(buf) > 0: hasher.update(buf) buf = file_handle.read(blocksize) return hasher.hexdigest()
def hashfile(path, targetfile): '''This function returns a hash from a given file.''' md5 = hashlib.md5() sha1 = hashlib.sha1() sha256 = hashlib.sha256() sha512 = hashlib.sha512() fullfile = path + "/" + targetfile with open(fullfile, 'rb') as f: while True: data = f.read(BUF_SIZE) if not data: break md5.update(data) sha1.update(data) sha256.update(data) sha512.update(data) hdict = { 'fileformat': magic.from_file(fullfile, mime=True), 'filename': str(targetfile), 'filesize': os.path.getsize(fullfile), 'md5': md5.hexdigest(), 'sha1': sha1.hexdigest(), 'sha256': sha256.hexdigest(), 'sha512': sha512.hexdigest(), 'ssdeep': pydeep.hash_file(fullfile) } return hdict
def buf_hashes(buf, algo='sha256'): if not buf: return None if algo == 'crc32': return "%X" % (zlib.crc32(buf) & 0xffffffff) elif algo == 'adler32': return "%X" % (zlib.adler32(buf) & 0xffffffff) elif algo == 'md5': hasher = hashlib.md5() elif algo == 'sha1': hasher = hashlib.sha1() elif algo == 'sha224': hasher = hashlib.sha224() elif algo == 'sha256': hasher = hashlib.sha256() elif algo == 'sha384': hasher = hashlib.sha384() elif algo == 'sha512': hasher = hashlib.sha512() elif algo == 'ssdeep': return pydeep.hash_file(filename) else: return None hasher.update(buf) return hasher.hexdigest()
def get_ssdeep(self): if not HAVE_SSDEEP: return None try: return pydeep.hash_file(self.file_path) except Exception: return None
def get_ssdeep(self): if not HAVE_SSDEEP: return '' try: return pydeep.hash_file(self.path).decode() except Exception: return ''
def get_ssdeep(self): if not HAVE_SSDEEP: return '' try: return pydeep.hash_file(self.path) except Exception: return ''
def test_pydeep(): for test in testL: filename, filelen, filehash = test data = io.open(filename, 'rb').read() hash_buf = pydeep.hash_buf(data) hash_file = pydeep.hash_file(filename) assert len(data) == filelen, "File length error" assert hash_buf == filehash, "Error hashing %s using hash_buf"%filename assert hash_file == filehash, "Error hashing %s using hash_file"%filename
def get_ssdeep(self): """ Obtem o hash SSDEEP do arquivo """ if not HAVE_SSDEEP: return '' if self.file_data is None: return '' try: return pydeep.hash_file(self.file_path) except Exception: return ''
def ssdeep(self, args, file, opts): document = db.file_collection.select(file.sha256_digest) if 'ssdeep' not in document: fuzzy = str(pydeep.hash_file(file.file_path), encoding="utf-8") data = {'ssdeep': fuzzy} if not db.file_collection.update(file.sha256_digest, data): raise error.MongoError( 'error adding ssdeep hash into file document %s' % file.sha256_digest) document = db.file_collection.select(file.sha256_digest) return {'ssdeep': document['ssdeep']}
def get_ssdeep(self): if not HAVE_SSDEEP: return None try: if Config().api.use_aws: return pydeep.hash_buffer(self.data) else: return pydeep.hash_file(self.path) except Exception: return None
def get_ssdeep(self): """Get SSDEEP. @return: SSDEEP. """ if not HAVE_SSDEEP: return None try: return pydeep.hash_file(self.file_path) except Exception: return None
def get_ssdeep(self): """Get SSDEEP. @return: SSDEEP. """ if not HAVE_PYDEEP: if not File.notified_pydeep: File.notified_pydeep = True log.warning("Unable to import pydeep (install with `pip install pydeep`)") return None try: return pydeep.hash_file(self.file_path) except Exception: return None
def run(analyzer_name, job_id, filepath, filename, md5, additional_config_params): logger.info("started analyzer {} job_id {}" "".format(analyzer_name, job_id)) report = general.get_basic_report_template(analyzer_name) try: results = {} results["magic"] = magic.from_file(filepath) results["mimetype"] = magic.from_file(filepath, mime=True) results["filetype"] = pyexifinfo.fileType(filepath) exif_report = pyexifinfo.get_json(filepath) if exif_report: exif_report_cleaned = { key: value for key, value in exif_report[0].items() if not (key.startswith("File") or key.startswith("SourceFile")) } results["exiftool"] = exif_report_cleaned binary = general.get_binary(job_id) results["md5"] = hashlib.md5(binary).hexdigest() results["sha1"] = hashlib.sha1(binary).hexdigest() results["sha256"] = hashlib.sha256(binary).hexdigest() results["ssdeep"] = pydeep.hash_file(filepath).decode() report["report"] = results except AnalyzerRunException as e: error_message = ( "job_id:{} analyzer:{} md5:{} filename: {} Analyzer Error {}" "".format(job_id, analyzer_name, md5, filename, e)) logger.error(error_message) report["errors"].append(error_message) report["success"] = False except Exception as e: traceback.print_exc() error_message = ( "job_id:{} analyzer:{} md5:{} filename: {} Unexpected Error {}" "".format(job_id, analyzer_name, md5, filename, e)) logger.exception(error_message) report["errors"].append(str(e)) report["success"] = False else: report["success"] = True general.set_report_and_cleanup(job_id, report) logger.info("ended analyzer {} job_id {}" "".format(analyzer_name, job_id)) return report
def get_ssdeep(self): """Get SSDEEP. @return: SSDEEP. """ if not HAVE_PYDEEP: if not File.notified_pydeep: File.notified_pydeep = True log.warning("Unable to import pydeep (install with `pip3 install pydeep`)") return None try: return pydeep.hash_file(self.file_path).decode("utf-8") except Exception: return None
def analyze(self, config, filename): """Analyze the file.""" # sanity check to make sure we can run if self.is_activated == False: return False log = logging.getLogger('Mastiff.Plugins.' + self.name) log.info('Starting execution.') log.info('Generating fuzzy hash.') try: my_fuzzy = pydeep.hash_file(filename) except pydeep.error, err: log.error('Could not generate fuzzy hash: %s', err) return False
def get_PE_Hashes(pe, filename): sdhash = None imph = pe.get_imphash() my_fuzzy = pydeep.hash_file(filename) pehash = peHash.get_peHash(filename) fh = open(filename, 'rb') m = hashlib.md5() d = fh.read() sdhash = fuzzyhashlib.sdhash(d).hexdigest() md5 = hashlib.md5(d).hexdigest() slashed = filename.split('/') filename = slashed[len(slashed) - 1] hashes ={"Filename":filename, "MD5":md5,"Imphash":imph,\ "peHash":pehash,"Fuzzy Hash": my_fuzzy, "sdhash": sdhash} return hashes
def run(self): results = {} results["magic"] = magic.from_file(self.filepath) results["mimetype"] = magic.from_file(self.filepath, mime=True) results["filetype"] = pyexifinfo.fileType(self.filepath) exif_report = pyexifinfo.get_json(self.filepath) if exif_report: exif_report_cleaned = { key: value for key, value in exif_report[0].items() if not (key.startswith("File") or key.startswith("SourceFile")) } results["exiftool"] = exif_report_cleaned binary = get_binary(self.job_id) results["md5"] = hashlib.md5(binary).hexdigest() results["sha1"] = hashlib.sha1(binary).hexdigest() results["sha256"] = hashlib.sha256(binary).hexdigest() results["ssdeep"] = pydeep.hash_file(self.filepath).decode() return results
def get_PE_Hashes(pe, filename): imph = pe.get_imphash() resfuzzy = "" for section in pe.sections: scn = section.Name if scn == ".rsrc": resfuzzy = section.get_hash_ssdeep() my_fuzzy = pydeep.hash_file(filename) pehash = peHash.get_peHash(filename) fh = open(filename, 'rb') m = hashlib.md5() s = hashlib.sha1() s256 = hashlib.sha256() while True: data = fh.read(8192) if not data: break m.update(data) s.update(data) s256.update(data) md5 = m.hexdigest() sha1 = s.hexdigest() sha256 = s256.hexdigest() slashed = filename.split('/') filename = slashed[len(slashed) - 1] hashes = { "Filename": filename, "MD5": md5, "SHA1": sha1, "SHA256": sha256, "Imphash": imph, "peHash": pehash, "Fuzzy Hash": my_fuzzy, "ResFuzzy": resfuzzy } return hashes
def get_ssdeep(path): try: return pydeep.hash_file(path) except Exception: return ''
def ssdeep(filename): return pydeep.hash_file(filename)
#!/usr/bin/env python #ssdeep insto hashin'! - since ssdeep is natively an unwieldy PITA. #https://pypi.python.org/pypi/pydeep #PhG import sys, pydeep pd1 = pydeep.hash_file(sys.argv[1]) pd2 = pydeep.hash_file(sys.argv[2]) percent = str(pydeep.compare(pd1, pd2)) print "SSDeep has determined that these files are " + percent +"% alike." #pd0 = pydeep.hash_file("VirusShare_ab208f0b517ba9850f1551c9555b5313") #pd1 = pydeep.hash_file("VirusShare_6570163cd34454b3d1476c134d44b9d9")
#!/usr/bin/env python #ssdeep massive comparision engine (fancy way of saying... shitty script) #there's no reason to do this... idk wtf I was doing. This makes no sense. Broken. import sys import pydeep pd0 = pydeep.hash_file(sys.argv[1]) pd1 = sys.argv[2] #Import our ssdeep file. def deepImp(pd1) : deepDict = [] with open(pd1, 'r') as deepFiles: for row in deepFiles: deepDict.append(row[1]) return ssdeepDir deepImpList = deepImp(pd1) x = str(pydeep.compare(pd0, pd1)) print "SSDeep has determined that the DeepImportHash between" + (sys.argv[1]) + " and " + deepImpList + " are " + x +"% alike."
def ssdeepMe(path): if path is not None: return pydeep.hash_file(path) else: print("Ssdeep: provide a file path") return ""
def get_ssdeep(malware_path): return pydeep.hash_file(malware_path)
def getSsdeep(path): try: import pydeep except ImportError as exc: return "pydeep:" + str(exc) return pydeep.hash_file(path)
def ssdeep(self): return pydeep.hash_file(self.path)
def each(self, target): self.results = dict() h = pydeep.hash_file(target) self.results['ssdeep'] = h return True
args = parser.parse_args() tohash = args.filepath md5hash = hashlib.md5() sha1hash = hashlib.sha1() sha256hash = hashlib.sha256() try: with open(tohash, 'rb') as afile: buf = afile.read() md5hash.update(buf) sha1hash.update(buf) sha256hash.update(buf) afile.close() ssdeephash = pydeep.hash_file(tohash) print "md5 hash:", md5hash.hexdigest() print "sha1 hash:",sha1hash.hexdigest() print "sha256 hash:",sha256hash.hexdigest() print "ssdeep hash:",ssdeephash try: url = "https://www.virustotal.com/vtapi/v2/file/report" parameters = {"resource": sha256hash.hexdigest(),"apikey": "<your_vt_api_key_here>"} data = urllib.urlencode(parameters) req = urllib2.Request(url, data) response = urllib2.urlopen(req) json = json.loads(response.read()) count = 0
def ssdeep_cluster(root_paths, recursive=False, dontcompute=False, calculate_sha256=False, should_print=False, score_threshold=0): paths = enumerate_paths(root_paths, recursive) hashes = {} sha256s = {} integerdb = {} matches = {} scores = {} def add_to_integer_db(block_size, chunk, path): if block_size not in integerdb: integerdb[block_size] = {} similar_to = set() for i in chunk: if i not in integerdb[block_size]: integerdb[block_size][i] = set() else: similar_to |= integerdb[block_size][i] integerdb[block_size][i].add(path) return similar_to if dontcompute: real_paths = set() for path in paths: with open(path, "r") as f: for line in f: line = line.strip() if len(line) == 0: continue real_paths.add(line) paths = list(real_paths) for path in paths: if not dontcompute: hashes[path] = pydeep.hash_file(path) if calculate_sha256: sha256s[path] = hashlib.sha256(file(path, 'rb').read()).hexdigest() else: if "," in path: shash, path = path.split(",", 1) path = path.strip('"') else: shash = path hashes[path] = shash if calculate_sha256: sha256s[path] = \ hashlib.sha256(file(path, 'rb').read()).hexdigest() if isfile(path)\ else hashlib.sha256(path).hexdigest() block_size, chunk, double_chunk = preprocess_hash(hashes[path]) similar_to = add_to_integer_db(block_size, chunk, path) | add_to_integer_db( block_size * 2, double_chunk, path) h = hashes[path] matches[path] = set() for other in similar_to: score = pydeep.compare(h, hashes[other]) if score > score_threshold: matches[path].add(other) matches[other].add(path) if path not in scores: scores[path] = {} if other not in scores[path]: scores[path][other] = score if other not in scores: scores[other] = {} if path not in scores[other]: scores[other][path] = score if should_print: print "{0}\tSHA256: {1}\tssdeep: {2}".format( path, sha256s.get(path), hashes[path]) groups = [] for path in matches.keys(): in_a_group = False for g in xrange(len(groups)): if path in groups[g]: in_a_group = True continue should_add = True for h in groups[g]: if h not in matches[path]: should_add = False if should_add: groups[g].append(path) in_a_group = True if not in_a_group: groups.append([path]) for g in xrange(len(groups)): groups[g].sort() return groups, hashes, scores, sha256s
def get_ssdeep(self): try: return pydeep.hash_file(self.file_path) except Exception: return None
import pydeep file1 = 'calc.exe' file2 = 'notepad.exe' file1hash = '1536:JEl14rQcWAkN7GAlqbkfAGQGV8aMbrNyrf1w+noPvLV6eBsCXKc:JYmZWXyaiedMbrN6pnoXL1BsC' file2hash = '1536:0awOnbNQKLjWDyy1o5RefYMJUEbooPRrKKRl1P3:0YNQKPWDyDRefVJltZrpRl1P3' data1 = open(file1).read() data2 = open(file2).read() assert len(data1) == 114688, "File length error" assert len(data2) == 69120, "File lendth error" hash01 = pydeep.hash_buf(data1) hash02 = pydeep.hash_buf(data2) assert hash01 == file1hash, "Error hashing file1" assert hash02 == file2hash, "Error hashing file2" hash1 = pydeep.hash_file(file1) hash2 = pydeep.hash_file(file2) assert hash1 == file1hash, "Error hashing file1" assert hash2 == file2hash, "Error hashing file2" assert pydeep.compare(hash1, hash2) == 0, "Error fuzzy compare value" print 'Stuff looks fine..'
import pydeep file1 = 'calc.exe' file2 = 'notepad.exe' file3 = 'bc' file1hash = '1536:JEl14rQcWAkN7GAlqbkfAGQGV8aMbrNyrf1w+noPvLV6eBsCXKc:JYmZWXyaiedMbrN6pnoXL1BsC' file2hash = '1536:0awOnbNQKLjWDyy1o5RefYMJUEbooPRrKKRl1P3:0YNQKPWDyDRefVJltZrpRl1P3' file3hash = '1536:MsjYdR3Bul8hcURWhEcg4/btZzDcQflbCUPEBEh8wkcGDioxMYeo7:TYf8l8htRWA4ztZsGlWUPEBEh8wmxMYe' data1 = open(file1).read() data2 = open(file2).read() data3 = open(file3).read() assert len(data1) == 114688, "File length error" assert len(data2) == 69120, "File length error" assert len(data3) == 77168, "File length error" hash01 = pydeep.hash_buf(data1) hash02 = pydeep.hash_buf(data2) hash03 = pydeep.hash_buf(data3) assert hash01 == file1hash, "Error hashing file1" assert hash02 == file2hash, "Error hashing file2" assert hash03 == file3hash, "Error hashing file2" hash1 = pydeep.hash_file(file1) hash2 = pydeep.hash_file(file2) hash3 = pydeep.hash_file(file3) assert hash1 == file1hash, "Error hashing file1" assert hash2 == file2hash, "Error hashing file2" assert hash3 == file3hash, "Error hashing file3" assert pydeep.compare(hash1,hash2) == 0, "Error fuzzy compare value" print 'Stuff looks fine..'
def getSsdeep(path): return pydeep.hash_file(path)