def file_hashes(filename, algo="sha256", blocksize=65536):
if filename and filename != "":
file_handle = open(filename, "rb")
data = file_handle.read(blocksize)
if algo == "crc32":
return int("%d" %
(zlib.crc32(open(filename, "rb").read()) & 0xffffffff))
elif algo == "adler32":
return "%d" % (zlib.adler32(open(filename, "rb").read())
& 0xffffffff)
elif algo == "md5":
hasher = hashlib.md5()
elif algo == "sha1":
hasher = hashlib.sha1()
elif algo == "sha224":
hasher = hashlib.sha224()
elif algo == "sha256":
hasher = hashlib.sha256()
elif algo == "sha384":
hasher = hashlib.sha384()
elif algo == "sha512":
hasher = hashlib.sha512()
elif algo == "ssdeep":
return pydeep.hash_file(filename)
else:
return None
while len(data) > 0:
hasher.update(data)
data = file_handle.read(blocksize)
return hasher.hexdigest()
def run(self):
results = {}
results["magic"] = magic.from_file(self.filepath)
results["mimetype"] = magic.from_file(self.filepath, mime=True)
binary = get_binary(self.job_id)
results["md5"] = hashlib.md5(binary).hexdigest()
results["sha1"] = hashlib.sha1(binary).hexdigest()
results["sha256"] = hashlib.sha256(binary).hexdigest()
results["ssdeep"] = pydeep.hash_file(self.filepath).decode()
try:
with ExifTool(self.exiftool_path) as et:
exif_report = et.execute_json(self.filepath)
if exif_report:
exif_single_report = exif_report[0]
exif_report_cleaned = {
key: value
for key, value in exif_single_report.items()
if not (key.startswith("File") or key.startswith("SourceFile"))
}
# compatibility with the previous version of this analyzer
results["filetype"] = exif_single_report.get("File:FileType", "")
results["exiftool"] = exif_report_cleaned
except Exception as e:
logger.exception(e)
return results
def file_hashes(filename, algo='sha256', blocksize=65536):
file_handle = open(filename, 'rb')
buf = file_handle.read(blocksize)
if algo == 'crc32':
return "%X" % (zlib.crc32(open(filename,"rb").read()) & 0xffffffff)
elif algo == 'adler32':
return "%X" % (zlib.adler32(open(filename,"rb").read()) & 0xffffffff)
elif algo == 'md5':
hasher = hashlib.md5()
elif algo == 'sha1':
hasher = hashlib.sha1()
elif algo == 'sha224':
hasher = hashlib.sha224()
elif algo == 'sha256':
hasher = hashlib.sha256()
elif algo == 'sha384':
hasher = hashlib.sha384()
elif algo == 'sha512':
hasher = hashlib.sha512()
elif algo == 'ssdeep':
return pydeep.hash_file(filename)
else:
return None
while len(buf) > 0:
hasher.update(buf)
buf = file_handle.read(blocksize)
return hasher.hexdigest()
def hashfile(path, targetfile):
'''This function returns a hash from a given file.'''
md5 = hashlib.md5()
sha1 = hashlib.sha1()
sha256 = hashlib.sha256()
sha512 = hashlib.sha512()
fullfile = path + "/" + targetfile
with open(fullfile, 'rb') as f:
while True:
data = f.read(BUF_SIZE)
if not data:
break
md5.update(data)
sha1.update(data)
sha256.update(data)
sha512.update(data)
hdict = {
'fileformat': magic.from_file(fullfile, mime=True),
'filename': str(targetfile),
'filesize': os.path.getsize(fullfile),
'md5': md5.hexdigest(),
'sha1': sha1.hexdigest(),
'sha256': sha256.hexdigest(),
'sha512': sha512.hexdigest(),
'ssdeep': pydeep.hash_file(fullfile)
}
return hdict
def buf_hashes(buf, algo='sha256'):
if not buf:
return None
if algo == 'crc32':
return "%X" % (zlib.crc32(buf) & 0xffffffff)
elif algo == 'adler32':
return "%X" % (zlib.adler32(buf) & 0xffffffff)
elif algo == 'md5':
hasher = hashlib.md5()
elif algo == 'sha1':
hasher = hashlib.sha1()
elif algo == 'sha224':
hasher = hashlib.sha224()
elif algo == 'sha256':
hasher = hashlib.sha256()
elif algo == 'sha384':
hasher = hashlib.sha384()
elif algo == 'sha512':
hasher = hashlib.sha512()
elif algo == 'ssdeep':
return pydeep.hash_file(filename)
else:
return None
hasher.update(buf)
return hasher.hexdigest()
def hashfile(path, targetfile):
'''This function returns a hash from a given file.'''
md5 = hashlib.md5()
sha1 = hashlib.sha1()
sha256 = hashlib.sha256()
sha512 = hashlib.sha512()
fullfile = path + "/" + targetfile
with open(fullfile, 'rb') as f:
while True:
data = f.read(BUF_SIZE)
if not data:
break
md5.update(data)
sha1.update(data)
sha256.update(data)
sha512.update(data)
hdict = {
'fileformat': magic.from_file(fullfile, mime=True),
'filename': str(targetfile),
'filesize': os.path.getsize(fullfile),
'md5': md5.hexdigest(),
'sha1': sha1.hexdigest(),
'sha256': sha256.hexdigest(),
'sha512': sha512.hexdigest(),
'ssdeep': pydeep.hash_file(fullfile)
}
return hdict
def get_ssdeep(self):
if not HAVE_SSDEEP:
return None
try:
return pydeep.hash_file(self.file_path)
except Exception:
return None
def get_ssdeep(self):
if not HAVE_SSDEEP:
return ''
try:
return pydeep.hash_file(self.path).decode()
except Exception:
return ''
def get_ssdeep(self):
if not HAVE_SSDEEP:
return ''
try:
return pydeep.hash_file(self.path)
except Exception:
return ''
def test_pydeep():
for test in testL:
filename, filelen, filehash = test
data = io.open(filename, 'rb').read()
hash_buf = pydeep.hash_buf(data)
hash_file = pydeep.hash_file(filename)
assert len(data) == filelen, "File length error"
assert hash_buf == filehash, "Error hashing %s using hash_buf"%filename
assert hash_file == filehash, "Error hashing %s using hash_file"%filename
def get_ssdeep(self):
""" Obtem o hash SSDEEP do arquivo """
if not HAVE_SSDEEP:
return ''
if self.file_data is None:
return ''
try:
return pydeep.hash_file(self.file_path)
except Exception:
return ''
def ssdeep(self, args, file, opts):
document = db.file_collection.select(file.sha256_digest)
if 'ssdeep' not in document:
fuzzy = str(pydeep.hash_file(file.file_path), encoding="utf-8")
data = {'ssdeep': fuzzy}
if not db.file_collection.update(file.sha256_digest, data):
raise error.MongoError(
'error adding ssdeep hash into file document %s' %
file.sha256_digest)
document = db.file_collection.select(file.sha256_digest)
return {'ssdeep': document['ssdeep']}
def get_ssdeep(self):
if not HAVE_SSDEEP:
return None
try:
if Config().api.use_aws:
return pydeep.hash_buffer(self.data)
else:
return pydeep.hash_file(self.path)
except Exception:
return None
def get_ssdeep(self):
"""Get SSDEEP.
@return: SSDEEP.
"""
if not HAVE_SSDEEP:
return None
try:
return pydeep.hash_file(self.file_path)
except Exception:
return None
def get_ssdeep(self):
"""Get SSDEEP.
@return: SSDEEP.
"""
if not HAVE_PYDEEP:
if not File.notified_pydeep:
File.notified_pydeep = True
log.warning("Unable to import pydeep (install with `pip install pydeep`)")
return None
try:
return pydeep.hash_file(self.file_path)
except Exception:
return None
def run(analyzer_name, job_id, filepath, filename, md5,
additional_config_params):
logger.info("started analyzer {} job_id {}"
"".format(analyzer_name, job_id))
report = general.get_basic_report_template(analyzer_name)
try:
results = {}
results["magic"] = magic.from_file(filepath)
results["mimetype"] = magic.from_file(filepath, mime=True)
results["filetype"] = pyexifinfo.fileType(filepath)
exif_report = pyexifinfo.get_json(filepath)
if exif_report:
exif_report_cleaned = {
key: value
for key, value in exif_report[0].items()
if not (key.startswith("File") or key.startswith("SourceFile"))
}
results["exiftool"] = exif_report_cleaned
binary = general.get_binary(job_id)
results["md5"] = hashlib.md5(binary).hexdigest()
results["sha1"] = hashlib.sha1(binary).hexdigest()
results["sha256"] = hashlib.sha256(binary).hexdigest()
results["ssdeep"] = pydeep.hash_file(filepath).decode()
report["report"] = results
except AnalyzerRunException as e:
error_message = (
"job_id:{} analyzer:{} md5:{} filename: {} Analyzer Error {}"
"".format(job_id, analyzer_name, md5, filename, e))
logger.error(error_message)
report["errors"].append(error_message)
report["success"] = False
except Exception as e:
traceback.print_exc()
error_message = (
"job_id:{} analyzer:{} md5:{} filename: {} Unexpected Error {}"
"".format(job_id, analyzer_name, md5, filename, e))
logger.exception(error_message)
report["errors"].append(str(e))
report["success"] = False
else:
report["success"] = True
general.set_report_and_cleanup(job_id, report)
logger.info("ended analyzer {} job_id {}" "".format(analyzer_name, job_id))
return report
def get_ssdeep(self):
"""Get SSDEEP.
@return: SSDEEP.
"""
if not HAVE_PYDEEP:
if not File.notified_pydeep:
File.notified_pydeep = True
log.warning("Unable to import pydeep (install with `pip3 install pydeep`)")
return None
try:
return pydeep.hash_file(self.file_path).decode("utf-8")
except Exception:
return None
def analyze(self, config, filename):
"""Analyze the file."""
# sanity check to make sure we can run
if self.is_activated == False:
return False
log = logging.getLogger('Mastiff.Plugins.' + self.name)
log.info('Starting execution.')
log.info('Generating fuzzy hash.')
try:
my_fuzzy = pydeep.hash_file(filename)
except pydeep.error, err:
log.error('Could not generate fuzzy hash: %s', err)
return False
def get_PE_Hashes(pe, filename):
sdhash = None
imph = pe.get_imphash()
my_fuzzy = pydeep.hash_file(filename)
pehash = peHash.get_peHash(filename)
fh = open(filename, 'rb')
m = hashlib.md5()
d = fh.read()
sdhash = fuzzyhashlib.sdhash(d).hexdigest()
md5 = hashlib.md5(d).hexdigest()
slashed = filename.split('/')
filename = slashed[len(slashed) - 1]
hashes ={"Filename":filename, "MD5":md5,"Imphash":imph,\
"peHash":pehash,"Fuzzy Hash": my_fuzzy, "sdhash": sdhash}
return hashes
def run(self):
results = {}
results["magic"] = magic.from_file(self.filepath)
results["mimetype"] = magic.from_file(self.filepath, mime=True)
results["filetype"] = pyexifinfo.fileType(self.filepath)
exif_report = pyexifinfo.get_json(self.filepath)
if exif_report:
exif_report_cleaned = {
key: value
for key, value in exif_report[0].items()
if not (key.startswith("File") or key.startswith("SourceFile"))
}
results["exiftool"] = exif_report_cleaned
binary = get_binary(self.job_id)
results["md5"] = hashlib.md5(binary).hexdigest()
results["sha1"] = hashlib.sha1(binary).hexdigest()
results["sha256"] = hashlib.sha256(binary).hexdigest()
results["ssdeep"] = pydeep.hash_file(self.filepath).decode()
return results
def get_PE_Hashes(pe, filename):
imph = pe.get_imphash()
resfuzzy = ""
for section in pe.sections:
scn = section.Name
if scn == ".rsrc":
resfuzzy = section.get_hash_ssdeep()
my_fuzzy = pydeep.hash_file(filename)
pehash = peHash.get_peHash(filename)
fh = open(filename, 'rb')
m = hashlib.md5()
s = hashlib.sha1()
s256 = hashlib.sha256()
while True:
data = fh.read(8192)
if not data:
break
m.update(data)
s.update(data)
s256.update(data)
md5 = m.hexdigest()
sha1 = s.hexdigest()
sha256 = s256.hexdigest()
slashed = filename.split('/')
filename = slashed[len(slashed) - 1]
hashes = {
"Filename": filename,
"MD5": md5,
"SHA1": sha1,
"SHA256": sha256,
"Imphash": imph,
"peHash": pehash,
"Fuzzy Hash": my_fuzzy,
"ResFuzzy": resfuzzy
}
return hashes
def get_ssdeep(path):
try:
return pydeep.hash_file(path)
except Exception:
return ''
def ssdeep(filename):
return pydeep.hash_file(filename)
#!/usr/bin/env python
#ssdeep insto hashin'! - since ssdeep is natively an unwieldy PITA.
#https://pypi.python.org/pypi/pydeep
#PhG
import sys, pydeep
pd1 = pydeep.hash_file(sys.argv[1])
pd2 = pydeep.hash_file(sys.argv[2])
percent = str(pydeep.compare(pd1, pd2))
print "SSDeep has determined that these files are " + percent +"% alike."
#pd0 = pydeep.hash_file("VirusShare_ab208f0b517ba9850f1551c9555b5313")
#pd1 = pydeep.hash_file("VirusShare_6570163cd34454b3d1476c134d44b9d9")
#!/usr/bin/env python
#ssdeep massive comparision engine (fancy way of saying... shitty script)
#there's no reason to do this... idk wtf I was doing. This makes no sense. Broken.
import sys
import pydeep
pd0 = pydeep.hash_file(sys.argv[1])
pd1 = sys.argv[2]
#Import our ssdeep file.
def deepImp(pd1) :
deepDict = []
with open(pd1, 'r') as deepFiles:
for row in deepFiles:
deepDict.append(row[1])
return ssdeepDir
deepImpList = deepImp(pd1)
x = str(pydeep.compare(pd0, pd1))
print "SSDeep has determined that the DeepImportHash between" + (sys.argv[1]) + " and " + deepImpList + " are " + x +"% alike."
def ssdeepMe(path):
if path is not None:
return pydeep.hash_file(path)
else:
print("Ssdeep: provide a file path")
return ""
def get_ssdeep(malware_path):
return pydeep.hash_file(malware_path)
def getSsdeep(path):
try:
import pydeep
except ImportError as exc:
return "pydeep:" + str(exc)
return pydeep.hash_file(path)
def ssdeep(self):
return pydeep.hash_file(self.path)
def each(self, target):
self.results = dict()
h = pydeep.hash_file(target)
self.results['ssdeep'] = h
return True
args = parser.parse_args()
tohash = args.filepath
md5hash = hashlib.md5()
sha1hash = hashlib.sha1()
sha256hash = hashlib.sha256()
try:
with open(tohash, 'rb') as afile:
buf = afile.read()
md5hash.update(buf)
sha1hash.update(buf)
sha256hash.update(buf)
afile.close()
ssdeephash = pydeep.hash_file(tohash)
print "md5 hash:", md5hash.hexdigest()
print "sha1 hash:",sha1hash.hexdigest()
print "sha256 hash:",sha256hash.hexdigest()
print "ssdeep hash:",ssdeephash
try:
url = "https://www.virustotal.com/vtapi/v2/file/report"
parameters = {"resource": sha256hash.hexdigest(),"apikey": "<your_vt_api_key_here>"}
data = urllib.urlencode(parameters)
req = urllib2.Request(url, data)
response = urllib2.urlopen(req)
json = json.loads(response.read())
count = 0
def ssdeep(filename): return pydeep.hash_file(filename)
def get_ssdeep(malware_path): return pydeep.hash_file(malware_path)
def ssdeep_cluster(root_paths,
recursive=False,
dontcompute=False,
calculate_sha256=False,
should_print=False,
score_threshold=0):
paths = enumerate_paths(root_paths, recursive)
hashes = {}
sha256s = {}
integerdb = {}
matches = {}
scores = {}
def add_to_integer_db(block_size, chunk, path):
if block_size not in integerdb:
integerdb[block_size] = {}
similar_to = set()
for i in chunk:
if i not in integerdb[block_size]:
integerdb[block_size][i] = set()
else:
similar_to |= integerdb[block_size][i]
integerdb[block_size][i].add(path)
return similar_to
if dontcompute:
real_paths = set()
for path in paths:
with open(path, "r") as f:
for line in f:
line = line.strip()
if len(line) == 0:
continue
real_paths.add(line)
paths = list(real_paths)
for path in paths:
if not dontcompute:
hashes[path] = pydeep.hash_file(path)
if calculate_sha256:
sha256s[path] = hashlib.sha256(file(path,
'rb').read()).hexdigest()
else:
if "," in path:
shash, path = path.split(",", 1)
path = path.strip('"')
else:
shash = path
hashes[path] = shash
if calculate_sha256:
sha256s[path] = \
hashlib.sha256(file(path, 'rb').read()).hexdigest() if isfile(path)\
else hashlib.sha256(path).hexdigest()
block_size, chunk, double_chunk = preprocess_hash(hashes[path])
similar_to = add_to_integer_db(block_size, chunk,
path) | add_to_integer_db(
block_size * 2, double_chunk, path)
h = hashes[path]
matches[path] = set()
for other in similar_to:
score = pydeep.compare(h, hashes[other])
if score > score_threshold:
matches[path].add(other)
matches[other].add(path)
if path not in scores:
scores[path] = {}
if other not in scores[path]:
scores[path][other] = score
if other not in scores:
scores[other] = {}
if path not in scores[other]:
scores[other][path] = score
if should_print:
print "{0}\tSHA256: {1}\tssdeep: {2}".format(
path, sha256s.get(path), hashes[path])
groups = []
for path in matches.keys():
in_a_group = False
for g in xrange(len(groups)):
if path in groups[g]:
in_a_group = True
continue
should_add = True
for h in groups[g]:
if h not in matches[path]:
should_add = False
if should_add:
groups[g].append(path)
in_a_group = True
if not in_a_group:
groups.append([path])
for g in xrange(len(groups)):
groups[g].sort()
return groups, hashes, scores, sha256s
def get_ssdeep(self):
try:
return pydeep.hash_file(self.file_path)
except Exception:
return None
import pydeep file1 = 'calc.exe' file2 = 'notepad.exe' file1hash = '1536:JEl14rQcWAkN7GAlqbkfAGQGV8aMbrNyrf1w+noPvLV6eBsCXKc:JYmZWXyaiedMbrN6pnoXL1BsC' file2hash = '1536:0awOnbNQKLjWDyy1o5RefYMJUEbooPRrKKRl1P3:0YNQKPWDyDRefVJltZrpRl1P3' data1 = open(file1).read() data2 = open(file2).read() assert len(data1) == 114688, "File length error" assert len(data2) == 69120, "File lendth error" hash01 = pydeep.hash_buf(data1) hash02 = pydeep.hash_buf(data2) assert hash01 == file1hash, "Error hashing file1" assert hash02 == file2hash, "Error hashing file2" hash1 = pydeep.hash_file(file1) hash2 = pydeep.hash_file(file2) assert hash1 == file1hash, "Error hashing file1" assert hash2 == file2hash, "Error hashing file2" assert pydeep.compare(hash1, hash2) == 0, "Error fuzzy compare value" print 'Stuff looks fine..'
import pydeep file1 = 'calc.exe' file2 = 'notepad.exe' file3 = 'bc' file1hash = '1536:JEl14rQcWAkN7GAlqbkfAGQGV8aMbrNyrf1w+noPvLV6eBsCXKc:JYmZWXyaiedMbrN6pnoXL1BsC' file2hash = '1536:0awOnbNQKLjWDyy1o5RefYMJUEbooPRrKKRl1P3:0YNQKPWDyDRefVJltZrpRl1P3' file3hash = '1536:MsjYdR3Bul8hcURWhEcg4/btZzDcQflbCUPEBEh8wkcGDioxMYeo7:TYf8l8htRWA4ztZsGlWUPEBEh8wmxMYe' data1 = open(file1).read() data2 = open(file2).read() data3 = open(file3).read() assert len(data1) == 114688, "File length error" assert len(data2) == 69120, "File length error" assert len(data3) == 77168, "File length error" hash01 = pydeep.hash_buf(data1) hash02 = pydeep.hash_buf(data2) hash03 = pydeep.hash_buf(data3) assert hash01 == file1hash, "Error hashing file1" assert hash02 == file2hash, "Error hashing file2" assert hash03 == file3hash, "Error hashing file2" hash1 = pydeep.hash_file(file1) hash2 = pydeep.hash_file(file2) hash3 = pydeep.hash_file(file3) assert hash1 == file1hash, "Error hashing file1" assert hash2 == file2hash, "Error hashing file2" assert hash3 == file3hash, "Error hashing file3" assert pydeep.compare(hash1,hash2) == 0, "Error fuzzy compare value" print 'Stuff looks fine..'
def getSsdeep(path): return pydeep.hash_file(path)
