def calc_hashes(self):
"""Calculate all possible hashes for this file."""
crc = 0
md5 = hashlib.md5()
sha1 = hashlib.sha1()
sha256 = hashlib.sha256()
sha512 = hashlib.sha512()
sha3_384 = hashlib.sha3_384()
if HAVE_TLSH:
tlsh_hash = tlsh.Tlsh()
for chunk in self.get_chunks():
crc = binascii.crc32(chunk, crc)
md5.update(chunk)
sha1.update(chunk)
sha256.update(chunk)
sha512.update(chunk)
sha3_384.update(chunk)
if HAVE_TLSH:
tlsh_hash.update(chunk)
self._crc32 = "".join(f"{(crc >> i) & 0xFF:02X}"
for i in [24, 16, 8, 0])
self._md5 = md5.hexdigest()
self._sha1 = sha1.hexdigest()
self._sha256 = sha256.hexdigest()
self._sha512 = sha512.hexdigest()
self._sha3_384 = sha3_384.hexdigest()
if HAVE_TLSH:
with contextlib.suppress(ValueError):
tlsh_hash.final()
self._tlsh_hash = tlsh_hash.hexdigest()
def compute_2(path):
h = tlsh.Tlsh()
with open(path, 'rb') as f:
for buf in iter(lambda: f.read(512), b''):
h.update(buf)
h.final()
return h
def calc_hashes(self):
"""Calculate all possible hashes for this file."""
crc = 0
md5 = hashlib.md5()
sha1 = hashlib.sha1()
sha256 = hashlib.sha256()
sha512 = hashlib.sha512()
sha3_384 = hashlib.sha3_384()
if HAVE_TLSH:
tlsh_hash = tlsh.Tlsh()
for chunk in self.get_chunks():
crc = binascii.crc32(chunk, crc)
md5.update(chunk)
sha1.update(chunk)
sha256.update(chunk)
sha512.update(chunk)
sha3_384.update(chunk)
if HAVE_TLSH:
tlsh_hash.update(chunk)
self._crc32 = "".join("%02X" % ((crc >> i) & 0xFF) for i in [24, 16, 8, 0])
self._md5 = md5.hexdigest()
self._sha1 = sha1.hexdigest()
self._sha256 = sha256.hexdigest()
self._sha512 = sha512.hexdigest()
self._sha3_384 = sha3_384.hexdigest()
if HAVE_TLSH:
try:
tlsh_hash.final()
self._tlsh_hash = tlsh_hash.hexdigest()
except ValueError:
print("TLSH: less than 50 of input, ignoring")
def main(argv):
nargs = len(argv)
if nargs != 3:
print("usage: tlsh_parts.py -d digest")
sys.exit()
# end if
digest = ""
if argv[1] == "-d":
digest = argv[2]
# end if
if (digest == ""):
print("usage: tlsh_parts.py -d digest")
sys.exit()
# end if
h1 = tlsh.Tlsh()
try:
h1.fromTlshStr(digest)
except:
print("invalid digest: " + digest)
sys.exit()
# end try
print("checksum: " + str(h1.checksum(0)))
print("lvalue: " + str(h1.lvalue))
print("q1ratio: " + str(h1.q1ratio))
print("q2ratio: " + str(h1.q2ratio))
for bi in range(0, 128):
print("bucket " + str(bi) + ": " + str(h1.bucket_value(bi)))
def fuzzy_hash(self):
if not hasattr(self, '_fuzzy_hash'):
# tlsh is not meaningful with files smaller than 512 bytes
if os.stat(self.path).st_size >= 512:
h = tlsh.Tlsh()
with open(self.path, 'rb') as f:
for buf in iter(lambda: f.read(32768), b''):
h.update(buf)
h.final()
self._fuzzy_hash = h.hexdigest()
else:
self._fuzzy_hash = None
return self._fuzzy_hash
def fuzzy_hash(self):
if not hasattr(self, '_fuzzy_hash'):
# tlsh is not meaningful with files smaller than 512 bytes
if os.stat(self.path).st_size >= 512:
h = tlsh.Tlsh()
with open(self.path, 'rb') as f:
for buf in iter(lambda: f.read(32768), b''):
h.update(buf)
h.final()
try:
self._fuzzy_hash = h.hexdigest()
except ValueError:
# File must contain a certain amount of randomness.
self._fuzzy_hash = None
else:
self._fuzzy_hash = None
return self._fuzzy_hash
def compute_fuzzy_hash(file, path):
"""
Compute the fuzzy hash of the given file
cf diffoscope/comparators/utils/file.py
"""
# tlsh is not meaningful with files smaller than 512 bytes
if get_file_size(path) >= 512:
h = tlsh.Tlsh()
for buf in file._read(path):
h.update(buf)
h.final()
try:
return h.hexdigest()
except ValueError:
# File must contain a certain amount of randomness.
return None
return None
def __compute_hashes(self):
if self.size == 0: # Can't mmap empty file
self.metadata["md5"] = "d41d8cd98f00b204e9800998ecf8427e"
self.metadata["sha1"] = "da39a3ee5e6b4b0d3255bfef95601890afd80709"
self.metadata["sha256"] = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
self.metadata["sha512"] = "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e"
return
tl = tlsh.Tlsh()
md5 = hashlib.md5()
sha1 = hashlib.sha1()
sha256 = hashlib.sha256()
sha512 = hashlib.sha512()
with self.location.open("rb") as fd:
buffer = fd.read(4096)
while buffer:
tl.update(buffer)
md5.update(buffer)
sha1.update(buffer)
sha256.update(buffer)
sha512.update(buffer)
buffer = fd.read(4096)
try:
tl.final()
self.metadata["tlsh"] = tl.hexdigest()
except ValueError: # TLSH needs at least 256 bytes
pass
self.metadata["md5"] = md5.hexdigest()
self.metadata["sha1"] = sha1.hexdigest()
self.metadata["sha256"] = sha256.hexdigest()
self.metadata["sha512"] = sha512.hexdigest()
def process_release(firstdir, seconddir, releasenr):
firstfile = os.path.join(firstdir, "%d.xml" % releasenr)
if not os.path.exists(firstfile):
return
secondfile = os.path.join(seconddir, "%d.xml" % releasenr)
if not os.path.exists(secondfile):
return
firstdata = open(firstfile, 'rb').read()
firstxmldom = defusedxml.minidom.parseString(firstdata)
seconddata = open(secondfile, 'rb').read()
secondxmldom = defusedxml.minidom.parseString(seconddata)
firstrelease = firstxmldom.getElementsByTagName('release')[0]
secondrelease = secondxmldom.getElementsByTagName('release')[0]
firstchilds = firstrelease.childNodes
secondchilds = secondrelease.childNodes
# store all differences found
differences = []
# check if any nodes were added or removed
firstchildnames = set()
secondchildnames = set()
for ch in firstchilds:
if ch.nodeName == 'videos':
continue
firstchildnames.add(ch.nodeName)
for ch in secondchilds:
if ch.nodeName == 'videos':
continue
secondchildnames.add(ch.nodeName)
for n in firstchildnames - secondchildnames:
differences.append(('removed', n))
for n in secondchildnames - firstchildnames:
differences.append(('added', n))
# store the total TLSH score
total_tlsh = 0
# see if any nodes were changed by pretty printing
# to XML first and then comparing the XML *shudder*
for ch in firstchilds:
if ch.nodeName == 'videos':
continue
for s in secondchilds:
if ch.nodeName != s.nodeName:
continue
chxml = ch.toxml()
sxml = s.toxml()
if chxml != sxml:
differences.append(('changed', ch.nodeName))
firsttlsh = tlsh.Tlsh()
firsttlsh.update(chxml.encode())
try:
firsttlsh.final()
except:
break
secondtlsh = tlsh.Tlsh()
secondtlsh.update(sxml.encode())
try:
secondtlsh.final()
except:
break
distance = secondtlsh.diff(firsttlsh)
total_tlsh += distance
break
return (differences, total_tlsh, releasenr)
def initialize(self):
self.tlsh_hash = tlsh.Tlsh()
def setup(self, arg):
self.h = tlsh.Tlsh()
h = tlsh.Tlsh()
with open(path, "rb") as f:
for buf in iter(lambda: f.read(512), b""):
h.update(buf)
h.final()
return h
hex1 = compute_1(sys.argv[1])
print("tlsh.hash hex1", hex1)
hex2 = compute_1(sys.argv[2])
print("tlsh.hash hex2", hex2)
print("tlsh.diff(hex1, hex2)", tlsh.diff(hex1, hex2))
print("tlsh.diff(hex2, hex1)", tlsh.diff(hex2, hex1))
h1 = compute_2(sys.argv[1])
hex1 = h1.hexdigest()
print("tlsh.Tlsh hex1", hex1)
h2 = compute_2(sys.argv[2])
hex2 = h2.hexdigest()
print("tlsh.Tlsh hex2", hex2)
print("h1.diff(h2)", h1.diff(h2))
print("h2.diff(h1)", h2.diff(h1))
print("h1.diff(hex2)", h1.diff(hex2))
print("h2.diff(hex1)", h2.diff(hex1))
h3 = tlsh.Tlsh()
h3.fromTlshStr(hex2)
print("tlsh.Tlsh.fromTlshStr", hex2)
print("h3.diff(h2)", h3.diff(h2))
