def test_xxh3_64(self):
self.assertEqual(xxhash.xxh3_64('a').intdigest(), 16629034431890738719)
self.assertEqual(
xxhash.xxh3_64('a', 0).intdigest(), 16629034431890738719)
self.assertEqual(
xxhash.xxh3_64('a', 1).intdigest(), 15201566949650179872)
self.assertEqual(
xxhash.xxh3_64('a', 2**64 - 1).intdigest(), 4875116479388997462)
def _hash_single_obj(cls, obj_tuple):
h = xxhash.xxh3_64() if _HAS_XXHASH else hashlib.sha1()
for obj in obj_tuple:
if isinstance(obj, str):
obj = obj.encode(errors='ignore')
h.update(obj)
return struct.unpack('<Q', h.digest()[:8])[0]
def test_xxh3_64_seed_reset(self):
seed = random.randint(0, 2**64 - 1)
x = xxhash.xxh3_64(seed=seed)
h = x.intdigest()
x.update('x' * 10240)
x.reset()
self.assertEqual(h, x.intdigest())
def test_xxh3_64_reset(self):
x = xxhash.xxh3_64()
h = x.intdigest()
x.update('x' * 10240)
x.reset()
self.assertEqual(h, x.intdigest())
def HashFile(file):
# 32MB buffer size
BUFFER_SIZE = 32 * 1024 * 1024
x = xxhash.xxh3_64(seed=0)
b = bytearray(BUFFER_SIZE)
mv = memoryview(b)
with open(file, 'rb') as f:
while n := f.readinto(mv):
x.update(mv[:n])
async def get_key(self, response, encoding):
# We cache as we will use in set likely
if inspect.iscoroutinefunction(response.get_data):
data = await response.get_data()
else:
data = response.get_data()
digest = xxhash.xxh3_64(data, seed=0).hexdigest()
filename = encoding + "_" + digest + \
mimetypes.guess_extension(response.mimetype)
return filename
def checksum_xxhash(file_path, block_size=65536):
"""Get xxhash checksum for a file"""
if xxhash is None:
raise Exception(
"xxhash not available on this platform. Try 'pip install xxhash'")
else:
h = xxhash.xxh3_64()
with open(file_path, "rb") as f:
for chunk in iter(lambda: f.read(block_size), b""):
h.update(chunk)
return h.hexdigest()
def get_source_hashes_xxhash_4096(what):
blocksize = 1024 * 4096
xxh = xxhash.xxh3_64()
for i in what:
xxh.reset()
with Path(i[0]).open("rb") as file:
while True:
buf = file.read(blocksize)
if not buf:
break
xxh.update(buf)
crcvalue = xxh.hexdigest()
i[6] = crcvalue
def test_xxh3_64_copy(self):
a = xxhash.xxh3_64()
a.update('xxhash')
b = a.copy()
self.assertEqual(a.digest(), b.digest())
self.assertEqual(a.intdigest(), b.intdigest())
self.assertEqual(a.hexdigest(), b.hexdigest())
b.update('xxhash')
self.assertNotEqual(a.digest(), b.digest())
self.assertNotEqual(a.intdigest(), b.intdigest())
self.assertNotEqual(a.hexdigest(), b.hexdigest())
a.update('xxhash')
self.assertEqual(a.digest(), b.digest())
self.assertEqual(a.intdigest(), b.intdigest())
self.assertEqual(a.hexdigest(), b.hexdigest())
def test_xxh3_64_reset_more(self):
x = xxhash.xxh3_64()
h = x.intdigest()
for i in range(random.randint(100, 200)):
x.reset()
self.assertEqual(h, x.intdigest())
for i in range(10, 1000):
x.update(os.urandom(i))
x.reset()
self.assertEqual(h, x.intdigest())
for i in range(10, 1000):
x.update(os.urandom(100))
x.reset()
self.assertEqual(h, x.intdigest())
def test_xxh3_64_overflow(self):
s = 'I want an unsigned 64-bit seed!'
a = xxhash.xxh3_64(s, seed=0)
b = xxhash.xxh3_64(s, seed=2**64)
self.assertEqual(a.seed, b.seed)
self.assertEqual(a.intdigest(), b.intdigest())
self.assertEqual(a.hexdigest(), b.hexdigest())
self.assertEqual(a.digest(), b.digest())
self.assertEqual(a.intdigest(), xxhash.xxh3_64_intdigest(s, seed=0))
self.assertEqual(a.intdigest(), xxhash.xxh3_64_intdigest(s,
seed=2**64))
self.assertEqual(a.digest(), xxhash.xxh3_64_digest(s, seed=0))
self.assertEqual(a.digest(), xxhash.xxh3_64_digest(s, seed=2**64))
self.assertEqual(a.hexdigest(), xxhash.xxh3_64_hexdigest(s, seed=0))
self.assertEqual(a.hexdigest(), xxhash.xxh3_64_hexdigest(s,
seed=2**64))
a = xxhash.xxh3_64(s, seed=1)
b = xxhash.xxh3_64(s, seed=2**64 + 1)
self.assertEqual(a.seed, b.seed)
self.assertEqual(a.intdigest(), b.intdigest())
self.assertEqual(a.hexdigest(), b.hexdigest())
self.assertEqual(a.digest(), b.digest())
self.assertEqual(a.intdigest(), xxhash.xxh3_64_intdigest(s, seed=1))
self.assertEqual(a.intdigest(),
xxhash.xxh3_64_intdigest(s, seed=2**64 + 1))
self.assertEqual(a.digest(), xxhash.xxh3_64_digest(s, seed=1))
self.assertEqual(a.digest(), xxhash.xxh3_64_digest(s, seed=2**64 + 1))
self.assertEqual(a.hexdigest(), xxhash.xxh3_64_hexdigest(s, seed=1))
self.assertEqual(a.hexdigest(),
xxhash.xxh3_64_hexdigest(s, seed=2**64 + 1))
a = xxhash.xxh3_64(s, seed=2**65 - 1)
b = xxhash.xxh3_64(s, seed=2**66 - 1)
self.assertEqual(a.seed, b.seed)
self.assertEqual(a.intdigest(), b.intdigest())
self.assertEqual(a.hexdigest(), b.hexdigest())
self.assertEqual(a.digest(), b.digest())
self.assertEqual(a.intdigest(),
xxhash.xxh3_64_intdigest(s, seed=2**65 - 1))
self.assertEqual(a.intdigest(),
xxhash.xxh3_64_intdigest(s, seed=2**66 - 1))
self.assertEqual(a.digest(), xxhash.xxh3_64_digest(s, seed=2**65 - 1))
self.assertEqual(a.digest(), xxhash.xxh3_64_digest(s, seed=2**66 - 1))
self.assertEqual(a.hexdigest(),
xxhash.xxh3_64_hexdigest(s, seed=2**65 - 1))
self.assertEqual(a.hexdigest(),
xxhash.xxh3_64_hexdigest(s, seed=2**66 - 1))
def test_xxh3_64_update(self):
x = xxhash.xxh3_64()
x.update('a')
self.assertEqual(xxhash.xxh3_64('a').digest(), x.digest())
self.assertEqual(xxhash.xxh3_64_digest('a'), x.digest())
x.update('b')
self.assertEqual(xxhash.xxh3_64('ab').digest(), x.digest())
self.assertEqual(xxhash.xxh3_64_digest('ab'), x.digest())
x.update('c')
self.assertEqual(xxhash.xxh3_64('abc').digest(), x.digest())
self.assertEqual(xxhash.xxh3_64_digest('abc'), x.digest())
seed = random.randint(0, 2**64)
x = xxhash.xxh3_64(seed=seed)
x.update('a')
self.assertEqual(xxhash.xxh3_64('a', seed).digest(), x.digest())
self.assertEqual(xxhash.xxh3_64_digest('a', seed), x.digest())
x.update('b')
self.assertEqual(xxhash.xxh3_64('ab', seed).digest(), x.digest())
self.assertEqual(xxhash.xxh3_64_digest('ab', seed), x.digest())
x.update('c')
self.assertEqual(xxhash.xxh3_64('abc', seed).digest(), x.digest())
self.assertEqual(xxhash.xxh3_64_digest('abc', seed), x.digest())
def xxh3_64(password):
return xxhash.xxh3_64(password).hexdigest()
def _hash_config(self, config: dict):
return xxhash.xxh3_64(
self._dump_config(config).encode('utf8')).hexdigest()
