def test_xxh3_64(self): self.assertEqual(xxhash.xxh3_64('a').intdigest(), 16629034431890738719) self.assertEqual( xxhash.xxh3_64('a', 0).intdigest(), 16629034431890738719) self.assertEqual( xxhash.xxh3_64('a', 1).intdigest(), 15201566949650179872) self.assertEqual( xxhash.xxh3_64('a', 2**64 - 1).intdigest(), 4875116479388997462)
def _hash_single_obj(cls, obj_tuple): h = xxhash.xxh3_64() if _HAS_XXHASH else hashlib.sha1() for obj in obj_tuple: if isinstance(obj, str): obj = obj.encode(errors='ignore') h.update(obj) return struct.unpack('<Q', h.digest()[:8])[0]
def test_xxh3_64_seed_reset(self): seed = random.randint(0, 2**64 - 1) x = xxhash.xxh3_64(seed=seed) h = x.intdigest() x.update('x' * 10240) x.reset() self.assertEqual(h, x.intdigest())
def test_xxh3_64_reset(self): x = xxhash.xxh3_64() h = x.intdigest() x.update('x' * 10240) x.reset() self.assertEqual(h, x.intdigest())
def HashFile(file): # 32MB buffer size BUFFER_SIZE = 32 * 1024 * 1024 x = xxhash.xxh3_64(seed=0) b = bytearray(BUFFER_SIZE) mv = memoryview(b) with open(file, 'rb') as f: while n := f.readinto(mv): x.update(mv[:n])
async def get_key(self, response, encoding): # We cache as we will use in set likely if inspect.iscoroutinefunction(response.get_data): data = await response.get_data() else: data = response.get_data() digest = xxhash.xxh3_64(data, seed=0).hexdigest() filename = encoding + "_" + digest + \ mimetypes.guess_extension(response.mimetype) return filename
def checksum_xxhash(file_path, block_size=65536): """Get xxhash checksum for a file""" if xxhash is None: raise Exception( "xxhash not available on this platform. Try 'pip install xxhash'") else: h = xxhash.xxh3_64() with open(file_path, "rb") as f: for chunk in iter(lambda: f.read(block_size), b""): h.update(chunk) return h.hexdigest()
def get_source_hashes_xxhash_4096(what): blocksize = 1024 * 4096 xxh = xxhash.xxh3_64() for i in what: xxh.reset() with Path(i[0]).open("rb") as file: while True: buf = file.read(blocksize) if not buf: break xxh.update(buf) crcvalue = xxh.hexdigest() i[6] = crcvalue
def test_xxh3_64_copy(self): a = xxhash.xxh3_64() a.update('xxhash') b = a.copy() self.assertEqual(a.digest(), b.digest()) self.assertEqual(a.intdigest(), b.intdigest()) self.assertEqual(a.hexdigest(), b.hexdigest()) b.update('xxhash') self.assertNotEqual(a.digest(), b.digest()) self.assertNotEqual(a.intdigest(), b.intdigest()) self.assertNotEqual(a.hexdigest(), b.hexdigest()) a.update('xxhash') self.assertEqual(a.digest(), b.digest()) self.assertEqual(a.intdigest(), b.intdigest()) self.assertEqual(a.hexdigest(), b.hexdigest())
def test_xxh3_64_reset_more(self): x = xxhash.xxh3_64() h = x.intdigest() for i in range(random.randint(100, 200)): x.reset() self.assertEqual(h, x.intdigest()) for i in range(10, 1000): x.update(os.urandom(i)) x.reset() self.assertEqual(h, x.intdigest()) for i in range(10, 1000): x.update(os.urandom(100)) x.reset() self.assertEqual(h, x.intdigest())
def test_xxh3_64_overflow(self): s = 'I want an unsigned 64-bit seed!' a = xxhash.xxh3_64(s, seed=0) b = xxhash.xxh3_64(s, seed=2**64) self.assertEqual(a.seed, b.seed) self.assertEqual(a.intdigest(), b.intdigest()) self.assertEqual(a.hexdigest(), b.hexdigest()) self.assertEqual(a.digest(), b.digest()) self.assertEqual(a.intdigest(), xxhash.xxh3_64_intdigest(s, seed=0)) self.assertEqual(a.intdigest(), xxhash.xxh3_64_intdigest(s, seed=2**64)) self.assertEqual(a.digest(), xxhash.xxh3_64_digest(s, seed=0)) self.assertEqual(a.digest(), xxhash.xxh3_64_digest(s, seed=2**64)) self.assertEqual(a.hexdigest(), xxhash.xxh3_64_hexdigest(s, seed=0)) self.assertEqual(a.hexdigest(), xxhash.xxh3_64_hexdigest(s, seed=2**64)) a = xxhash.xxh3_64(s, seed=1) b = xxhash.xxh3_64(s, seed=2**64 + 1) self.assertEqual(a.seed, b.seed) self.assertEqual(a.intdigest(), b.intdigest()) self.assertEqual(a.hexdigest(), b.hexdigest()) self.assertEqual(a.digest(), b.digest()) self.assertEqual(a.intdigest(), xxhash.xxh3_64_intdigest(s, seed=1)) self.assertEqual(a.intdigest(), xxhash.xxh3_64_intdigest(s, seed=2**64 + 1)) self.assertEqual(a.digest(), xxhash.xxh3_64_digest(s, seed=1)) self.assertEqual(a.digest(), xxhash.xxh3_64_digest(s, seed=2**64 + 1)) self.assertEqual(a.hexdigest(), xxhash.xxh3_64_hexdigest(s, seed=1)) self.assertEqual(a.hexdigest(), xxhash.xxh3_64_hexdigest(s, seed=2**64 + 1)) a = xxhash.xxh3_64(s, seed=2**65 - 1) b = xxhash.xxh3_64(s, seed=2**66 - 1) self.assertEqual(a.seed, b.seed) self.assertEqual(a.intdigest(), b.intdigest()) self.assertEqual(a.hexdigest(), b.hexdigest()) self.assertEqual(a.digest(), b.digest()) self.assertEqual(a.intdigest(), xxhash.xxh3_64_intdigest(s, seed=2**65 - 1)) self.assertEqual(a.intdigest(), xxhash.xxh3_64_intdigest(s, seed=2**66 - 1)) self.assertEqual(a.digest(), xxhash.xxh3_64_digest(s, seed=2**65 - 1)) self.assertEqual(a.digest(), xxhash.xxh3_64_digest(s, seed=2**66 - 1)) self.assertEqual(a.hexdigest(), xxhash.xxh3_64_hexdigest(s, seed=2**65 - 1)) self.assertEqual(a.hexdigest(), xxhash.xxh3_64_hexdigest(s, seed=2**66 - 1))
def test_xxh3_64_update(self): x = xxhash.xxh3_64() x.update('a') self.assertEqual(xxhash.xxh3_64('a').digest(), x.digest()) self.assertEqual(xxhash.xxh3_64_digest('a'), x.digest()) x.update('b') self.assertEqual(xxhash.xxh3_64('ab').digest(), x.digest()) self.assertEqual(xxhash.xxh3_64_digest('ab'), x.digest()) x.update('c') self.assertEqual(xxhash.xxh3_64('abc').digest(), x.digest()) self.assertEqual(xxhash.xxh3_64_digest('abc'), x.digest()) seed = random.randint(0, 2**64) x = xxhash.xxh3_64(seed=seed) x.update('a') self.assertEqual(xxhash.xxh3_64('a', seed).digest(), x.digest()) self.assertEqual(xxhash.xxh3_64_digest('a', seed), x.digest()) x.update('b') self.assertEqual(xxhash.xxh3_64('ab', seed).digest(), x.digest()) self.assertEqual(xxhash.xxh3_64_digest('ab', seed), x.digest()) x.update('c') self.assertEqual(xxhash.xxh3_64('abc', seed).digest(), x.digest()) self.assertEqual(xxhash.xxh3_64_digest('abc', seed), x.digest())
def xxh3_64(password): return xxhash.xxh3_64(password).hexdigest()
def _hash_config(self, config: dict): return xxhash.xxh3_64( self._dump_config(config).encode('utf8')).hexdigest()