def decode(nxt, quiet=False): '''Decompress a stream of bytes compressed by "lz.encode". Consume bytes. Produce decompressed bytes. When finished: Treat remaining bytes as a lone prefix. (If any leftover) Print a message to stderr. (Set quiet to True to disable) ''' table = {} # map blockids to known chunks try: for blockid in itertools.count(): blockm = bytearray() # accumulate the next block for _ in range(ints.bytewidth(blockid) + 1): blockm.append((yield)) block = bytes(blockm) # decompress the block to a chunk pointerb, newbyte = block[:-1], block[-1:] pointer = ints.frombytes(pointerb) prefix = b'' if pointer == blockid else table[pointer] # remember the chunk table[blockid] = prefix + newbyte # send the chunk nxt.send(table[blockid]) finally: if blockm: pointerb = bytes(blockm) pointer = ints.frombytes(pointerb) nxt.send(table[pointer]) if not quiet: ct = blockid + (0.5 if blockm else 0) print('lz.decoder: {} blocks done'.format(ct), file=sys.stderr)
def encode(nxt, quiet=False): '''Compress a stream of bytes according to LempelZiv. Consume bytes. Produce pointer-newbyte bytes. When finished: Send buffer as a lone prefix. (If any leftover) Print a message to stderr. (Set quiet to True to disable) ''' table = {} # map known chunks to blockids try: for blockid in itertools.count(): chunkm = bytearray() chunk = None # accumulate an unfamiliar chunk while chunk in table or chunk is None: chunkm.append((yield)) chunk = bytes(chunkm) # remember the chunk table[chunk] = blockid # compress the chunk to a block prefix, newbyte = chunk[:-1], chunk[-1:] pointer = table[prefix] if prefix else blockid pointerb = ints.tobytes(pointer, length=ints.bytewidth(blockid)) # send the block nxt.send(pointerb + newbyte) finally: if chunkm: pointer = table[chunk] pointerb = ints.tobytes(pointer, length=ints.bytewidth(blockid)) # send partial block nxt.send(pointerb) if not quiet: ct = blockid + (0.5 if chunkm else 0) print('lz.encoder: {} blocks done'.format(ct), file=sys.stderr)