def _fit(pieces, preprocessor, packer, filler): # Pulls bytes from `filler` and adds them to `pad` until it ends in `key`. # Returns the index of `key` in `pad`. pad = [] def fill(key): key = list(key) while len(pad) < len(key) or pad[-len(key):] != key: pad.append(filler.next()) return len(pad) - len(key) # Key conversion: # - convert str/unicode keys to offsets # - convert large int (no null-bytes in a machine word) keys to offsets pieces_ = dict() large_key = 2**(context.word_size - 8) for k, v in pieces.items(): if isinstance(k, (int, long)): if k >= large_key: k = fill(pack(k)) elif isinstance(k, unicode): k = fill(k.encode('utf8')) elif isinstance(k, bytearray): k = fill(str(k)) elif isinstance(k, str): k = fill(k) else: raise TypeError( "flat(): offset must be of type int or str, but got '%s'" % type(k)) if k in pieces_: raise ValueError("flag(): multiple values at offset %d" % k) pieces_[k] = v pieces = pieces_ # We must "roll back" `filler` so each recursive call to `_flat` gets it in # the right position filler = iters.chain(pad, filler) # Build output out = '' for k, v in sorted(pieces.items()): if k < len(out): raise ValueError( "flat(): data at offset %d overlaps with previous data which ends at offset %d" % (k, len(out))) # Fill up to offset while len(out) < k: out += filler.next() # Recursively flatten data out += _flat([v], preprocessor, packer, filler) return filler, out
def _fit(pieces, preprocessor, packer, filler, stacklevel=1): # Pulls bytes from `filler` and adds them to `pad` until it ends in `key`. # Returns the index of `key` in `pad`. pad = bytearray() def fill(key): key = bytearray(key) offset = pad.find(key) while offset == -1: pad.append(next(filler)) offset = pad.find(key, -len(key)) return offset # Key conversion: # - convert str/unicode keys to offsets # - convert large int (no null-bytes in a machine word) keys to offsets pieces_ = dict() large_key = 2**(context.word_size - 8) for k, v in pieces.items(): if isinstance(k, six.integer_types): if k >= large_key: k = fill(pack(k)) elif isinstance(k, (six.text_type, bytearray, bytes)): k = fill(_need_bytes(k, stacklevel, 0x80)) else: raise TypeError( "flat(): offset must be of type int or str, but got '%s'" % type(k)) if k in pieces_: raise ValueError("flag(): multiple values at offset %d" % k) pieces_[k] = v pieces = pieces_ # We must "roll back" `filler` so each recursive call to `_flat` gets it in # the right position filler = iters.chain(pad, filler) # Build output out = b'' # Negative indices need to be removed and then re-submitted negative = { k: v for k, v in pieces.items() if isinstance(k, int) and k < 0 } for k in negative: del pieces[k] # Positive output for k, v in sorted(pieces.items()): if k < len(out): raise ValueError( "flat(): data at offset %d overlaps with previous data which ends at offset %d" % (k, len(out))) # Fill up to offset while len(out) < k: out += p8(next(filler)) # Recursively flatten data out += _flat([v], preprocessor, packer, filler, stacklevel + 1) # Now do negative indices out_negative = b'' if negative: most_negative = min(negative.keys()) for k, v in sorted(negative.items()): k += -most_negative if k < len(out_negative): raise ValueError( "flat(): data at offset %d overlaps with previous data which ends at offset %d" % (k, len(out))) # Fill up to offset while len(out_negative) < k: out_negative += p8(next(filler)) # Recursively flatten data out_negative += _flat([v], preprocessor, packer, filler, stacklevel + 1) return filler, out_negative + out