def _fit(pieces, preprocessor, packer, filler):
# Pulls bytes from `filler` and adds them to `pad` until it ends in `key`.
# Returns the index of `key` in `pad`.
pad = []
def fill(key):
key = list(key)
while len(pad) < len(key) or pad[-len(key):] != key:
pad.append(filler.next())
return len(pad) - len(key)
# Key conversion:
# - convert str/unicode keys to offsets
# - convert large int (no null-bytes in a machine word) keys to offsets
pieces_ = dict()
large_key = 2**(context.word_size - 8)
for k, v in pieces.items():
if isinstance(k, (int, long)):
if k >= large_key:
k = fill(pack(k))
elif isinstance(k, unicode):
k = fill(k.encode('utf8'))
elif isinstance(k, bytearray):
k = fill(str(k))
elif isinstance(k, str):
k = fill(k)
else:
raise TypeError(
"flat(): offset must be of type int or str, but got '%s'" %
type(k))
if k in pieces_:
raise ValueError("flag(): multiple values at offset %d" % k)
pieces_[k] = v
pieces = pieces_
# We must "roll back" `filler` so each recursive call to `_flat` gets it in
# the right position
filler = iters.chain(pad, filler)
# Build output
out = ''
for k, v in sorted(pieces.items()):
if k < len(out):
raise ValueError(
"flat(): data at offset %d overlaps with previous data which ends at offset %d"
% (k, len(out)))
# Fill up to offset
while len(out) < k:
out += filler.next()
# Recursively flatten data
out += _flat([v], preprocessor, packer, filler)
return filler, out
def _fit(pieces, preprocessor, packer, filler, stacklevel=1):
# Pulls bytes from `filler` and adds them to `pad` until it ends in `key`.
# Returns the index of `key` in `pad`.
pad = bytearray()
def fill(key):
key = bytearray(key)
offset = pad.find(key)
while offset == -1:
pad.append(next(filler))
offset = pad.find(key, -len(key))
return offset
# Key conversion:
# - convert str/unicode keys to offsets
# - convert large int (no null-bytes in a machine word) keys to offsets
pieces_ = dict()
large_key = 2**(context.word_size - 8)
for k, v in pieces.items():
if isinstance(k, six.integer_types):
if k >= large_key:
k = fill(pack(k))
elif isinstance(k, (six.text_type, bytearray, bytes)):
k = fill(_need_bytes(k, stacklevel, 0x80))
else:
raise TypeError(
"flat(): offset must be of type int or str, but got '%s'" %
type(k))
if k in pieces_:
raise ValueError("flag(): multiple values at offset %d" % k)
pieces_[k] = v
pieces = pieces_
# We must "roll back" `filler` so each recursive call to `_flat` gets it in
# the right position
filler = iters.chain(pad, filler)
# Build output
out = b''
# Negative indices need to be removed and then re-submitted
negative = {
k: v
for k, v in pieces.items() if isinstance(k, int) and k < 0
}
for k in negative:
del pieces[k]
# Positive output
for k, v in sorted(pieces.items()):
if k < len(out):
raise ValueError(
"flat(): data at offset %d overlaps with previous data which ends at offset %d"
% (k, len(out)))
# Fill up to offset
while len(out) < k:
out += p8(next(filler))
# Recursively flatten data
out += _flat([v], preprocessor, packer, filler, stacklevel + 1)
# Now do negative indices
out_negative = b''
if negative:
most_negative = min(negative.keys())
for k, v in sorted(negative.items()):
k += -most_negative
if k < len(out_negative):
raise ValueError(
"flat(): data at offset %d overlaps with previous data which ends at offset %d"
% (k, len(out)))
# Fill up to offset
while len(out_negative) < k:
out_negative += p8(next(filler))
# Recursively flatten data
out_negative += _flat([v], preprocessor, packer, filler,
stacklevel + 1)
return filler, out_negative + out
