def flat(*args, **kwargs):
r"""flat(\*args, preprocessor = None, length = None, filler = de_bruijn(),
word_size = None, endianness = None, sign = None) -> str
Flattens the arguments into a string.
This function takes an arbitrary number of arbitrarily nested lists, tuples
and dictionaries. It will then find every string and number inside those
and flatten them out. Strings are inserted directly while numbers are
packed using the :func:`pack` function. Unicode strings are UTF-8 encoded.
Dictionary keys give offsets at which to place the corresponding values
(which are recursively flattened). Offsets are relative to where the
flattened dictionary occurs in the output (i.e. ``{0: 'foo'}`` is equivalent
to ``'foo'``). Offsets can be integers, unicode strings or regular strings.
Integer offsets >= ``2**(word_size-8)`` are converted to a string using
:func:`pack`. Unicode strings are UTF-8 encoded. After these conversions
offsets are either integers or strings. In the latter case, the offset will
be the lowest index at which the string occurs in `filler`. See examples
below.
Space between pieces of data is filled out using the iterable `filler`. The
`n`'th byte in the output will be byte at index ``n % len(iterable)`` byte
in `filler` if it has finite length or the byte at index `n` otherwise.
If `length` is given, the output will be padded with bytes from `filler` to
be this size. If the output is longer than `length`, a :py:exc:`ValueError`
exception is raised.
The three kwargs `word_size`, `endianness` and `sign` will default to using
values in :mod:`pwnlib.context` if not specified as an argument.
Arguments:
args: Values to flatten
preprocessor (function): Gets called on every element to optionally
transform the element before flattening. If :const:`None` is
returned, then the original value is used.
length: The length of the output.
filler: Iterable to use for padding.
word_size (int): Word size of the converted integer.
endianness (str): Endianness of the converted integer ("little"/"big").
sign (str): Signedness of the converted integer (False/True)
Examples:
(Test setup, please ignore)
>>> context.clear()
Basic usage of :meth:`flat` works similar to the pack() routines.
>>> flat(4)
b'\x04\x00\x00\x00'
:meth:`flat` works with strings, bytes, lists, and dictionaries.
>>> flat(b'X')
b'X'
>>> flat([1,2,3])
b'\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00\x00\x00'
>>> flat({4:b'X'})
b'aaaaX'
:meth:`.flat` flattens all of the values provided, and allows nested lists
and dictionaries.
>>> flat([{4:b'X'}] * 2)
b'aaaaXaaacX'
>>> flat([[[[[[[[[1]]]], 2]]]]])
b'\x01\x00\x00\x00\x02\x00\x00\x00'
You can also provide additional arguments like endianness, word-size, and
whether the values are treated as signed or not.
>>> flat(1, b"test", [[[b"AB"]*2]*3], endianness = 'little', word_size = 16, sign = False)
b'\x01\x00testABABABABABAB'
A preprocessor function can be provided in order to modify the values in-flight.
This example converts increments each value by 1, then converts to a byte string.
>>> flat([1, [2, 3]], preprocessor = lambda x: str(x+1).encode())
b'234'
Using dictionaries is a fast way to get specific values at specific offsets,
without having to do ``data += "foo"`` repeatedly.
>>> flat({12: 0x41414141,
... 24: b'Hello',
... })
b'aaaabaaacaaaAAAAeaaafaaaHello'
Dictionary usage permits directly using values derived from :func:`.cyclic`.
See :func:`.cyclic`, :function:`pwnlib.context.context.cyclic_alphabet`, and :data:`.context.cyclic_size`
for more options.
The cyclic pattern can be provided as either the text or hexadecimal offset.
>>> flat({ 0x61616162: b'X'})
b'aaaaX'
>>> flat({'baaa': b'X'})
b'aaaaX'
Fields do not have to be in linear order, and can be freely mixed.
This also works with cyclic offsets.
>>> flat({2: b'A', 0:b'B'})
b'BaA'
>>> flat({0x61616161: b'x', 0x61616162: b'y'})
b'xaaay'
>>> flat({0x61616162: b'y', 0x61616161: b'x'})
b'xaaay'
Fields do not have to be in order, and can be freely mixed.
>>> flat({'caaa': b'XXXX', 16: b'\x41', 20: 0xdeadbeef})
b'aaaabaaaXXXXdaaaAaaa\xef\xbe\xad\xde'
>>> flat({ 8: [0x41414141, 0x42424242], 20: b'CCCC'})
b'aaaabaaaAAAABBBBeaaaCCCC'
>>> fit({
... 0x61616161: b'a',
... 1: b'b',
... 0x61616161+2: b'c',
... 3: b'd',
... })
b'abadbaaac'
By default, gaps in the data are filled in with the :meth:`.cyclic` pattern.
You can customize this by providing an iterable or method for the ``filler``
argument.
>>> flat({12: b'XXXX'}, filler = b'_', length = 20)
b'____________XXXX____'
>>> flat({12: b'XXXX'}, filler = b'AB', length = 20)
b'ABABABABABABXXXXABAB'
Nested dictionaries also work as expected.
>>> flat({4: {0: b'X', 4: b'Y'}})
b'aaaaXaaaY'
>>> fit({4: {4: b'XXXX'}})
b'aaaabaaaXXXX'
Negative indices are also supported, though this only works for integer
keys.
>>> flat({-4: b'x', -1: b'A', 0: b'0', 4: b'y'})
b'xaaA0aaay'
"""
# HACK: To avoid circular imports we need to delay the import of `cyclic`
from pwnlib.util import cyclic
preprocessor = kwargs.pop('preprocessor', lambda x: None)
filler = kwargs.pop('filler', cyclic.de_bruijn())
length = kwargs.pop('length', None)
stacklevel = kwargs.pop('stacklevel', 0)
if isinstance(filler, (str, six.text_type)):
filler = bytearray(_need_bytes(filler))
if kwargs != {}:
raise TypeError("flat() does not support argument %r" %
kwargs.popitem()[0])
filler = iters.cycle(filler)
out = _flat(args, preprocessor, make_packer(), filler, stacklevel + 2)
if length:
if len(out) > length:
raise ValueError(
"flat(): Arguments does not fit within `length` (= %d) bytes" %
length)
out += b''.join(p8(next(filler)) for _ in range(length - len(out)))
return out
def fit(pieces=None, **kwargs):
"""fit(pieces, filler = de_bruijn(), length = None, preprocessor = None) -> str
Generates a string from a dictionary mapping offsets to data to place at
that offset.
For each key-value pair in `pieces`, the key is either an offset or a byte
sequence. In the latter case, the offset will be the lowest index at which
the sequence occurs in `filler`. See examples below.
Each piece of data is passed to :meth:`flat` along with the keyword
arguments `word_size`, `endianness` and `sign`.
Space between pieces of data is filled out using the iterable `filler`. The
`n`'th byte in the output will be byte at index ``n % len(iterable)`` byte
in `filler` if it has finite length or the byte at index `n` otherwise.
If `length` is given, the output will padded with bytes from `filler` to be
this size. If the output is longer than `length`, a :py:exc:`ValueError`
exception is raised.
If entries in `pieces` overlap, a :py:exc:`ValueError` exception is
raised.
Arguments:
pieces: Offsets and values to output.
length: The length of the output.
filler: Iterable to use for padding.
preprocessor (function): Gets called on every element to optionally
transform the element before flattening. If :const:`None` is
returned, then the original value is used.
word_size (int): Word size of the converted integer.
endianness (str): Endianness of the converted integer ("little"/"big").
sign (str): Signedness of the converted integer (False/True)
Examples:
>>> fit({12: 0x41414141,
... 24: 'Hello',
... })
'aaaabaaacaaaAAAAeaaafaaaHello'
>>> fit({'caaa': ''})
'aaaabaaa'
>>> fit({12: 'XXXX'}, filler = 'AB', length = 20)
'ABABABABABABXXXXABAB'
>>> fit({ 8: [0x41414141, 0x42424242],
... 20: 'CCCC'})
'aaaabaaaAAAABBBBeaaaCCCC'
"""
# HACK: To avoid circular imports we need to delay the import of `cyclic`
from pwnlib.util import cyclic
filler = kwargs.pop('filler', cyclic.de_bruijn())
length = kwargs.pop('length', None)
preprocessor = kwargs.pop('preprocessor', lambda x: None)
word_size = kwargs.pop('word_size', None)
endianness = kwargs.pop('endianness', None)
sign = kwargs.pop('sign', None)
if kwargs != {}:
raise TypeError("fit() does not support argument %r" % kwargs.popitem()[0])
packer = make_packer()
filler = iters.cycle(filler)
out = ''
if not length and not pieces:
return ''
if not pieces:
return ''.join(filler.next() for f in range(length))
# convert str keys to offsets
pieces_ = dict()
for k, v in pieces.items():
if isinstance(k, (int, long)):
pass
elif isinstance(k, str):
while k not in out:
out += filler.next()
k = out.index(k)
else:
raise TypeError("fit(): offset must be of type int or str, but got '%s'" % type(k))
pieces_[k] = v
pieces = pieces_
# convert values to their flattened forms
for k,v in pieces.items():
pieces[k] = _flat([v], preprocessor, packer)
# if we were provided a length, make sure everything fits
last = max(pieces)
if length and last and (last + len(pieces[last]) > length):
raise ValueError("fit(): Pieces do not fit within `length` (= %d) bytes" % length)
# insert data into output
out = list(out)
l = 0
for k, v in sorted(pieces.items()):
if k < l:
raise ValueError("fit(): data at offset %d overlaps with previous data which ends at offset %d" % (k, l))
while len(out) < k:
out.append(filler.next())
v = _flat([v], preprocessor, packer)
l = k + len(v)
# consume the filler for each byte of actual data
for i in range(len(out), l):
filler.next()
out[k:l] = v
# truncate/pad output
if length:
if l > length:
raise ValueError("fit(): Pieces does not fit within `length` (= %d) bytes" % length)
while len(out) < length:
out.append(filler.next())
else:
out = out[:l]
return ''.join(out)
def flat(*args, **kwargs):
r"""flat(\*args, preprocessor = None, length = None, filler = de_bruijn(),
word_size = None, endianness = None, sign = None) -> str
Flattens the arguments into a string.
This function takes an arbitrary number of arbitrarily nested lists, tuples
and dictionaries. It will then find every string and number inside those
and flatten them out. Strings are inserted directly while numbers are
packed using the :func:`pack` function. Unicode strings are UTF-8 encoded.
Dictionary keys give offsets at which to place the corresponding values
(which are recursively flattened). Offsets are relative to where the
flattened dictionary occurs in the output (i.e. `{0: 'foo'}` is equivalent
to `'foo'`). Offsets can be integers, unicode strings or regular strings.
Integer offsets >= ``2**(word_size-8)`` are converted to a string using
`:func:pack`. Unicode strings are UTF-8 encoded. After these conversions
offsets are either integers or strings. In the latter case, the offset will
be the lowest index at which the string occurs in `filler`. See examples
below.
Space between pieces of data is filled out using the iterable `filler`. The
`n`'th byte in the output will be byte at index ``n % len(iterable)`` byte
in `filler` if it has finite length or the byte at index `n` otherwise.
If `length` is given, the output will be padded with bytes from `filler` to
be this size. If the output is longer than `length`, a :py:exc:`ValueError`
exception is raised.
The three kwargs `word_size`, `endianness` and `sign` will default to using
values in :mod:`pwnlib.context` if not specified as an argument.
Arguments:
args: Values to flatten
preprocessor (function): Gets called on every element to optionally
transform the element before flattening. If :const:`None` is
returned, then the original value is used.
length: The length of the output.
filler: Iterable to use for padding.
word_size (int): Word size of the converted integer.
endianness (str): Endianness of the converted integer ("little"/"big").
sign (str): Signedness of the converted integer (False/True)
Examples:
>>> flat(1, "test", [[["AB"]*2]*3], endianness = 'little', word_size = 16, sign = False)
b'\x01\x00testABABABABABAB'
>>> flat([1, [2, 3]], preprocessor = lambda x: str(x+1))
b'234'
>>> flat({12: 0x41414141,
... 24: 'Hello',
... })
b'aaaabaaacaaaAAAAeaaafaaaHello'
>>> flat({'caaa': ''})
b'aaaabaaa'
>>> flat({12: 'XXXX'}, filler = (ord('A'), ord('B')), length = 20)
b'ABABABABABABXXXXABAB'
>>> flat({ 8: [0x41414141, 0x42424242],
... 20: 'CCCC'})
b'aaaabaaaAAAABBBBeaaaCCCC'
>>> flat({ 0x61616162: 'X'})
b'aaaaX'
>>> flat({4: {0: 'X', 4: 'Y'}})
b'aaaaXaaaY'
"""
# HACK: To avoid circular imports we need to delay the import of `cyclic`
from pwnlib.util import cyclic
preprocessor = kwargs.pop('preprocessor', lambda x: None)
filler = kwargs.pop('filler', cyclic.de_bruijn())
length = kwargs.pop('length', None)
if kwargs != {}:
raise TypeError("flat() does not support argument %r" %
kwargs.popitem()[0])
filler = iters.cycle(filler)
out = _flat(args, preprocessor, make_packer(), filler)
if length:
if len(out) > length:
raise ValueError(
"flat(): Arguments does not fit within `length` (= %d) bytes" %
length)
out += b''.join(p8(next(filler)) for _ in range(length - len(out)))
return out
def fit(pieces=None, **kwargs):
"""fit(pieces, filler = de_bruijn(), length = None, preprocessor = None) -> str
Generates a string from a dictionary mapping offsets to data to place at
that offset.
For each key-value pair in `pieces`, the key is either an offset or a byte
sequence. In the latter case, the offset will be the lowest index at which
the sequence occurs in `filler`. See examples below.
Each piece of data is passed to :meth:`flat` along with the keyword
arguments `word_size`, `endianness` and `sign`.
Space between pieces of data is filled out using the iterable `filler`. The
`n`'th byte in the output will be byte at index ``n % len(iterable)`` byte
in `filler` if it has finite length or the byte at index `n` otherwise.
If `length` is given, the output will padded with bytes from `filler` to be
this size. If the output is longer than `length`, a :py:exc:`ValueError`
exception is raised.
If entries in `pieces` overlap, a :py:exc:`ValueError` exception is
raised.
Arguments:
pieces: Offsets and values to output.
length: The length of the output.
filler: Iterable to use for padding.
preprocessor (function): Gets called on every element to optionally
transform the element before flattening. If :const:`None` is
returned, then the original value is used.
word_size (int): Word size of the converted integer.
endianness (str): Endianness of the converted integer ("little"/"big").
sign (str): Signedness of the converted integer (False/True)
Examples:
>>> fit({12: 0x41414141,
... 24: 'Hello',
... })
'aaaabaaacaaaAAAAeaaafaaaHello'
>>> fit({'caaa': ''})
'aaaabaaa'
>>> fit({12: 'XXXX'}, filler = 'AB', length = 20)
'ABABABABABABXXXXABAB'
>>> fit({ 8: [0x41414141, 0x42424242],
... 20: 'CCCC'})
'aaaabaaaAAAABBBBeaaaCCCC'
>>> fit({ 0x61616162: 'X'})
'aaaaX'
"""
# HACK: To avoid circular imports we need to delay the import of `cyclic`
from pwnlib.util import cyclic
filler = kwargs.pop('filler', cyclic.de_bruijn())
length = kwargs.pop('length', None)
preprocessor = kwargs.pop('preprocessor', lambda x: None)
if kwargs != {}:
raise TypeError("fit() does not support argument %r" % kwargs.popitem()[0])
packer = make_packer()
filler = iters.cycle(filler)
out = ''
if not length and not pieces:
return ''
if not pieces:
return ''.join(filler.next() for f in range(length))
def fill(out, value):
while value not in out:
out += filler.next()
return out, out.index(value)
# convert str keys to offsets
# convert large int keys to offsets
pieces_ = dict()
for k, v in pieces.items():
if isinstance(k, (int, long)):
# cyclic() generally starts with 'aaaa'
if k >= 0x61616161:
out, k = fill(out, pack(k))
elif isinstance(k, str):
out, k = fill(out, k)
else:
raise TypeError("fit(): offset must be of type int or str, but got '%s'" % type(k))
pieces_[k] = v
pieces = pieces_
# convert values to their flattened forms
for k,v in pieces.items():
pieces[k] = _flat([v], preprocessor, packer)
# if we were provided a length, make sure everything fits
last = max(pieces)
if length and last and (last + len(pieces[last]) > length):
raise ValueError("fit(): Pieces do not fit within `length` (= %d) bytes" % length)
# insert data into output
out = list(out)
l = 0
for k, v in sorted(pieces.items()):
if k < l:
raise ValueError("fit(): data at offset %d overlaps with previous data which ends at offset %d" % (k, l))
while len(out) < k:
out.append(filler.next())
v = _flat([v], preprocessor, packer)
l = k + len(v)
# consume the filler for each byte of actual data
for i in range(len(out), l):
filler.next()
out[k:l] = v
# truncate/pad output
if length:
if l > length:
raise ValueError("fit(): Pieces does not fit within `length` (= %d) bytes" % length)
while len(out) < length:
out.append(filler.next())
else:
out = out[:l]
return ''.join(out)