Example #1
0
def flat(*args, **kwargs):
    r"""flat(\*args, preprocessor = None, length = None, filler = de_bruijn(),
     word_size = None, endianness = None, sign = None) -> str

    Flattens the arguments into a string.

    This function takes an arbitrary number of arbitrarily nested lists, tuples
    and dictionaries.  It will then find every string and number inside those
    and flatten them out.  Strings are inserted directly while numbers are
    packed using the :func:`pack` function.  Unicode strings are UTF-8 encoded.

    Dictionary keys give offsets at which to place the corresponding values
    (which are recursively flattened).  Offsets are relative to where the
    flattened dictionary occurs in the output (i.e. ``{0: 'foo'}`` is equivalent
    to ``'foo'``).  Offsets can be integers, unicode strings or regular strings.
    Integer offsets >= ``2**(word_size-8)`` are converted to a string using
    :func:`pack`.  Unicode strings are UTF-8 encoded.  After these conversions
    offsets are either integers or strings.  In the latter case, the offset will
    be the lowest index at which the string occurs in `filler`.  See examples
    below.

    Space between pieces of data is filled out using the iterable `filler`.  The
    `n`'th byte in the output will be byte at index ``n % len(iterable)`` byte
    in `filler` if it has finite length or the byte at index `n` otherwise.

    If `length` is given, the output will be padded with bytes from `filler` to
    be this size.  If the output is longer than `length`, a :py:exc:`ValueError`
    exception is raised.

    The three kwargs `word_size`, `endianness` and `sign` will default to using
    values in :mod:`pwnlib.context` if not specified as an argument.

    Arguments:
      args: Values to flatten
      preprocessor (function): Gets called on every element to optionally
         transform the element before flattening. If :const:`None` is
         returned, then the original value is used.
      length: The length of the output.
      filler: Iterable to use for padding.
      word_size (int): Word size of the converted integer.
      endianness (str): Endianness of the converted integer ("little"/"big").
      sign (str): Signedness of the converted integer (False/True)

    Examples:

        (Test setup, please ignore)
    
        >>> context.clear()

        Basic usage of :meth:`flat` works similar to the pack() routines.

        >>> flat(4)
        b'\x04\x00\x00\x00'

        :meth:`flat` works with strings, bytes, lists, and dictionaries.

        >>> flat(b'X')
        b'X'
        >>> flat([1,2,3])
        b'\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00\x00\x00'
        >>> flat({4:b'X'})
        b'aaaaX'

        :meth:`.flat` flattens all of the values provided, and allows nested lists
        and dictionaries.

        >>> flat([{4:b'X'}] * 2)
        b'aaaaXaaacX'
        >>> flat([[[[[[[[[1]]]], 2]]]]])
        b'\x01\x00\x00\x00\x02\x00\x00\x00'

        You can also provide additional arguments like endianness, word-size, and
        whether the values are treated as signed or not.

        >>> flat(1, b"test", [[[b"AB"]*2]*3], endianness = 'little', word_size = 16, sign = False)
        b'\x01\x00testABABABABABAB'

        A preprocessor function can be provided in order to modify the values in-flight.
        This example converts increments each value by 1, then converts to a byte string.

        >>> flat([1, [2, 3]], preprocessor = lambda x: str(x+1).encode())
        b'234'

        Using dictionaries is a fast way to get specific values at specific offsets,
        without having to do ``data += "foo"`` repeatedly.

        >>> flat({12: 0x41414141,
        ...       24: b'Hello',
        ...      })
        b'aaaabaaacaaaAAAAeaaafaaaHello'

        Dictionary usage permits directly using values derived from :func:`.cyclic`.
        See :func:`.cyclic`, :function:`pwnlib.context.context.cyclic_alphabet`, and :data:`.context.cyclic_size`
        for more options.  

        The cyclic pattern can be provided as either the text or hexadecimal offset.

        >>> flat({ 0x61616162: b'X'})
        b'aaaaX'
        >>> flat({'baaa': b'X'})
        b'aaaaX'

        Fields do not have to be in linear order, and can be freely mixed.
        This also works with cyclic offsets.

        >>> flat({2: b'A', 0:b'B'})
        b'BaA'
        >>> flat({0x61616161: b'x', 0x61616162: b'y'})
        b'xaaay'
        >>> flat({0x61616162: b'y', 0x61616161: b'x'})
        b'xaaay'

        Fields do not have to be in order, and can be freely mixed.

        >>> flat({'caaa': b'XXXX', 16: b'\x41', 20: 0xdeadbeef})
        b'aaaabaaaXXXXdaaaAaaa\xef\xbe\xad\xde'
        >>> flat({ 8: [0x41414141, 0x42424242], 20: b'CCCC'})
        b'aaaabaaaAAAABBBBeaaaCCCC'
        >>> fit({
        ...     0x61616161: b'a',
        ...     1: b'b',
        ...     0x61616161+2: b'c',
        ...     3: b'd',
        ... })
        b'abadbaaac'

        By default, gaps in the data are filled in with the :meth:`.cyclic` pattern.
        You can customize this by providing an iterable or method for the ``filler``
        argument.

        >>> flat({12: b'XXXX'}, filler = b'_', length = 20)
        b'____________XXXX____'
        >>> flat({12: b'XXXX'}, filler = b'AB', length = 20)
        b'ABABABABABABXXXXABAB'

        Nested dictionaries also work as expected.

        >>> flat({4: {0: b'X', 4: b'Y'}})
        b'aaaaXaaaY'
        >>> fit({4: {4: b'XXXX'}})
        b'aaaabaaaXXXX'

        Negative indices are also supported, though this only works for integer
        keys.
    
        >>> flat({-4: b'x', -1: b'A', 0: b'0', 4: b'y'})
        b'xaaA0aaay'
    """
    # HACK: To avoid circular imports we need to delay the import of `cyclic`
    from pwnlib.util import cyclic

    preprocessor = kwargs.pop('preprocessor', lambda x: None)
    filler = kwargs.pop('filler', cyclic.de_bruijn())
    length = kwargs.pop('length', None)
    stacklevel = kwargs.pop('stacklevel', 0)

    if isinstance(filler, (str, six.text_type)):
        filler = bytearray(_need_bytes(filler))

    if kwargs != {}:
        raise TypeError("flat() does not support argument %r" %
                        kwargs.popitem()[0])

    filler = iters.cycle(filler)
    out = _flat(args, preprocessor, make_packer(), filler, stacklevel + 2)

    if length:
        if len(out) > length:
            raise ValueError(
                "flat(): Arguments does not fit within `length` (= %d) bytes" %
                length)
        out += b''.join(p8(next(filler)) for _ in range(length - len(out)))

    return out
Example #2
0
def fit(pieces=None, **kwargs):
    """fit(pieces, filler = de_bruijn(), length = None, preprocessor = None) -> str

    Generates a string from a dictionary mapping offsets to data to place at
    that offset.

    For each key-value pair in `pieces`, the key is either an offset or a byte
    sequence.  In the latter case, the offset will be the lowest index at which
    the sequence occurs in `filler`.  See examples below.

    Each piece of data is passed to :meth:`flat` along with the keyword
    arguments `word_size`, `endianness` and `sign`.

    Space between pieces of data is filled out using the iterable `filler`.  The
    `n`'th byte in the output will be byte at index ``n % len(iterable)`` byte
    in `filler` if it has finite length or the byte at index `n` otherwise.

    If `length` is given, the output will padded with bytes from `filler` to be
    this size.  If the output is longer than `length`, a :py:exc:`ValueError`
    exception is raised.

    If entries in `pieces` overlap, a :py:exc:`ValueError` exception is
    raised.

    Arguments:
      pieces: Offsets and values to output.
      length: The length of the output.
      filler: Iterable to use for padding.
      preprocessor (function): Gets called on every element to optionally
         transform the element before flattening. If :const:`None` is
         returned, then the original value is used.
      word_size (int): Word size of the converted integer.
      endianness (str): Endianness of the converted integer ("little"/"big").
      sign (str): Signedness of the converted integer (False/True)

    Examples:
      >>> fit({12: 0x41414141,
      ...      24: 'Hello',
      ...     })
      'aaaabaaacaaaAAAAeaaafaaaHello'
      >>> fit({'caaa': ''})
      'aaaabaaa'
      >>> fit({12: 'XXXX'}, filler = 'AB', length = 20)
      'ABABABABABABXXXXABAB'
      >>> fit({ 8: [0x41414141, 0x42424242],
      ...      20: 'CCCC'})
      'aaaabaaaAAAABBBBeaaaCCCC'

    """
    # HACK: To avoid circular imports we need to delay the import of `cyclic`
    from pwnlib.util import cyclic

    filler       = kwargs.pop('filler', cyclic.de_bruijn())
    length       = kwargs.pop('length', None)
    preprocessor = kwargs.pop('preprocessor', lambda x: None)
    word_size    = kwargs.pop('word_size', None)
    endianness   = kwargs.pop('endianness', None)
    sign         = kwargs.pop('sign', None)

    if kwargs != {}:
        raise TypeError("fit() does not support argument %r" % kwargs.popitem()[0])

    packer = make_packer()
    filler = iters.cycle(filler)
    out = ''

    if not length and not pieces:
        return ''

    if not pieces:
        return ''.join(filler.next() for f in range(length))

    # convert str keys to offsets
    pieces_ = dict()
    for k, v in pieces.items():
        if isinstance(k, (int, long)):
            pass
        elif isinstance(k, str):
            while k not in out:
                out += filler.next()
            k = out.index(k)
        else:
            raise TypeError("fit(): offset must be of type int or str, but got '%s'" % type(k))
        pieces_[k] = v
    pieces = pieces_

    # convert values to their flattened forms
    for k,v in pieces.items():
        pieces[k] = _flat([v], preprocessor, packer)

    # if we were provided a length, make sure everything fits
    last = max(pieces)
    if length and last and (last + len(pieces[last]) > length):
        raise ValueError("fit(): Pieces do not fit within `length` (= %d) bytes" % length)

    # insert data into output
    out = list(out)
    l = 0
    for k, v in sorted(pieces.items()):
        if k < l:
            raise ValueError("fit(): data at offset %d overlaps with previous data which ends at offset %d" % (k, l))
        while len(out) < k:
            out.append(filler.next())
        v = _flat([v], preprocessor, packer)
        l = k + len(v)

        # consume the filler for each byte of actual data
        for i in range(len(out), l):
            filler.next()

        out[k:l] = v

    # truncate/pad output
    if length:
        if l > length:
            raise ValueError("fit(): Pieces does not fit within `length` (= %d) bytes" % length)
        while len(out) < length:
            out.append(filler.next())
    else:
        out = out[:l]

    return ''.join(out)
Example #3
0
def flat(*args, **kwargs):
    r"""flat(\*args, preprocessor = None, length = None, filler = de_bruijn(),
     word_size = None, endianness = None, sign = None) -> str

    Flattens the arguments into a string.

    This function takes an arbitrary number of arbitrarily nested lists, tuples
    and dictionaries.  It will then find every string and number inside those
    and flatten them out.  Strings are inserted directly while numbers are
    packed using the :func:`pack` function.  Unicode strings are UTF-8 encoded.

    Dictionary keys give offsets at which to place the corresponding values
    (which are recursively flattened).  Offsets are relative to where the
    flattened dictionary occurs in the output (i.e. `{0: 'foo'}` is equivalent
    to `'foo'`).  Offsets can be integers, unicode strings or regular strings.
    Integer offsets >= ``2**(word_size-8)`` are converted to a string using
    `:func:pack`.  Unicode strings are UTF-8 encoded.  After these conversions
    offsets are either integers or strings.  In the latter case, the offset will
    be the lowest index at which the string occurs in `filler`.  See examples
    below.

    Space between pieces of data is filled out using the iterable `filler`.  The
    `n`'th byte in the output will be byte at index ``n % len(iterable)`` byte
    in `filler` if it has finite length or the byte at index `n` otherwise.

    If `length` is given, the output will be padded with bytes from `filler` to
    be this size.  If the output is longer than `length`, a :py:exc:`ValueError`
    exception is raised.

    The three kwargs `word_size`, `endianness` and `sign` will default to using
    values in :mod:`pwnlib.context` if not specified as an argument.

    Arguments:
      args: Values to flatten
      preprocessor (function): Gets called on every element to optionally
         transform the element before flattening. If :const:`None` is
         returned, then the original value is used.
      length: The length of the output.
      filler: Iterable to use for padding.
      word_size (int): Word size of the converted integer.
      endianness (str): Endianness of the converted integer ("little"/"big").
      sign (str): Signedness of the converted integer (False/True)

    Examples:
      >>> flat(1, "test", [[["AB"]*2]*3], endianness = 'little', word_size = 16, sign = False)
      b'\x01\x00testABABABABABAB'
      >>> flat([1, [2, 3]], preprocessor = lambda x: str(x+1))
      b'234'
      >>> flat({12: 0x41414141,
      ...       24: 'Hello',
      ...      })
      b'aaaabaaacaaaAAAAeaaafaaaHello'
      >>> flat({'caaa': ''})
      b'aaaabaaa'
      >>> flat({12: 'XXXX'}, filler = (ord('A'), ord('B')), length = 20)
      b'ABABABABABABXXXXABAB'
      >>> flat({ 8: [0x41414141, 0x42424242],
      ...       20: 'CCCC'})
      b'aaaabaaaAAAABBBBeaaaCCCC'
      >>> flat({ 0x61616162: 'X'})
      b'aaaaX'
      >>> flat({4: {0: 'X', 4: 'Y'}})
      b'aaaaXaaaY'

    """
    # HACK: To avoid circular imports we need to delay the import of `cyclic`
    from pwnlib.util import cyclic

    preprocessor = kwargs.pop('preprocessor', lambda x: None)
    filler = kwargs.pop('filler', cyclic.de_bruijn())
    length = kwargs.pop('length', None)

    if kwargs != {}:
        raise TypeError("flat() does not support argument %r" %
                        kwargs.popitem()[0])

    filler = iters.cycle(filler)
    out = _flat(args, preprocessor, make_packer(), filler)

    if length:
        if len(out) > length:
            raise ValueError(
                "flat(): Arguments does not fit within `length` (= %d) bytes" %
                length)
        out += b''.join(p8(next(filler)) for _ in range(length - len(out)))

    return out
Example #4
0
def fit(pieces=None, **kwargs):
    """fit(pieces, filler = de_bruijn(), length = None, preprocessor = None) -> str

    Generates a string from a dictionary mapping offsets to data to place at
    that offset.

    For each key-value pair in `pieces`, the key is either an offset or a byte
    sequence.  In the latter case, the offset will be the lowest index at which
    the sequence occurs in `filler`.  See examples below.

    Each piece of data is passed to :meth:`flat` along with the keyword
    arguments `word_size`, `endianness` and `sign`.

    Space between pieces of data is filled out using the iterable `filler`.  The
    `n`'th byte in the output will be byte at index ``n % len(iterable)`` byte
    in `filler` if it has finite length or the byte at index `n` otherwise.

    If `length` is given, the output will padded with bytes from `filler` to be
    this size.  If the output is longer than `length`, a :py:exc:`ValueError`
    exception is raised.

    If entries in `pieces` overlap, a :py:exc:`ValueError` exception is
    raised.

    Arguments:
      pieces: Offsets and values to output.
      length: The length of the output.
      filler: Iterable to use for padding.
      preprocessor (function): Gets called on every element to optionally
         transform the element before flattening. If :const:`None` is
         returned, then the original value is used.
      word_size (int): Word size of the converted integer.
      endianness (str): Endianness of the converted integer ("little"/"big").
      sign (str): Signedness of the converted integer (False/True)

    Examples:
      >>> fit({12: 0x41414141,
      ...      24: 'Hello',
      ...     })
      'aaaabaaacaaaAAAAeaaafaaaHello'
      >>> fit({'caaa': ''})
      'aaaabaaa'
      >>> fit({12: 'XXXX'}, filler = 'AB', length = 20)
      'ABABABABABABXXXXABAB'
      >>> fit({ 8: [0x41414141, 0x42424242],
      ...      20: 'CCCC'})
      'aaaabaaaAAAABBBBeaaaCCCC'
      >>> fit({ 0x61616162: 'X'})
      'aaaaX'

    """
    # HACK: To avoid circular imports we need to delay the import of `cyclic`
    from pwnlib.util import cyclic

    filler       = kwargs.pop('filler', cyclic.de_bruijn())
    length       = kwargs.pop('length', None)
    preprocessor = kwargs.pop('preprocessor', lambda x: None)

    if kwargs != {}:
        raise TypeError("fit() does not support argument %r" % kwargs.popitem()[0])

    packer = make_packer()
    filler = iters.cycle(filler)
    out = ''

    if not length and not pieces:
        return ''

    if not pieces:
        return ''.join(filler.next() for f in range(length))

    def fill(out, value):
        while value not in out:
            out += filler.next()
        return out, out.index(value)

    # convert str keys to offsets
    # convert large int keys to offsets
    pieces_ = dict()
    for k, v in pieces.items():
        if isinstance(k, (int, long)):
            # cyclic() generally starts with 'aaaa'
            if k >= 0x61616161:
                out, k = fill(out, pack(k))
        elif isinstance(k, str):
            out, k = fill(out, k)
        else:
            raise TypeError("fit(): offset must be of type int or str, but got '%s'" % type(k))
        pieces_[k] = v
    pieces = pieces_

    # convert values to their flattened forms
    for k,v in pieces.items():
        pieces[k] = _flat([v], preprocessor, packer)

    # if we were provided a length, make sure everything fits
    last = max(pieces)
    if length and last and (last + len(pieces[last]) > length):
        raise ValueError("fit(): Pieces do not fit within `length` (= %d) bytes" % length)

    # insert data into output
    out = list(out)
    l = 0
    for k, v in sorted(pieces.items()):
        if k < l:
            raise ValueError("fit(): data at offset %d overlaps with previous data which ends at offset %d" % (k, l))
        while len(out) < k:
            out.append(filler.next())
        v = _flat([v], preprocessor, packer)
        l = k + len(v)

        # consume the filler for each byte of actual data
        for i in range(len(out), l):
            filler.next()

        out[k:l] = v

    # truncate/pad output
    if length:
        if l > length:
            raise ValueError("fit(): Pieces does not fit within `length` (= %d) bytes" % length)
        while len(out) < length:
            out.append(filler.next())
    else:
        out = out[:l]

    return ''.join(out)