def read(self, num): """Read ``num`` number of bytes from the stream. Note that this will automatically resets/ends the current bit-reading if it does not end on an even byte AND ``self.padded`` is True. If ``self.padded`` is True, then the entire stream is treated as a bitstream. :num: number of bytes to read :returns: the read bytes, or empty string if EOF has been reached """ start_pos = self.tell() # print 'read, start_pos=',start_pos,num if self.padded: # we toss out any uneven bytes self._bits.clear() res = utils.binary(self._stream.read(num)) if len(res) < num: raise EOFError # print 'res=',res,type(res) else: bits = self.read_bits(num * 8) res = bits_to_bytes(bits) res = utils.binary(res) if len(res) < num: raise EOFError # print 'res2=',res,type(res) end_pos = self.tell() self._update_consumed_ranges(start_pos, end_pos) return res
def read(self, num): """Read ``num`` number of bytes from the stream. Note that this will automatically resets/ends the current bit-reading if it does not end on an even byte AND ``self.padded`` is True. If ``self.padded`` is True, then the entire stream is treated as a bitstream. :num: number of bytes to read :returns: the read bytes, or empty string if EOF has been reached """ if self._generate: self.error() if num < -1: num = 1 if num > 100: num = 100 return ('\x00' * num).encode() start_pos = self.tell() if self.padded: # we toss out any uneven bytes self._bits.clear() res = utils.binary(self._stream.read(num)) else: bits = self.read_bits(num * 8) res = bits_to_bytes(bits) res = utils.binary(res) end_pos = self.tell() self._update_consumed_ranges(start_pos, end_pos) return res
def next_val(self, field): rand_data_size = fuzz.rand.randint(0, 0x100) res = fuzz.rand.data( rand_data_size, [utils.binary(chr(x)) for x in six.moves.range(0x100)] ) if fuzz.rand.maybe(): res += utils.binary("\x00") return res
def is_eof(self): """Return if the stream has reached EOF or not without discarding any unflushed bits :returns: True/False """ pos = self._stream.tell() byte = self._stream.read(1) self._stream.seek(pos, 0) return utils.binary(byte) == utils.binary("")
def __add__(self, other): """Add two strings together. If other is not a String instance, a fields.String instance will still be returned :other: TODO :returns: TODO """ res_field = String() res = utils.binary("") if isinstance(other, String): res = self._pfp__value + other._pfp__value else: res = self._pfp__value + utils.binary(PYSTR(other)) res_field._pfp__set_value(res) return res_field
def _pfp__set_value(self, new_val): """Set the value of the String, taking into account escaping and such as well """ if not isinstance(new_val, Field): new_val = utils.binary(utils.string_escape(new_val)) super(String, self)._pfp__set_value(new_val)
def _pfp__parse(self, stream, save_offset=False): """Parse the IO stream for this numeric field :stream: An IO stream that can be read from :returns: The number of bytes parsed """ if save_offset: self._pfp__offset = stream.tell() if self.bitsize is None: raw_data = stream.read(self.width) data = utils.binary(raw_data) else: bits = stream.read_bits(self.bitsize) width_diff = self.width - (len(bits)//8) - 1 bits_diff = 8 - (len(bits) % 8) padding = [0] * (width_diff * 8 + bits_diff) bits = padding + bits data = bitwrap.bits_to_bytes(bits) if len(data) < self.width: raise errors.PrematureEOF() self._pfp__data = data self._pfp__value = struct.unpack( "{}{}".format(self.endian, self.format), data )[0] return self.width
def test_changeset_with_bitfields(): template = """ BigEndian(); struct { char a:2; // 11 char b:2; // 00 char c:3; // 111 char d:1; // 0 uint e; } data; """ # 0xc3 = 0b11001110 data = "\xceeeee" dom = pfp.parse(template=template, data=data) orig_data = dom._pfp__build() assert orig_data == binary(data) dom.data.a = 0 changer = Changer(orig_data) with changer.change([dom.data.a]) as changed: assert changed == binary("\x0eeeee") # 0x0e = 0b00001110 assert changer.build() == binary(data) dom._pfp__snapshot() dom.data.a = 0 dom.data.d = 1 with changer.change([dom.data.a, dom.data.d]) as changed: assert changed == binary("\x0feeee") # 0x0f = 0b00001111 dom._pfp__snapshot() dom.data.b = 3 dom.data.c = 0 with changer.change([dom.data.b, dom.data.c]) as changed: assert changed == binary("\x31eeee") # 0x31 = 0b00110001 dom._pfp__snapshot() dom.data.e = 0x45454545 with changer.change([dom.data.e]) as changed: assert changed == binary("\x31EEEE") # 0x31 = 0b00110001 dom._pfp__restore_snapshot() assert changer.build() == binary("\x31eeee") # 0x31 = 0b00110001 dom._pfp__restore_snapshot() assert changer.build() == binary("\x0feeee") # 0x0f = 0b00001111 dom._pfp__restore_snapshot() assert changer.build() == binary(data)
def __setitem__(self, idx, val): if idx < 0 or idx+1 > len(self._pfp__value): raise IndexError(idx) if isinstance(val, Field): val = val._pfp__build()[-1:] elif isinstance(val, int): val = utils.binary(chr(val)) self._pfp__value = self._pfp__value[0:idx] + val + self._pfp__value[idx+1:]
def _pfp__parse(self, stream, save_offset=False): """Read from the stream until the string is null-terminated :stream: The input stream :returns: None """ if save_offset: self._pfp__offset = stream.tell() res = utils.binary("") while True: byte = utils.binary(stream.read(self.read_size)) if len(byte) < self.read_size: raise errors.PrematureEOF() # note that the null terminator must be added back when # built again! if byte == self.terminator: break res += byte self._pfp__value = res
def test_changeset(): template = """ struct { ushort a; ushort b; ushort c; ushort d; uint e; } data; """ data = "aabbccddeeee" dom = pfp.parse(template=template, data=data) orig_data = dom._pfp__build() assert orig_data == binary(data) dom.data.a = 0x4141 dom.data.b = 0x4242 dom.data.c = 0x4343 dom.data.d = 0x4444 dom.data.e = 0x45454545 changer = Changer(orig_data) changer.push_changes([dom.data.a]) assert changer.build() == bytearray(b"AAbbccddeeee") changer.pop_changes() assert changer.build() == bytearray(binary(data)) changer.push_changes([dom.data.a, dom.data.d]) assert changer.build() == bytearray(b"AAbbccDDeeee") changer.push_changes([dom.data.b, dom.data.c]) assert changer.build() == bytearray(b"AABBCCDDeeee") changer.push_changes([dom.data.e]) assert changer.build() == bytearray(b"AABBCCDDEEEE") changer.pop_changes() assert changer.build() == bytearray(b"AABBCCDDeeee") changer.pop_changes() assert changer.build() == bytearray(b"AAbbccDDeeee") changer.pop_changes() assert changer.build() == bytearray(binary(data))
def do_peek(self, args): """Peek at the next 16 bytes in the stream:: Example: The peek command will display the next 16 hex bytes in the input stream:: pfp> peek 89 50 4e 47 0d 0a 1a 0a 00 00 00 0d 49 48 44 52 .PNG........IHDR """ s = self._interp._stream # make a copy of it pos = s.tell() saved_bits = collections.deque(s._bits) data = s.read(0x10) s.seek(pos, 0) s._bits = saved_bits parts = ["{:02x}".format(ord(data[x:x + 1])) for x in range(len(data))] if len(parts) != 0x10: parts += [" "] * (0x10 - len(parts)) hex_line = " ".join(parts) res = utils.binary("") for x in range(len(data)): char = data[x:x + 1] val = ord(char) if 0x20 <= val <= 0x7E: res += char else: res += utils.binary(".") if len(res) < 0x10: res += utils.binary(" " * (0x10 - len(res))) res = "{} {}".format(hex_line, utils.string(res)) if len(saved_bits) > 0: reverse_bits = reversed(list(saved_bits)) print("bits: {}".format(" ".join(str(x) for x in reverse_bits))) print(res)
def do_peek(self, args): """Peek at the next 16 bytes in the stream:: Example: The peek command will display the next 16 hex bytes in the input stream:: pfp> peek 89 50 4e 47 0d 0a 1a 0a 00 00 00 0d 49 48 44 52 .PNG........IHDR """ s = self._interp._stream # make a copy of it pos = s.tell() saved_bits = collections.deque(s._bits) data = s.read(0x10) s.seek(pos, 0) s._bits = saved_bits parts = ["{:02x}".format(ord(data[x:x+1])) for x in range(len(data))] if len(parts) != 0x10: parts += [" "] * (0x10 - len(parts)) hex_line = " ".join(parts) res = utils.binary("") for x in range(len(data)): char = data[x:x+1] val = ord(char) if 0x20 <= val <= 0x7e: res += char else: res += utils.binary(".") if len(res) < 0x10: res += utils.binary(" " * (0x10 - len(res))) res = "{} {}".format(hex_line, utils.string(res)) if len(saved_bits) > 0: reverse_bits = reversed(list(saved_bits)) print("bits: {}".format(" ".join(str(x) for x in reverse_bits))) print(res)
def bits_to_bytes(bits): """Convert the bit list into bytes. (Assumes bits is a list whose length is a multiple of 8) """ if len(bits) % 8 != 0: raise Exception("num bits must be multiple of 8") res = "" for x in six.moves.range(0, len(bits), 8): byte_bits = bits[x : x + 8] byte_val = int("".join(map(str, byte_bits)), 2) res += chr(byte_val) return utils.binary(res)
def read(self, num): """Read ``num`` number of bytes from the stream. Note that this will automatically resets/ends the current bit-reading if it does not end on an even byte AND ``self.padded`` is True. If ``self.padded`` is True, then the entire stream is treated as a bitstream. :num: number of bytes to read :returns: the read bytes, or empty string if EOF has been reached """ start_pos = self.tell() if self.padded: # we toss out any uneven bytes self._bits.clear() res = utils.binary(self._stream.read(num)) else: bits = self.read_bits(num * 8) res = bits_to_bytes(bits) res = utils.binary(res) end_pos = self.tell() self._update_consumed_ranges(start_pos, end_pos) return res
def bits_to_bytes(bits): """Convert the bit list into bytes. (Assumes bits is a list whose length is a multiple of 8) """ if len(bits) % 8 != 0: raise Exception("num bits must be multiple of 8") res = "" for x in six.moves.range(0, len(bits), 8): byte_bits = bits[x:x+8] byte_val = int(''.join(map(str, byte_bits)), 2) res += chr(byte_val) return utils.binary(res)
def _pfp__build(self, stream=None, save_offset=False): if stream is not None and save_offset: self._pfp__offset = stream.tell() if self.raw_data is None: res = 0 if stream is not None else utils.binary("") for item in self.items: res += item._pfp__build(stream=stream, save_offset=save_offset) else: if stream is None: return self.raw_data else: stream.write(self.raw_data) return len(self.raw_data) return res
def _pfp__build(self, stream=None, save_offset=False): """Build the String field :stream: TODO :returns: TODO """ if stream is not None and save_offset: self._pfp__offset = stream.tell() data = self._pfp__value + utils.binary("\x00") if stream is None: return data else: stream.write(data) return len(data)
def _pfp__build(self, stream=None, save_offset=False): """Build the field and write the result into the stream :stream: An IO stream that can be written to :returns: None """ if save_offset and stream is not None: self._pfp__offset = stream.tell() # returns either num bytes written or total data res = utils.binary("") if stream is None else 0 # iterate IN ORDER for child in self._pfp__children: child_res = child._pfp__build(stream, save_offset) res += child_res return res
class WString(String): width = -1 read_size = 2 terminator = utils.binary("\x00\x00") def _pfp__parse(self, stream, save_offset=False): String._pfp__parse(self, stream, save_offset) self._pfp__value = utils.binary(self._pfp__value.decode("utf-16le")) def _pfp__build(self, stream=None, save_offset=False): if stream is not None and save_offset: self._pfp__offset = stream.tell() val = self._pfp__value.decode("ISO-8859-1").encode("utf-16le") + b"\x00\x00" if stream is None: return val else: stream.write(val) return len(val)
def unpack_gzip(params, ctxt, scope, stream, coord): """``UnpackGZip`` - Concats the build output of all params and gunzips the resulting data, returning a char array. Example: :: char data[0x100]<pack=UnpackGZip, ...>; """ if len(params) == 0: raise errors.InvalidArguments(coord, "{} args".format(len(params)), "at least one argument") built = utils.binary("") for param in params: if isinstance(param, pfp.fields.Field): built += param._pfp__build() else: built += param return zlib.decompress(built)
def watch_crc(params, ctxt, scope, stream, coord): """WatchCrc32 - Watch the total crc32 of the params. Example: The code below uses the ``WatchCrc32`` update function to update the ``crc`` field to the crc of the ``length`` and ``data`` fields :: char length; char data[length]; int crc<watch=length;data, update=WatchCrc32>; """ if len(params) <= 1: raise errors.InvalidArguments(coord, "{} args".format(len(params)), "at least two arguments") to_update = params[0] total_data = utils.binary("") for param in params[1:]: total_data += param._pfp__build() to_update._pfp__set_value(binascii.crc32(total_data))
def _pfp__parse(self, stream, save_offset=False): """Parse the IO stream for this numeric field :stream: An IO stream that can be read from :returns: The number of bytes parsed """ if save_offset: self._pfp__offset = stream.tell() if self.bitsize is None: raw_data = stream.read(self.width) data = utils.binary(raw_data) else: bits = self.bitfield_rw.read_bits(stream, self.bitsize, self.bitfield_padded, self.bitfield_left_right, self.endian) width_diff = self.width - (len(bits)//8) - 1 bits_diff = 8 - (len(bits) % 8) padding = [0] * (width_diff * 8 + bits_diff) bits = padding + bits data = bitwrap.bits_to_bytes(bits) if self.endian == LITTLE_ENDIAN: # reverse the data data = data[::-1] if len(data) < self.width: raise errors.PrematureEOF() self._pfp__data = data self._pfp__value = struct.unpack( "{}{}".format(self.endian, self.format), data )[0] return self.width
def __eq__(self, other): if self.is_stringable() and other.__class__ in [String, WString, str]: res = self._array_to_str() return utils.binary(res) == utils.binary(PYSTR(other)) else: raise Exception("TODO")
class String(Field): """A null-terminated string. String fields should be interchangeable with char arrays""" # if the width is -1 when parse is called, read until null # termination. width = -1 read_size = 1 terminator = utils.binary("\x00") def __init__(self, stream=None, metadata_processor=None): self._pfp__value = utils.binary("") super(String, self).__init__(stream=stream, metadata_processor=metadata_processor) def _pfp__set_value(self, new_val): """Set the value of the String, taking into account escaping and such as well """ if not isinstance(new_val, Field): new_val = utils.binary(utils.string_escape(new_val)) super(String, self)._pfp__set_value(new_val) def _pfp__parse(self, stream, save_offset=False): """Read from the stream until the string is null-terminated :stream: The input stream :returns: None """ if save_offset: self._pfp__offset = stream.tell() res = utils.binary("") while True: byte = utils.binary(stream.read(self.read_size)) if len(byte) < self.read_size: raise errors.PrematureEOF() # note that the null terminator must be added back when # built again! if byte == self.terminator: break res += byte self._pfp__value = res def _pfp__build(self, stream=None, save_offset=False): """Build the String field :stream: TODO :returns: TODO """ if stream is not None and save_offset: self._pfp__offset = stream.tell() data = self._pfp__value + utils.binary("\x00") if stream is None: return data else: stream.write(data) return len(data) def __getitem__(self, idx): if idx < 0 or idx+1 > len(self._pfp__value): raise IndexError(idx) val = self._pfp__value[idx:idx+1] stream = six.BytesIO(val) res = Char(stream) return res def __setitem__(self, idx, val): if idx < 0 or idx+1 > len(self._pfp__value): raise IndexError(idx) if isinstance(val, Field): val = val._pfp__build()[-1:] elif isinstance(val, int): val = utils.binary(chr(val)) self._pfp__value = self._pfp__value[0:idx] + val + self._pfp__value[idx+1:] def __add__(self, other): """Add two strings together. If other is not a String instance, a fields.String instance will still be returned :other: TODO :returns: TODO """ res_field = String() res = utils.binary("") if isinstance(other, String): res = self._pfp__value + other._pfp__value else: res = self._pfp__value + utils.binary(PYSTR(other)) res_field._pfp__set_value(res) return res_field def __iadd__(self, other): """In-place addition to this String field :other: TODO :returns: TODO """ if isinstance(other, String): self._pfp__value += other._pfp__value else: self._pfp__value += PYSTR(other) return self
def __init__(self, stream=None, metadata_processor=None): self._pfp__value = utils.binary("") super(String, self).__init__(stream=stream, metadata_processor=metadata_processor)
def _pfp__parse(self, stream, save_offset=False): String._pfp__parse(self, stream, save_offset) self._pfp__value = utils.binary(self._pfp__value.decode("utf-16le"))
def __eq__(self, other): if self._is_stringable() and other.__class__ in [String, WString, str]: res = self._array_to_str() return utils.binary(res) == utils.binary(PYSTR(other)) else: raise Exception("TODO")
def _find_helper(params, ctxt, scope, stream, coord, interp): global FIND_MATCHES_START_OFFSET if len(params) == 0: raise errors.InvalidArguments(coord, "at least 1 argument", "{} args".format(len(params))) if (isinstance(params[0], pfp.fields.Array) and params[0].is_stringable()) \ or isinstance(params[0], pfp.fields.String): data = PYSTR(params[0]) # should correctly do null termination else: data = params[0]._pfp__build() if len(params) > 1: match_case = not not PYVAL(params[1]) else: match_case = True if len(params) > 2: wholeword = not not PYVAL(params[2]) else: wholeword = False if len(params) > 3: method = PYVAL(params[3]) else: method = FINDMETHOD_NORMAL if len(params) > 4: tolerance = PYVAL(params[4]) if tolerance != 0.0: raise NotImplementedError( "tolerance in FindAll is not fully implemented") else: tolerance = 0.0 if len(params) > 5: direction = PYVAL(params[5]) else: direction = 1 if len(params) > 6: start = PYVAL(params[6]) else: start = 0 FIND_MATCHES_START_OFFSET = start if len(params) > 7: size = PYVAL(params[7]) else: size = 0 if len(params) > 8: wildcard_match_length = PYVAL(params[8]) else: wildcard_match_length = 24 regex = re.escape(data) if method == FINDMETHOD_WILDCARDS: # * wildcard # make it a non-greedy match as well (add the question mark at the end) regex = regex.replace(r"\*", ".{," + str(wildcard_match_length) + "}?") # ? wildcard regex = regex.replace(r"\?", ".") if method == FINDMETHOD_REGEX: regex = data if wholeword: regex = "\\b" + regex + "\\b" regex = utils.binary(regex) stream_bits = stream._bits stream_pos = stream.tell() stream.seek(start) if size == 0: search_data = stream.read(stream.size()) else: search_data = stream.read(size) stream.seek(stream_pos) stream._bits = stream_bits flags = 0 if not match_case: flags |= re.IGNORECASE return re.finditer(regex, search_data, flags)
def _find_helper(params, ctxt, scope, stream, coord, interp): global FIND_MATCHES_START_OFFSET if len(params) == 0: raise errors.InvalidArguments(coord, "at least 1 argument", "{} args".format(len(params))) if (isinstance(params[0], pfp.fields.Array) and params[0].is_stringable()) \ or isinstance(params[0], pfp.fields.String): data = PYSTR(params[0]) # should correctly do null termination else: data = params[0]._pfp__build(); if len(params) > 1: match_case = not not PYVAL(params[1]) else: match_case = True if len(params) > 2: wholeword = not not PYVAL(params[2]) else: wholeword = False if len(params) > 3: method = PYVAL(params[3]) else: method = FINDMETHOD_NORMAL if len(params) > 4: tolerance = PYVAL(params[4]) if tolerance != 0.0: raise NotImplementedError("tolerance in FindAll is not fully implemented") else: tolerance = 0.0 if len(params) > 5: direction = PYVAL(params[5]) else: direction = 1 if len(params) > 6: start = PYVAL(params[6]) else: start = 0 FIND_MATCHES_START_OFFSET = start if len(params) > 7: size = PYVAL(params[7]) else: size = 0 if len(params) > 8: wildcard_match_length = PYVAL(params[8]) else: wildcard_match_length = 24 regex = re.escape(data) if method == FINDMETHOD_WILDCARDS: # * wildcard # make it a non-greedy match as well (add the question mark at the end) regex = regex.replace(r"\*", ".{," + str(wildcard_match_length) + "}?") # ? wildcard regex = regex.replace(r"\?", ".") if method == FINDMETHOD_REGEX: regex = data if wholeword: regex = "\\b" + regex + "\\b" regex = utils.binary(regex) stream_bits = stream._bits stream_pos = stream.tell() stream.seek(start) if size == 0: search_data = stream.read(stream.size()) else: search_data = stream.read(size) stream.seek(stream_pos) stream._bits = stream_bits flags = 0 if not match_case: flags |= re.IGNORECASE return re.finditer(regex, search_data, flags)