Пример #1
0
def parse(data=None, template=None, data_file=None, template_file=None, interp=None, debug=False, predefines=True, int3=True, cpp_path="cpp", cpp_args="-xc++", keep_successful=False):
	"""Parse the data stream using the supplied template. The data stream
	WILL NOT be automatically closed.

	:data: Input stream (yes, a STREAM, not str)
	:template: template contents (str)
	:data_file: path to the data to be used as the input stream
	:template_file: template file path
	:interp: the interpretor to be used (a default one will be created if ``None``)
	:debug: if debug information should be printed while interpreting the template (false)
	:predefines: if built-in type information should be inserted (true)
	:int3: if debugger breaks are allowed while interpreting the template (true)
	:cpp_path: the path to the ``cpp`` binary, used to strip comments ("cpp")
	:cpp_args: the args to the ``cpp`` binary to strip comments. Defaults to "", but "-xc++" might be useful on macs.
	:keep_successful: return any succesfully parsed data instead of raising an error. If an error occurred and ``keep_successful`` is True, then ``_pfp__error`` will be contain the exception object
	:returns: pfp DOM
	"""
	if data is None and data_file is None:
		raise Exception("No input data was specified")
	
	if data is not None and data_file is not None:
		raise Exception("Only one input data may be specified")
	
	if data_file is not None:
		data = open(os.path.expanduser(data_file), "rb")

	if template is None and template_file is None:
		raise Exception("No template specified!")
	
	if template is not None and template_file is not None:
		raise Exception("Only one template may be specified!")
	
	orig_filename = "string"
	if template_file is not None:
		orig_filename = template_file
		try:
			with open(os.path.expanduser(template_file), "r") as f:
				template = f.read()
		except Exception as e:
			raise Exception("Could not open template file '{}'".format(template_file))

	# the user may specify their own instance of PfpInterp to be
	# used
	if interp is None:
		interp = pfp.interp.PfpInterp(debug=debug, parser=PARSER, int3=int3, cpp_path=cpp_path, cpp_args=cpp_args)
	
	# so we can consume single bits at a time
	data = BitwrappedStream(data)

	dom = interp.parse(data, template, predefines=predefines, orig_filename=orig_filename, keep_successful=keep_successful)

	# close the data stream if a data_file was specified
	if data_file is not None:
		data.close()

	return dom
Пример #2
0
def parse(data=None, template=None, data_file=None, template_file=None, interp=None, debug=False, predefines=True, int3=True, cpp_path="cpp", cpp_args="-xc++", keep_successful=False):
	"""Parse the data stream using the supplied template. The data stream
	WILL NOT be automatically closed.

	:data: Input stream (yes, a STREAM, not str)
	:template: template contents (str)
	:data_file: path to the data to be used as the input stream
	:template_file: template file path
	:interp: the interpretor to be used (a default one will be created if ``None``)
	:debug: if debug information should be printed while interpreting the template (false)
	:predefines: if built-in type information should be inserted (true)
	:int3: if debugger breaks are allowed while interpreting the template (true)
	:cpp_path: the path to the ``cpp`` binary, used to strip comments ("cpp")
	:cpp_args: the args to the ``cpp`` binary to strip comments. Defaults to "", but "-xc++" might be useful on macs.
	:keep_successful: return any succesfully parsed data instead of raising an error. If an error occurred and ``keep_successful`` is True, then ``_pfp__error`` will be contain the exception object
	:returns: pfp DOM
	"""
	if data is None and data_file is None:
		raise Exception("No input data was specified")
	
	if data is not None and data_file is not None:
		raise Exception("Only one input data may be specified")
	
	if data_file is not None:
		data = open(os.path.expanduser(data_file), "rb")

	if template is None and template_file is None:
		raise Exception("No template specified!")
	
	if template is not None and template_file is not None:
		raise Exception("Only one template may be specified!")
	
	orig_filename = "string"
	if template_file is not None:
		orig_filename = template_file
		try:
			with open(os.path.expanduser(template_file), "r") as f:
				template = f.read()
		except Exception as e:
			raise Exception("Could not open template file '{}'".format(template_file))

	# the user may specify their own instance of PfpInterp to be
	# used
	if interp is None:
		interp = pfp.interp.PfpInterp(debug=debug, parser=PARSER, int3=int3, cpp_path=cpp_path, cpp_args=cpp_args)
	
	# so we can consume single bits at a time
	data = BitwrappedStream(data)

	dom = interp.parse(data, template, predefines=predefines, orig_filename=orig_filename, keep_successful=keep_successful)

	# close the data stream if a data_file was specified
	if data_file is not None:
		data.close()

	return dom
Пример #3
0
	def test_bits_read2_padded2(self):
		stream = six.BytesIO(pfp.utils.binary(chr(int("11110000",2)) + chr(int("10101010", 2))))
		bitwrapped = BitwrappedStream(stream)
		bitwrapped.padded = True

		res = bitwrapped.read_bits(4)
		self.assertEqual([1,1,1,1], res)

		next_byte = bitwrapped.read(1)
		self.assertEqual(pfp.utils.binary(chr(int("10101010", 2))), next_byte)
Пример #4
0
    def test_unconsumed_ranges3(self):
        stream = six.BytesIO(pfp.utils.binary("A" * 100))
        bitwrapped = BitwrappedStream(stream)

        bitwrapped.read(10)

        # it should not need a second read to add the
        # unconsumed range
        uranges = bitwrapped.unconsumed_ranges()

        self.assertEqual(len(uranges), 0)
Пример #5
0
	def test_unconsumed_ranges3(self):
		stream = six.BytesIO(pfp.utils.binary("A" * 100))
		bitwrapped = BitwrappedStream(stream)

		bitwrapped.read(10)

		# it should not need a second read to add the
		# unconsumed range
		uranges = bitwrapped.unconsumed_ranges()

		self.assertEqual(len(uranges), 0)
Пример #6
0
    def test_bits_read2_padded2(self):
        stream = six.BytesIO(
            pfp.utils.binary(
                chr(int("11110000", 2)) + chr(int("10101010", 2))))
        bitwrapped = BitwrappedStream(stream)
        bitwrapped.padded = True

        res = bitwrapped.read_bits(4)
        self.assertEqual([1, 1, 1, 1], res)

        next_byte = bitwrapped.read(1)
        self.assertEqual(pfp.utils.binary(chr(int("10101010", 2))), next_byte)
Пример #7
0
	def test_bits_read_unpadded(self):
		stream = six.BytesIO(pfp.utils.binary(chr(int("11110000",2)) + chr(int("10101010", 2))))
		bitwrapped = BitwrappedStream(stream)
		bitwrapped.padded = False

		res = bitwrapped.read_bits(4)
		self.assertEqual([1,1,1,1], res)

		res = bitwrapped.read(1)
		self.assertEqual(pfp.utils.binary(chr(int("00001010", 2))), res)

		res = bitwrapped.read_bits(4)
		self.assertEqual([1,0,1,0], res)
Пример #8
0
    def test_bits_write_padded(self):
        stream = six.BytesIO()
        bitwrapped = BitwrappedStream(stream)
        bitwrapped.padded = True

        bitwrapped.write_bits([1, 1, 0, 1])
        # should go to a new byte now, zero padded after the
        # 1101 bits
        bitwrapped.write(pfp.utils.binary("hello"))

        self.assertEqual(stream.getvalue(),
                         pfp.utils.binary(chr(int("11010000", 2)) + "hello"))
Пример #9
0
    def _handle_bitfield(self, field):
        """Find the field's first evenly-aligned previous sibling that is
        also a bitfield, as well as all subsequent siblings until a full
        bit "class" is reached. Build the entire set of bitfields as a group.

        E.g.:

            ushort a:1;
            ushort b:3;
            ushort c:10;
            ushort d:2;

        This entire group should be built
        """
        total_bits = field.width * 8
        bit_offset = lambda x: total_bits - x._pfp__offset_bits

        fields_to_build = []
        curr_field = field._pfp__prev_sibling
        # previous siblings
        while curr_field is not None and bit_offset(curr_field) >= 0:
            fields_to_build.append(curr_field)
            curr_field = curr_field._pfp__prev_sibling

        fields_to_build = list(reversed(fields_to_build))
        fields_to_build.append(field)

        # next siblings
        curr_field = field._pfp__next_sibling
        while (curr_field is not None
               and isinstance(curr_field, field.__class__)
               and curr_field.bitsize is not None):
            if bit_offset(curr_field) + curr_field.bitsize > total_bits:
                break
            fields_to_build.append(curr_field)
            curr_field = curr_field._pfp__next_sibling

        core_stream = six.BytesIO(b"")
        bit_stream = BitwrappedStream(core_stream)
        bitfield_rw = BitfieldRW(None, field.__class__)
        bitfield_rw.reserved_bits = field.bitfield_rw.reserved_bits

        for to_build in fields_to_build:
            old_bitfield_rw = to_build.bitfield_rw
            to_build.bitfield_rw = bitfield_rw
            to_build._pfp__build(bit_stream)
            to_build.bitfield_rw = old_bitfield_rw

        return core_stream.getvalue()
Пример #10
0
    def test_unconsumed_ranges2(self):
        stream = six.BytesIO(pfp.utils.binary("A" * 100))
        bitwrapped = BitwrappedStream(stream)

        bitwrapped.read(10)
        bitwrapped.seek(bitwrapped.tell() + 10)

        # it should not need a second read to add the
        # unconsumed range
        uranges = bitwrapped.unconsumed_ranges()

        self.assertEqual(len(uranges), 1)

        # test (11,20]
        self.assertEqual(len(uranges[11]), 1)
        self.assertEqual(len(uranges[10]), 0)
        self.assertEqual(len(uranges[19]), 1)
        self.assertEqual(len(uranges[20]), 0)
Пример #11
0
	def test_bits_write_padded(self):
		stream = six.BytesIO()
		bitwrapped = BitwrappedStream(stream)
		bitwrapped.padded = True

		bitwrapped.write_bits([1,1,0,1])
		# should go to a new byte now, zero padded after the
		# 1101 bits
		bitwrapped.write(pfp.utils.binary("hello"))

		self.assertEqual(stream.getvalue(), pfp.utils.binary(chr(int("11010000", 2)) + "hello"))
Пример #12
0
    def test_bits_read2_padded1(self):
        stream = six.BytesIO(
            pfp.utils.binary(
                chr(int("11110000", 2)) + chr(int("10101010", 2))))
        bitwrapped = BitwrappedStream(stream)
        bitwrapped.padded = True

        res = bitwrapped.read_bits(4)
        self.assertEqual([1, 1, 1, 1], res)

        res = bitwrapped.read_bits(3)
        self.assertEqual([0, 0, 0], res)

        res = bitwrapped.read_bits(4)
        self.assertEqual([0, 1, 0, 1], res)

        res = bitwrapped.read_bits(5)
        self.assertEqual([0, 1, 0, 1, 0], res)
Пример #13
0
	def test_unconsumed_ranges2(self):
		stream = six.BytesIO(pfp.utils.binary("A" * 100))
		bitwrapped = BitwrappedStream(stream)

		bitwrapped.read(10)
		bitwrapped.seek(bitwrapped.tell()+10)

		# it should not need a second read to add the
		# unconsumed range
		uranges = bitwrapped.unconsumed_ranges()

		self.assertEqual(len(uranges), 1)

		# test (11,20]
		self.assertEqual(len(uranges[11]), 1)
		self.assertEqual(len(uranges[10]), 0)
		self.assertEqual(len(uranges[19]), 1)
		self.assertEqual(len(uranges[20]), 0)
Пример #14
0
    def test_bits_read_unpadded(self):
        stream = six.BytesIO(
            pfp.utils.binary(
                chr(int("11110000", 2)) + chr(int("10101010", 2))))
        bitwrapped = BitwrappedStream(stream)
        bitwrapped.padded = False

        res = bitwrapped.read_bits(4)
        self.assertEqual([1, 1, 1, 1], res)

        res = bitwrapped.read(1)
        self.assertEqual(pfp.utils.binary(chr(int("00001010", 2))), res)

        res = bitwrapped.read_bits(4)
        self.assertEqual([1, 0, 1, 0], res)
Пример #15
0
	def test_bits_read2_padded1(self):
		stream = six.BytesIO(pfp.utils.binary(chr(int("11110000",2)) + chr(int("10101010", 2))))
		bitwrapped = BitwrappedStream(stream)
		bitwrapped.padded = True

		res = bitwrapped.read_bits(4)
		self.assertEqual([1,1,1,1], res)

		res = bitwrapped.read_bits(3)
		self.assertEqual([0,0,0], res)

		res = bitwrapped.read_bits(4)
		self.assertEqual([0,1,0,1], res)

		res = bitwrapped.read_bits(5)
		self.assertEqual([0,1,0,1,0], res)
Пример #16
0
def parse(
    data=None,
    template=None,
    data_file=None,
    template_file=None,
    interp=None,
    debug=False,
    predefines=True,
    int3=True,
    keep_successful=False,
    printf=True,
):
    """Parse the data stream using the supplied template. The data stream
    WILL NOT be automatically closed.

    :data: Input data, can be either a string or a file-like object (StringIO, file, etc)
    :template: template contents (str)
    :data_file: PATH to the data to be used as the input stream
    :template_file: template file path
    :interp: the interpretor to be used (a default one will be created if ``None``)
    :debug: if debug information should be printed while interpreting the template (false)
    :predefines: if built-in type information should be inserted (true)
    :int3: if debugger breaks are allowed while interpreting the template (true)
    :keep_successful: return any succesfully parsed data instead of raising an error. If an error occurred and ``keep_successful`` is True, then ``_pfp__error`` will be contain the exception object
    :printf: if ``False``, all calls to ``Printf`` (:any:`pfp.native.compat_interface.Printf`) will be noops. (default=``True``)
    :returns: pfp DOM
    """
    if data is None and data_file is None:
        raise Exception("No input data was specified")

    if data is not None and data_file is not None:
        raise Exception("Only one input data may be specified")

    if isinstance(data, six.string_types):
        data = six.StringIO(data)

    if data_file is not None:
        data = open(os.path.expanduser(data_file), "rb")

    if template is None and template_file is None:
        raise Exception("No template specified!")

    if template is not None and template_file is not None:
        raise Exception("Only one template may be specified!")

    orig_filename = "string"
    if template_file is not None:
        orig_filename = template_file
        try:
            with open(os.path.expanduser(template_file), "r") as f:
                template = f.read()
        except Exception as e:
            raise Exception(
                "Could not open template file '{}'".format(template_file)
            )

    # the user may specify their own instance of PfpInterp to be
    # used
    if interp is None:
        interp = pfp.interp.PfpInterp(debug=debug, parser=PARSER, int3=int3)

    # so we can consume single bits at a time
    data = BitwrappedStream(data)

    dom = interp.parse(
        data,
        template,
        predefines=predefines,
        orig_filename=orig_filename,
        keep_successful=keep_successful,
        printf=printf,
    )

    # close the data stream if a data_file was specified
    if data_file is not None:
        data.close()

    return dom
Пример #17
0
 def test_bits_read1(self):
     stream = six.BytesIO(pfp.utils.binary(chr(int("01010101", 2))))
     bitwrapped = BitwrappedStream(stream)
     res = bitwrapped.read_bits(8)
     self.assertEqual([0, 1, 0, 1, 0, 1, 0, 1], res)
Пример #18
0
 def test_bytes_read(self):
     stream = six.BytesIO(pfp.utils.binary("abcd"))
     bitwrapped = BitwrappedStream(stream)
     res = bitwrapped.read(4)
     self.assertEqual(pfp.utils.binary("abcd"), res)
Пример #19
0
    def test_tell_bits(self):
        stream = six.BytesIO(pfp.utils.binary("\x41" + chr(0b11001100)))
        bitwrapped = BitwrappedStream(stream)

        res = bitwrapped.read(1)
        self.assertEqual(res, b"\x41")

        self.assertEqual(bitwrapped.tell(), 1)
        self.assertEqual(bitwrapped.tell_bits(), 0)

        bits = bitwrapped.read_bits(1)
        self.assertEqual(bits, [1])
        self.assertEqual(bitwrapped.tell_bits(), 1)

        bits = bitwrapped.read_bits(1)
        self.assertEqual(bits, [1])
        self.assertEqual(bitwrapped.tell_bits(), 2)

        bits = bitwrapped.read_bits(1)
        self.assertEqual(bits, [0])
        self.assertEqual(bitwrapped.tell_bits(), 3)
Пример #20
0
    def test_unconsumed_ranges1(self):
        stream = six.BytesIO(pfp.utils.binary("A" * 100))
        bitwrapped = BitwrappedStream(stream)

        bitwrapped.read(10)
        bitwrapped.seek(bitwrapped.tell() + 10)
        bitwrapped.read(10)
        bitwrapped.seek(bitwrapped.tell() + 10)
        bitwrapped.read(10)

        uranges = bitwrapped.unconsumed_ranges()

        # test (11,20]
        self.assertEqual(len(uranges[11]), 1)
        self.assertEqual(len(uranges[10]), 0)
        self.assertEqual(len(uranges[19]), 1)
        self.assertEqual(len(uranges[20]), 0)

        # test (31,40]
        self.assertEqual(len(uranges[31]), 1)
        self.assertEqual(len(uranges[30]), 0)
        self.assertEqual(len(uranges[39]), 1)
        self.assertEqual(len(uranges[40]), 0)
Пример #21
0
	def test_unconsumed_ranges1(self):
		stream = six.BytesIO(pfp.utils.binary("A" * 100))
		bitwrapped = BitwrappedStream(stream)

		bitwrapped.read(10)
		bitwrapped.seek(bitwrapped.tell()+10)
		bitwrapped.read(10)
		bitwrapped.seek(bitwrapped.tell()+10)
		bitwrapped.read(10)

		uranges = bitwrapped.unconsumed_ranges()

		# test (11,20]
		self.assertEqual(len(uranges[11]), 1)
		self.assertEqual(len(uranges[10]), 0)
		self.assertEqual(len(uranges[19]), 1)
		self.assertEqual(len(uranges[20]), 0)

		# test (31,40]
		self.assertEqual(len(uranges[31]), 1)
		self.assertEqual(len(uranges[30]), 0)
		self.assertEqual(len(uranges[39]), 1)
		self.assertEqual(len(uranges[40]), 0)
Пример #22
0
	def test_bits_read1(self):
		stream = six.BytesIO(pfp.utils.binary(chr(int("01010101", 2))))
		bitwrapped = BitwrappedStream(stream)
		res = bitwrapped.read_bits(8)
		self.assertEqual([0,1,0,1,0,1,0,1], res)
Пример #23
0
	def test_bytes_read(self):
		stream = six.BytesIO(pfp.utils.binary("abcd"))
		bitwrapped = BitwrappedStream(stream)
		res = bitwrapped.read(4)
		self.assertEqual(pfp.utils.binary("abcd"), res)