def parse(data=None, template=None, data_file=None, template_file=None, interp=None, debug=False, predefines=True, int3=True, cpp_path="cpp", cpp_args="-xc++", keep_successful=False):
"""Parse the data stream using the supplied template. The data stream
WILL NOT be automatically closed.
:data: Input stream (yes, a STREAM, not str)
:template: template contents (str)
:data_file: path to the data to be used as the input stream
:template_file: template file path
:interp: the interpretor to be used (a default one will be created if ``None``)
:debug: if debug information should be printed while interpreting the template (false)
:predefines: if built-in type information should be inserted (true)
:int3: if debugger breaks are allowed while interpreting the template (true)
:cpp_path: the path to the ``cpp`` binary, used to strip comments ("cpp")
:cpp_args: the args to the ``cpp`` binary to strip comments. Defaults to "", but "-xc++" might be useful on macs.
:keep_successful: return any succesfully parsed data instead of raising an error. If an error occurred and ``keep_successful`` is True, then ``_pfp__error`` will be contain the exception object
:returns: pfp DOM
"""
if data is None and data_file is None:
raise Exception("No input data was specified")
if data is not None and data_file is not None:
raise Exception("Only one input data may be specified")
if data_file is not None:
data = open(os.path.expanduser(data_file), "rb")
if template is None and template_file is None:
raise Exception("No template specified!")
if template is not None and template_file is not None:
raise Exception("Only one template may be specified!")
orig_filename = "string"
if template_file is not None:
orig_filename = template_file
try:
with open(os.path.expanduser(template_file), "r") as f:
template = f.read()
except Exception as e:
raise Exception("Could not open template file '{}'".format(template_file))
# the user may specify their own instance of PfpInterp to be
# used
if interp is None:
interp = pfp.interp.PfpInterp(debug=debug, parser=PARSER, int3=int3, cpp_path=cpp_path, cpp_args=cpp_args)
# so we can consume single bits at a time
data = BitwrappedStream(data)
dom = interp.parse(data, template, predefines=predefines, orig_filename=orig_filename, keep_successful=keep_successful)
# close the data stream if a data_file was specified
if data_file is not None:
data.close()
return dom
def parse(data=None, template=None, data_file=None, template_file=None, interp=None, debug=False, predefines=True, int3=True, cpp_path="cpp", cpp_args="-xc++", keep_successful=False):
"""Parse the data stream using the supplied template. The data stream
WILL NOT be automatically closed.
:data: Input stream (yes, a STREAM, not str)
:template: template contents (str)
:data_file: path to the data to be used as the input stream
:template_file: template file path
:interp: the interpretor to be used (a default one will be created if ``None``)
:debug: if debug information should be printed while interpreting the template (false)
:predefines: if built-in type information should be inserted (true)
:int3: if debugger breaks are allowed while interpreting the template (true)
:cpp_path: the path to the ``cpp`` binary, used to strip comments ("cpp")
:cpp_args: the args to the ``cpp`` binary to strip comments. Defaults to "", but "-xc++" might be useful on macs.
:keep_successful: return any succesfully parsed data instead of raising an error. If an error occurred and ``keep_successful`` is True, then ``_pfp__error`` will be contain the exception object
:returns: pfp DOM
"""
if data is None and data_file is None:
raise Exception("No input data was specified")
if data is not None and data_file is not None:
raise Exception("Only one input data may be specified")
if data_file is not None:
data = open(os.path.expanduser(data_file), "rb")
if template is None and template_file is None:
raise Exception("No template specified!")
if template is not None and template_file is not None:
raise Exception("Only one template may be specified!")
orig_filename = "string"
if template_file is not None:
orig_filename = template_file
try:
with open(os.path.expanduser(template_file), "r") as f:
template = f.read()
except Exception as e:
raise Exception("Could not open template file '{}'".format(template_file))
# the user may specify their own instance of PfpInterp to be
# used
if interp is None:
interp = pfp.interp.PfpInterp(debug=debug, parser=PARSER, int3=int3, cpp_path=cpp_path, cpp_args=cpp_args)
# so we can consume single bits at a time
data = BitwrappedStream(data)
dom = interp.parse(data, template, predefines=predefines, orig_filename=orig_filename, keep_successful=keep_successful)
# close the data stream if a data_file was specified
if data_file is not None:
data.close()
return dom
def test_bits_read2_padded2(self):
stream = six.BytesIO(pfp.utils.binary(chr(int("11110000",2)) + chr(int("10101010", 2))))
bitwrapped = BitwrappedStream(stream)
bitwrapped.padded = True
res = bitwrapped.read_bits(4)
self.assertEqual([1,1,1,1], res)
next_byte = bitwrapped.read(1)
self.assertEqual(pfp.utils.binary(chr(int("10101010", 2))), next_byte)
def test_unconsumed_ranges3(self):
stream = six.BytesIO(pfp.utils.binary("A" * 100))
bitwrapped = BitwrappedStream(stream)
bitwrapped.read(10)
# it should not need a second read to add the
# unconsumed range
uranges = bitwrapped.unconsumed_ranges()
self.assertEqual(len(uranges), 0)
def test_unconsumed_ranges3(self):
stream = six.BytesIO(pfp.utils.binary("A" * 100))
bitwrapped = BitwrappedStream(stream)
bitwrapped.read(10)
# it should not need a second read to add the
# unconsumed range
uranges = bitwrapped.unconsumed_ranges()
self.assertEqual(len(uranges), 0)
def test_bits_read2_padded2(self):
stream = six.BytesIO(
pfp.utils.binary(
chr(int("11110000", 2)) + chr(int("10101010", 2))))
bitwrapped = BitwrappedStream(stream)
bitwrapped.padded = True
res = bitwrapped.read_bits(4)
self.assertEqual([1, 1, 1, 1], res)
next_byte = bitwrapped.read(1)
self.assertEqual(pfp.utils.binary(chr(int("10101010", 2))), next_byte)
def test_bits_read_unpadded(self):
stream = six.BytesIO(pfp.utils.binary(chr(int("11110000",2)) + chr(int("10101010", 2))))
bitwrapped = BitwrappedStream(stream)
bitwrapped.padded = False
res = bitwrapped.read_bits(4)
self.assertEqual([1,1,1,1], res)
res = bitwrapped.read(1)
self.assertEqual(pfp.utils.binary(chr(int("00001010", 2))), res)
res = bitwrapped.read_bits(4)
self.assertEqual([1,0,1,0], res)
def test_bits_write_padded(self):
stream = six.BytesIO()
bitwrapped = BitwrappedStream(stream)
bitwrapped.padded = True
bitwrapped.write_bits([1, 1, 0, 1])
# should go to a new byte now, zero padded after the
# 1101 bits
bitwrapped.write(pfp.utils.binary("hello"))
self.assertEqual(stream.getvalue(),
pfp.utils.binary(chr(int("11010000", 2)) + "hello"))
def _handle_bitfield(self, field):
"""Find the field's first evenly-aligned previous sibling that is
also a bitfield, as well as all subsequent siblings until a full
bit "class" is reached. Build the entire set of bitfields as a group.
E.g.:
ushort a:1;
ushort b:3;
ushort c:10;
ushort d:2;
This entire group should be built
"""
total_bits = field.width * 8
bit_offset = lambda x: total_bits - x._pfp__offset_bits
fields_to_build = []
curr_field = field._pfp__prev_sibling
# previous siblings
while curr_field is not None and bit_offset(curr_field) >= 0:
fields_to_build.append(curr_field)
curr_field = curr_field._pfp__prev_sibling
fields_to_build = list(reversed(fields_to_build))
fields_to_build.append(field)
# next siblings
curr_field = field._pfp__next_sibling
while (curr_field is not None
and isinstance(curr_field, field.__class__)
and curr_field.bitsize is not None):
if bit_offset(curr_field) + curr_field.bitsize > total_bits:
break
fields_to_build.append(curr_field)
curr_field = curr_field._pfp__next_sibling
core_stream = six.BytesIO(b"")
bit_stream = BitwrappedStream(core_stream)
bitfield_rw = BitfieldRW(None, field.__class__)
bitfield_rw.reserved_bits = field.bitfield_rw.reserved_bits
for to_build in fields_to_build:
old_bitfield_rw = to_build.bitfield_rw
to_build.bitfield_rw = bitfield_rw
to_build._pfp__build(bit_stream)
to_build.bitfield_rw = old_bitfield_rw
return core_stream.getvalue()
def test_unconsumed_ranges2(self):
stream = six.BytesIO(pfp.utils.binary("A" * 100))
bitwrapped = BitwrappedStream(stream)
bitwrapped.read(10)
bitwrapped.seek(bitwrapped.tell() + 10)
# it should not need a second read to add the
# unconsumed range
uranges = bitwrapped.unconsumed_ranges()
self.assertEqual(len(uranges), 1)
# test (11,20]
self.assertEqual(len(uranges[11]), 1)
self.assertEqual(len(uranges[10]), 0)
self.assertEqual(len(uranges[19]), 1)
self.assertEqual(len(uranges[20]), 0)
def test_bits_write_padded(self):
stream = six.BytesIO()
bitwrapped = BitwrappedStream(stream)
bitwrapped.padded = True
bitwrapped.write_bits([1,1,0,1])
# should go to a new byte now, zero padded after the
# 1101 bits
bitwrapped.write(pfp.utils.binary("hello"))
self.assertEqual(stream.getvalue(), pfp.utils.binary(chr(int("11010000", 2)) + "hello"))
def test_bits_read2_padded1(self):
stream = six.BytesIO(
pfp.utils.binary(
chr(int("11110000", 2)) + chr(int("10101010", 2))))
bitwrapped = BitwrappedStream(stream)
bitwrapped.padded = True
res = bitwrapped.read_bits(4)
self.assertEqual([1, 1, 1, 1], res)
res = bitwrapped.read_bits(3)
self.assertEqual([0, 0, 0], res)
res = bitwrapped.read_bits(4)
self.assertEqual([0, 1, 0, 1], res)
res = bitwrapped.read_bits(5)
self.assertEqual([0, 1, 0, 1, 0], res)
def test_unconsumed_ranges2(self):
stream = six.BytesIO(pfp.utils.binary("A" * 100))
bitwrapped = BitwrappedStream(stream)
bitwrapped.read(10)
bitwrapped.seek(bitwrapped.tell()+10)
# it should not need a second read to add the
# unconsumed range
uranges = bitwrapped.unconsumed_ranges()
self.assertEqual(len(uranges), 1)
# test (11,20]
self.assertEqual(len(uranges[11]), 1)
self.assertEqual(len(uranges[10]), 0)
self.assertEqual(len(uranges[19]), 1)
self.assertEqual(len(uranges[20]), 0)
def test_bits_read_unpadded(self):
stream = six.BytesIO(
pfp.utils.binary(
chr(int("11110000", 2)) + chr(int("10101010", 2))))
bitwrapped = BitwrappedStream(stream)
bitwrapped.padded = False
res = bitwrapped.read_bits(4)
self.assertEqual([1, 1, 1, 1], res)
res = bitwrapped.read(1)
self.assertEqual(pfp.utils.binary(chr(int("00001010", 2))), res)
res = bitwrapped.read_bits(4)
self.assertEqual([1, 0, 1, 0], res)
def test_bits_read2_padded1(self):
stream = six.BytesIO(pfp.utils.binary(chr(int("11110000",2)) + chr(int("10101010", 2))))
bitwrapped = BitwrappedStream(stream)
bitwrapped.padded = True
res = bitwrapped.read_bits(4)
self.assertEqual([1,1,1,1], res)
res = bitwrapped.read_bits(3)
self.assertEqual([0,0,0], res)
res = bitwrapped.read_bits(4)
self.assertEqual([0,1,0,1], res)
res = bitwrapped.read_bits(5)
self.assertEqual([0,1,0,1,0], res)
def parse(
data=None,
template=None,
data_file=None,
template_file=None,
interp=None,
debug=False,
predefines=True,
int3=True,
keep_successful=False,
printf=True,
):
"""Parse the data stream using the supplied template. The data stream
WILL NOT be automatically closed.
:data: Input data, can be either a string or a file-like object (StringIO, file, etc)
:template: template contents (str)
:data_file: PATH to the data to be used as the input stream
:template_file: template file path
:interp: the interpretor to be used (a default one will be created if ``None``)
:debug: if debug information should be printed while interpreting the template (false)
:predefines: if built-in type information should be inserted (true)
:int3: if debugger breaks are allowed while interpreting the template (true)
:keep_successful: return any succesfully parsed data instead of raising an error. If an error occurred and ``keep_successful`` is True, then ``_pfp__error`` will be contain the exception object
:printf: if ``False``, all calls to ``Printf`` (:any:`pfp.native.compat_interface.Printf`) will be noops. (default=``True``)
:returns: pfp DOM
"""
if data is None and data_file is None:
raise Exception("No input data was specified")
if data is not None and data_file is not None:
raise Exception("Only one input data may be specified")
if isinstance(data, six.string_types):
data = six.StringIO(data)
if data_file is not None:
data = open(os.path.expanduser(data_file), "rb")
if template is None and template_file is None:
raise Exception("No template specified!")
if template is not None and template_file is not None:
raise Exception("Only one template may be specified!")
orig_filename = "string"
if template_file is not None:
orig_filename = template_file
try:
with open(os.path.expanduser(template_file), "r") as f:
template = f.read()
except Exception as e:
raise Exception(
"Could not open template file '{}'".format(template_file)
)
# the user may specify their own instance of PfpInterp to be
# used
if interp is None:
interp = pfp.interp.PfpInterp(debug=debug, parser=PARSER, int3=int3)
# so we can consume single bits at a time
data = BitwrappedStream(data)
dom = interp.parse(
data,
template,
predefines=predefines,
orig_filename=orig_filename,
keep_successful=keep_successful,
printf=printf,
)
# close the data stream if a data_file was specified
if data_file is not None:
data.close()
return dom
def test_bits_read1(self):
stream = six.BytesIO(pfp.utils.binary(chr(int("01010101", 2))))
bitwrapped = BitwrappedStream(stream)
res = bitwrapped.read_bits(8)
self.assertEqual([0, 1, 0, 1, 0, 1, 0, 1], res)
def test_bytes_read(self):
stream = six.BytesIO(pfp.utils.binary("abcd"))
bitwrapped = BitwrappedStream(stream)
res = bitwrapped.read(4)
self.assertEqual(pfp.utils.binary("abcd"), res)
def test_tell_bits(self):
stream = six.BytesIO(pfp.utils.binary("\x41" + chr(0b11001100)))
bitwrapped = BitwrappedStream(stream)
res = bitwrapped.read(1)
self.assertEqual(res, b"\x41")
self.assertEqual(bitwrapped.tell(), 1)
self.assertEqual(bitwrapped.tell_bits(), 0)
bits = bitwrapped.read_bits(1)
self.assertEqual(bits, [1])
self.assertEqual(bitwrapped.tell_bits(), 1)
bits = bitwrapped.read_bits(1)
self.assertEqual(bits, [1])
self.assertEqual(bitwrapped.tell_bits(), 2)
bits = bitwrapped.read_bits(1)
self.assertEqual(bits, [0])
self.assertEqual(bitwrapped.tell_bits(), 3)
def test_unconsumed_ranges1(self):
stream = six.BytesIO(pfp.utils.binary("A" * 100))
bitwrapped = BitwrappedStream(stream)
bitwrapped.read(10)
bitwrapped.seek(bitwrapped.tell() + 10)
bitwrapped.read(10)
bitwrapped.seek(bitwrapped.tell() + 10)
bitwrapped.read(10)
uranges = bitwrapped.unconsumed_ranges()
# test (11,20]
self.assertEqual(len(uranges[11]), 1)
self.assertEqual(len(uranges[10]), 0)
self.assertEqual(len(uranges[19]), 1)
self.assertEqual(len(uranges[20]), 0)
# test (31,40]
self.assertEqual(len(uranges[31]), 1)
self.assertEqual(len(uranges[30]), 0)
self.assertEqual(len(uranges[39]), 1)
self.assertEqual(len(uranges[40]), 0)
def test_unconsumed_ranges1(self):
stream = six.BytesIO(pfp.utils.binary("A" * 100))
bitwrapped = BitwrappedStream(stream)
bitwrapped.read(10)
bitwrapped.seek(bitwrapped.tell()+10)
bitwrapped.read(10)
bitwrapped.seek(bitwrapped.tell()+10)
bitwrapped.read(10)
uranges = bitwrapped.unconsumed_ranges()
# test (11,20]
self.assertEqual(len(uranges[11]), 1)
self.assertEqual(len(uranges[10]), 0)
self.assertEqual(len(uranges[19]), 1)
self.assertEqual(len(uranges[20]), 0)
# test (31,40]
self.assertEqual(len(uranges[31]), 1)
self.assertEqual(len(uranges[30]), 0)
self.assertEqual(len(uranges[39]), 1)
self.assertEqual(len(uranges[40]), 0)
def test_bits_read1(self):
stream = six.BytesIO(pfp.utils.binary(chr(int("01010101", 2))))
bitwrapped = BitwrappedStream(stream)
res = bitwrapped.read_bits(8)
self.assertEqual([0,1,0,1,0,1,0,1], res)
def test_bytes_read(self):
stream = six.BytesIO(pfp.utils.binary("abcd"))
bitwrapped = BitwrappedStream(stream)
res = bitwrapped.read(4)
self.assertEqual(pfp.utils.binary("abcd"), res)