def replaceField(self, name, new_fields):
# TODO: Check in self and not self.field
# Problem is that "generator is already executing"
if name not in self._fields:
raise ParserError("Unable to replace %s: field doesn't exist!" %
name)
assert 1 <= len(new_fields)
old_field = self[name]
total_size = sum((field.size for field in new_fields))
if old_field.size != total_size:
raise ParserError(
"Unable to replace %s: "
"new field(s) hasn't same size (%u bits instead of %u bits)!" %
(name, total_size, old_field.size))
field = new_fields[0]
if field._name.endswith("[]"):
self.setUniqueFieldName(field)
field._address = old_field.address
if field.name != name and field.name in self._fields:
raise ParserError(
"Unable to replace %s: name \"%s\" is already used!" %
(name, field.name))
self._fields.replace(name, field.name, field)
self.raiseEvent("field-replaced", old_field, field)
if 1 < len(new_fields):
index = self._fields.index(new_fields[0].name) + 1
address = field.address + field.size
for field in new_fields[1:]:
if field._name.endswith("[]"):
self.setUniqueFieldName(field)
field._address = address
if field.name in self._fields:
raise ParserError(
"Unable to replace %s: name \"%s\" is already used!" %
(name, field.name))
self._fields.insert(index, field.name, field)
self.raiseEvent("field-inserted", index, field)
index += 1
address += field.size
def parseGraphicControl(parent):
yield UInt8(parent, "size", "Block size (4)")
yield Bit(parent, "has_transp", "Has transparency")
yield Bit(parent, "user_input", "User input")
yield Enum(Bits(parent, "disposal_method", 3), DISPOSAL_METHOD)
yield NullBits(parent, "reserved[]", 3)
if parent["size"].value != 4:
raise ParserError("Invalid graphic control size")
yield displayHandler(UInt16(parent, "delay", "Delay time in millisecond"), humanDuration)
yield UInt8(parent, "transp", "Transparent color index")
yield NullBytes(parent, "terminator", 1, "Terminator (0)")
def createFields(self):
yield UInt16(self, "signature", "Signature (0x0000)")
yield Enum(UInt16(self, "type", "Resource type"), self.TYPE_NAME)
yield UInt16(self, "nb_items", "Number of items")
items = []
for index in range(self["nb_items"].value):
item = IconHeader(self, "icon_header[]")
yield item
items.append(item)
for header in items:
if header["offset"].value * 8 != self.current_size:
raise ParserError("Icon: Problem with icon data offset.")
yield IconData(self, "icon_data[]", header)
def createFields(self):
yield Bytes(self, "id", 4, "Tcpdump identifier")
yield UInt16(self, "maj_ver", "Major version")
yield UInt16(self, "min_ver", "Minor version")
yield Int32(self, "this_zone", "GMT to local time zone correction")
yield Int32(self, "sigfigs", "accuracy of timestamps")
yield UInt32(self, "snap_len", "max length saved portion of each pkt")
yield Enum(UInt32(self, "link_type", "data link type"), self.LINK_TYPE_DESC)
link = self["link_type"].value
if link not in self.LINK_TYPE:
raise ParserError("Unknown link type: %s" % link)
name, parser = self.LINK_TYPE[link]
while self.current_size < self.size:
yield Packet(self, "packet[]", parser, name)
def createFields(self):
# Code based on function get_dqt() (jdmarker.c from libjpeg62)
yield Bits(self, "is_16bit", 4)
yield Bits(self, "index", 4)
if self["index"].value >= 4:
raise ParserError("Invalid quantification index (%s)" %
self["index"].value)
if self["is_16bit"].value:
coeff_type = UInt16
else:
coeff_type = UInt8
for index in range(64):
natural = JPEG_NATURAL_ORDER[index]
yield coeff_type(self, "coeff[%u]" % natural)
def __init__(self, parent, name, description=None):
Bits.__init__(self, parent, name, 8, description)
stream = parent.stream
addr = self.absolute_address
value = 0
while True:
bits = stream.readBits(addr, 8, parent.endian)
value = (value << 7) + (bits & 127)
if not (bits & 128):
break
addr += 8
self._size += 8
if 32 < self._size:
raise ParserError("Integer size is bigger than 32-bit")
self.createValue = lambda: value
def createFields(self):
yield PropID(self, "id")
while not self.eof:
uid = ReadNextByte(self)
if uid == kEnd:
yield PropID(self, "end")
break
elif uid == kPackInfo:
yield PackInfo(self, "pack_info", PROP_DESC[uid])
elif uid == kUnPackInfo:
yield UnpackInfo(self, "unpack_info", PROP_DESC[uid])
elif uid == kSubStreamsInfo:
yield SubStreamInfo(self, "substreams_info", PROP_DESC[uid])
else:
raise ParserError("Unexpected ID (%i)" % uid)
def addChunk(self, new_chunk):
index = 0
# Find first chunk whose value is bigger
while index < len(self.chunks):
offset = self.chunks[index].offset
if offset < new_chunk.offset:
if not self.canHouse(new_chunk, index):
raise ParserError("Chunk '%s' doesn't fit!" %
new_chunk.name)
self.chunks.insert(index, new_chunk)
return
index += 1
# Not found or empty
# We could at least check that it fits in the memory
self.chunks.append(new_chunk)
def createFields(self):
m2ts = self.is_m2ts()
while not self.eof:
current = self.current_size
next_sync = current
if m2ts:
next_sync += 4 * 8
sync = self.stream.searchBytes(b"\x47", current,
current + MAX_PACKET_SIZE * 8)
if sync is None:
raise ParserError("Unable to find synchronization byte")
elif sync > next_sync:
yield RawBytes(self, "incomplete_packet[]",
(sync - current) // 8)
yield Packet(self, "packet[]", m2ts=m2ts)
def parseFields(parser):
# Determine field names
ext = EXTENSIONS[parser["block_type"].value]
if ext is None:
raise ParserError("Unknown parent '%s'" % parser["block_type"].value)
# Parse fields
addr = parser.absolute_address + parser.current_size
while not parser.eof and parser.stream.readBytes(addr, 4) in ext:
field = MPField(parser, "field[]", ext)
yield field
addr += field._size
# Abort on unknown codes
parser.info("End of extension '%s' when finding '%s'" %
(parser["block_type"].value, parser.stream.readBytes(addr, 4)))
def createFields(self):
yield PropID(self, "id")
while not self.eof:
uid = ReadNextByte(self)
if uid == kEnd:
yield PropID(self, "end")
break
elif uid == kArchiveProperties:
yield ArchiveProperties(self, "props", PROP_DESC[uid])
elif uid == kAdditionalStreamsInfo:
yield StreamsInfo(self, "additional_streams", PROP_DESC[uid])
elif uid == kMainStreamsInfo:
yield StreamsInfo(self, "main_streams", PROP_DESC[uid])
elif uid == kFilesInfo:
yield FilesInfo(self, "files_info", PROP_DESC[uid])
else:
raise ParserError("Unexpected ID %u" % uid)
def __init__(self, parent, name, tree, description=None):
Field.__init__(self, parent, name, 0, description)
endian = self.parent.endian
stream = self.parent.stream
addr = self.absolute_address
value = 0
while (self.size, value) not in tree:
if self.size > 256:
raise ParserError("Huffman code too long!")
bit = stream.readBits(addr, 1, endian)
value <<= 1
value += bit
self._size += 1
addr += 1
self.huffvalue = value
self.realvalue = tree[(self.size, value)]
def __init__(self,
parent,
name,
nb_items,
item_class,
item_name="item",
description=None):
# Sanity checks
assert issubclass(item_class, Field)
assert isinstance(item_class.static_size, int)
if not (0 < nb_items):
raise ParserError('Unable to create empty vector "%s" in %s' %
(name, parent.path))
size = nb_items * item_class.static_size
self.__nb_items = nb_items
self._item_class = item_class
self._item_name = item_name
FieldSet.__init__(self, parent, name, description, size=size)
def createFields(self):
yield textHandler(UInt8(self, "header", "Header"), hexadecimal)
if self["header"].value != 0xFF:
raise ParserError("JPEG: Invalid chunk header!")
yield textHandler(UInt8(self, "type", "Type"), hexadecimal)
tag = self["type"].value
# D0 - D7 inclusive are the restart markers
if tag in [self.TAG_SOI, self.TAG_EOI] + list(range(0xD0, 0xD8)):
return
yield UInt16(self, "size", "Size")
size = (self["size"].value - 2)
if 0 < size:
if self._parser:
yield self._parser(self,
"content",
"Chunk content",
size=size * 8)
else:
yield RawBytes(self, "data", size, "Data")
def createFields(self):
if False:
yield UInt32(self, "width0")
yield UInt32(self, "height0")
yield PaddingBytes(self, "reserved[]", 7)
yield UInt32(self, "width")
yield UInt32(self, "height")
yield PaddingBytes(self, "reserved[]", 2)
yield UInt16(self, "depth")
yield Enum(String(self, "codec", 4, charset="ASCII"), video_fourcc_name)
yield NullBytes(self, "padding", 20)
else:
yield UInt32(self, "width")
yield UInt32(self, "height")
yield PaddingBytes(self, "reserved[]", 1)
yield UInt16(self, "format_data_size")
if self["format_data_size"].value < 40:
raise ParserError("Unknown format data size")
yield BitmapInfoHeader(self, "bmp_info", use_fourcc=True)
def __init__(self, parent, name, description=None):
Bits.__init__(self, parent, name, 8, description)
stream = self._parent.stream
addr = self.absolute_address
size = 8
value = 0
byte = stream.readBits(addr, 8, BIG_ENDIAN)
value = byte & 127
while 128 <= byte:
addr += 8
size += 8
if 64 < size:
# Arbitrary limit to catch errors
raise ParserError(
"ASN.1: Object identifier is limited 64 bits")
byte = stream.readBits(addr, 8, BIG_ENDIAN)
value = (value << 7) + (byte & 127)
self._size = size
self.createValue = lambda: value
def createFields(self):
yield String(self, "jfif", 5, "JFIF string", charset="ASCII")
if self["jfif"].value != "JFIF\0":
raise ParserError(
"Stream doesn't look like JPEG chunk (wrong JFIF signature)")
yield UInt8(self, "ver_maj", "Major version")
yield UInt8(self, "ver_min", "Minor version")
yield Enum(UInt8(self, "units", "Units"), self.UNIT_NAME)
if self["units"].value == 0:
yield UInt16(self, "aspect_x", "Aspect ratio (X)")
yield UInt16(self, "aspect_y", "Aspect ratio (Y)")
else:
yield UInt16(self, "x_density", "X density")
yield UInt16(self, "y_density", "Y density")
yield UInt8(self, "thumb_w", "Thumbnail width")
yield UInt8(self, "thumb_h", "Thumbnail height")
thumb_size = self["thumb_w"].value * self["thumb_h"].value
if thumb_size != 0:
yield PaletteRGB(self, "thumb_palette", 256)
yield RawBytes(self, "thumb_data", thumb_size, "Thumbnail data")
def createFields(self):
end = False
while not end:
marker = self.stream.readBits(
self.absolute_address + self.current_size, 48, self.endian)
if marker == self.START_BLOCK:
yield Bzip2Block(self, "block[]")
elif marker == self.END_STREAM:
yield textHandler(
Bits(self, "stream_end", 48, "End-of-stream marker"),
hexadecimal)
yield textHandler(
UInt32(self, "crc32", "CRC32 for entire stream"),
hexadecimal)
padding = paddingSize(self.current_size, 8)
if padding:
yield PaddingBits(self, "padding[]", padding)
end = True
else:
raise ParserError("Invalid marker 0x%02X!" % marker)
def createFields(self):
yield textHandler(UInt8(self, "signature", "IPTC signature (0x1c)"),
hexadecimal)
if self["signature"].value != 0x1C:
raise ParserError("Wrong IPTC signature")
yield textHandler(UInt8(self, "dataset_nb", "Dataset number"),
hexadecimal)
yield UInt8(self, "tag", "Tag")
yield IPTC_Size(self, "size", "Content size")
size = self["size"].value
if 0 < size:
if self.dataset_info:
cls = self.dataset_info[2]
else:
cls = None
if cls:
yield cls(self, "content")
else:
yield RawBytes(self, "content", size)
def createFields(self):
yield PropID(self, "id")
yield SZUInt64(self, "pack_pos", "File offset to the packed data")
num = SZUInt64(self, "num_pack_streams", "Number of packed streams")
yield num
while not self.eof:
uid = ReadNextByte(self)
if uid == kEnd:
yield PropID(self, "end_marker")
break
elif uid == kSize:
yield PropID(self, "size_marker")
for index in range(num.value):
yield SZUInt64(self, "pack_size[]")
elif uid == kCRC:
yield Digests(self, "digests", num.value)
else:
raise ParserError("Unexpected ID (%i)" % uid)
def __init__(self, parent, name, description=None):
Field.__init__(self, parent, name, 8, description)
endian = self._parent.endian
stream = self._parent.stream
addr = self.absolute_address
value = 0
byte = stream.readBits(addr, 8, endian)
while byte & 0x80:
value <<= 7
value += (byte & 0x7f)
self._size += 8
if 64 < self._size:
raise ParserError("CHM: CWord is limited to 64 bits")
addr += 8
byte = stream.readBits(addr, 8, endian)
value <<= 7
value += byte
self.createValue = lambda: value
def createFields(self):
yield PascalString32UTF16(self, "filename")
yield Bytes(self, "property", 4)
yield Bytes(self, "type", 4)
type = self['type'].value
if type == 'long':
yield UInt32(self, "value")
elif type == 'shor':
yield NullBytes(self, "padding", 2)
yield UInt16(self, "value")
elif type == 'bool':
yield UInt8(self, "value")
elif type == 'blob':
yield UInt32(self, "size")
yield SubFile(self, "value", self['size'].value)
elif type == 'type':
yield Bytes(self, "value", 4)
elif type == 'ustr':
yield PascalString32UTF16(self, "value")
else:
raise ParserError("Unknown record type %s" % type)
def createFields(self):
yield UInt16(self, "signature", "PE optional header signature (0x010b)")
# TODO: Support PE32+ (signature=0x020b)
if self["signature"].value != 0x010b:
raise ParserError("Invalid PE optional header signature")
yield UInt8(self, "maj_lnk_ver", "Major linker version")
yield UInt8(self, "min_lnk_ver", "Minor linker version")
yield filesizeHandler(UInt32(self, "size_code", "Size of code"))
yield filesizeHandler(UInt32(self, "size_init_data", "Size of initialized data"))
yield filesizeHandler(UInt32(self, "size_uninit_data", "Size of uninitialized data"))
yield textHandler(UInt32(self, "entry_point", "Address (RVA) of the code entry point"), hexadecimal)
yield textHandler(UInt32(self, "base_code", "Base (RVA) of code"), hexadecimal)
yield textHandler(UInt32(self, "base_data", "Base (RVA) of data"), hexadecimal)
yield textHandler(UInt32(self, "image_base", "Image base (RVA)"), hexadecimal)
yield filesizeHandler(UInt32(self, "sect_align", "Section alignment"))
yield filesizeHandler(UInt32(self, "file_align", "File alignment"))
yield UInt16(self, "maj_os_ver", "Major OS version")
yield UInt16(self, "min_os_ver", "Minor OS version")
yield UInt16(self, "maj_img_ver", "Major image version")
yield UInt16(self, "min_img_ver", "Minor image version")
yield UInt16(self, "maj_subsys_ver", "Major subsystem version")
yield UInt16(self, "min_subsys_ver", "Minor subsystem version")
yield NullBytes(self, "reserved", 4)
yield filesizeHandler(UInt32(self, "size_img", "Size of image"))
yield filesizeHandler(UInt32(self, "size_hdr", "Size of headers"))
yield textHandler(UInt32(self, "checksum"), hexadecimal)
yield Enum(UInt16(self, "subsystem"), self.SUBSYSTEM_NAME)
yield UInt16(self, "dll_flags")
yield filesizeHandler(UInt32(self, "size_stack_reserve"))
yield filesizeHandler(UInt32(self, "size_stack_commit"))
yield filesizeHandler(UInt32(self, "size_heap_reserve"))
yield filesizeHandler(UInt32(self, "size_heap_commit"))
yield UInt32(self, "loader_flags")
yield UInt32(self, "nb_directory", "Number of RVA and sizes")
for index in range(self["nb_directory"].value):
try:
name = self.DIRECTORY_NAME[index]
except KeyError:
name = "data_dir[%u]" % index
yield DataDirectory(self, name)
def _createFields(self, field_generator):
if self._first is None:
for field in field_generator:
if self._first is not None:
break
yield field
else:
raise ParserError("Fragment.setLinks not called")
else:
field = None
if self._first is not self:
link = Link(self, "first", None)
link._getValue = lambda: self._first
yield link
if self._next:
link = Link(self, "next", None)
link.createValue = self._getNext
yield link
if field:
yield field
for field in field_generator:
yield field
def __init__(self, parent, name, description=None):
FieldSet.__init__(self, parent, name, description)
self._size = (self["size"].value + 3 * 4) * 8
if MAX_CHUNK_SIZE < (self._size // 8):
raise ParserError("PNG: Chunk is too big (%s)" %
humanFilesize(self._size // 8))
tag = self["tag"].value
self.desc_func = None
self.value_func = None
if tag in self.TAG_INFO:
self._name, self.parse_func, desc, value_func = self.TAG_INFO[tag]
if value_func:
self.value_func = value_func
self.createValue = self.createValueFunc
if desc:
if isinstance(desc, str):
self._description = desc
else:
self.desc_func = desc
else:
self._description = ""
self.parse_func = None
def __init__(self, parent, name, desc=None):
Field.__init__(self, parent, name, description=desc)
if parent.stream.readBytes(self.absolute_address, 1) != b'/':
raise ParserError(
"Unknown PDFName '%s'" %
parent.stream.readBytes(self.absolute_address, 10))
size = getElementEnd(parent, offset=1)
# other_size = getElementEnd(parent, '[')-1
# if size is None or (other_size is not None and other_size < size):
# size = other_size
for limit in self.LIMITS:
other_size = getElementEnd(parent, limit, 1)
if other_size is not None:
other_size -= 1
if size is None or other_size < size:
# self.info("New size: %u" % other_size)
size = other_size
self._size = 8 * (size + 1)
# Value should be without the initial '/' and final ' '
self.createValue = lambda: parent.stream.readBytes(
self.absolute_address + 8, size).strip(b' ')
def parseAVIStreamHeader(self):
if self["size"].value != 56:
raise ParserError("Invalid stream header size")
yield String(self,
"stream_type",
4,
"Stream type four character code",
charset="ASCII")
field = String(self,
"fourcc",
4,
"Stream four character code",
strip=" \0",
charset="ASCII")
if self["stream_type"].value == "vids":
yield Enum(field, video_fourcc_name, lambda text: text.upper())
else:
yield field
yield UInt32(self, "flags", "Stream flags")
yield UInt16(self, "priority", "Stream priority")
yield String(self,
"language",
2,
"Stream language",
charset="ASCII",
strip="\0")
yield UInt32(self, "init_frames", "InitialFrames")
yield UInt32(self, "scale", "Time scale")
yield UInt32(self, "rate", "Divide by scale to give frame rate")
yield UInt32(self, "start", "Stream start time (unit: rate/scale)")
yield UInt32(self, "length", "Stream length (unit: rate/scale)")
yield UInt32(self, "buf_size", "Suggested buffer size")
yield UInt32(self, "quality", "Stream quality")
yield UInt32(self, "sample_size", "Size of samples")
yield UInt16(self, "left", "Destination rectangle (left)")
yield UInt16(self, "top", "Destination rectangle (top)")
yield UInt16(self, "right", "Destination rectangle (right)")
yield UInt16(self, "bottom", "Destination rectangle (bottom)")
def createFields(self):
yield UInt32(self, "width", "Width in pixel")
yield UInt32(self, "height", "Height in pixel")
yield UInt32(self, "offset", "Offset")
offset = self["offset"].value
if offset == 0:
return
data_offsets = []
while (self.absolute_address + self.current_size) // 8 < offset:
chunk = UInt32(self, "data_offset[]", "Data offset")
yield chunk
if chunk.value == 0:
break
data_offsets.append(chunk)
if (self.absolute_address + self.current_size) // 8 != offset:
raise ParserError("Problem with level offset.")
previous = offset
for chunk in data_offsets:
data_offset = chunk.value
size = data_offset - previous
yield RawBytes(self, "data[]", size,
"Data content of %s" % chunk.name)
previous = data_offset
def createFields(self):
yield UInt32(self, "unknown", description="Always 1")
yield BudHeader(self, "header")
self.seekByte(self['header/allocator_offset'].value + 4)
yield BudAllocator(self, "allocator", size=self['header/allocator_size'].value * 8)
for dir in self['allocator'].array('dir'):
if dir['name'].value == 'DSDB':
break
else:
raise ParserError("DSDB not found.")
offs, size = self.getBlock(dir['block'].value)
self.seekByte(offs + 4)
yield DSDB(self, "dsdb", size=size * 8)
blocks = [self['dsdb/root_block'].value]
while blocks:
block = blocks.pop()
offs, size = self.getBlock(block)
self.seekByte(offs + 4)
node = BTNode(self, "node[%d]" % block, size=size * 8)
yield node
if node['last_block'].value != 0:
new_blocks = []
for block in node.array('child_block'):
new_blocks.append(block.value)
new_blocks.append(node['last_block'].value)
blocks.extend(reversed(new_blocks)) # dfs
# blocks = new_blocks[::-1] + blocks # bfs
for i, fl in enumerate(self['allocator'].array('freelist')):
if fl['count'].value == 0:
continue
for offs in fl.array('offset'):
size = min(1 << i, self.size // 8 - offs.value - 4)
if size > 0:
self.seekByte(offs.value + 4)
yield RawBytes(self, "free[]", size)
def createFields(self):
yield UInt16(self, "count", "Number of entries")
count = self["count"].value
if count == 0:
raise ParserError("IFDs cannot be empty.")
for i in range(count):
yield self.EntryClass(self, "entry[]")
yield UInt32(self, "next", "Offset to next IFD")
for i in range(count):
entry = self['entry[%d]' % i]
if 'offset' not in entry:
continue
self.seekByte(entry['offset'].value + self.base_addr // 8,
relative=False)
count = entry['count'].value
name = "value[%s]" % i
if issubclass(entry.value_cls, Bytes):
yield entry.value_cls(self, name, count)
elif entry['tag'].display == "XMPPacket":
yield String(self, name, count)
elif count == 1:
yield entry.value_cls(self, name)
else:
yield ValueArray(self, name, entry.value_cls, count)
