def __init__(self, reader: StructReader):
reader.bigendian = True
entry_start_offset = reader.tell()
self.size_of_entry = reader.i32()
self.offset = reader.i32()
self.size_of_compressed_data = reader.i32()
self.size_od_uncompressed_data = reader.i32()
self.is_compressed = bool(reader.read_byte())
entry_type = bytes(reader.read(1))
name_length = self.size_of_entry - reader.tell() + entry_start_offset
if name_length > 0x1000:
raise RuntimeError(
F'Refusing to process TOC entry with name of size {name_length}.'
)
name, *_ = bytes(reader.read(name_length)).partition(B'\0')
try:
name = name.decode('utf8', 'backslashreplace')
except Exception:
name = None
if not all(part.isprintable() for part in re.split('\\s*', name)):
raise RuntimeError(
'Refusing to process TOC entry with non-printable name.')
name = name or str(uuid.uuid4())
if entry_type == B'Z':
entry_type = B'z'
try:
self.type = PiType(entry_type)
except ValueError:
xtpyi.logger.error(F'unknown type {entry_type!r} in field {name}')
self.type = PiType.UNKNOWN
self.name = name
def _decompress_mszip(self,
reader: StructReader,
writer: MemoryFile,
target: Optional[int] = None):
header = bytes(reader.read(2))
if header != B'CK':
raise ValueError(
F'chunk did not begin with CK header, got {header!r} instead')
decompress = zlib.decompressobj(-zlib.MAX_WBITS,
zdict=writer.getbuffer())
writer.write(decompress.decompress(reader.read()))
writer.write(decompress.flush())
def __init__(self, reader: StructReader):
if reader.read(4) != self.SIGNATURE:
raise ValueError
self.disk_number = reader.u16()
self.start_disk_number = reader.u16()
self.entries_on_disk = reader.u16()
self.entries_in_directory = reader.u16()
self.directory_size = reader.u32()
self.directory_offset = reader.u32()
try:
cl = reader.u32()
self.comment = cl and reader.read(cl) or None
except EOFError:
self.comment = None
def __init__(self, reader: StructReader, version: str):
reader.bigendian = True
self.base = reader.tell()
signature = reader.read(4)
if signature != self.MagicSignature:
raise ValueError('invalid magic')
magic = bytes(reader.read(4))
with contextlib.suppress(KeyError):
version = xtpyi._xdis.magics.versions[magic]
vtuple = version2tuple(version)
padding_size = 4
if vtuple >= (3, 3):
padding_size += 4
if vtuple >= (3, 7):
padding_size += 4
self.version = version
self.magic = magic + padding_size * b'\0'
self.toc_offset = reader.i32()
self.reader = reader
self.entries: List[PiMeta] = []
def __init__(self, reader: StructReader):
reader.bigendian = True
self.max_stack = reader.u16()
self.max_locals = reader.u16()
self.disassembly: List[JvOpCode] = []
with StructReader(reader.read(reader.u32())) as code:
code.bigendian = True
while not code.eof:
self.disassembly.append(JvOpCode(code, pool=self.pool))
self.exceptions = [JvException(reader) for _ in range(reader.u16())]
self.attributes = [JvAttribute(reader) for _ in range(reader.u16())]
def __init__(self, reader: StructReader):
if reader.read(4) != self.SIGNATURE:
raise ValueError
self.version_made_by = reader.u16()
self.version_to_extract = reader.u16()
self.flags = reader.u16()
self.compression = reader.u16()
self.date = datefix.dostime(reader.u32())
self.crc32 = reader.u32()
self.compressed_size = reader.u32()
self.decompressed_size = reader.u32()
len_filename = reader.u16()
len_extra = reader.u16()
len_comment = reader.u16()
self.disk_nr_start = reader.u16()
self.internal_attributes = reader.u16()
self.external_attributes = reader.u32()
self.header_offset = reader.u32()
self.filename = len_filename and reader.read(len_filename) or None
self.extra = len_extra and reader.read(len_extra) or None
self.comment = len_comment and reader.read(len_comment) or None
def __init__(self, reader: StructReader):
def readint(length: int):
return int(bytes(reader.read(length * 2)), 16)
self.signature = reader.read(6)
if self.signature != b'070701':
raise ValueError('invalid CPIO header signature')
self.inode = readint(4)
self.mode = readint(4)
self.uid = readint(4)
self.gid = readint(4)
self.nlinks = readint(4)
mtime = readint(4)
self.mtime = datetime.utcfromtimestamp(mtime)
self.size = readint(4)
self.dev = readint(4), readint(4)
self.rdev = readint(4), readint(4)
namesize = readint(4)
self.checksum = readint(4)
self.name = bytes(reader.read(namesize)).decode('ascii').rstrip('\0')
reader.byte_align(4)
self.data = reader.read(self.size)
reader.byte_align(4)
def __init__(self, reader: StructReader, calculate_checks=False):
if reader.read(4) != b'dex\n':
raise ValueError('Invalid Signature')
with StreamDetour(reader, 0x28):
endian_test_data = reader.u32()
if endian_test_data == 0x78563412:
reader.bigendian = True
self.version = reader.read(4).rstrip(b'\0')
self.checksum = reader.u32()
if calculate_checks:
with StreamDetour(reader):
self.calculated_checksum = zlib.adler32(reader.read())
else:
self.calculated_checksum = None
self.signature = reader.read(20)
if calculate_checks:
with StreamDetour(reader):
self.calculated_signature = hashlib.sha1(
reader.read()).digest()
else:
self.calculated_signature = None
self.size_of_file = reader.u32()
self.size_of_header = reader.u32()
if reader.u32() != 0x12345678:
raise ValueError('Invalid Endian Tag')
self.link_size = reader.u32()
self.link_offset = reader.u32()
self.map_offset = reader.u32()
self.strings: List[str] = list(
self._read_strings(reader, reader.u32(), reader.u32()))
def __init__(self, reader: StructReader):
with StreamDetour(reader):
self.code = opc(reader.read_byte())
self.table: Optional[Dict[int, int]] = None
try:
fmt = self.OPC_ARGMAP[self.code]
except KeyError:
self.arguments = []
else:
self.arguments = list(reader.read_struct(fmt))
if self.code == opc.newarray:
self.arguments = [JvBaseType(reader.read_byte())]
elif self.code in self.OPC_CONSTPOOL:
try:
self.arguments[0] = self.pool[self.arguments[0] - 1]
except (AttributeError, IndexError):
pass
elif self.code == opc.lookupswitch:
reader.byte_align(blocksize=4)
default, npairs = reader.read_struct('LL')
pairs = reader.read_struct(F'{npairs*2}L')
self.table = dict(zip(*([iter(pairs)] * 2)))
self.table[None] = default
elif self.code == opc.tableswitch:
reader.byte_align(blocksize=4)
default, low, high = reader.read_struct('LLL')
assert low <= high
offsets = reader.read_struct(F'{high-low+1}L')
self.table = {k + low: offset for k, offset in enumerate(offsets)}
self.table[None] = default
elif self.code == opc.wide:
argop = opc(reader.get_byte())
self.arguments = (argop, reader.u16())
if argop == opc.iinc:
self.arguments += reader.i16(),
else:
assert argop in (
opc.iload, opc.istore,
opc.fload, opc.fstore,
opc.aload, opc.astore,
opc.lload, opc.lstore,
opc.dload, opc.dstore,
opc.ret)
offset = reader.tell()
self.raw = bytes(reader.read(offset - reader.tell()))
def __init__(self, reader: StructReader):
reader.bigendian = True
if reader.read(4).hex() != 'cafebabe':
raise ValueError('class file magic missing.')
minor = reader.u16()
major = reader.u16()
self.version = (major, minor)
self.pool: List[Union[Struct, int, float, str]] = []
self._read_pool(reader)
self.strings: List[str] = {
s.value for s in self.pool if isinstance(s, Struct) and s.tag == JvConstType.String}
self.access = JvAccessFlags(reader)
self.this = reader.u16()
self.parent = reader.u16()
try:
self.interfaces = [self.pool[reader.u16()]
for _ in range(reader.u16())]
except IndexError:
raise ValueError('Failed parsing Interfaces.')
try:
self.fields = [JvClassMember(reader, pool=self.pool)
for _ in range(reader.u16())]
except IndexError:
raise ValueError('Failed parsing Fields.')
try:
self.methods = [JvClassMember(reader, pool=self.pool)
for _ in range(reader.u16())]
except IndexError:
raise ValueError('Failed parsing Methods.')
try:
self.attributes = [JvAttribute(reader, pool=self.pool)
for _ in range(reader.u16())]
except IndexError:
raise ValueError('Failed parsing Attributes.')
def test_bitreader_structured(self):
items = (
0b1100101, # noqa
-0x1337, # noqa
0xDEFACED, # noqa
0xC0CAC01A, # noqa
-0o1337, # noqa
2076.171875, # noqa
math.pi # noqa
)
data = struct.pack('<bhiLqfd', *items)
sr = StructReader(data)
self.assertEqual(sr.read_nibble(), 0b101)
self.assertRaises(sr.Unaligned, lambda: sr.read(2))
sr.seek(0)
self.assertEqual(sr.read_byte(), 0b1100101)
self.assertEqual(sr.i16(), -0x1337)
self.assertEqual(sr.i32(), 0xDEFACED)
self.assertEqual(sr.u32(), 0xC0CAC01A)
self.assertEqual(sr.i64(), -0o1337)
self.assertAlmostEqual(sr.read_struct('f', True), 2076.171875)
self.assertAlmostEqual(sr.read_struct('d', True), math.pi)
self.assertTrue(sr.eof)
def _decompress_xpress(self,
reader: StructReader,
writer: MemoryFile,
target: Optional[int] = None) -> bytearray:
if target is not None:
target += writer.tell()
flags = BitBufferedReader(reader)
nibble_cache = None
while not reader.eof:
if target is not None and writer.tell() >= target:
return
if not flags.next():
writer.write(reader.read(1))
continue
offset, length = divmod(reader.u16(), 8)
offset += 1
if length == 7:
length = nibble_cache
if length is None:
length_pair = reader.u8()
nibble_cache = length_pair >> 4
length = length_pair & 0xF
else:
nibble_cache = None
if length == 15:
length = reader.u8()
if length == 0xFF:
length = reader.u16() or reader.u32()
length -= 22
if length < 0:
raise RuntimeError(
F'Invalid match length of {length} for long delta sequence'
)
length += 15
length += 7
length += 3
writer.replay(offset, length)
def process(self, data: bytearray):
formatter = string.Formatter()
until = self.args.until
until = until and PythonExpression(until, all_variables_allowed=True)
reader = StructReader(memoryview(data))
mainspec = self.args.spec
byteorder = mainspec[:1]
if byteorder in '<!=@>':
mainspec = mainspec[1:]
else:
byteorder = '='
def fixorder(spec):
if spec[0] not in '<!=@>':
spec = byteorder + spec
return spec
it = itertools.count() if self.args.multi else (0, )
for index in it:
if reader.eof:
break
if index >= self.args.count:
break
meta = metavars(data, ghost=True)
meta['index'] = index
args = []
last = None
checkpoint = reader.tell()
try:
for prefix, name, spec, conversion in formatter.parse(
mainspec):
if prefix:
args.extend(reader.read_struct(fixorder(prefix)))
if name is None:
continue
if conversion:
reader.byte_align(
PythonExpression.evaluate(conversion, meta))
if spec:
spec = meta.format_str(spec, self.codec, args)
if spec != '':
try:
spec = PythonExpression.evaluate(spec, meta)
except ParserError:
pass
if spec == '':
last = value = reader.read()
elif isinstance(spec, int):
last = value = reader.read_bytes(spec)
else:
value = reader.read_struct(fixorder(spec))
if not value:
self.log_warn(F'field {name} was empty, ignoring.')
continue
if len(value) > 1:
self.log_info(
F'parsing field {name} produced {len(value)} items reading a tuple'
)
else:
value = value[0]
args.append(value)
if name == _SHARP:
raise ValueError(
'Extracting a field with name # is forbidden.')
elif name.isdecimal():
index = int(name)
limit = len(args) - 1
if index > limit:
self.log_warn(
F'cannot assign index field {name}, the highest index is {limit}'
)
else:
args[index] = value
continue
elif name:
meta[name] = value
if until and not until(meta):
self.log_info(
F'the expression ({until}) evaluated to zero; aborting.'
)
break
with StreamDetour(reader, checkpoint) as detour:
full = reader.read(detour.cursor - checkpoint)
if last is None:
last = full
outputs = []
for template in self.args.outputs:
used = set()
outputs.append(
meta.format(template,
self.codec, [full, *args], {_SHARP: last},
True,
used=used))
for key in used:
meta.pop(key, None)
for output in outputs:
chunk = self.labelled(output, **meta)
chunk.set_next_batch(index)
yield chunk
except EOF:
leftover = repr(SizeInt(len(reader) - checkpoint)).strip()
self.log_info(F'discarding {leftover} left in buffer')
break
def __init__(self, reader: StructReader):
self.name = reader.u16()
self.data = reader.read(reader.u32())
