"interval_version"/Int32ul,
"game_options_version"/Int32ul,
"dlc_count"/Int32ul,
"dlc_ids"/Array(lambda ctx: ctx.dlc_count, Int32ul),
"dataset_ref"/Int32ul,
Peek("difficulty_id"/Int32ul),
DifficultyEnum("difficulty"/Int32ul),
"selected_map_id"/Int32ul,
"resolved_map_id"/Int32ul,
"reveal_map"/Int32ul,
Peek("victory_type_id"/Int32ul),
VictoryEnum("victory_type"/Int32ul),
Peek("starting_resources_id"/Int32ul),
ResourceLevelEnum("starting_resources"/Int32ul),
"starting_age_id"/Int32ul,
"starting_age"/AgeEnum(Computed(lambda ctx: ctx.starting_age_id - 2)),
"ending_age_id"/Int32ul,
"ending_age"/AgeEnum(Computed(lambda ctx: ctx.ending_age_id - 2)),
"game_type"/Int32ul,
separator,
separator,
"speed"/Float32l,
"treaty_length"/Int32ul,
"population_limit"/Int32ul,
"num_players"/Int32ul,
"unused_player_color"/Int32ul,
"victory_amount"/Int32ul,
separator,
"trade_enabled"/Flag,
"team_bonus_disabled"/Flag,
"random_positions"/Flag,
UnpackedPayload = Reparsed(
Struct(
"inner_header" / InnerHeader,
"xml" / Unprotect(
this.inner_header.protected_stream_id.data,
this.inner_header.protected_stream_key.data,
XML(GreedyBytes)
)
)
)
# -------------------- Main KDBX Structure --------------------
Body = Struct(
"transformed_key" / Computed(compute_transformed),
"master_key" / Computed(compute_master),
"sha256" / Checksum(
Bytes(32),
lambda data: hashlib.sha256(data).digest(),
this._.header.data,
# exception=HeaderChecksumError,
),
"hmac" / Checksum(
Bytes(32),
compute_header_hmac_hash,
this,
# exception=CredentialsError,
),
"payload" / UnpackedPayload(
IfThenElse(
def _encode(self, obj, context):
raise RuntimeError('Not implemented')
ChuangmiIrSignal = Struct(
Const(0xa567,
Int16ul), 'edge_count' / Rebuild(Int16ul,
len_(this.edge_pairs) * 2 - 1),
'times_index' / Array(16, Int32ul), 'edge_pairs' / Array(
(this.edge_count + 1) / 2,
BitStruct(
'gap' / BitsInteger(4),
'pulse' / BitsInteger(4),
)))
ProntoBurstPair = Struct(
'pulse' / ProntoPulseAdapter(Int16ub),
'gap' / ProntoPulseAdapter(Int16ub),
)
Pronto = Struct(
Const(0, Int16ub),
'_ticks' / Int16ub,
'modulation_period' / Computed(this._ticks * 0.241246),
'frequency' / Computed(1000000 / this.modulation_period),
'intro_len' / Int16ub,
'repeat_len' / Int16ub,
'intro' / Array(this.intro_len, ProntoBurstPair),
'repeat' / Array(this.repeat_len, ProntoBurstPair),
)
kcdata_types_enum.STACKSHOT_KCTYPE_KERN_STACKLR64: 'kernel_stack_frames',
kcdata_types_enum.KCDATA_TYPE_LIBRARY_LOADINFO64: 'dyld_load_info',
kcdata_types_enum.STACKSHOT_KCTYPE_USER_STACKLR64: 'user_stack_frames',
kcdata_types_enum.STACKSHOT_KCTYPE_JETSAM_COALITION_SNAPSHOT:
'jetsam_coalition_snapshot',
kcdata_types_enum.STACKSHOT_KCTYPE_DONATING_PIDS: 'donating_pids',
kcdata_types_enum.STACKSHOT_KCTYPE_THREAD_DISPATCH_QUEUE_LABEL:
'dispatch_queue_label',
kcdata_types_enum.KCDATA_BUFFER_BEGIN_STACKSHOT: 'kcdata_stackshot',
kcdata_types_enum.STACKSHOT_KCTYPE_STACKSHOT_FAULT_STATS:
'stackshot_fault_stats',
kcdata_types_enum.STACKSHOT_KCTYPE_STACKSHOT_DURATION:
'stackshot_duration',
}
predefined_name_substruct = 'name' / Computed(
lambda ctx: predefined_names[ctx._.type])
uint32_desc = Struct(
'name' / Padded(32, CString('utf8')),
'obj' / Int32ul,
)
uint64_desc = Struct(
'name' / Padded(32, CString('utf8')),
'obj' / Int64ul,
)
jetsam_level = Struct(predefined_name_substruct, 'obj' / Int32ul)
thread_policy_version = Struct(predefined_name_substruct, 'obj' / Int32ul)
kernel_page_size = Struct(predefined_name_substruct, 'obj' / Int32ul)
osversion = Struct(predefined_name_substruct, 'obj' / CString('utf8'))
# mapping
object_types[0x05] = Struct(
"monkey_" / Int8ul, # Usually 0
"items" / PrefixedArray(Int32ul, KeyValuePair),
)
Version = Struct(
"major" / Int16ul,
"minor" / Int16ul,
"patch" / Int16ul,
"even_" / Int16ul, # Sometimes 1
"odd_" / Int8ul, # Usually 0
)
OldScriptDat = Struct(
"_type" / Computed(lambda this: "script"),
"version" / Version,
"data" / PrefixedArray(
Int32ul,
Struct("name" / PascalString(Int8ul, "latin-1"), "dump" / PascalString(
FactorioInt32ul, "latin-1"), "tabletop_" / Int8ul)),
Terminated,
)
script_object_types = {}
ScriptSerializedObject = Struct(
"type" / Int8ul,
"data" / Switch(this.type, script_object_types),
)
return int(obj * context._.modulation_period)
def _encode(self, obj, context, path):
raise RuntimeError("Not implemented")
ChuangmiIrSignal = Struct(
Const(0xA567, Int16ul),
"edge_count" / Rebuild(Int16ul,
len_(this.edge_pairs) * 2 - 1),
"times_index" / Array(16, Int32ul),
"edge_pairs" / Array(
(this.edge_count + 1) // 2,
BitStruct("gap" / BitsInteger(4), "pulse" / BitsInteger(4)),
),
)
ProntoBurstPair = Struct("pulse" / ProntoPulseAdapter(Int16ub),
"gap" / ProntoPulseAdapter(Int16ub))
Pronto = Struct(
Const(0, Int16ub),
"_ticks" / Int16ub,
"modulation_period" / Computed(this._ticks * 0.241246),
"frequency" / Computed(1000000 / this.modulation_period),
"intro_len" / Int16ub,
"repeat_len" / Int16ub,
"intro" / Array(this.intro_len, ProntoBurstPair),
"repeat" / Array(this.repeat_len, ProntoBurstPair),
)
struct at this point will parse the embedded header.
This section is a work in progress.
"""
from construct import (Array, Computed, Embedded, GreedyBytes, If, Int16ul,
Int32ul, Padding, Peek, String, Struct, Switch)
from mgz import subheader
# pylint: disable=invalid-name
# Embedded chat message
chat = Struct(
"subtype" / Computed("chat"), "data" / Struct(
"length" / Computed(lambda ctx: ctx._._._.op),
"text" / String(lambda ctx: ctx._._._.op,
padchar=b'\x00',
trimdir='right',
encoding='latin1'),
))
# Embedded header (aka saved chapter)
header = Struct(
"subtype" / Computed("savedchapter"), "data" / Struct(
"header_length" / Computed(lambda ctx: ctx._._._.op - ctx._._._.start),
Embedded(subheader)))
# Unknown embedded structure - looks like a partial action?
other = Struct(
StatusRequestStruct = "StatusTransaction" / Struct(
"data" / Array(15, OneOf(Int32ub, [0])))
StatusResponseStruct = "StatusTransaction" / Struct(
"data" / Array(15, Int32ub))
"""
Struct detailing the Status Action logic
"""
ResendStruct = "ResendTransaction" / Struct()
"""
Struct detailing the Resend Action logic
"""
IPBusWords = "IPBusWords" / Struct("data" / GreedyRange(Int32ub))
IPBusConstruct = "IPBusPacket" / Struct(
"pointer" / Pointer(3, Int8ub),
"bigendian" / Computed(this.pointer == 0xf0),
"header" / IfThenElse(
this.bigendian, PacketHeaderStruct,
ByteSwapped(PacketHeaderStruct)), # defined as 'header' in context
"transactions" / If(lambda ctx: ctx.header.type_id == "CONTROL",
GreedyRange(ControlStruct)),
"status" /
If(lambda ctx: ctx.header.type_id == "STATUS", StatusRequestStruct),
"resend" / If(lambda ctx: ctx.header.type_id == "RESEND", ResendStruct),
Terminated)
"""
Top-level IPBus Construct which is a packet parser/builder
"""
"""Stores the last-seen command byte in the parsing context.
Bit of a hack to make running status support work.
"""
setattr(ctx._root, '_last_command_byte', obj)
class Struct(BaseStruct):
"""Adds `create()`, a friendlier `build()` method."""
def create(self, **kwargs):
return self.build(kwargs)
AppleMIDIExchangePacket = Struct(
'_name' / Computed('AppleMIDIExchangePacket'),
'preamble' / Const(b'\xff\xff'),
'command' / Bytes(2),
'protocol_version' / Int32ub,
'initiator_token' / Int32ub,
'ssrc' / Int32ub,
'name' / Optional(CString('utf8')),
)
AppleMIDITimestampPacket = Struct(
'_name' / Computed('AppleMIDITimestampPacket'),
'preamble' / Const(b'\xff\xff'),
'command' / Bytes(2),
'ssrc' / Int32ub,
'count' / Int8ub,
'padding' / Padding(3),
from construct import Bitwise, Struct, this, Flag, Computed, Default
from paradox.hardware.evo.adapters import DictArray
TestParser = Bitwise(
DictArray(8, 1, Struct(
"_index" / Computed(this._index + 1),
"enabled" / Default(Flag, False)
))
)
TestParser2 = Bitwise(
DictArray(8, 65, Struct(
"_index" / Computed(this._index + 65),
"enabled" / Default(Flag, False)
))
)
TestPickerParser = Bitwise(
DictArray(8, 1, Struct(
"_index" / Computed(this._index + 1),
"enabled" / Default(Flag, False)
), pick_key="enabled")
)
def test_index_parsing():
r = TestParser.parse(b'\x02')
assert r[8].enabled is False
assert r[7].enabled is True
5236: Struct(
"Title" / Utf8(CString("utf16")),
Seek(256), "Initial Folder" / Utf8(CString("utf16")),
Seek(776), "COM Object Path" / Utf8(CString("utf16")),
Seek(1296), "Script Engine" / Utf8(CString("utf16")),
Seek(1816), "Leave Data Files" / Flag,
Seek(1820), "Leave Script Files" / Flag,
Seek(1824), "Leave COM Objects" / Flag,
Seek(1828), "Run Parallel" / Flag,
Seek(1832), "Folder Settings" / Int32ul,
Seek(1836), "UserID" / Utf8(CString("utf16")),
Seek(1964), "Password" / Password(Bytes(64)),
Seek(2092), "Alternate Credentials" / Runmode(Int32ul),
Seek(2096), "Trial Mode" / Int2Bool(Int32ul),
Seek(2100), "Options" / Int32ul,
"MTA Mode" / Computed(this.Options & 1 == 1),
"Execution restrictions" / Computed(this.Options & 2 == 2),
"Allow only single instance" / Computed(this.Options & 4 == 4),
"Check EnableScriptBlockLogging" / Computed(this.Options & 8 == 8),
"Disable EnableScriptBlockLogging" / Computed(this.Options & 32 == 32),
"Disable EnableTranscripting" / Computed(this.Options & 64 == 64),
"Use AES" / Computed(this.Options & 512 == 512),
Seek(2104), "Trial Mode Day" / Int32ul,
Seek(2108), "Trial Mode Month" / Int32ul,
Seek(2112), "Trial Mode Year" / Int32ul,
Seek(2116), "Script name" / Utf8(CString("utf16")),
Seek(2636), "Allowed OS version" / Utf8(CString("utf16")),
Seek(3156), "Allowed users" / Utf8(CString("utf16")),
Seek(3676), "Allowed hostnames" / Utf8(CString("utf16")),
Seek(4196), "Allowed MAC addresses" / Utf8(CString("utf16")),
Seek(4716), "Allowed domains" / Utf8(CString("utf16")),
from mgz.util import MgzPrefixed, ZlibCompressed, Version, VersionAdapter, get_version
from mgz.header.ai import ai
from mgz.header.replay import replay
from mgz.header.map_info import map_info
from mgz.header.initial import initial
from mgz.header.achievements import achievements
from mgz.header.scenario import scenario
from mgz.header.lobby import lobby
from mgz.header.de import de
from mgz.header.hd import hd
compressed_header = Struct(
"game_version"/CString(encoding='latin1'),
"save_version"/VersionAdapter(Float32l),
"version"/Computed(lambda ctx: get_version(ctx.game_version, ctx.save_version, None)),
"hd"/If(lambda ctx: ctx.version == Version.HD and ctx.save_version > 12.34, hd),
"de"/If(lambda ctx: ctx.version == Version.DE, de),
ai,
replay,
map_info,
initial,
achievements,
scenario,
lobby,
Terminated
)
subheader = Struct(
"check"/Peek(Int32ul),
'r' / Int8ul,
'g' / Int8ul,
'b' / Int8ul,
'a' / Int8ul
)
struct_blit_image = Struct(
'header' / Const(b'SPRITE'),
'type' / PaddedString(2, 'ASCII'),
'size' / Rebuild(Int32ul, len_(this.data) + (this.palette_entries * 4) + 18),
'width' / Int16ul,
'height' / Int16ul,
'format' / Const(0x02, Int8ul),
'palette_entries' / PaletteCountAdapter(Int8ul),
'palette' / Array(this.palette_entries, struct_blit_pixel),
'bit_length' / Computed(compute_bit_length),
'data_length' / Computed(compute_data_length),
'data' / Array(this.data_length, Int8ul)
)
struct_blit_meta = Struct(
'header' / Const(b'BLITMETA'),
'data' / Prefixed(Int16ul, Struct(
'checksum' / Checksum(
Int32ul,
lambda data: binascii.crc32(data),
this._._.bin.data
),
'date' / PaddedString(16, 'ascii'),
'title' / PaddedString(25, 'ascii'),
'description' / PaddedString(129, 'ascii'),
"unk0" / Int16sl,
"unk1" / Int16sl,
If(
lambda ctx: ctx._._._.save_version >= 13.03 or ctx._._.
check.val > 1000, "unk2" / Int16sl))),
Embedded(
IfThenElse(
lambda ctx: ctx.terrain_type == 255,
Struct("terrain_type" / Byte, "elevation" / Byte,
Padding(1)), Struct("elevation" / Byte))))))
# Map size and terrain.
map_info = "map_info" / Struct(
"size_x" / Int32ul,
"size_y" / Int32ul,
"tile_num" / Computed(lambda ctx: ctx.size_x * ctx.size_y),
"zone_num" / Int32ul,
Array(
lambda ctx: ctx.zone_num,
Struct(
IfThenElse(lambda ctx: ctx._._.version == Version.DE,
Padding(lambda ctx: 2048 + (ctx._.tile_num * 2)),
Padding(lambda ctx: 1275 + ctx._.tile_num)),
"num_floats" / Int32ul, Padding(lambda ctx: ctx.num_floats * 4),
Padding(4))),
"all_visible" / Flag,
"fog_of_war" / Flag,
"check" /
Peek(Struct(Padding(lambda ctx: ctx._.tile_num * 7),
"val" / Int32ul)), # DE 12.97 fix
Array(lambda ctx: ctx.tile_num, tile),
def do_dbg_info(self):
if not '.debug_info' in self.sections:
return
section = self.sections['.debug_info']
image = io.BytesIO(section.image)
length = len(section.image)
DbgInfo = Struct(
"start" / Tell,
"unit_length" / self.u32,
"version" / self.u16,
"debug_abbrev_offset" / self.u32,
"address_size" / self.u8,
"stop" / Tell,
"size" / Computed(this.stop - this.start),
)
Attribute = Struct(
"start" / Tell,
"attr" / ULEB,
"stop" / Tell,
"size" / Computed(this.stop - this.start),
)
offset = 0
finished = False
while True:
pos = image.tell()
if pos >= length - 1:
break
dbgInfo = DbgInfo.parse_stream(image) # CU
print(" Compilation Unit @ offset 0x{:x}:".format(dbgInfo.start))
print(" Start: {:08x}".format(dbgInfo.start))
print(" Stop: {:08x}".format(dbgInfo.stop))
print(" Size: {:08x}".format(dbgInfo.size))
print(" Length: 0x{:x} (32-bit)".format(
dbgInfo.unit_length))
print(" Version: 2".format(dbgInfo.version))
print(" Abbrev Offset: 0x{:x}".format(
dbgInfo.debug_abbrev_offset))
print(" Pointer Size: {}".format(dbgInfo.address_size))
pos = 0
offset += dbgInfo.stop - dbgInfo.start
formReaders = self.get_form_readers(dbgInfo.address_size)
print("Pos, Length", pos, dbgInfo.unit_length)
if pos >= dbgInfo.unit_length:
break
while True:
start = image.tell()
attr = Attribute.parse_stream(image)
abbr = self.abbreviations.get(
(dbgInfo.debug_abbrev_offset, attr.attr))
if not abbr:
print("<{:02x}>: {}".format(start, "Abbrev Number: 0"))
else:
#print("<{:02x}>: Abbrev Number: {} ({})".format(start, enc.name, value))
for enc, form in abbr.attrs:
reader = formReaders.get(form)
if reader is None:
print("*EF", enc, form, start, attr, abbr)
start = image.tell()
value = reader.parse_stream(image)
print(" <{:02x}> {}: {}".format(start, enc, value))
stop = image.tell()
offset += (attr.stop - attr.start)
pos = image.tell()
#if pos >= 0x8727:
# print("chk")
if pos >= dbgInfo.unit_length:
image.seek(image.tell() + 1)
break
"Keyword" / Int64ul)
EventHeader = Struct("marker" / wmi_trace_marker(EventHeaderType),
"flags" / EventHeaderFlag,
"event_property" / EventHeaderPropertyFlag,
"thread_id" / Int32ul, "process_id" / Int32ul,
"timestamp" / Int64ul, "provider_id" / Guid,
"event_descriptor" / EventDescriptor,
"processor_time" / Int64ul, "activity_id" / Guid)
EventHeaderExtendedDataItem = Struct(
"reserved1" / Int16ul, "ext_type" / Int16ul, "reserved2" / Int16ul,
"data_size" / Int16ul, "data_item" / Bytes(lambda this: this.data_size))
EventRecord = AlignedStruct(
8, "mark1" / Computed(lambda this: this._io.tell()),
"event_header" / EventHeader, "extended_data" / If(
lambda this: this.event_header.flags.EVENT_HEADER_FLAG_EXTENDED_INFO,
RepeatUntil(lambda el, lst, this: not lst[-1].reserved2 & 0x1,
Aligned(8, EventHeaderExtendedDataItem))),
"mark2" / Computed(lambda this: this._io.tell()),
"user_data" / Bytes(lambda this: this.event_header.marker.version -
(this.mark2 - this.mark1)))
class Event:
"""
This is a python wrapper around construct struct to access interesting fields
"""
def __init__(self, source):
self.source = source
"""
TimeZone Information definition in Windows Internal
"""
TimeZoneInformation = Struct("bias" / Int32sl, "standard_name" / Byte[64],
"standard_date" / SystemTime,
"standard_bias" / Int32sl,
"delight_name" / Byte[64],
"delight_date" / SystemTime,
"delight_bias" / Int32sl)
PerfinfoGroupMask = Struct("masks" / Int32ul[8])
"""
Wide string windows style
"""
WString = Struct(
"type" / Computed("WString"), "string" / RepeatUntil(
lambda x, lst, ctx: len(lst) % 2 == 0 and lst[-2:] == [0, 0], Byte))
"""
C string style
"""
CString = Struct(
"type" / Computed("CString"),
"string" / RepeatUntil(lambda x, lst, ctx: lst[-1:] == [0], Byte))
def check_enum(enum: Enum) -> Struct:
"""
Enforce an enum value to be in enum range
:param enum: source enum
:return: Struct
:raise: construct.core.CheckError
return b""
return obj.encode("UTF-16le", "replace")
def PaddedUTF16StringBestEffort(length):
return UTF16EncodedBestEffort(
FixedSized(length, NullStripped(GreedyBytes, pad="\x00\x00")))
FixedNullTerminatedUTF16String = Struct( # I don't use PascalString because it's a null terminated string.
"string_size" / Int32ul, "string" / IfThenElse(
lambda this: this.string_size,
FixedSized(
lambda this: this.string_size,
NullTerminated(GreedyString("UTF_16_le"),
term="\x00".encode("utf-16le"))), Computed('')))
class FiletimeAdapter(Adapter):
def _decode(self, obj, context, path):
if 0 == obj:
return None # 0 is not really a date
secs = int(obj // int(1e7))
nanosecs = int(obj - int(secs * int(1e7))) * 100
# I use numpy's datetime64 instead of datetime.datetime because filetime have 100 nanoseconds precision.
return datetime64('1601-01-01') + timedelta64(secs, 's') + timedelta64(
nanosecs, 'ns')
def _encode(self, obj, context, path):
return int(uint64((obj - datetime64('1601-01-01')).astype('O'))) // 100
floatField = Struct(keylength=Int32un,
key=PaddedString(this.keylength, "utf8"),
dataLength=Int32un,
value=Float64n)
intField = Struct(keylength=Int32un,
key=PaddedString(this.keylength, "utf8"),
dataLength=Int32un,
value=BytesInteger(this.dataLength,
signed=False,
swapped=True))
lazyIntField = Struct("keylength" / Int32un,
"key" / PaddedString(this.keylength, "utf8"),
"dataLength" / Int32un, "offset" / Tell,
"end" / Computed(this.offset + this.dataLength),
"value" / Lazy(Bytes(this.dataLength)), Seek(this.end))
date = Struct(year=Int16un,
month=Int16un,
dow=Int16un,
day=Int16un,
hour=Int16un,
minute=Int16un,
second=Int16un,
millisecond=Int16un)
dateField = Struct(keylength=Int32un,
key=PaddedString(this.keylength, "utf8"),
dataLength=Int32un,
value=date)
import zlib
from construct import Construct, Struct, Tell, Computed, Seek, this
# used for unpacking zlib block and return in context
class ZlibContext(Construct):
def __init__(self):
super(ZlibContext, self).__init__()
def _parse(self, stream, ctx, path):
ctx.decompressed_data, ctx.size_of_unused_data = self._zlib_decompress(stream.getvalue()[ctx.start_offset:])
def _zlib_decompress(self, data):
zdo = zlib.decompressobj()
decompressed_data = zdo.decompress(data)
size_of_unused_data = len(zdo.unused_data)
return decompressed_data, size_of_unused_data
# only one 'real' field is `decompressed_body`, other only for changing offset
zlib_stream = "zlib_stream" / Struct(
"start_offset" / Tell,
ZlibContext(),
"unused_size" / Computed(this.size_of_unused_data),
"global_file_size" / Seek(0, 2),
"decompressed_body" / Computed(this.decompressed_data),
"end_offset" / Computed(this.global_file_size - this.unused_size),
Seek(this.end_offset)
)
"order": actions.order,
"droprelic": actions.droprelic,
"gatherpoint": actions.gatherpoint,
"townbell": actions.townbell,
"resign": actions.resign,
"tribute": actions.tribute,
"queue": actions.queue,
"multiqueue": actions.multiqueue,
"research": actions.research,
"sell": actions.sell,
"buy": actions.buy,
"backtowork": actions.backtowork,
"de": actions.de,
"postgame": actions.postgame
}, default=Struct(
"unk_action"/Computed(lambda ctx: ctx._.type),
"bytes"/Bytes(lambda ctx: ctx._._.length - 1)
))),
Padding(4)
)
# Action - length followed by data.
action = "action"/Struct(
"length"/Int32ul,
action_data
)
# Synchronization.
sync = "sync"/Struct(
"strings" / PrefixedArray(
Int32ul,
Struct(
"offset" / Int32ul,
"string" / Pointer(
lambda this: this._._.table_offset + this.offset,
FixedNullTerminatedUTF16String
)
)
)
)
RawProcessStruct = """
Struct that describes a process.
""" * Struct(
"strings_table" / Computed(lambda ctx: ctx._.strings_table), # keep the reference to the strings table
"process_index" / Int32ul,
"process_id" / Int32ul,
"parent_process_id" / Int32ul,
"reserved1" / Int32ul * "!!Unknown field!!",
"authentication_id" / Int32ul,
"reserved2" / Int32ul * "!!Unknown field!!",
"session" / Int32ul,
"reserved3" / Array(5, Int32ul) * "!!Unknown field!!",
"virtualized" / Int32ul,
"is_64bit" / Int32ul,
"integrity" / StringIndex,
"user" / StringIndex,
"process_name" / StringIndex,
"image_path" / StringIndex,
"command_line" / StringIndex,
{
# string longer than 127 bytes, prefixed with 3 bytes length
0x40:
FocusedSeq(
"text", "actual_length" /
ExprAdapter(Int16ul, lambda o, c: o - 4, lambda o, c: o + 4),
Padding(1),
"text" / PaddedString(this.actual_length, encoding="ascii")),
# utf-16 text
0x90:
FocusedSeq(
"text", "actual_length" /
ExprAdapter(Int16ul, lambda o, c: o - 4, lambda o, c: o + 4),
"text" / PaddedString(this.actual_length, "utf-16-be")),
},
default= # just ascii text
FocusedSeq(
"text",
"actual_length" / Computed((this._.padded_length - 1) // 2 - 1),
"text" / PaddedString(this.actual_length, encoding="ascii"))))
# parses a PioString relative to entry start using an str_idx array
def OffsetPioString(index):
return Pointer(this.entry_start + index, PioString)
# parses a PioString relative to entry start using an str_idx array
def IndexedPioString(index):
return Pointer(this.entry_start + this.str_idx[index], PioString)
'enable_experiment'/Flag,
'short_y'/Flag
)
DateVersionK = BitSplitter(
Int32ul,
minute=(0, 6),
hour=(6, 5),
day=(11, 5),
month=(16, 4),
year=(20, 12)
)
DateVersionM = Struct(
'year1900'/Int16sl, # XXX: guessing that we count years since 1900
'year'/Computed(this.year1900 + 1900),
'month'/Byte,
'day'/Byte,
'hour'/Byte,
'minute'/Byte
)
HeaderVersionK = Struct(
'experiment_type'/Byte,
'exp'/Byte,
'npts'/Int32sl,
'first'/Float64l,
'last'/Float64l,
'nsub'/Int32sl,
'xtype'/Byte,
'ytype'/Byte,
Int64ul, Computed, Embedded, IfThenElse)
from mgz.enums import VictoryEnum, ResourceLevelEnum, AgeEnum, PlayerTypeEnum, DifficultyEnum
from mgz.util import find_save_version
separator = Const(b"\xa3_\x02\x00")
hd_string = Struct("length" / Int16ul, Const(b"\x60\x0A"),
"value" / Bytes(lambda ctx: ctx.length))
test_57 = "test_57" / Struct(
"check" / Int32ul, Padding(4),
If(lambda ctx: ctx._._.version >= 1006, Bytes(1)), Padding(15), hd_string,
Padding(1), If(lambda ctx: ctx._._.version >= 1005, hd_string), hd_string,
Padding(16), "test" / Int32ul,
"is_57" / Computed(lambda ctx: ctx.check == ctx.test))
player = Struct(
"dlc_id" / Int32ul, "color_id" / Int32ul,
"unk1_1006" / If(lambda ctx: ctx._._.version >= 1006, Bytes(1)),
"unk" / Bytes(2), "dat_crc" / Bytes(4), "mp_game_version" / Byte,
"team_index" / Int32ul, "civ_id" / Int32ul, "ai_type" / hd_string,
"ai_civ_name_index" / Byte,
"ai_name" / If(lambda ctx: ctx._._.version >= 1005, hd_string),
"name" / hd_string, "type" / PlayerTypeEnum(Int32ul), "steam_id" / Int64ul,
"player_number" / Int32sl,
Embedded(
If(lambda ctx: ctx._._.version >= 1006 and not ctx._.test_57.is_57,
Struct(
"hd_rm_rating" / Int32ul,
"hd_dm_rating" / Int32ul,
EtlChunk = Struct(
"header" / WmiBufferHeader,
"payload" / Bytes(lambda this: this.header.wnode.saved_offset -
WmiBufferHeader.sizeof()),
"padding" / Bytes(lambda this: this.header.wnode.buffer_size - this.header.
wnode.saved_offset))
# An ETL file is structured by ETL chunks until the end
EtlLogFile = GreedyRange(EtlChunk)
# This is a common way to select any type of chunks
# We add name selecting as computed to handle typing during parsing
Chunk = Aligned(
8,
Select(
Struct("type" / Computed("PerfInfoTraceRecord"),
"value" / PerfInfoTraceRecord),
Struct("type" / Computed("EventRecord"), "value" / EventRecord),
Struct("type" / Computed("TraceRecord"), "value" / TraceRecord),
Struct("type" / Computed("SystemTraceRecord"),
"value" / SystemTraceRecord),
Struct("type" / Computed("WinTraceRecord"), "value" / WinTraceRecord)))
ChunkParser = RepeatUntil(
lambda x, lst, ctx: len(x._io.getbuffer()) == x._io.tell(), Chunk)
class IEtlFileObserver(metaclass=ABCMeta):
"""
This is etl file observer
Parse sequentially an etl file and commit event when found a particular event
ReverseIndexedEntry = FocusedSeq(1,
"entry_offset" / Int16ul,
"entry" / Pointer(this._._.entries_start+this.entry_offset,
Switch(lambda ctx: "strange" if ctx._._.is_strange_page else ctx._._.page_type, {
"block_tracks": Track,
"block_artists": Artist,
"block_albums": Album,
"block_playlists": Playlist,
"block_playlist_map": PlaylistMap,
"block_artwork": Artwork,
"block_colors": Color,
"block_genres": Genre,
"block_keys": Key,
"block_labels": Label,
#"strange": StrangePage,
}, default = Computed("page type not implemented")),
)
)
# unfortunately, the entry_enabled field contains even unexistant entries:
# entry_enabled[:-1] matches revidx[:-1],
# but len(entry_enabled)==16 while len(revidx)<=16
FirstReverseIndexArray = Struct(
"entries" / Array(this._._.entry_count%16, ReverseIndexedEntry),
"entry_enabled" / ByteSwapped(Bitwise(Array(16, Flag))), # may start with unexistant entries, see above!
"entry_enabled_override" / ByteSwapped(Bitwise(Array(16, Flag))) # may start with unexistant entries, see above!
)
FullReverseIndexArray = Struct(
"entries" / Array(16, ReverseIndexedEntry),
Padding(44),
'blocks' / Array(this.num_blocks, Block_v101))
def _unsupported_version(ctx):
v = b'1' if ctx.version == b' ' else ctx.version
raise NotImplementedError('Bruker RAW version %r is not implemented' % v)
RAW = Struct(
Const(b'RAW'), 'version' / Bytes(1),
'body' / Switch(this.version, {
b'2': RAW_v2,
b'1': RAW_v101,
},
default=Computed(_unsupported_version)))
def parse_raw(fh):
# Parser requires binary file mode
if hasattr(fh, 'mode') and 'b' not in fh.mode:
fh = open(fh.name, 'rb')
data = RAW.parse_stream(fh)
assert data.body.num_blocks == 1, 'Bruker RAW files w/ only 1 block supported'
block, = data.body.blocks
n = block.num_steps
if data.version == b'1':
x_start = block.start_2theta
x_stop = x_start + n * block.step_size
elif data.version == b'2':
x_start = block.x_start
def _EOSEventRecord(self):
'''Return desired endianness for a single EOS Event Record'''
return Struct(
'Bytes' / self._UInt32,
Embedded(
Struct(
'EventCode' / self._EOSEventCode,
'Record' / Switch(
lambda ctx: ctx.EventCode,
{
'AvailListChanged':
Embedded(
Struct(
'PropertyCode' / self._EOSPropertyCode,
'Enumeration' / Array(
# TODO: Verify if this is actually an
# enumeration.
lambda ctx: ctx._._.Bytes - 12,
self._UInt8))),
'DevicePropChanged':
Embedded(
Struct(
'PropertyCode' / self._EOSPropertyCode,
'DataTypeCode' /
Computed(lambda ctx: self._EOSDataTypeCode[
ctx.PropertyCode]),
'Value' /
Switch(lambda ctx: ctx.DataTypeCode, {
None:
Array(lambda ctx: ctx._._.Bytes - 12,
self._UInt8)
},
default=self._DataType),
)),
# TODO: 'EmptyEvent',
# TODO: 'RequestGetEvent',
# TODO: 'ObjectAdded',
# TODO: 'ObjectRemoved',
# TODO: 'RequestGetObjectInfoEx',
# TODO: 'StorageStatusChanged',
# TODO: 'StorageInfoChanged',
# TODO: 'RequestObjectTransfer',
# TODO: 'ObjectInfoChangedEx',
# TODO: 'ObjectContentChanged',
# TODO: 'DevicePropChanged',
# TODO: 'AvailListChanged',
# TODO: 'CameraStatusChanged',
# TODO: 'WillSoonShutdown',
# TODO: 'ShutdownTimerUpdated',
# TODO: 'RequestCancelTransfer',
# TODO: 'RequestObjectTransferDT',
# TODO: 'RequestCancelTransferDT',
# TODO: 'StoreAdded',
# TODO: 'StoreRemoved',
# TODO: 'BulbExposureTime',
# TODO: 'RecordingTime',
# TODO: 'InnerDevelopParam',
# TODO: 'RequestObjectTransferDevelop',
# TODO: 'GPSLogOutputProgress',
# TODO: 'GPSLogOutputComplete',
# TODO: 'TouchTrans',
# TODO: 'RequestObjectTransferExInfo',
# TODO: 'PowerZoomInfoChanged',
# TODO: 'RequestPushMode',
# TODO: 'RequestObjectTransferTS',
# TODO: 'AfResult',
# TODO: 'CTGInfoCheckComplete',
# TODO: 'OLCInfoChanged',
# TODO: 'ObjectAddedEx64',
# TODO: 'ObjectInfoChangedEx64',
# TODO: 'RequestObjectTransfer64',
# TODO: 'RequestObjectTransferFTP64',
# TODO: 'ImportFailed',
# TODO: 'BlePairing',
# TODO: 'RequestObjectTransferFTP',
# TODO: 'Unknown',
},
default=Array(lambda ctx: ctx._.Bytes - 8,
self._UInt8)))))
Struct("_free" / Padding(1),
"bus-module_trouble" / StatusAdapter(Bytes(63)))
}
LiveEvent = Struct(
"fields" / RawCopy(
Struct(
"po" / BitStruct(
"command" / Const(0xE, Nibble), "status" /
Struct("reserved" / Flag, "alarm_reporting_pending" / Flag,
"Winload_connected" / Flag, "NeWare_connected" / Flag)),
"event_source" / Const(0xFF, Int8ub),
"event_nr" / Int16ub,
"time" / DateAdapter(Bytes(6)),
"event" / EventAdapter(Bytes(4)),
"partition" / Computed(this.event.partition),
"module_serial" / Bytes(4),
"label_type" / Bytes(1),
"label" / Bytes(16),
"_not_used0" / Bytes(1),
)), "checksum" / Checksum(
Bytes(1), lambda data: calculate_checksum(data), this.fields.data))
# "Event 1 requested (in compressed format) 1 (12 bytes)
# Byte 00: [7-3]: Day, [2-0]: Month (MSB)
# Byte 01: [7]: Month (LSB), [6-0]: Century
# Byte 02: [7-1]: Year, [0]: Hour (MSB)
# Byte 03: [7-4]: Hour (LSB), [3-0]: Minutes (MSB)
# Byte 04: [7-6]: Minutes (LSB), [5-0]: Event Group
# Byte 05: [7-4]: Partition, [3-0]: Event Number High Nibble
# Byte 06: Event Number 1
