class Request(object):
"""Parse the Socks4 request from the client.
"""
SOCKS_4_REQUEST_STRUCT = Struct(
"cd" / Byte,
"dstport" / Int16ub,
"dstip" / Int32ub,
"userid" / RepeatUntil(lambda b, _1, _2: b == 0, Byte),
)
def __init__(self, cd, dstport, dstip, userid):
self.vn = VERSION
self.cd = cd
self.dstport = dstport
self.dstip = dstip
self.userid = userid
@classmethod
def parse_stream(cls, stream):
try:
req = cls.SOCKS_4_REQUEST_STRUCT.parse_stream(stream)
return cls(cd=req.cd,
dstport=req.dstport,
dstip=ipaddress.ip_address(req.dstip),
userid=req.userid[:-1])
except StreamError as ex:
raise ex
raise SocksError("Error parsing socks 4 request")
def __repr__(self):
return repr(self.__dict__)
def _gen_fmh3_struct(int_type, pointer_type, codepoint_type, addr_mode='rel'):
return Struct(
"pointer_offset" / (Computed(0) if addr_mode == 'abs' else Tell),
"signature" / Const(b'FMH3'),
Padding(4),
"fonts_count" / Padded(pointer_type.sizeof(), int_type),
"fonts_pointers_offset" / pointer_type,
"fonts" / Pointer(
lambda this: this.fonts_pointers_offset + this.pointer_offset,
RepeatUntil(
lambda obj, lst, ctx: ctx._index >= ctx.fonts_count - 1,
Struct(
"pointer" / pointer_type,
"data" / Pointer(
lambda this: this.pointer + this._.pointer_offset,
Struct(
"id" / int_type,
"advance_width" / Byte,
"line_height" / Byte,
"box_width" / Byte,
"box_height" / Byte,
"layout_param_1" / Byte,
"layout_param_2_numerator" / Byte,
"layout_param_2_denominator" / Byte,
Padding(1),
"other_params?" / int_type,
"tex_size_chars" / int_type,
"chars_count" / int_type,
"chars_pointer" / pointer_type,
"chars" / Pointer(
lambda this: this.chars_pointer + this._._.
pointer_offset,
RepeatUntil(
lambda obj, lst, ctx: ctx._index >= ctx.
chars_count - 1,
Struct(
"codepoint" / codepoint_type,
"halfwidth" / Flag,
Padding(1),
"tex_col" / Byte,
"tex_row" / Byte,
"glyph_x" / Byte,
"glyph_width" / Byte,
))),
)),
))),
)
def DelimitedField(stop):
return _StripDelimiter(
Sequence(
RepeatUntil(
lambda x, lst, ctx: lst[-len(stop):] == [int(c) for c in stop],
Byte), Computed(this[0][:-len(stop)]),
Seek(-len(stop), whence=1)))
def __init__(self, idx_field, subcon):
super().__init__(RepeatUntil(
lambda obj, values, ctx: obj["idx"] < 0,
Struct(
"idx" / idx_field,
Check(lambda this: abs(this.idx) == this._index + 1),
"data" / subcon,
),
))
def Label(name):
'''
return a construct to parse/build a DNS label.
Example:
>> Label('qname').parse('\x05hello\x00')
>> ['hello', '']
:param name: the name of the field
:type name: string
:returns: a construct to parse/build a DNS label
:rtype: construct.core.RepeatUntil
'''
return RepeatUntil(
lambda obj, ctx: obj == '',
PascalString(name),
)
Bytes(10),
)),
"fog_of_war" / Flag,
"cheat_notifications" / Flag,
"colored_chat" / Flag,
Bytes(4),
"ranked" / Flag,
"allow_specs" / Flag,
Bytes(3),
separator,
Bytes(12),
"strings" / Array(
23,
Struct(
"string" / de_string,
RepeatUntil(lambda x, lst, ctx: lst[-1] not in [3, 21, 42],
Int32ul))),
"strategic_numbers" / Array(59, Int32sl),
"num_ai_files" / Int32ul,
Const(b"\x00\x00\x00\x00"),
"ai_files" / Array(lambda ctx: ctx.num_ai_files,
Struct(
Bytes(4),
"name" / de_string,
Bytes(4),
)),
"guid" / Bytes(16),
"lobby_name" / de_string,
de_string,
Bytes(19),
de_string,
Bytes(5),
from crypto.aeswrap import AESUnwrap
from zipfile import crc32
import struct
Dkey = 0x446B6579
EMF = 0x454D4621
BAG1 = 0x42414731
DONE = 0x444f4e45 #locker sentinel
#**** = 0x2A2A2A2A #wildcard for erase
#MAGIC (kL) | LEN (2bytes) | TAG (4) | DATA (LEN)
Locker = Struct("Locker", String("magic", 2), ULInt16("length"),
Union("tag", ULInt32("int"), String("tag", 4)),
String("data", lambda ctx: ctx["length"]))
Lockers = RepeatUntil(lambda obj, ctx: obj.tag.int == DONE, Locker)
def xor_strings(s, key):
res = ""
for i in xrange(len(s)):
res += chr(ord(s[i]) ^ ord(key[i % len(key)]))
return res
def check_effaceable_header(plog):
z = xor_strings(plog[:16], plog[16:32])
if z[:4] != "ecaF":
return False
plog_generation = struct.unpack("<L", plog[0x38:0x3C])[0]
print "Effaceable generation", plog_generation
class DurationAdapter(Adapter):
def _decode(self, obj, context, path):
secs = obj // (10**7)
nanosecs = (obj - secs * (10**7)) * 100
return timedelta64(secs, 's') + timedelta64(nanosecs, 'ns')
def _encode(self, obj, context, path):
return int(uint64(obj.astype('O')))
Duration = DurationAdapter(Int64ul)
PVoid = IfThenElse(lambda ctx: ctx.is_64bit, Int64ul, Int32ul)
UTF16MultiSz = ExprAdapter(
RepeatUntil(lambda x, lst, ctx: not x, CString("UTF_16_le")),
lambda obj, ctx: list(obj[:-1]), # last element is the null
lambda obj, ctx: obj + [''])
def SizedUTF16MultiSz(size_func):
return ExprAdapter(
FixedSized(size_func, GreedyRange(CString("UTF_16_le"))),
lambda obj, ctx: list(obj[:-1]), # last element is the null
lambda obj, ctx: obj + [''])
class CheckCustom(Check):
def __init__(self, func, exc_type, msg):
super(CheckCustom, self).__init__(func)
self.exc_type = exc_type
SurfaceResult = Struct("type" / Const(1, Int8ul), "status" / Int32ul)
DecodingResult = Struct("type" / Const(2, Int8ul), "status" / Int32ul)
RectResult = Struct("type" / Const(3, Int8ul), "status" / Int32ul)
"""
This struct encompass all calista stream
"""
Bitmap = Struct("type" / Const(4, Int8ul), "index" / Int32ul,
"length" / Int32ul, "data" / Bytes(lambda this: this.length),
"info" / RdpBitmapinfo, "have_point" / Int8ul,
"point" / If(lambda this: this.have_point != 0, Int64ul))
CacdecDump = Struct(
"magic" / Const(b"CacDec01"), "objects" / RepeatUntil(
lambda x, lst, ctx: len(x._io.getbuffer()) == x._io.tell(),
Select(SurfaceContext, SurfaceResult, DecodingResult, RectResult,
Bitmap)))
def write_bmp(file_path, width, height, data):
"""
Write a BMP file a output
:param file_path: path of file
:param width: with of image
:param height: height of image
:param data: raw image data
"""
with open(file_path, "wb") as f:
f.write(b"BM" + pack("<L",
len(data) + 0x36) +
b"\x00\x00\x00\x00\x32\x00\x00\x00\x28\x00\x00\x00" +
"object_type" / Int16ul, "sprite" / Int16ul, "garrisoned_in_id" / Int32sl,
"hitpoints" / Float32l, "object_state" / Byte, "sleep" / Flag,
"doppleganger" / Flag, "go_to_sleep" / Flag, "object_id" / Int32ul,
"facet" / Byte, "x" / Float32l, "y" / Float32l, "z" / Float32l,
"screen_offset_x" / Int16ul, "screen_offset_y" / Int16ul,
"shadow_offset_x" / Int16ul, "shadow_offset_y" / Int16ul,
"selected_group" / If(lambda ctx: find_version(ctx) == Version.AOK, Byte),
ResourceEnum("resource_type" / Int16sl), "amount" / Float32l,
"worker_count" / Byte, "current_damage" / Byte, "damaged_lately_timer" /
Byte, "under_attack" / Byte, "pathing_group_len" / Int32ul, "pathing_group"
/ Array(lambda ctx: ctx.pathing_group_len, "object_id" / Int32ul),
"group_id" / Int32sl, "roo_already_called" / Byte, "de_static_unk1" /
If(lambda ctx: find_version(ctx) == Version.DE, Bytes(19)),
"has_sprite_list" / Byte, "sprite_list" /
If(lambda ctx: ctx.has_sprite_list != 0,
RepeatUntil(lambda x, lst, ctx: lst[-1].type == 0, sprite_list)),
"de_effect_block" / If(
lambda ctx: find_version(ctx) == Version.DE,
Struct(
Padding(4), "has_effect" / Byte, "effect" / If(
lambda ctx: ctx.has_effect == 1,
Struct(Padding(3), "length" / Int16ul,
"name" / Bytes(lambda ctx: ctx.length),
If(lambda ctx: ctx.length > 0, Padding(33)))),
If(
lambda ctx: ctx.effect is None or
(ctx.effect and ctx.effect.length > 0), Byte), Padding(4))))
animated = "animated" / Struct(Embedded(static), "turn_speed" / Float32l)
path_data = "path_data" / Struct(
"items",
SBInt16("primary"),
If(
lambda context: context["primary"] >= 0,
Embed(
Struct(
"item_information",
UBInt8("count"),
UBInt16("secondary"),
)),
),
)
# Metadata subconstruct.
metadata = StringAdapter(
RepeatUntil(lambda obj, ctx: obj == "\x7f", Field("metadata", 1)))
# Build faces, used during dig and build.
faces = {
"noop": -1,
"-y": 0,
"+y": 1,
"-z": 2,
"+z": 3,
"-x": 4,
"+x": 5,
}
face = Enum(SBInt8("face"), **faces)
packets = {
0:
WIPTag = Struct(
'WIPTag',
'name' / WIPString,
'type' / WIPTagType,
'data_start' / Int64sl,
'data_end' / Int64sl,
# 'start'/Tell,
# OneOf(Computed(this.start == this.data_start), [True]),
'data_size' / Computed(this.data_end - this.data_start),
'data' / If(
this.data_end > this.data_start,
Switch(
this.type,
dict(
WIP_TAG_LIST=RepeatUntil(
lambda obj, ctx: obj.data_end >= ctx.data_end,
LazyBound(lambda: WIPTag)),
WIP_TAG_EXTENDED=Bytes(10),
WIP_TAG_DOUBLE=Float64l,
WIP_TAG_FLOAT=Float32l,
WIP_TAG_INT64=Int64sl,
WIP_TAG_INT32=Array(this.data_size // 4, Int32sl),
WIP_TAG_UINT32=Array(this.data_size // 4, Int32ul),
WIP_TAG_CHAR=Array(this.data_size, Int8ul),
WIP_TAG_BOOL=Int8ul,
WIP_TAG_STRING=WIPString,
))),
'end' / Tell,
# pad to get to data_end
Padding(this.data_end - this.end),
)
def __init__(self):
super(VariableLengthIDAdapter, self).__init__(RepeatUntil(lambda obj, lst, ctx: obj & 0xF0 == 0x00, Byte))
)
orientation = Struct("orientation", BFloat32("rotation"), BFloat32("pitch"))
# Notchian item packing
items = Struct("items",
SBInt16("primary"),
If(lambda context: context["primary"] >= 0,
Embed(Struct("item_information",
UBInt8("count"),
UBInt16("secondary"),
)),
),
)
# Metadata subconstruct.
metadata = StringAdapter(RepeatUntil(lambda obj, ctx: obj == "\x7f",
Field("metadata", 1)))
# Build faces, used during dig and build.
faces = {
"noop": -1,
"-y": 0,
"+y": 1,
"-z": 2,
"+z": 3,
"-x": 4,
"+x": 5,
}
face = Enum(SBInt8("face"), **faces)
packets = {
0: Struct("ping"),
"ObjectId" / Int32ul,
"MetadataId" / Int32ul,
"Values" / MemberValues(True))
ObjectNull = Struct(
"RecordTypeEnum" / RecordTypeEnum)
Record = Struct(
"RecordTypeEnum" / Peek(RecordTypeEnum),
"Obj" / Switch(this.RecordTypeEnum, {
RecordTypeEnum.SerializedStreamHeader: SerializationHeaderRecord,
RecordTypeEnum.BinaryObjectString: BinaryObjectString,
RecordTypeEnum.BinaryLibrary: BinaryLibrary,
RecordTypeEnum.ClassWithMembersAndTypes: ClassWithMembersAndTypes,
RecordTypeEnum.MemberReference: MemberReference,
RecordTypeEnum.SystemClassWithMembers: SystemClassWithMembers,
RecordTypeEnum.ArraySinglePrimitive: ArraySinglePrimitive,
RecordTypeEnum.ClassWithId: ClassWithId,
RecordTypeEnum.ObjectNull: ObjectNull,
RecordTypeEnum.BinaryArray: BinaryArray,
RecordTypeEnum.MemberPrimitiveTyped: MemberPrimitiveTyped
}
))
AllRecords = RepeatUntil(lambda obj,lst,ctx: obj.RecordTypeEnum == RecordTypeEnum.MessageEnd, Record)
def parse(data):
MemberValues._allClses = {}
return AllRecords.parse(data)
SBInt32("y"),
SBInt32("z"),
),
}
# Metadata subconstruct.
entity_metadata = MetadataAdapter(
Struct(
"metadata",
RepeatUntil(
lambda obj, context: obj["peeked"] == 0x7f,
Struct(
"data",
BitStruct(
"id",
BitField("data_type", 3), # first
BitField("identifier", 5), # second
),
Switch("value", lambda context: context["id"]["data_type"],
metadata_switch),
Peek(SBInt8("peeked")),
),
),
Const(UBInt8("terminator"), 0x7f),
), )
# The actual packet list.
packets = {
0:
Struct(
"keep alive",
SBInt32("pid"),
"""
TimeZone Information definition in Windows Internal
"""
TimeZoneInformation = Struct("bias" / Int32sl, "standard_name" / Byte[64],
"standard_date" / SystemTime,
"standard_bias" / Int32sl,
"delight_name" / Byte[64],
"delight_date" / SystemTime,
"delight_bias" / Int32sl)
PerfinfoGroupMask = Struct("masks" / Int32ul[8])
"""
Wide string windows style
"""
WString = Struct(
"type" / Computed("WString"), "string" / RepeatUntil(
lambda x, lst, ctx: len(lst) % 2 == 0 and lst[-2:] == [0, 0], Byte))
"""
C string style
"""
CString = Struct(
"type" / Computed("CString"),
"string" / RepeatUntil(lambda x, lst, ctx: lst[-1:] == [0], Byte))
def check_enum(enum: Enum) -> Struct:
"""
Enforce an enum value to be in enum range
:param enum: source enum
:return: Struct
:raise: construct.core.CheckError
"""
"damaged_lately_timer"/Byte,
"under_attack"/Byte,
"pathing_group_len"/Int32ul,
"pathing_group"/Array(lambda ctx: ctx.pathing_group_len, "object_id"/Int32ul),
"group_id"/Int32sl,
"roo_already_called"/Byte,
"de_static_unk1"/If(lambda ctx: find_version(ctx) == Version.DE, Bytes(17)),
If(lambda ctx: find_save_version(ctx) >= 26.16, Byte),
"de_has_object_props"/If(lambda ctx: find_version(ctx) == Version.DE, Int16ul),
"de_object_props"/IfThenElse(lambda ctx: ctx.de_has_object_props == 1, Struct(
Bytes(162),
Find(b'`\n\x00\x00`\n\x00\x00', 1000), # Skip a large unparseable block that (likely) contains RMS-modified object data
Bytes(2)
), Struct(
"has_sprite_list"/Byte,
"sprite_list"/If(lambda ctx: ctx.has_sprite_list != 0, RepeatUntil(lambda x,lst,ctx: lst[-1].type == 0, sprite_list)),
"de_extension"/If(lambda ctx: find_version(ctx) == Version.DE, Struct(
"particles"/Array(5, particle),
If(lambda ctx: find_save_version(ctx) >= 13.15, Bytes(5)),
If(lambda ctx: find_save_version(ctx) >= 13.17, Bytes(2)),
If(lambda ctx: find_save_version(ctx) >= 13.34, Struct(
Bytes(2),
de_string,
de_string,
Bytes(2)
))
))
)),
"de_extension_p2"/If(lambda ctx: find_version(ctx) == Version.DE, Struct(
# not the right way to do this, needs improvement
If(lambda ctx: find_save_version(ctx) >= 20.16, Struct(
UBInt32("y"),
UBInt32("z"),
),
}
# Metadata subconstruct.
metadata = MetadataAdapter(
Struct(
"metadata",
RepeatUntil(
lambda obj, context: obj["peeked"] == 0x7f,
Struct(
"data",
BitStruct(
"id",
BitField("first", 3),
BitField("second", 5),
),
Switch("value", lambda context: context["id"]["first"],
metadata_switch),
Peek(UBInt8("peeked")),
),
),
Const(UBInt8("terminator"), 0x7f),
), )
# Build faces, used during dig and build.
faces = {
"noop": -1,
"-y": 0,
"+y": 1,
"-z": 2,
Int32ul, Const(b"\x00\x00\x00\x00"), "player_number" / Int32sl,
"hd_rm_elo" / Int32ul, "hd_dm_elo" / Int32ul,
"animated_destruction_enabled" / Flag, "custom_ai" / Flag)),
"fog_of_war" / Flag,
"cheat_notifications" / Flag,
"colored_chat" / Flag,
Bytes(9),
separator,
Bytes(12),
If(lambda ctx: ctx._.save_version >= 13.13, Bytes(5)),
"strings" / Array(
23,
Struct(
"string" / de_string,
RepeatUntil(
lambda x, lst, ctx: lst[-1] not in [3, 21, 23, 42, 44, 45],
Int32ul))),
"strategic_numbers" / Array(59, Int32sl),
"num_ai_files" / Int64ul,
"ai_files" / Array(lambda ctx: ctx.num_ai_files,
Struct(
Bytes(4),
"name" / de_string,
Bytes(4),
)),
"guid" / Bytes(16),
"lobby_name" / de_string,
"modded_dataset" / de_string,
Bytes(19),
If(lambda ctx: ctx._.save_version >= 13.13, Bytes(5)),
If(lambda ctx: ctx._.save_version >= 13.17, Bytes(9)),
'ReservedSpace' / Int16ul,
# Only look for a Value if this isn't the END pseudo-parameter.
'Value' / If(
this.Name != 'END',
Switch(this.Type, {
'INT32': Int32ul,
'REAL64': Float64l
},
default=FixedSizeCString(this.ReservedSpace * 2))))
def is_ParameterList(block):
return block.BlockType.param != 0
ParameterList = RepeatUntil(obj_.Name == 'END', Parameter)
FloatData = Array(this.BlockLength, Float32l)
StringData = String(this.BlockLength * 4)
DirectoryEntry = Struct(
'BlockType' / BlockType, 'BlockLength' / Int32ul, 'DataPtr' / Int32ul,
'Block' / Pointer(
this.DataPtr,
FunctionSwitch(
[(is_ParameterList, ParameterList),
(lambda ctx: ctx.BlockType.extend != 0, OnDemand(StringData)),
(lambda ctx: ctx.BlockType.data not in
(0, 13), OnDemand(FloatData))])))
# The entire file.
OpusFile = Struct(
"fog_of_war"/Flag,
"cheat_notifications"/Flag,
"colored_chat"/Flag,
separator,
"ranked"/Flag,
"allow_specs"/Flag,
"lobby_visibility"/Int32ul,
"hidden_civs"/Flag,
"matchmaking"/Flag,
"spec_delay"/Int32ul,
"scenario_civ"/If(lambda ctx: find_save_version(ctx) >= 13.13, Byte),
"rms_crc"/If(lambda ctx: find_save_version(ctx) >= 13.13, Bytes(4)),
"strings"/Array(23,
Struct(
"string"/de_string,
"nums"/RepeatUntil(lambda x, lst, ctx: lst[-1] not in [3, 21, 23, 42, 44, 45, 46, 47], Int32ul)
)
),
# There's probably a right way to do this, but this is not it.
"num_sn"/Computed(lambda ctx: ctx.strings[22].nums[1]),
"strategic_numbers"/Array(lambda ctx: ctx.num_sn if find_save_version(ctx) >= 25.22 else 59, Int32sl),
"num_ai_files"/Int64ul,
"ai_files"/Array(lambda ctx: ctx.num_ai_files, Struct(
Bytes(4),
"name"/de_string,
Bytes(4),
)),
If(lambda ctx: find_save_version(ctx) >= 25.02, Bytes(8)),
"guid"/Bytes(16),
"lobby_name"/de_string,
If(lambda ctx: find_save_version(ctx) >= 25.22, Bytes(8)),
'stream_start_bytes': 9,
'protected_stream_id': 10,
}),
"data" / Prefixed(
Int16ul,
Switch(this.id, {
'compression_flags': CompressionFlags,
'cipher_id': CipherId,
'transform_rounds': Int32ul,
'protected_stream_id': ProtectedStreamId
},
default=GreedyBytes)),
)
DynamicHeader = DynamicDict(
'id', RepeatUntil(lambda item, a, b: item.id == 'end', DynamicHeaderItem))
# -------------------- Payload Verification --------------------
# encrypted payload is split into multiple data blocks with hashes
PayloadBlock = Struct(
"block_index" / Checksum(Int32ul, lambda this: this._index, this),
"block_hash_offset" / Tell,
Padding(32),
"block_data" / Prefixed(Int32ul, GreedyBytes),
# block_hash has to be at the end with a pointer because it needs to
# come after other fields
"block_hash" / Pointer(
this.block_hash_offset,
IfThenElse(
len_(this.block_data) == 0,
"flags" / EventHeaderFlag,
"event_property" / EventHeaderPropertyFlag,
"thread_id" / Int32ul, "process_id" / Int32ul,
"timestamp" / Int64ul, "provider_id" / Guid,
"event_descriptor" / EventDescriptor,
"processor_time" / Int64ul, "activity_id" / Guid)
EventHeaderExtendedDataItem = Struct(
"reserved1" / Int16ul, "ext_type" / Int16ul, "reserved2" / Int16ul,
"data_size" / Int16ul, "data_item" / Bytes(lambda this: this.data_size))
EventRecord = AlignedStruct(
8, "mark1" / Computed(lambda this: this._io.tell()),
"event_header" / EventHeader, "extended_data" / If(
lambda this: this.event_header.flags.EVENT_HEADER_FLAG_EXTENDED_INFO,
RepeatUntil(lambda el, lst, this: not lst[-1].reserved2 & 0x1,
Aligned(8, EventHeaderExtendedDataItem))),
"mark2" / Computed(lambda this: this._io.tell()),
"user_data" / Bytes(lambda this: this.event_header.marker.version -
(this.mark2 - this.mark1)))
class Event:
"""
This is a python wrapper around construct struct to access interesting fields
"""
def __init__(self, source):
self.source = source
def get_process_id(self):
"""
Return the process id of issuer
from construct import Computed, ExprAdapter, FocusedSeq, Int8ul, Int24ul, Pointer, RepeatUntil, String, Switch, this
#PioString = Struct(
PioString = FocusedSeq(
1,
"padded_length" / RepeatUntil(lambda x, lst, ctx: x != 0, Int8ul),
"data" / Switch(
this.padded_length[-1],
{
# string longer than 127 bytes, prefixed with 3 bytes length
0x40:
FocusedSeq(
1, "actual_length" /
ExprAdapter(Int24ul, lambda o, c: o + 4, lambda o, c: o - 4),
"text" / String(this.actual_length, encoding="ascii")),
# iso-8859 text with \x00 between every character (like utf-16, but its iso-8859)
0x90:
FocusedSeq(
1, "actual_length" /
ExprAdapter(Int24ul, lambda o, c: o + 4, lambda o, c: o - 4),
"text" / ExprAdapter(
String(this.actual_length, encoding="iso-8859-1"),
lambda o, c: "".join(x + "\x00"
for x in o), lambda o, c: o[::2])),
},
default= # just ascii text
FocusedSeq(
1,
"actual_length" / Computed((this.padded_length[-1] - 1) // 2 - 1),
"text" / String(this.actual_length, encoding="ascii"))))
EtlLogFile = GreedyRange(EtlChunk)
# This is a common way to select any type of chunks
# We add name selecting as computed to handle typing during parsing
Chunk = Aligned(
8,
Select(
Struct("type" / Computed("PerfInfoTraceRecord"),
"value" / PerfInfoTraceRecord),
Struct("type" / Computed("EventRecord"), "value" / EventRecord),
Struct("type" / Computed("TraceRecord"), "value" / TraceRecord),
Struct("type" / Computed("SystemTraceRecord"),
"value" / SystemTraceRecord),
Struct("type" / Computed("WinTraceRecord"), "value" / WinTraceRecord)))
ChunkParser = RepeatUntil(
lambda x, lst, ctx: len(x._io.getbuffer()) == x._io.tell(), Chunk)
class IEtlFileObserver(metaclass=ABCMeta):
"""
This is etl file observer
Parse sequentially an etl file and commit event when found a particular event
"""
@abstractmethod
def on_event_record(self, event: Event):
"""
Raise when an event record is parsed
Mostly use by ETW and tracelogging
If you search classic provider you are at the correct place
:param event: the event record
"""
Switch(
this.type, {
0x04: Int32ul,
0x05: Int64ul,
0x08: Flag,
0x0C: Int32sl,
0x0D: Int64sl,
0x42: GreedyBytes,
0x18: GreedyString('utf-8')
})), "next_byte" / Peek(Byte))
# new dynamic dictionary structure added in KDBX4
VariantDictionary = Struct(
"version" / Bytes(2), "dict" / DynamicDict(
'key',
RepeatUntil(lambda item, a, b: item.next_byte == 0x00,
VariantDictionaryItem)),
Padding(1) * "null padding")
# -------------------- Dynamic Header --------------------
# https://github.com/dlech/KeePass2.x/blob/dbb9d60095ef39e6abc95d708fb7d03ce5ae865e/KeePassLib/Serialization/KdbxFile.cs#L234-L246
DynamicHeaderItem = Struct(
"id" / Mapping(
Byte, {
'end': 0,
'comment': 1,
'cipher_id': 2,
'compression_flags': 3,
'master_seed': 4,
'encryption_iv': 7,
def __setup_constructors(self):
'''Set endianness and create transport-specific constructors.'''
# Set endianness of constructors before using them.
self._set_endian('little')
self.__Length = Int32ul
self.__Type = Enum(
Int32ul,
Undefined=0x00000000,
InitCommand=0x00000001,
InitCommandAck=0x00000002,
InitEvent=0x00000003,
InitEventAck=0x00000004,
InitFail=0x00000005,
Command=0x00000006,
Response=0x00000007,
Event=0x00000008,
StartData=0x00000009,
Data=0x0000000A,
Cancel=0x0000000B,
EndData=0x0000000C,
Ping=0x0000000D,
Pong=0x0000000E,
)
self.__Header = Struct(
'Length' / self.__Length,
'Type' / self.__Type,
)
self.__Param = Range(0, 5, self._Parameter)
self.__EventParam = Range(0, 3, self._Parameter)
self.__PacketBase = Struct(
Embedded(self.__Header),
'Payload' / Bytes(
lambda ctx, h=self.__Header: ctx.Length - h.sizeof()),
)
self.__Packet = ExprAdapter(
self.__PacketBase,
encoder=lambda obj, ctx, h=self.__Header: Container(
Length=len(obj.Payload) + h.sizeof(),
**obj
),
decoder=lambda obj, ctx: obj,
)
# Yet another arbitrary string type. Max-length CString utf8-encoded
self.__PTPIPString = ExprAdapter(
RepeatUntil(
lambda obj, ctx, lst:
six.unichr(obj) in '\x00' or len(lst) == 40, Int16ul
),
encoder=lambda obj, ctx:
[] if len(obj) == 0 else[ord(c) for c in six.text_type(obj)]+[0],
decoder=lambda obj, ctx:
u''.join(
[six.unichr(o) for o in obj]
).split('\x00')[0],
)
# PTP/IP packets
# Command
self.__ProtocolVersion = Struct(
'Major' / Int16ul,
'Minor' / Int16ul,
)
self.__InitCommand = Embedded(Struct(
'InitiatorGUID' / Array(16, Int8ul),
'InitiatorFriendlyName' / self.__PTPIPString,
'InitiatorProtocolVersion' / self.__ProtocolVersion,
))
self.__InitCommandACK = Embedded(Struct(
'ConnectionNumber' / Int32ul,
'ResponderGUID' / Array(16, Int8ul),
'ResponderFriendlyName' / self.__PTPIPString,
'ResponderProtocolVersion' / self.__ProtocolVersion,
))
# Event
self.__InitEvent = Embedded(Struct(
'ConnectionNumber' / Int32ul,
))
# Common to Events and Command requests
self.__Reason = Enum(
# TODO: Verify these codes...
Int32ul,
Undefined=0x0000,
RejectedInitiator=0x0001,
Busy=0x0002,
Unspecified=0x0003,
)
self.__InitFail = Embedded(Struct(
'Reason' / self.__Reason,
))
self.__DataphaseInfo = Enum(
Int32ul,
Undefined=0x00000000,
In=0x00000001,
Out=0x00000002,
)
self.__Command = Embedded(Struct(
'DataphaseInfo' / self.__DataphaseInfo,
'OperationCode' / self._OperationCode,
'TransactionID' / self._TransactionID,
'Parameter' / self.__Param,
))
self.__Response = Embedded(Struct(
'ResponseCode' / self._ResponseCode,
'TransactionID' / self._TransactionID,
'Parameter' / self.__Param,
))
self.__Event = Embedded(Struct(
'EventCode' / self._EventCode,
'TransactionID' / self._TransactionID,
'Parameter' / self.__EventParam,
))
self.__StartData = Embedded(Struct(
'TransactionID' / self._TransactionID,
'TotalDataLength' / Int64ul,
))
# TODO: Fix packing and unpacking dataphase data
self.__Data = Embedded(Struct(
'TransactionID' / self._TransactionID,
'Data' / Bytes(
lambda ctx:
ctx._.Length -
self.__Header.sizeof() -
self._TransactionID.sizeof()
),
))
self.__EndData = Embedded(Struct(
'TransactionID' / self._TransactionID,
'Data' / Bytes(
lambda ctx:
ctx._.Length -
self.__Header.sizeof() -
self._TransactionID.sizeof()
),
))
self.__Cancel = Embedded(Struct(
'TransactionID' / self._TransactionID,
))
# Convenience construct for parsing packets
self.__PacketPayload = Debugger(Struct(
'Header' / Embedded(self.__Header),
'Payload' / Embedded(Switch(
lambda ctx: ctx.Type,
{
'InitCommand': self.__InitCommand,
'InitCommandAck': self.__InitCommandACK,
'InitEvent': self.__InitEvent,
'InitFail': self.__InitFail,
'Command': self.__Command,
'Response': self.__Response,
'Event': self.__Event,
'StartData': self.__StartData,
'Data': self.__Data,
'EndData': self.__EndData,
},
default=Pass,
))
))
if _construct_version:
if (_construct_version[0] < 2) or ((_construct_version[0] == 2) and
(_construct_version[1] < 9)):
raise Exception(
'Construct version too low, please install version 2.9+')
from construct import Struct, Const, Int32ub, Int32sb, RepeatUntil, CString, Pointer, Bytes, Padding, BitStruct, Flag, IfThenElse, Seek, Tell
_FArc_format = Struct(
"signature" / Const(b'FArc'),
"header_size" / Int32ub, # doesn't include signature or header_size
"alignment" / Int32sb,
"files" / RepeatUntil(
lambda obj, lst, ctx: ctx._io.tell() - 7 > ctx.header_size,
Struct(
"name" / CString("utf8"), "pointer" / Int32ub, "size" / Int32ub,
"data" / Pointer(lambda this: this.pointer,
Bytes(lambda this: this.size)))),
#Padding(lambda this: this.alignment - (this._io.tell() % this.alignment) if this._io.tell() % this.alignment else 0)
)
_FArC_format = Struct(
"signature" / Const(b'FArC'),
"header_size" / Int32ub, # doesn't include signature or header_size
"alignment" / Int32sb,
"files" / RepeatUntil(
lambda obj, lst, ctx: ctx._io.tell() - 7 > ctx.header_size,
Struct(
"name" / CString("utf8"), "pointer" / Int32ub,
"compressed_size" / Int32ub, "uncompressed_size" / Int32ub,
"data" / Pointer(lambda this: this.pointer,
# unfortunately, the entry_enabled field contains unexistant entries for the last entry
# entry_enabled[:-1] matches revidx[:-1],
# but len(entry_enabled)==16 while len(revidx)<=16
ReverseIndexArray = Struct(
"entry_count" /
Computed(lambda ctx: min([16, ctx._.entry_count - 16 * ctx._._index])),
Seek(-4 - 2 * this.entry_count, 1), # jump back the size of this struct
"entries" / Array(this.entry_count, ReverseIndexedEntry),
"entry_enabled" / ByteSwapped(Bitwise(Array(16, Flag))),
"entry_enabled_override" / ByteSwapped(Bitwise(Array(16, Flag))),
Seek(-36 if this.entry_count == 16 else 0,
1) # jump back once again for the next read or 0 if finished
)
PageFooter = RepeatUntil(lambda x, lst, ctx: len(lst) * 16 > ctx.entry_count,
ReverseIndexArray)
PageHeader = Struct( # 40 bytes
Padding(4), # always 0
"index" / Int32ul, # in units of 4096 bytes
"page_type" / PageTypeEnum,
"next_index" /
Int32ul, # in units of 4096 bytes, finally points to empty page, even outside of file
"u1" / Int32ul, # sequence number (0->1: 8->13, 1->2: 22, 2->3: 27)
Padding(4),
"entry_count_small" / Int8ul,
"u3" / Int8ul, # a bitmask (1st track: 32)
"u4" /
Int8ul, # often 0, sometimes larger, esp. for pages with high entry_count_small (e.g. 12 for 101 entries)
"u5" / Int8ul, # strange pages: 0x44, 0x64; otherwise seen: 0x24, 0x34
"free_size" / Int16ul, # excluding data at page end
