def _gen_fmh3_struct(int_type, pointer_type, codepoint_type, addr_mode='rel'):
return Struct(
"pointer_offset" / (Computed(0) if addr_mode == 'abs' else Tell),
"signature" / Const(b'FMH3'),
Padding(4),
"fonts_count" / Padded(pointer_type.sizeof(), int_type),
"fonts_pointers_offset" / pointer_type,
"fonts" / Pointer(
lambda this: this.fonts_pointers_offset + this.pointer_offset,
RepeatUntil(
lambda obj, lst, ctx: ctx._index >= ctx.fonts_count - 1,
Struct(
"pointer" / pointer_type,
"data" / Pointer(
lambda this: this.pointer + this._.pointer_offset,
Struct(
"id" / int_type,
"advance_width" / Byte,
"line_height" / Byte,
"box_width" / Byte,
"box_height" / Byte,
"layout_param_1" / Byte,
"layout_param_2_numerator" / Byte,
"layout_param_2_denominator" / Byte,
Padding(1),
"other_params?" / int_type,
"tex_size_chars" / int_type,
"chars_count" / int_type,
"chars_pointer" / pointer_type,
"chars" / Pointer(
lambda this: this.chars_pointer + this._._.
pointer_offset,
RepeatUntil(
lambda obj, lst, ctx: ctx._index >= ctx.
chars_count - 1,
Struct(
"codepoint" / codepoint_type,
"halfwidth" / Flag,
Padding(1),
"tex_col" / Byte,
"tex_row" / Byte,
"glyph_x" / Byte,
"glyph_width" / Byte,
))),
)),
))),
)
def perf_event_header():
return Embedded(Struct(None,
Enum(UNInt32("type"),
MMAP = 1,
LOST = 2,
COMM = 3,
EXIT = 4,
THROTTLE = 5,
UNTHROTTLE = 6,
FORK = 7,
READ = 8,
SAMPLE = 9,
MMAP2 = 10,
RECORD_AUX = 11,
ITRACE_START = 12,
LOST_SAMPLES = 13,
SWITCH = 14,
SWITCH_CPU_WIDE = 15,
NAMESPACES = 16,
KSYMBOL = 17,
BPF_EVENT = 18,
CGROUP = 19,
HEADER_ATTR = 64,
HEADER_EVENT_TYPE = 65,
TRACING_DATA = 66,
HEADER_BUILD_ID = 67,
FINISHED_ROUND = 68,
ID_INDEX = 69,
AUXTRACE_INFO = 70,
AUXTRACE = 71,
AUXTRACE_ERROR = 72,
THREAD_MAP = 73,
CPU_MAP = 74,
STAT_CONFIG = 75,
STAT = 76,
STAT_ROUND = 77,
EVENT_UPDATE = 78,
TIME_CONV = 79,
HEADER_FEATURE = 80,
COMPRESSED = 81),
Embedded(BitStruct(None,
Padding(1),
Enum(BitField("cpumode", 7),
UNKNOWN = 0,
KERNEL = 1,
USER = 2,
HYPERVISOR = 3,
GUEST_KERNEL = 4,
GUEST_USER = 5),
Flag("ext_reserved"),
Flag("exact_ip"),
Flag("mmap_data"),
Padding(5))),
UNInt16("size"),
If(has_sample_id_all,
Pointer(lambda ctx: ctx.start + ctx.size - 8,
UNInt64("end_id"))),
Value("attr", lookup_event_attr)))
def mmap():
return Struct("mmap",
SNInt32("pid"),
SNInt32("tid"),
UNInt64("addr"),
UNInt64("len"),
UNInt64("pgoff"),
Anchor("start_of_filename"),
CString("filename"),
Anchor("end_of_filename"),
# hack for now. this shouldn't be needed.
If(lambda ctx: True, # XXX
Embedded(Pointer(lambda ctx:
ctx.size + ctx.start -
sample_id_size(ctx),
sample_id()))))
def _generate_struct_from_headers(
self, header_tables): # type: (Container) -> Struct
"""
Generate ``construct`` Struct for this file
:param header_tables: contains elf_header, program_headers, section_headers
:return: Struct of the whole file
"""
elf_header = header_tables.elf_header
program_headers = header_tables.program_headers
section_headers = header_tables.section_headers
assert program_headers or section_headers
string_table_sh = None
load_segment_subcons = []
note_segment_subcons = []
# Here we point back to make segments know their program headers
for i, ph in enumerate(program_headers):
args = [
'ph' / Pointer(elf_header.e_phoff + i * ProgramHeader.sizeof(),
ProgramHeader),
'data' / Pointer(ph.p_offset, Bytes(ph.p_filesz)),
]
if ph.p_vaddr == 0 and ph.p_type == self.PT_NOTE:
args.append('note_secs' / Pointer(ph.p_offset, NoteSections))
note_segment_subcons.append(Struct(*args))
elif ph.p_vaddr != 0:
load_segment_subcons.append(Struct(*args))
section_subcons = []
for i, sh in enumerate(section_headers):
if sh.sh_type == self.SHT_STRTAB and i == elf_header.e_shstrndx:
string_table_sh = sh
elif sh.sh_addr != 0 and sh.sh_type == self.SHT_PROGBITS:
section_subcons.append(
Struct(
'sh' / Pointer(
elf_header.e_shoff + i * SectionHeader.sizeof(),
SectionHeader),
'data' / Pointer(sh.sh_offset, Bytes(sh.sh_size)),
))
args = [
'elf_header' / ElfHeader,
'load_segments' / Sequence(*load_segment_subcons),
'note_segments' / Sequence(*note_segment_subcons),
'sections' / Sequence(*section_subcons),
]
if string_table_sh is not None:
args.append('string_table' / Pointer(
string_table_sh.sh_offset, Bytes(string_table_sh.sh_size)))
return Struct(*args)
def perf_event_header():
return Embedded(
Struct(
None,
Enum(UNInt32("type"),
MMAP=1,
LOST=2,
COMM=3,
EXIT=4,
THROTTLE=5,
UNTHROTTLE=6,
FORK=7,
READ=8,
SAMPLE=9,
MMAP2=10,
TRACING_DATA=66,
FINISHED_ROUND=68,
ID_INDEX=69,
AUXTRACE_INFO=70,
AUXTRACE=71,
AUXTRACE_ERROR=72),
Embedded(
BitStruct(
None, Padding(1),
Enum(BitField("cpumode", 7),
UNKNOWN=0,
KERNEL=1,
USER=2,
HYPERVISOR=3,
GUEST_KERNEL=4,
GUEST_USER=5), Flag("ext_reserved"), Flag("exact_ip"),
Flag("mmap_data"), Padding(5))), UNInt16("size"),
If(
has_sample_id_all,
Pointer(lambda ctx: ctx.start + ctx.size - 8,
UNInt64("end_id"))), Value("attr", lookup_event_attr)))
UNInt64("size"),
UNInt64("attr_size"),
perf_file_section("attrs", perf_file_attr),
perf_file_section("data", perf_data),
perf_file_section("event_types", perf_event_types),
# little endian
Embedded(
BitStruct(None, Flag("nrcpus"), Flag("arch"), Flag("version"),
Flag("osrelease"), Flag("hostname"), Flag("build_id"),
Flag("tracing_data"), Flag("reserved"), Flag("branch_stack"),
Flag("numa_topology"), Flag("cpu_topology"),
Flag("event_desc"), Flag("cmdline"), Flag("total_mem"),
Flag("cpuid"), Flag("cpudesc"),
Padding(6), Flag("group_desc"), Flag("pmu_mappings"),
Padding(256 - 3 * 8))),
Pointer(lambda ctx: ctx.data.offset + ctx.data.size, perf_features()),
Padding(3 * 8))
def get_events(h):
return h.data.perf_data
if __name__ == '__main__':
import argparse
args = argparse.ArgumentParser()
args.add_argument('file',
help='perf.data to read',
default='perf.data',
nargs='?')
"u2" / Int16ul, # always 8191?
),
Array(1004, Int32ul),
Padding(20)
)
ReverseIndexedEntry = FocusedSeq(1,
"entry_offset" / Int16ul,
"entry" / Pointer(this._._.entries_start+this.entry_offset,
Switch(lambda ctx: "strange" if ctx._._.is_strange_page else ctx._._.page_type, {
"block_tracks": Track,
"block_artists": Artist,
"block_albums": Album,
"block_playlists": Playlist,
"block_playlist_map": PlaylistMap,
"block_artwork": Artwork,
"block_colors": Color,
"block_genres": Genre,
"block_keys": Key,
"block_labels": Label,
#"strange": StrangePage,
}, default = Computed("page type not implemented")),
)
)
# unfortunately, the entry_enabled field contains even unexistant entries:
# entry_enabled[:-1] matches revidx[:-1],
# but len(entry_enabled)==16 while len(revidx)<=16
FirstReverseIndexArray = Struct(
"entries" / Array(this._._.entry_count%16, ReverseIndexedEntry),
"symbolTableIndex" / Int32ul,
"addressExtension" / Int16ul,
"relocationType" / Int16ul,
#relocationType
)
fileHeader = Struct(
"versionID" / Int16ul,
"numSectionHeaders" / Int16ul,
UTCTimeStamp("timestamp"),
"filePoimter" / Int32ul,
"numSymbols" / Int32ul,
"symbols" / Pointer(this.filePoimter,
Bytes(this.numSymbols * symbol.sizeof()),
),
"stringTable" / Pointer( (this.filePoimter + (this.numSymbols * symbol.sizeof()) + 0),
ListToBytesAdapter(GreedyRange(Byte))
),
"sizeOptHeader" / Int16ul,
"flags" / FlagsEnum(Int16ul,
F_RELFLG = 0x0001,
F_EXEC = 0x0002,
F_LNNO = 0x0004,
F_LSYMS = 0x0008,
'next' / Float32l,
'nois' / Float32l, 'npts' / Int32sl, 'scan' / Int32sl, 'wlevel' / Float32l,
Padding(4), 'float_y' / Computed(this.exp == 128), 'num_pts' / Computed(
lambda ctx: ctx.npts if ctx.npts > 0 else ctx._.Header.header.npts),
'exponent' / Computed(lambda ctx: ctx.exp
if 0 < ctx.exp < 128 else ctx._.Header.header.exp),
'raw_x' / If(this._.Header.TFlags.use_subfile_xs,
LazyField(Array(this.num_pts, Int32sl))),
'raw_y' / IfThenElse(this.float_y, LazyField(Array(
this.num_pts, Float32l)), LazyField(Array(this.num_pts, Int32sl))))
LogData = Struct(
'log_start' / Tell, 'sizd' / Int32sl, 'sizm' / Int32sl,
'text_offset' / Int32sl, 'bins' / Int32sl, 'dsks' / Int32sl, Padding(44),
'content' /
Pointer(this.log_start + this.text_offset, LazyField(Bytes(this.sizd))))
# The entire file.
SPCFile = Struct(
'Header' / Header,
'xvals' / If(this.Header.TFlags.has_xs,
LazyField(Array(this.Header.header.npts, Float32l))),
'Subfile' / Array(this.Header.header.nsub, Subfile),
'LogData' / If(this.Header.header.log_offset != 0,
Pointer(this.Header.header.log_offset, LogData)))
def prettyprint(data):
np.set_printoptions(precision=4, suppress=True)
version = data.Header.version
h = data.Header.header
def __init__(self, endianity='<'):
if endianity == '<':
Int8u = Int8ul
Int16u = Int16ul
Int32u = Int32ul
else:
Int8u = Int8ub
Int16u = Int16ub
Int32u = Int32ub
self.Elf_Class = Enum(
Hex(Int8u),
ELFCLASSNONE=elf_consts.ELFCLASSNONE,
ELFCLASS32=elf_consts.ELFCLASS32,
ELFCLASS64=elf_consts.ELFCLASS64,
ELFCLASSNUM=elf_consts.ELFCLASSNUM,
)
self.Elf_Data = Enum(
Hex(Int8u),
ELFDATANONE=elf_consts.ELFDATANONE,
ELFDATA2LSB=elf_consts.ELFDATA2LSB,
ELFDATA2MSB=elf_consts.ELFDATA2MSB,
)
self.Elf_Machine = Enum(
Int16u,
EM_NONE=elf_consts.EM_NONE,
EM_M32=elf_consts.EM_M32,
EM_SPARC=elf_consts.EM_SPARC,
EM_386=elf_consts.EM_386,
EM_68K=elf_consts.EM_68K,
EM_88K=elf_consts.EM_88K,
EM_IAMCU=elf_consts.EM_IAMCU,
EM_860=elf_consts.EM_860,
EM_MIPS=elf_consts.EM_MIPS,
EM_S370=elf_consts.EM_S370,
EM_MIPS_RS3_LE=elf_consts.EM_MIPS_RS3_LE,
EM_PARISC=elf_consts.EM_PARISC,
EM_VPP500=elf_consts.EM_VPP500,
EM_SPARC32PLUS=elf_consts.EM_SPARC32PLUS,
EM_960=elf_consts.EM_960,
EM_PPC=elf_consts.EM_PPC,
EM_PPC64=elf_consts.EM_PPC64,
EM_S390=elf_consts.EM_S390,
EM_SPU=elf_consts.EM_SPU,
EM_V800=elf_consts.EM_V800,
EM_FR20=elf_consts.EM_FR20,
EM_RH32=elf_consts.EM_RH32,
EM_RCE=elf_consts.EM_RCE,
EM_ARM=elf_consts.EM_ARM,
EM_ALPHA=elf_consts.EM_ALPHA,
EM_SH=elf_consts.EM_SH,
EM_SPARCV9=elf_consts.EM_SPARCV9,
EM_TRICORE=elf_consts.EM_TRICORE,
EM_ARC=elf_consts.EM_ARC,
EM_H8_300=elf_consts.EM_H8_300,
EM_H8_300H=elf_consts.EM_H8_300H,
EM_H8S=elf_consts.EM_H8S,
EM_H8_500=elf_consts.EM_H8_500,
EM_IA_64=elf_consts.EM_IA_64,
EM_MIPS_X=elf_consts.EM_MIPS_X,
EM_COLDFIRE=elf_consts.EM_COLDFIRE,
EM_68HC12=elf_consts.EM_68HC12,
EM_MMA=elf_consts.EM_MMA,
EM_PCP=elf_consts.EM_PCP,
EM_NCPU=elf_consts.EM_NCPU,
EM_NDR1=elf_consts.EM_NDR1,
EM_STARCORE=elf_consts.EM_STARCORE,
EM_ME16=elf_consts.EM_ME16,
EM_ST100=elf_consts.EM_ST100,
EM_TINYJ=elf_consts.EM_TINYJ,
EM_X86_64=elf_consts.EM_X86_64,
EM_PDSP=elf_consts.EM_PDSP,
EM_PDP10=elf_consts.EM_PDP10,
EM_PDP11=elf_consts.EM_PDP11,
EM_FX66=elf_consts.EM_FX66,
EM_ST9PLUS=elf_consts.EM_ST9PLUS,
EM_ST7=elf_consts.EM_ST7,
EM_68HC16=elf_consts.EM_68HC16,
EM_68HC11=elf_consts.EM_68HC11,
EM_68HC08=elf_consts.EM_68HC08,
EM_68HC05=elf_consts.EM_68HC05,
EM_SVX=elf_consts.EM_SVX,
EM_ST19=elf_consts.EM_ST19,
EM_VAX=elf_consts.EM_VAX,
EM_CRIS=elf_consts.EM_CRIS,
EM_JAVELIN=elf_consts.EM_JAVELIN,
EM_FIREPATH=elf_consts.EM_FIREPATH,
EM_ZSP=elf_consts.EM_ZSP,
EM_MMIX=elf_consts.EM_MMIX,
EM_HUANY=elf_consts.EM_HUANY,
EM_PRISM=elf_consts.EM_PRISM,
EM_AVR=elf_consts.EM_AVR,
EM_FR30=elf_consts.EM_FR30,
EM_D10V=elf_consts.EM_D10V,
EM_D30V=elf_consts.EM_D30V,
EM_V850=elf_consts.EM_V850,
EM_M32R=elf_consts.EM_M32R,
EM_MN10300=elf_consts.EM_MN10300,
EM_MN10200=elf_consts.EM_MN10200,
EM_PJ=elf_consts.EM_PJ,
EM_OPENRISC=elf_consts.EM_OPENRISC,
EM_ARC_COMPACT=elf_consts.EM_ARC_COMPACT,
EM_XTENSA=elf_consts.EM_XTENSA,
EM_VIDEOCORE=elf_consts.EM_VIDEOCORE,
EM_TMM_GPP=elf_consts.EM_TMM_GPP,
EM_NS32K=elf_consts.EM_NS32K,
EM_TPC=elf_consts.EM_TPC,
EM_SNP1K=elf_consts.EM_SNP1K,
EM_ST200=elf_consts.EM_ST200,
EM_IP2K=elf_consts.EM_IP2K,
EM_MAX=elf_consts.EM_MAX,
EM_CR=elf_consts.EM_CR,
EM_F2MC16=elf_consts.EM_F2MC16,
EM_MSP430=elf_consts.EM_MSP430,
EM_BLACKFIN=elf_consts.EM_BLACKFIN,
EM_SE_C33=elf_consts.EM_SE_C33,
EM_SEP=elf_consts.EM_SEP,
EM_ARCA=elf_consts.EM_ARCA,
EM_UNICORE=elf_consts.EM_UNICORE,
EM_EXCESS=elf_consts.EM_EXCESS,
EM_DXP=elf_consts.EM_DXP,
EM_ALTERA_NIOS2=elf_consts.EM_ALTERA_NIOS2,
EM_CRX=elf_consts.EM_CRX,
EM_XGATE=elf_consts.EM_XGATE,
EM_C166=elf_consts.EM_C166,
EM_M16C=elf_consts.EM_M16C,
EM_DSPIC30F=elf_consts.EM_DSPIC30F,
EM_CE=elf_consts.EM_CE,
EM_M32C=elf_consts.EM_M32C,
EM_TSK3000=elf_consts.EM_TSK3000,
EM_RS08=elf_consts.EM_RS08,
EM_SHARC=elf_consts.EM_SHARC,
EM_ECOG2=elf_consts.EM_ECOG2,
EM_SCORE7=elf_consts.EM_SCORE7,
EM_DSP24=elf_consts.EM_DSP24,
EM_VIDEOCORE3=elf_consts.EM_VIDEOCORE3,
EM_LATTICEMICO32=elf_consts.EM_LATTICEMICO32,
EM_SE_C17=elf_consts.EM_SE_C17,
EM_TI_C6000=elf_consts.EM_TI_C6000,
EM_TI_C2000=elf_consts.EM_TI_C2000,
EM_TI_C5500=elf_consts.EM_TI_C5500,
EM_TI_ARP32=elf_consts.EM_TI_ARP32,
EM_TI_PRU=elf_consts.EM_TI_PRU,
EM_MMDSP_PLUS=elf_consts.EM_MMDSP_PLUS,
EM_CYPRESS_M8C=elf_consts.EM_CYPRESS_M8C,
EM_R32C=elf_consts.EM_R32C,
EM_TRIMEDIA=elf_consts.EM_TRIMEDIA,
EM_QDSP6=elf_consts.EM_QDSP6,
EM_8051=elf_consts.EM_8051,
EM_STXP7X=elf_consts.EM_STXP7X,
EM_NDS32=elf_consts.EM_NDS32,
EM_ECOG1=elf_consts.EM_ECOG1,
EM_ECOG1X=elf_consts.EM_ECOG1X,
EM_MAXQ30=elf_consts.EM_MAXQ30,
EM_XIMO16=elf_consts.EM_XIMO16,
EM_MANIK=elf_consts.EM_MANIK,
EM_CRAYNV2=elf_consts.EM_CRAYNV2,
EM_RX=elf_consts.EM_RX,
EM_METAG=elf_consts.EM_METAG,
EM_MCST_ELBRUS=elf_consts.EM_MCST_ELBRUS,
EM_ECOG16=elf_consts.EM_ECOG16,
EM_CR16=elf_consts.EM_CR16,
EM_ETPU=elf_consts.EM_ETPU,
EM_SLE9X=elf_consts.EM_SLE9X,
EM_L10M=elf_consts.EM_L10M,
EM_K10M=elf_consts.EM_K10M,
EM_AVR32=elf_consts.EM_AVR32,
EM_STM8=elf_consts.EM_STM8,
EM_TILE64=elf_consts.EM_TILE64,
EM_TILEPRO=elf_consts.EM_TILEPRO,
EM_MICROBLAZE=elf_consts.EM_MICROBLAZE,
EM_CUDA=elf_consts.EM_CUDA,
EM_TILEGX=elf_consts.EM_TILEGX,
EM_CLOUDSHIELD=elf_consts.EM_CLOUDSHIELD,
EM_COREA_1ST=elf_consts.EM_COREA_1ST,
EM_COREA_2ND=elf_consts.EM_COREA_2ND,
EM_ARC_COMPACT2=elf_consts.EM_ARC_COMPACT2,
EM_OPEN8=elf_consts.EM_OPEN8,
EM_RL78=elf_consts.EM_RL78,
EM_VIDEOCORE5=elf_consts.EM_VIDEOCORE5,
EM_78KOR=elf_consts.EM_78KOR,
EM_56800EX=elf_consts.EM_56800EX,
EM_BA1=elf_consts.EM_BA1,
EM_BA2=elf_consts.EM_BA2,
EM_XCORE=elf_consts.EM_XCORE,
EM_MCHP_PIC=elf_consts.EM_MCHP_PIC,
EM_INTEL205=elf_consts.EM_INTEL205,
EM_INTEL206=elf_consts.EM_INTEL206,
EM_INTEL207=elf_consts.EM_INTEL207,
EM_INTEL208=elf_consts.EM_INTEL208,
EM_INTEL209=elf_consts.EM_INTEL209,
EM_KM32=elf_consts.EM_KM32,
EM_KMX32=elf_consts.EM_KMX32,
EM_KMX16=elf_consts.EM_KMX16,
EM_KMX8=elf_consts.EM_KMX8,
EM_KVARC=elf_consts.EM_KVARC,
EM_CDP=elf_consts.EM_CDP,
EM_COGE=elf_consts.EM_COGE,
EM_COOL=elf_consts.EM_COOL,
EM_NORC=elf_consts.EM_NORC,
EM_CSR_KALIMBA=elf_consts.EM_CSR_KALIMBA,
EM_Z80=elf_consts.EM_Z80,
EM_VISIUM=elf_consts.EM_VISIUM,
EM_FT32=elf_consts.EM_FT32,
EM_MOXIE=elf_consts.EM_MOXIE,
EM_AMDGPU=elf_consts.EM_AMDGPU,
EM_RISCV=elf_consts.EM_RISCV,
)
self.Elf_SegmentType = Enum(
Hex(Int32u),
PT_NULL=elf_consts.PT_NULL,
PT_LOAD=elf_consts.PT_LOAD,
PT_DYNAMIC=elf_consts.PT_DYNAMIC,
PT_INTER=elf_consts.PT_INTER,
PT_NOTE=elf_consts.PT_NOTE,
PT_SHLIB=elf_consts.PT_SHLIB,
PT_PHDR=elf_consts.PT_PHDR,
PT_TLS=elf_consts.PT_TLS,
PT_LOOS=elf_consts.PT_LOOS,
PT_HIOS=elf_consts.PT_HIOS,
PT_LOPROC=elf_consts.PT_LOPROC,
PT_HIPROC=elf_consts.PT_HIPROC,
PT_GNU_EH_FRAME=elf_consts.PT_GNU_EH_FRAME,
)
self.Elf_Type = Enum(
Hex(Int16u),
ET_NONE=elf_consts.ET_NONE,
ET_REL=elf_consts.ET_REL,
ET_EXEC=elf_consts.ET_EXEC,
ET_DYN=elf_consts.ET_DYN,
ET_CORE=elf_consts.ET_CORE,
ET_LOPROC=elf_consts.ET_LOPROC,
ET_HIPROC=elf_consts.ET_HIPROC,
)
self.Elf_Version = Enum(
Hex(Int8u),
EV_NONE=elf_consts.EV_NONE,
EV_CURRENT=elf_consts.EV_CURRENT,
EV_NUM=elf_consts.EV_NUM,
)
self.Elf_Version2 = Enum(
Hex(Int32u),
EV_NONE=elf_consts.EV_NONE,
EV_CURRENT=elf_consts.EV_CURRENT,
EV_NUM=elf_consts.EV_NUM,
)
self.Elf_OsAbi = Enum(
Hex(Int8u),
ELFOSABI_NONE=elf_consts.ELFOSABI_NONE,
ELFOSABI_LINUX=elf_consts.ELFOSABI_LINUX,
)
# special section indexes
self.Elf_SectionIndex = Enum(
Hex(Int32u),
SHN_UNDEF=elf_consts.SHN_UNDEF,
SHN_LORESERVE=elf_consts.SHN_LORESERVE,
SHN_LOPROC=elf_consts.SHN_LOPROC,
SHN_HIPROC=elf_consts.SHN_HIPROC,
SHN_LIVEPATCH=elf_consts.SHN_LIVEPATCH,
SHN_ABS=elf_consts.SHN_ABS,
SHN_COMMON=elf_consts.SHN_COMMON,
SHN_HIRESERVE=elf_consts.SHN_HIRESERVE,
)
self.Elf_SectionType = Enum(
Hex(Int32u),
SHT_NULL=elf_consts.SHT_NULL,
SHT_PROGBITS=elf_consts.SHT_PROGBITS,
SHT_SYMTAB=elf_consts.SHT_SYMTAB,
SHT_STRTAB=elf_consts.SHT_STRTAB,
SHT_RELA=elf_consts.SHT_RELA,
SHT_HASH=elf_consts.SHT_HASH,
SHT_DYNAMIC=elf_consts.SHT_DYNAMIC,
SHT_NOTE=elf_consts.SHT_NOTE,
SHT_NOBITS=elf_consts.SHT_NOBITS,
SHT_REL=elf_consts.SHT_REL,
SHT_SHLIB=elf_consts.SHT_SHLIB,
SHT_DYNSYM=elf_consts.SHT_DYNSYM,
SHT_NUM=elf_consts.SHT_NUM,
SHT_LOPROC=elf_consts.SHT_LOPROC,
SHT_HIPROC=elf_consts.SHT_HIPROC,
SHT_LOUSER=elf_consts.SHT_LOUSER,
SHT_HIUSER=elf_consts.SHT_HIUSER,
)
self.Elf32_Phdr = Struct(
'p_type' / self.Elf_SegmentType, 'p_offset' / Hex(Int32u),
'p_vaddr' / Hex(Int32u), 'p_paddr' / Hex(Int32u),
'p_filesz' / Hex(Int32u), 'p_memsz' / Hex(Int32u),
'p_flags' / Hex(Int32u), 'p_align' / Hex(Int32u),
'data' / If(this.p_type == self.Elf_SegmentType.PT_LOAD,
Pointer(this.p_offset, Bytes(this.p_filesz))))
self.Elf32_Ehdr = Struct(
'e_ident' / Struct(
'magic' / Const(elf_consts.ELFMAG),
'class' / self.Elf_Class,
'data' / self.Elf_Data,
'version' /
Default(self.Elf_Version, self.Elf_Version.EV_CURRENT),
'osabi' / self.Elf_OsAbi,
'pad' / Padding(8),
),
'e_type' / Default(self.Elf_Type, self.Elf_Type.ET_EXEC),
'e_machine' / Hex(self.Elf_Machine),
'e_version' /
Default(self.Elf_Version2, self.Elf_Version2.EV_CURRENT),
'e_entry' / Hex(Int32u),
'e_phoff' / Hex(Int32u),
'e_shoff' / Hex(Int32u),
'e_flags' / Default(Hex(Int32u), 0),
'e_ehsize' / Hex(Int16u),
'e_phentsize' / Hex(Int16u),
'e_phnum' / Hex(Int16u),
'e_shentsize' / Hex(Int16u),
'e_shnum' / Hex(Int16u),
'e_shstrndx' / Hex(Int16u),
)
self.Elf32_Shdr = Struct(
'sh_name' / self.Elf_SectionIndex,
'sh_type' / self.Elf_SectionType, 'sh_flags' / Hex(Int32u),
'sh_addr' / Hex(Int32u), 'sh_offset' / Hex(Int32u),
'sh_size' / Hex(Int32u), 'sh_link' / Hex(Int32u),
'sh_info' / Hex(Int32u), 'sh_addralign' / Hex(Int32u),
'sh_entsize' / Hex(Int32u),
'data' / If(this.sh_type != self.Elf_SectionType.SHT_NOBITS,
Pointer(this.sh_offset, Bytes(this.sh_size))))
self.Elf32 = Struct(
'header' / self.Elf32_Ehdr,
'segments' / Pointer(this.header.e_phoff,
Array(this.header.e_phnum, self.Elf32_Phdr)),
'sections' / Pointer(this.header.e_shoff,
Array(this.header.e_shnum, self.Elf32_Shdr)),
)
def __init__(self, offsetfunc, subcon, offset=None, whence=os.SEEK_CUR):
Pointer.__init__(self, offsetfunc, subcon)
self._ptr_offset = offset
self._ptr_whence = whence
'raw_x'/If(this._.Header.TFlags.use_subfile_xs,
OnDemand(Array(this.num_pts, Int32sl))),
'raw_y'/IfThenElse(this.float_y,
OnDemand(Array(this.num_pts, Float32l)),
OnDemand(Array(this.num_pts, Int32sl)))
)
LogData = Struct(
'log_start'/Tell,
'sizd'/Int32sl,
'sizm'/Int32sl,
'text_offset'/Int32sl,
'bins'/Int32sl,
'dsks'/Int32sl,
Padding(44),
'content'/Pointer(this.log_start + this.text_offset,
OnDemand(String(this.sizd)))
)
# The entire file.
SPCFile = Struct(
'Header'/Header,
'xvals'/If(this.Header.TFlags.has_xs,
OnDemand(Array(this.Header.npts, Float32l))),
'Subfile'/Array(this.Header.nsub, Subfile),
'LogData'/If(this.Header.log_offset != 0,
Pointer(this.Header.log_offset, LogData))
)
def prettyprint(data):
np.set_printoptions(precision=4, suppress=True)
CheckCustom(lambda this:
0 != this.events_offset and 0 != this.events_offsets_array_offset and 0 != this.process_table_offset and
0 != this.strings_table_offset and 0 != this.hosts_and_ports_tables_offset and 0 != this.unknown_table_offset,
RuntimeError, "Procmon was probably not closed properly.")
)
StringsTable = """
The table of all the strings needed for the logs.
""" * Struct(
"table_offset" / Tell,
"strings" / PrefixedArray(
Int32ul,
Struct(
"offset" / Int32ul,
"string" / Pointer(
lambda this: this._._.table_offset + this.offset,
FixedNullTerminatedUTF16String
)
)
)
)
RawProcessStruct = """
Struct that describes a process.
""" * Struct(
"strings_table" / Computed(lambda ctx: ctx._.strings_table), # keep the reference to the strings table
"process_index" / Int32ul,
"process_id" / Int32ul,
"parent_process_id" / Int32ul,
"reserved1" / Int32ul * "!!Unknown field!!",
"authentication_id" / Int32ul,
"reserved2" / Int32ul * "!!Unknown field!!",
ProgramHeader = Struct(
'p_type' / Int32ul,
'p_offset' / Int32ul,
'p_vaddr' / Int32ul,
'p_paddr' / Int32ul,
'p_filesz' / Int32ul,
'p_memsz' / Int32ul,
'p_flags' / Int32ul,
'p_align' / Int32ul,
)
ElfHeaderTables = Struct(
'elf_header' / ElfHeader,
'program_headers' /
Pointer(this.elf_header.e_phoff, ProgramHeader[this.elf_header.e_phnum]),
'section_headers' /
Pointer(this.elf_header.e_shoff, SectionHeader[this.elf_header.e_shnum]),
)
NoteSection = AlignedStruct(
4,
'namesz' / Int32ul,
'descsz' / Int32ul,
'type' / Int32ul,
'name' / Bytes(this.namesz),
'desc' / Bytes(this.descsz),
)
NoteSections = GreedyRange(NoteSection)
StatusRequestStruct = "StatusTransaction" / Struct(
"data" / Array(15, OneOf(Int32ub, [0])))
StatusResponseStruct = "StatusTransaction" / Struct(
"data" / Array(15, Int32ub))
"""
Struct detailing the Status Action logic
"""
ResendStruct = "ResendTransaction" / Struct()
"""
Struct detailing the Resend Action logic
"""
IPBusWords = "IPBusWords" / Struct("data" / GreedyRange(Int32ub))
IPBusConstruct = "IPBusPacket" / Struct(
"pointer" / Pointer(3, Int8ub),
"bigendian" / Computed(this.pointer == 0xf0),
"header" / IfThenElse(
this.bigendian, PacketHeaderStruct,
ByteSwapped(PacketHeaderStruct)), # defined as 'header' in context
"transactions" / If(lambda ctx: ctx.header.type_id == "CONTROL",
GreedyRange(ControlStruct)),
"status" /
If(lambda ctx: ctx.header.type_id == "STATUS", StatusRequestStruct),
"resend" / If(lambda ctx: ctx.header.type_id == "RESEND", ResendStruct),
Terminated)
"""
Top-level IPBus Construct which is a packet parser/builder
"""
code = OnDemand(Bytes("code", lambda ctx: ctx.len_code))
sunp_file = Struct(
"sunp_file",
Magic(b"SUNP BURN FILE"),
Padding(2, strict=True), # guessing...
ULInt32("file_size"),
ULInt32("off_burnhdr1"),
ULInt32("off_burnhdr2"),
ULInt32("off_code"),
ULInt32("unk_zero"),
ULInt32("off_burnhdr3"),
Value("len_code", lambda ctx: ctx.off_burnhdr3 - ctx.off_code),
#Padding(472),
Pointer(lambda ctx: ctx.off_burnhdr1, burnhdr1),
Pointer(lambda ctx: ctx.off_burnhdr2, burnhdr2),
Pointer(lambda ctx: ctx.off_code, code),
Pointer(lambda ctx: ctx.off_burnhdr3, burnhdr3),
)
if __name__ == '__main__':
import pprint
import argparse
parser = argparse.ArgumentParser(
description='Parse a SUNP BURN FILE (SPHOST.BRN)')
parser.add_argument("filename",
type=str,
help="input file, e.g. SPHOST.BRN")
parser.add_argument('--debug',
ConstructError,
Enum,
GreedyRange,
If,
Int16ul,
Int32ul,
PaddedString,
Pointer,
Seek,
Struct,
Tell,
this,
)
from winsign.asn1 import der_encode, make_authenticode_signeddata
dos_stub = Struct("magic" / Const(b"MZ"), "pe_offset" / Pointer(0x3C, Int16ul))
coff_header = Struct(
"signature" / Const(b"PE\x00\x00"),
"machine" / Int16ul,
"nsections" / Int16ul,
"timestamp" / Int32ul,
"symboltable_offset" / Int32ul,
"nsymbols" / Int32ul,
"optionalheader_size" / Int16ul,
"characteristics" / Int16ul,
)
pe_header = Struct(
"offset" / Tell,
def perf_file_section(name, target):
return Struct(name, UNInt64("offset"), UNInt64("size"),
Pointer(lambda ctx: ctx.offset, target))
"action" / RuleActionType,
"value_offset" / Tell, # NOT IN THE REAL FORMAT - USED FOR BUILDING ONLY
"value_length" / Default(Int32ul, 0),
"before_value_offset" /
Tell, # NOT IN THE REAL FORMAT - USED FOR BUILDING ONLY
"value" / FixedUTF16CString(lambda this: this.value_length, "value"),
"after_value_offset" /
Tell, # NOT IN THE REAL FORMAT - USED FOR BUILDING ONLY
"int_value" / Rebuild(
Int32ul, lambda this: get_rule_integer_value(this.column, this.value)),
"reserved2" / Default(Bytes(1), 0) * "!!Unknown field!!",
# To calculate value string in build time
"value_length" / Pointer(
lambda this: this.value_offset,
Default(
Int32ul,
lambda this: this.after_value_offset - this.before_value_offset)))
class RuleStructAdapter(Adapter):
def _decode(self, obj, context, path):
return Rule(column=obj["column"],
relation=obj["relation"],
value=obj["value"],
action=obj["action"])
def _encode(self, obj, context, path):
return {
"column": obj.column,
"relation": obj.relation,
"dir" / Struct(
"dir_size" / Int32ul,
"file_size_without_directory" / Int32ul,
"sec_cnt" / Int32ul,
"magic" / Const(b'\xc8\x00\x00\x00'),
Padding(16),
"uncompr_size" / Array(100, Int32ul),
"compr_size" / Array(100, Int32ul),
"id" / Array(100, Int32ul),
"is_compr" / Array(100, Int32ul),
"offset" / Array(100, Int32ul),
Padding(4),
),
# 2001
"x_view" / Pointer(this.dir.dir_size + this.dir.offset[0], Int32ul),
# 2002
"y_view" / Pointer(this.dir.dir_size + this.dir.offset[1], Int32ul),
# 2003
"random_state" /
Pointer(this.dir.dir_size + this.dir.offset[2],
FixedSized(this.dir.uncompr_size[2], GreedyBytes)),
# 2004
"bmap_size" / Pointer(
this.dir.dir_size + this.dir.offset[3],
Struct(
"uncompr_size" / Int32ul, "uncompr_size" /
Computed(this._.dir.uncompr_size[3]), "compr_size" / Int32ul,
def IndexedPioString(index):
return Pointer(this.entry_start + this.str_idx[index], PioString)
# -------------------- Payload Verification --------------------
# encrypted payload is split into multiple data blocks with hashes
PayloadBlock = Struct(
"block_index" / Checksum(Int32ul, lambda this: this._index, this),
"block_hash_offset" / Tell,
Padding(32),
"block_data" / Prefixed(Int32ul, GreedyBytes),
# block_hash has to be at the end with a pointer because it needs to
# come after other fields
"block_hash" / Pointer(
this.block_hash_offset,
IfThenElse(
len_(this.block_data) == 0,
Checksum(Bytes(32), lambda _: b'\x00' * 32, this),
Checksum(
Bytes(32),
lambda block_data: hashlib.sha256(block_data).digest(),
this.block_data,
# exception=PayloadChecksumError
))),
)
PayloadBlocks = RepeatUntil(
lambda item, a, b: len(item.block_data) ==
0, # and item.block_hash == b'\x00' * 32,
PayloadBlock)
# -------------------- Payload Decryption/Decompression --------------------
# Compressed Bytes <---> Stream Start Bytes, Decompressed XML
if (_construct_version[0] < 2) or ((_construct_version[0] == 2) and
(_construct_version[1] < 9)):
raise Exception(
'Construct version too low, please install version 2.9+')
from construct import Struct, Const, Int32ub, Int32sb, RepeatUntil, CString, Pointer, Bytes, Padding, BitStruct, Flag, IfThenElse, Seek, Tell
_FArc_format = Struct(
"signature" / Const(b'FArc'),
"header_size" / Int32ub, # doesn't include signature or header_size
"alignment" / Int32sb,
"files" / RepeatUntil(
lambda obj, lst, ctx: ctx._io.tell() - 7 > ctx.header_size,
Struct(
"name" / CString("utf8"), "pointer" / Int32ub, "size" / Int32ub,
"data" / Pointer(lambda this: this.pointer,
Bytes(lambda this: this.size)))),
#Padding(lambda this: this.alignment - (this._io.tell() % this.alignment) if this._io.tell() % this.alignment else 0)
)
_FArC_format = Struct(
"signature" / Const(b'FArC'),
"header_size" / Int32ub, # doesn't include signature or header_size
"alignment" / Int32sb,
"files" / RepeatUntil(
lambda obj, lst, ctx: ctx._io.tell() - 7 > ctx.header_size,
Struct(
"name" / CString("utf8"), "pointer" / Int32ub,
"compressed_size" / Int32ub, "uncompressed_size" / Int32ub,
"data" / Pointer(lambda this: this.pointer,
Bytes(lambda this: this.compressed_size)))),
#Padding(lambda this: this.alignment - (this._io.tell() % this.alignment) if this._io.tell() % this.alignment else 0)
return ctx.data_offset > 8 and ctx.data_offset > (ctx.signatures.offset_end + 8)
def _data_offset(ctx):
if not ctx.index.entries:
return ctx.header.index_offset
else:
# data offset can never be less than 8, which is the size of the MAR header
return max(ctx.index.entries[0].offset, 8)
mar = "mar" / Struct(
"header" / mar_header,
"index" / Pointer(this.header.index_offset, index_header),
# Helper attributes to assist with navigating the file
"data_offset" / Computed(_data_offset),
"data_length" / Computed(this.header.index_offset - this.data_offset),
# Pointer above restores our seek position to where it was before, i.e.
# after the mar header. In modern MAR files, the mar header is followed by
# the signatures and additional data sections. In older MAR files file data
# follows immediately after.
# These sections will not be present for older MAR files
# that don't have signature / extra sections
# Only add them if the earliest entry offset is greater than 8
"signatures" / If(_has_sigs, sigs_header),
"additional" / If(_has_extras, extras_header),
)
'next' / Float32l,
'nois' / Float32l, 'npts' / Int32sl, 'scan' / Int32sl, 'wlevel' / Float32l,
Padding(4), 'float_y' / Computed(this.exp == 128), 'num_pts' /
Computed(lambda ctx: ctx.npts if ctx.npts > 0 else ctx._.Header.npts),
'exponent' /
Computed(lambda ctx: ctx.exp if 0 < ctx.exp < 128 else ctx._.Header.exp),
'raw_x' / If(this._.Header.TFlags.use_subfile_xs,
OnDemand(Array(this.num_pts, Int32sl))),
'raw_y' / IfThenElse(this.float_y, OnDemand(Array(this.num_pts, Float32l)),
OnDemand(Array(this.num_pts, Int32sl))))
LogData = Struct(
'log_start' / Tell, 'sizd' / Int32sl, 'sizm' / Int32sl,
'text_offset' / Int32sl, 'bins' / Int32sl, 'dsks' / Int32sl, Padding(44),
'content' /
Pointer(this.log_start + this.text_offset, OnDemand(String(this.sizd))))
# The entire file.
SPCFile = Struct(
'Header' / Header, 'xvals' /
If(this.Header.TFlags.has_xs, OnDemand(Array(this.Header.npts, Float32l))),
'Subfile' / Array(this.Header.nsub, Subfile), 'LogData' /
If(this.Header.log_offset != 0, Pointer(this.Header.log_offset, LogData)))
def prettyprint(data):
np.set_printoptions(precision=4, suppress=True)
h = data.Header
d = h.Date
print('SPC file, version %s (%s)' % (h.version, VERSIONS[h.version]))
try:
def OffsetPioString(index):
return Pointer(this.entry_start + index, PioString)
def __init__(self, offsetfunc, subcon, offset=None, whence=os.SEEK_CUR):
Pointer.__init__(self, offsetfunc, subcon)
self._ptr_offset = offset
self._ptr_whence = whence
struct.pack('<I', len(this.block_data)) + this.block_data,
hashlib.sha256).digest()
# -------------------- Payload Decryption/Decompression --------------------
# encrypted payload is split into multiple data blocks with hashes
EncryptedPayloadBlock = Struct(
"hmac_hash_offset" / Tell,
Padding(32),
"block_data" / Prefixed(Int32ul, GreedyBytes),
# hmac_hash has to be at the end with a pointer because it needs to
# come after other fields
"hmac_hash" / Pointer(
this.hmac_hash_offset,
Checksum(
Bytes(32),
compute_payload_block_hash,
this,
# exception=PayloadChecksumError
)))
EncryptedPayload = Concatenated(
RepeatUntil(lambda item, a, b: len(item.block_data) == 0,
EncryptedPayloadBlock))
DecryptedPayload = Switch(
this._.header.value.dynamic_header.cipher_id.data, {
'aes256': AES256Payload(EncryptedPayload),
'chacha20': ChaCha20Payload(EncryptedPayload),
'twofish': TwoFishPayload(EncryptedPayload)
})
def is_ParameterList(block):
return block.BlockType.param != 0
ParameterList = RepeatUntil(obj_.Name == 'END', Parameter)
FloatData = Array(this.BlockLength, Float32l)
StringData = String(this.BlockLength * 4)
DirectoryEntry = Struct(
'BlockType' / BlockType, 'BlockLength' / Int32ul, 'DataPtr' / Int32ul,
'Block' / Pointer(
this.DataPtr,
FunctionSwitch(
[(is_ParameterList, ParameterList),
(lambda ctx: ctx.BlockType.extend != 0, OnDemand(StringData)),
(lambda ctx: ctx.BlockType.data not in
(0, 13), OnDemand(FloatData))])))
# The entire file.
OpusFile = Struct(
Const(b'\n\n\xfe\xfe'), # 0x0a0afefe magic
'Version' / Float64l,
'FirstDirPtr' / Int32ul,
'MaxDirSize' / Int32ul,
'CurrDirSize' / Int32ul,
'Directory' /
Pointer(this.FirstDirPtr, Array(this.MaxDirSize, DirectoryEntry)))
decrypted = decrypted.rstrip(b"\x00")
except Exception as ex:
_LOGGER.debug("Unable to decrypt, returning raw bytes.")
return obj
try:
jsoned = json.loads(decrypted.decode('utf-8'))
except:
_LOGGER.error("unable to parse json, was: %s", decrypted)
jsoned = b'{}'
raise
return jsoned
Message = Struct(
# for building we need data before anything else.
"data" / Pointer(32, RawCopy(EncryptionAdapter(GreedyBytes))),
"header" / RawCopy(
Struct(
Const(Int16ub,
0x2131), "length" / Rebuild(Int16ub, Utils.get_length),
"unknown" / Default(Int32ub, 0x00000000), "devtype" /
Enum(Default(Int16ub, 0x02c1), default=Pass, **xiaomi_devices),
"serial" / Default(Int16ub, 0xa40d),
"ts" / TimeAdapter(Default(Int32ub, datetime.datetime.utcnow())))),
"checksum" /
IfThenElse(Utils.is_hello, Bytes(16),
Checksum(Bytes(16), Utils.md5, Utils.checksum_field_bytes)),
)
LOCAL_SYMS_STRIPPED=0x0008,
AGGRESSIVE_WS_TRIM=0x0010,
LARGE_ADDRESS_AWARE=0x0020,
MACHINE_16BIT=0x0040,
BYTES_REVERSED_LO=0x0080,
MACHINE_32BIT=0x0100,
DEBUG_STRIPPED=0x0200,
REMOVABLE_RUN_FROM_SWAP=0x0400,
SYSTEM=0x1000,
DLL=0x2000,
UNIPROCESSOR_ONLY=0x4000,
BIG_ENDIAN_MACHINE=0x8000,
),
# symbol table
"symbol_table" / Pointer(this.symbol_table_pointer,
Array(this.number_of_symbols, symbol_table)))
def PEPlusField():
return IfThenElse(
this.pe_type == "PE32_plus",
Int64ul,
Int32ul,
)
optional_header = "optional_header" / Struct(
# standard fields
"pe_type" / Enum(
Int16ul,
PE32=0x10b,
