def read_ftr(filename, delete_keys=None):
offset_size = 1859
ftr_record_struct = Struct(
"record",
LFloat32("Datetime"),
LFloat32("PosX"),
LFloat32("PosY"),
LFloat32("Altitude"),
LFloat32("Qx"),
LFloat32("Qy"),
LFloat32("Qz"),
LFloat32("Qw"),
LFloat32("DistUnused"),
)
ftr_struct = Struct(
"ftr_header",
String("filetype", 4),
#Bytes("unknown00", 136),
#String("FirstName", 17),
Bytes("unknown00", 135),
String("FirstName", 17),
String("FamilyName", 17),
String("Country", 17),
String("RN", 8),
String("CN", 4),
Bytes("unknown02", 5),
String("Landscape", 17),
Bytes("unknown03", offset_size - 4 - 135 - 17 - 17 - 17 - 17 - 17),
ULInt32("length"), # uint32 (4 bytes) @ 1859
Array(lambda ctx: ctx.length, ftr_record_struct),
)
with open(filename, "rb") as fd:
dat = ftr_struct.parse_stream(fd)
df_ftr = pd.DataFrame(dat['record'])
df_ftr['Time'] = df_ftr['Datetime'] * 3600.0
df_ftr['Time'] = df_ftr['Time'] - df_ftr['Time'].irow(0)
df_ftr['Datetime'] = pd.to_datetime(df_ftr['Time'], unit='s')
df_ftr['Deltatime'] = df_ftr['Datetime'] - df_ftr['Datetime'].shift(1)
df_ftr['Deltatime'] = df_ftr['Deltatime'] / np.timedelta64(
1, 's') # Deltatime as seconds
#df_ftr['Vz'] = ((df_ftr['Altitude'] - df_ftr['Altitude'].shift(1)).fillna(0) / df_ftr['Deltatime']).fillna(0)
#df_ftr = df_ftr.set_index('Datetime', verify_integrity=True)
df_ftr = df_ftr.set_index('Time', verify_integrity=True) # Time (s)
dat['record'] = df_ftr
if delete_keys is not None:
for key in delete_keys:
if key in dat.keys():
del dat[key]
for key in ['FirstName', 'FamilyName', 'Country', 'Landscape', 'RN', 'CN']:
length = ord(dat[key][0])
s = dat[key][1:length + 1]
dat[key] = s.replace('\x00', '')
assert len(
dat[key]) == length, "Length error with %s len=%d should be %d" % (
s, len(s), length)
return (dat)
def build_id():
return Struct("build_id", Anchor("start"), UNInt32("type"),
UNInt16("misc"), UNInt16("size"), SNInt32("pid"),
HexDumpAdapter(String("build_id", 24)), CString("filename"),
Anchor("offset"), pad("size"))
"explored_percent"/Byte,
"research_count"/Byte,
"research_percent"/Byte
)
society = "society"/Struct(
"score"/Int16ul,
"total_wonders"/Byte,
"total_castles"/Byte,
"relics_captured"/Byte,
Padding(1),
"villager_high"/Int16ul,
)
achievements = "achievements"/Struct(
"player_name"/String(16, padchar=b'\x00', trimdir='right', encoding='latin1'),
"total_score"/Int16ul,
Array(8, "total_scores"/Int16ul),
"victory"/Flag,
"civilization"/Byte,
"color_id"/Byte,
"team"/Byte,
"ally_count"/Byte,
Padding(1),
"mvp"/Flag,
Padding(3),
"result"/Flag,
Padding(3),
military,
Padding(32),
economy,
import hashlib
"""
iOS >= 4 raw images
http://opensource.apple.com/source/xnu/xnu-1699.22.73/bsd/hfs/hfs_cprotect.c
http://opensource.apple.com/source/xnu/xnu-1699.22.73/bsd/sys/cprotect.h
"""
cp_root_xattr = Struct("cp_root_xattr", ULInt16("major_version"),
ULInt16("minor_version"), ULInt64("flags"),
ULInt32("reserved1"), ULInt32("reserved2"),
ULInt32("reserved3"), ULInt32("reserved4"))
cprotect_xattr = Struct("cprotect_xattr", ULInt16("xattr_major_version"),
ULInt16("xattr_minor_version"), ULInt32("flags"),
ULInt32("persistent_class"), ULInt32("key_size"),
String("persistent_key", length=0x28))
cprotect4_xattr = Struct(
"cprotect_xattr", ULInt16("xattr_major_version"),
ULInt16("xattr_minor_version"), ULInt32("flags"),
ULInt32("persistent_class"), ULInt32("key_size"), Padding(20),
String("persistent_key", length=lambda ctx: ctx["key_size"]))
#HAX: flags set in finderInfo[3] to tell if the image was already decrypted
FLAG_DECRYPTING = 0x454d4664 #EMFd big endian
FLAG_DECRYPTED = 0x454d4644 #EMFD big endian
class EMFFile(HFSFile):
def __init__(self, volume, hfsplusfork, fileID, filekey, deleted=False):
super(EMFFile, self).__init__(volume, hfsplusfork, fileID, deleted)
GUNS = list(range(GUNSTOTALNUM))
SEGD_FOLD = "F:/Faxian6_Backups/programming/2016YQ/"
os.chdir(SEGD_FOLD)
LINES = glob.glob('2016YQ*')
for LINE in LINES:
BINARYSAVE = "E:/mypython/" + LINE + "GLK4K_SEG-D_ExtHedSAVE02.txt"
os.chdir(SEGD_FOLD + LINE + "/")
SEGD_LIST = glob.glob('*.segd')
for SEGD_SHOTFILE in SEGD_LIST:
#print(SEGD_SHOTFILE)
BINARYFILE = SEGD_SHOTFILE
FD = open(BINARYFILE, 'rb')
SAVE = open(BINARYSAVE, 'at') #file appended ,text mode
FD.seek(96) #omit the first 96 bytes on begining of SEG-D file
BINDATA = FD.read(1514) #define the external header's length
BINSTR = String(1514, encoding="utf8").parse(BINDATA)
# print(BINSTR)
'''The offset 0 to 90 contained
the fixed GCS90(SYNTRAK) header information'''
SAVE.write('ID:'+BINSTR[0:6]+\
';Size:'+BINSTR[6:10]+'bytes'+\
';Line:'+BINSTR[10:18]+\
';ShotPoint:'+str(int(BINSTR[18:28]))+\
';ArrayMask:'+BINSTR[28:30]+\
';TriggerMode:'+BINSTR[30:31]+\
';GPS ShotDate:'+BINSTR[31:39]+\
';GPS ShotTime:'+BINSTR[40:48]+\
';Current Sequence:'+BINSTR[48:49]+\
';Strings:'+BINSTR[49:50]+\
';Guns in Array:'+BINSTR[50:52]+\
';Active Guns:'+BINSTR[52:54]+\
def test_parse_padded_left(self):
s = String("foo", 10, padchar="X", paddir="left")
self.assertEqual(s.parse("XXXXXhello"), "hello")
def test_parse(self):
s = String("foo", 5)
self.assertEqual(s.parse(b"hello"), b"hello")
from construct import Array, Const, Default, Enum, GreedyRange, Int8sb, Int8ub, Int16ub, Int32ub, Padding, PrefixedArray, String, Struct, Switch, this
# file format from https://reverseengineering.stackexchange.com/questions/4311/help-reversing-a-edb-database-file-for-pioneers-rekordbox-software
AnlzTagPath = Struct(
"payload_size" / Int32ub, # is 0 for some tag types
"path" / String(this.payload_size - 2, encoding="utf-16-be"),
Padding(2))
AnlzTagVbr = Struct(Padding(4), "idx" / Array(400, Int32ub),
"unknown" / Int32ub)
AnlzQuantizeTick = Struct(
"beat" / Int16ub,
"bpm_100" / Int16ub,
"time" / Int32ub # in ms from start
)
AnlzTagQuantize = Struct(Padding(4), "unknown" / Const(0x80000, Int32ub),
"entries" / PrefixedArray(Int32ub, AnlzQuantizeTick))
AnlzTagQuantize2 = Struct(
Padding(4),
"u1" /
Const(0x01000002,
Int32ub), # maybe this encodes the count of "bpm" objects below
Padding(4),
"bpm" / Array(2, AnlzQuantizeTick),
"entry_count" / Int32ub, # 680
"u3" / Int32ub,
"u4" / Int32ub,
def test_build_padded_left(self):
s = String("foo", 10, padchar=b"X", paddir="left")
self.assertEqual(s.build(b"hello"), b"XXXXXhello")
def test_build_padded_center(self):
s = String("foo", 10, padchar=b"X", paddir="center")
self.assertEqual(s.build(b"hello"), b"XXhelloXXX")
def test_build_padded(self):
s = String("foo", 10, padchar=b"X", paddir="right")
self.assertEqual(s.build(b"hello"), b"helloXXXXX")
def test_build(self):
s = String("foo", 5)
self.assertEqual(s.build(b"hello"), b"hello")
def test_parse_utf8(self):
s = String("foo", 12, encoding="utf8")
self.assertEqual(s.parse(b"hello joh\xd4\x83n"), six.u("hello joh\u0503n"))
# -*- coding: utf-8 -*-
"""
constructBytes.py
Created on Sat Apr 13 11:28:18 2019
@author: madhu
"""
from construct import Struct, Magic, UBInt32, Const, String
# adapted from code at https://github.com/construct
fmt = Struct('png', Magic(b'\x89PNG\r\n\x1a\n'), UBInt32('length'),
Const(String('type', 4), b'IHDR'), UBInt32('width'),
UBInt32('height'))
data = b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR' + \
b'\x00\x00\x00\x9a\x00\x00\x00\x8d\x08\x02\x00\x00\x00\xc0'
result = fmt.parse(data)
print(result)
print(result.width, result.height)
def test_parse(self):
s = String("foo", 5)
self.assertEqual(s.parse(six.b("hello")), six.b("hello"))
Int16ul, "movement_type" / Byte, "attribute_max" / Int16ul,
"attribute_rot" / Float32l, "area_effect_level" / Byte,
"combat_level" / Byte, "select_level" / Byte, "map_draw_level" / Byte,
"unit_level" / Byte, "multiple_attribute_mod" / Float32l,
"map_color" / Byte, "help_string_id" / Int32ul, "help_page_id" / Int32ul,
"hotkey_id" / Int32ul, "recyclable" / Byte, "track_as_resource" / Byte,
"create_doppleganger" / Byte, "resource_group" / Byte, "occlusion_mask" /
Byte, "obstruction_type" / Byte, "selection_shape" / Byte,
"object_flags" / If(lambda ctx: find_version(ctx) != Version.AOK, Int32ul),
"civilization" / Byte, "attribute_piece" / Byte,
"outline_radius" / Struct("x" / Float32l, "y" / Float32l, "z" / Float32l),
"attributes" / Array(3, attribute), "num_damage_sprites" / Byte,
"damage_sprites" / Array(this.num_damage_sprites, damage_sprite),
"selected_sound" / Int16ul, "death_sound" / Int16ul,
"attack_reaction" / Byte, "convert_terrain_flag" / Byte,
"name" / String(this.name_len), "copy_id" / Int16ul, "group" / Int16ul)
animated = "animated" / Struct(base, "speed" / Float32l)
moving = "moving" / Struct(
animated, "move_sprite" / Int16ul, "run_sprite" / Int16ul,
"turn_speed" / Float32l, "size_class" / Byte, "trailing_unit" / Int16ul,
"trailing_options" / Byte, "trailing_spacing" / Float32l,
"move_algorithm" / Byte, "turn_radius" / Float32l,
"turn_radius_speed" / Float32l, "maximum_yaw_per_second_moving" / Float32l,
"stationary_yaw_per_revolution_time" / Float32l,
"maximum_yaw_per_second_stationary" / Float32l)
action = "action" / Struct(moving, "default_task" / Int16ul,
"search_radius" / Float32l, "work_rate" / Float32l,
"drop_site" / Int16ul, "backup_drop_site" / Int16ul,
Padding(28), # spare
'comment'/FixedSizeCString(130),
'catxt'/FixedSizeCString(30),
# only one subfile supported by this version
'nsub'/Computed(1),
# log data is not supported by this version
'log_offset'/Computed(0)
)
def _wrong_version_error(ctx):
raise NotImplementedError('SPC version %s is not implemented' % ctx.version)
Header = Struct(
'TFlags'/TFlags,
'version'/String(1),
Embedded(Switch(this.version, {
'K': HeaderVersionK,
'L': Computed(_wrong_version_error),
'M': HeaderVersionM,
}, default=Computed(_wrong_version_error)))
)
Subfile = Struct(
'flags'/Byte,
'exp'/Byte,
'index'/Int16sl,
'time'/Float32l,
'next'/Float32l,
'nois'/Float32l,
'npts'/Int32sl,
import struct
struct.pack('>L', 154)
struct.pack('>L', 141)
struct.unpack('>2L', data[16:24])
struct.unpack('>16x2L6x', data)
from construct import Struct, Magic, UBInt32, Const, String
# adapted from code at https://github.com/construct
fmt = Struct('png',
Magic(b'\x89PNG\r\n\x1a\n'),
UBInt32('length'),
Const(String('type', 4), b'IHDR'),
UBInt32('width'),
UBInt32('height')
)
data = b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR' + \
b'\x00\x00\x00\x9a\x00\x00\x00\x8d\x08\x02\x00\x00\x00\xc0'
result = fmt.parse(data)
print(result)
print(result.width, result.height)
import binascii
valid_png_header = b'\x89PNG\r\n\x1a\n'
print(binascii.hexlify(valid_png_header))
print(binascii.unhexlify(b'89504e470d0a1a0a'))
"xvc_hash" / Bytes(HASH_SIZE), # 0x260
"xvd_type" / CEnum(Int32ul, **XvdType), # 0x280
"content_type" / Int32ul, # 0x284
"embedded_xvd_length" / Int32ul, # 0x288
"userdata_length" / Int32ul, # 0x28C
"xvc_length" / Int32ul, # 0x290
"dynamic_header_length" / Int32ul, # 0x294
"block_size" / Int32ul, # 0x298
"ext_entry" / Array(4, XvdExtEntry), # 0x29C
"xvd_capabilities" / Bytes(16), # 0x2FC
"pe_catalog_hash" / Bytes(HASH_SIZE), # 0x30C
"embedded_xvd_pduid" / UUIDAdapter(), # 0x32C
Padding(0x10), # 0x33C
"key_material" / Bytes(XVD_KEY_SIZE), # 0x34C
"user_data_hash" / Bytes(HASH_SIZE), # 0x36C
"sandbox_id" / String(SANDBOX_ID_SIZE), # 0x38C
"product_id" / UUIDAdapter(), # 0x39C
"build_id" / UUIDAdapter(), # 0x3AC
"package_version" / Int64ul, # 0x3BC
"pe_catalog_info" / Bytes(0xA0), # 0x3C4
"writeable_expiration_data" / Int32ul, # 0x464
"writeable_policy_flags" / Int32ul, # 0x468
"local_storage_size" / Int32ul, # 0x46C
Padding(0x1C), # 0x470
"sequence_number" / Int64sl, # 0x48C
"required_systemversion" / Int64ul, # 0x494
"odk_keyslot_id" / Int32ul, # 0x49C
"reserved" / Padding(0xB60) # 0x4A0
)
"""Scenario."""
from construct import (Array, Float32l, Int16ul, Int32sl, Int32ul, Padding,
Embedded, PascalString, Peek, String, Struct, Bytes, If,
IfThenElse)
from mgz.enums import DifficultyEnum, PlayerTypeEnum, AgeEnum
from mgz.util import Find, Version
# pylint: disable=invalid-name, bad-continuation
# Scenario header.
scenario_header = "scenario_header" / Struct(
"next_uid" / Int32ul,
"constant" / Bytes(4),
Array(16, "names" / String(256)),
Array(16, "player_ids" / Int32ul),
Array(
16,
"player_data" / Struct(
"active" / Int32ul,
"human" / Int32ul,
"civilization" / Int32ul,
"constant" / Int32ul, # 0x04 0x00 0x00 0x00
)),
Padding(5),
"elapsed_time" / Float32l,
"scenario_filename" /
PascalString(lengthfield="scenario_filename_length" / Int16ul),
If(lambda ctx: ctx._._.version == Version.DE, Padding(64)))
def test_parse_padded_center(self):
s = String("foo", 10, padchar=six.b("X"), paddir="center")
self.assertEqual(s.parse(six.b("XXhelloXXX")), six.b("hello"))
"""Lobby."""
from construct import Array, Byte, Flag, Int32ul, Padding, String, Struct
from mgz.enums import GameTypeEnum, RevealMapEnum
# pylint: disable=invalid-name
# Player inputs in the lobby, and several host settings.
lobby = "lobby" / Struct(
Array(8, "teams" / Byte), # team number selected by each player
Padding(1),
RevealMapEnum("reveal_map" / Int32ul),
Padding(4),
"map_size" / Int32ul,
"population_limit" / Int32ul, # multiply by 25 for UserPatch 1.4
GameTypeEnum("game_type" / Byte),
"lock_teams" / Flag,
"num_chat" / Int32ul,
Array(
lambda ctx: ctx.num_chat, # pre-game chat messages
"messages" / Struct(
"message_length" / Int32ul,
"message" / String(lambda ctx: ctx.message_length,
padchar=b'\x00',
encoding='latin1'))))
def test_build(self):
s = String("foo", 5)
self.assertEqual(s.build(six.b("hello")), six.b("hello"))
from construct import Struct, Union, Const, Computed, Short, Int, this, String, Byte, Optional, Single
color = Struct(
# A colorStruct represents one color in the palette.
"chunk_type" / Const(b"\x00\x01"),
"chunk_length" / Int, # Four bytes
"title_length" / Short, # Two bytes
"title" / String(this.title_length * 2, encoding="utf-16be"),
"color_mode" / String(4), # Four bytes as a string.
# # Float # Total Length # Twice title length # title_length+color_mode+swatch_type_index
"color_values" /
Single[(this.chunk_length -
(this.title_length * 2) - 2 - 4 - 2) / Single.sizeof()],
"swatch_type_index" / Short, # Two bytes
)
end_palette = Struct(
"chunk_type" / Const(b"\xc0\x02"),
"chunk_length" / Int,
)
palette = Struct(
"chunk_type" / Const(b"\xc0\x01"),
"chunk_length" / Int,
"title_length" / Short,
"title" / String(this.title_length * 2, encoding="utf-16be"),
"colors" / color[1:],
"_end" / end_palette,
)
header = Struct(
def test_build_utf8(self):
s = String("foo", 12, encoding="utf8")
self.assertEqual(s.build(six.u("hello joh\u0503n")), six.b("hello joh\xd4\x83n"))
def spawn_entity(name='spawn_entity'):
return Struct(
name,
GreedyRange(
Struct('entity', Byte('entity_type'), VLQ('payload_size'),
String('payload', lambda ctx: ctx.payload_size))))
def test_build_padded_center(self):
s = String("foo", 10, padchar="X", paddir="center", encoding="utf8")
self.assertEqual(s.build(six.u("hello")), six.b("XXhelloXXX"))
'Value' / If(
this.Name != 'END',
Switch(this.Type, {
'INT32': Int32ul,
'REAL64': Float64l
},
default=FixedSizeCString(this.ReservedSpace * 2))))
def is_ParameterList(block):
return block.BlockType.param != 0
ParameterList = RepeatUntil(obj_.Name == 'END', Parameter)
FloatData = Array(this.BlockLength, Float32l)
StringData = String(this.BlockLength * 4)
DirectoryEntry = Struct(
'BlockType' / BlockType, 'BlockLength' / Int32ul, 'DataPtr' / Int32ul,
'Block' / Pointer(
this.DataPtr,
FunctionSwitch(
[(is_ParameterList, ParameterList),
(lambda ctx: ctx.BlockType.extend != 0, OnDemand(StringData)),
(lambda ctx: ctx.BlockType.data not in
(0, 13), OnDemand(FloatData))])))
# The entire file.
OpusFile = Struct(
Const(b'\n\n\xfe\xfe'), # 0x0a0afefe magic
'Version' / Float64l,
def test_parse(self):
s = String("foo", 5)
self.assertEqual(s.parse(six.b("hello")), six.b("hello"))
def section_adapter(name, target):
return perf_file_section(
name, TunnelAdapter(String("data", lambda ctx: ctx.size), target))
from mgz.enums import MyDiplomacyEnum, TheirDiplomacyEnum
from mgz.header.objects import existing_object
from mgz.header.playerstats import player_stats
from mgz.util import Find, GotoObjectsEnd, RepeatUpTo
# Player attributes.
attributes = "attributes" / Struct(
Array(lambda ctx: ctx._._._.replay.num_players,
TheirDiplomacyEnum("their_diplomacy" / Byte)),
Array(9, MyDiplomacyEnum("my_diplomacy" / Int32sl)),
"allied_los" / Int32ul,
"allied_victory" / Flag,
"player_name_length" / Int16ul,
"player_name" / String(this.player_name_length,
padchar=b'\x00',
trimdir='right',
encoding='latin1'),
Padding(1), # 0x16
"num_header_data" / Int32ul, # always 198
Padding(1), # 0x21
player_stats,
Padding(1),
"camera_x" / Float32l,
"camera_y" / Float32l,
"end_of_camera" / Tell,
"num_saved_views" / Int32sl,
# present in resumed games
If(
lambda ctx: ctx.num_saved_views > 0,
Array(
lambda ctx: ctx.num_saved_views, "saved_view" /
stype_hello=0x25,
stype_number=0x26,
stype_mac=0x2c,
stype_ip=0x32,
stype_status=0x36,
stype_change=0x29,
stype_status_mixer=0x00 # djm 900 nxs sends this stype on type_status
)
DeviceType = Enum(Int8ub, djm=1, cdj=2, rekordbox=3)
PlayerNumberAssignment = Enum(Int8ub, auto=1, manual=2)
# received on udp port 50000
KeepAlivePacket = Struct(
"magic" / Const(b'Qspt1WmJOL', String(10)),
"type" / KeepAlivePacketType, # pairs with subtype
Padding(1),
"model" / Padded(20, CString(encoding="ascii")),
"u1" / Const(1, Int8ub),
"device_type" / Default(DeviceType, "cdj"),
Padding(1),
"subtype" / KeepAlivePacketSubtype,
Embedded(
Switch(
this.type,
{
# type=0x0a, request for other players to propose a player number?
"type_hello":
Struct("u2" /
Default(Int8ub, 1)), # cdjs send 1, djm900nxs sends 3
def GUID(name):
return Struct(name,
ULInt32("Data1"),
ULInt16("Data2"),
ULInt16("Data3"),
String("Data4", 8))
class VolumeBootRecord(BootRecord):
_NTFS_VBR_STRUCT = Struct(
'JumpOverBPB' / Bytes(3),
'OemId' / String(8),
'BiosParameterBlock' / Struct(
'SectorSize' / Int16ul,
'SectorsPerCluster' / Int8ul,
'Reserved1' / Bytes(2),
'MustBeZero1' / Bytes(3),
'MustBeZero2' / Bytes(2),
'MediaDescriptor' / Int8ul,
'MustBeZero3' / Bytes(2),
'SectorsPerTrack' / Int16ul,
'NumberOfHeads' / Int16ul,
'HiddenSectors' / Int32ul,
'NotUsed1' / Bytes(4),
'DriveNumber' / Const('80'.decode('hex')),
'Reserved2' / Bytes(3),
'TotalSectors' / Int64ul,
'MFTCluster' / Int64ul,
'MFTMirrCluster' / Int64ul,
'ClustersPerMFTRecord' / Int8sl,
'NotUsed2' / Bytes(3),
'ClustersPerIdxBuffer' / Int8sl,
'NotUsed3' / Bytes(3),
'VolumneSN' / Int64ul,
'NotUsed4' / Bytes(4),
),
'Code' / Bytes(426),
Const('55aa'.decode('hex')),
)
_BITLOCKER_VBR_STRUCT = Struct(
'JumpOverBPB' / Bytes(3),
'OemId' / Const('-FVE-FS-'.encode('utf8')),
'BiosParameterBlock' / Struct(
'SectorSize' / Int16ul,
'SectorsPerCluster' / Int8ul,
'Reserved1' / Bytes(2),
'MustBeZero1' / Bytes(3),
'MustBeZero2' / Bytes(2),
'MediaDescriptor' / Int8ul,
'MustBeZero3' / Bytes(2),
'SectorsPerTrack' / Int16ul,
'NumberOfHeads' / Int16ul,
'HiddenSectors' / Int32ul,
'TotalSectors' / Int32ul,
'SectorsPerFAT' / Int32ul,
'FATFlags' / Int16ul,
'Version' / Int16ul,
'RootDirCluster' / Int32ul,
'FSInfoSector' / Int16ul,
'BackupSector' / Int16ul,
'Reserved2' / Bytes(12),
'DriveNumber' / Const('80'.decode('hex')),
'Reserved3' / Bytes(1),
'ExtendedBootSignature' / Bytes(1),
'VolumneSN' / Int32ul,
'VolumeLabel' / Const('NO NAME '.encode('utf8')),
'SystemId' / Const('FAT32 '.encode('utf8')),
),
'Code1' / Bytes(70),
'BitlockerGUID' / Bytes(16),
'FVEMetadataBlockOffset1' / Int64ul,
'FVEMetadataBlockOffset2' / Int64ul,
'FVEMetadataBlockOffset3' / Int64ul,
'Code2' / Bytes(307),
'FirstStrOffset' / Int8ul,
'SecondStrOffset' / Int8ul,
'ThirdStrOffset' / Int8ul,
Const('55aa'.decode('hex')),
)
def __init__(self, filePath, size, offset=None, whitelist=()):
self._type = 'VBR'
super(VolumeBootRecord, self).__init__(filePath, size, offset,
whitelist)
def _parse(self):
"""
Main method in charge of parsing the VBR.
It will try to parse the boot record according to known structures (NTFS and Bitlocker supported).
It will then try to narrow down invariant code, hash it and match the hash against a whitelist.
If no match was found, it will try some simple heuristics to detect malicious behaviours.
Finally it will compare the HiddenSectors value in BPB to that of the record's dump offset.
Returns: nothing
"""
try:
# This will parse both NTFS and Vista bitlocker volumes since they only differ by their OEM ID
vbr = self._NTFS_VBR_STRUCT.parse(self._raw)
expectedLoader, invariantCode = self._getInvariantCode('NTFS', vbr)
except ConstructError as e1:
# Retry with Bitlocker (Win7+) volume header structure
try:
vbr = self._BITLOCKER_VBR_STRUCT.parse(self._raw)
expectedLoader, invariantCode = self._getInvariantCode(
'bitlocker', vbr)
except ConstructError as e2:
raise InvalidVBRError(
'Invalid VBR structure: e1={0}, e2={1}\n{2}'.format(
e1, e2, hexdump(self._raw)))
self._oemId = vbr.OemId
self._bpb = vbr.BiosParameterBlock
codeHash = hashlib.sha256(invariantCode)
self._matchHash(codeHash, expectedLoader)
# If no whitelisted signature matched, try some simple heuristics to flag this VBR as malicious
# Note that the self._checkCode method is only given the "invariant" code section to help with the
# disassembling. This will obviously leads to broken offsets, but it doesn't matter since the heuristics don't
# use them.
if len(self._signature) == 0:
self._checkCode(invariantCode)
# At last, compare the offset at which this VBR was found with the value of the BPB HiddenSectors
if self._offset is not None \
and (vbr.BiosParameterBlock.HiddenSectors * vbr.BiosParameterBlock.SectorSize) != self._offset:
self._suspiciousBehaviour.append(
'Suspicious HiddenSectors value: {0} ({1} bytes)'.format(
vbr.BiosParameterBlock.HiddenSectors,
vbr.BiosParameterBlock.HiddenSectors *
vbr.BiosParameterBlock.SectorSize))
def _getInvariantCode(self, vbrType, vbrStruct):
"""
Helper method that finds all the sections of the boot code that can be hashed and compared to a whitelist.
This means that localized strings and other variable parameters (BPB, etc...) are excluded.
Currently, this method only supports NTFS and Bitlocker VBR.
Args:
vbrType: unicode string corresponding to the VBR type ('NTFS' or 'bitlocker')
vbrStruct: construct.container of the VBR
Returns: 2-tuple (unicode string of expected loader, concatenated strings of invariant sections of code)
"""
codeStart = 0
codeEnd = None
invariantCode = str()
expectedLoader = None
if vbrType == 'NTFS':
# The first three bytes are a jump over the NTFS BPB to where the code really starts (0x54) and a NOP
invariantCode += vbrStruct.JumpOverBPB
codeStart = 0x54
# NTFS VBR contains localized strings which must be excluded from the hash computation.
# Before Windows 8, these strings are located at 4 different offsets which can be calculated by adding
# 0x100 to the values respectively stored in bytes 0x1f8, 0x1f9, 0x1fa and 0x1fb.
# Starting from Windows 8, these strings are located at 3 different offsets which are directly stored in
# little endian words respectively at 0x1f6, 0x1f8 and 0x1fa
# Since there is no easy way to tell which version of Windows we are dealing with beforehand, we first
# assume it is a Windows < 8 by testing 0x1f8 against all the known first offset. If all tests fail, assume
# it is Windows >= 8 and check 0x1f6 against the only known first offset (to date)
firstStrOffset = Int8ub.parse(self._raw[0x1f8])
# Windows NT5
if firstStrOffset == 0x83:
expectedLoader = 'NT5.1/NT5.2 VBR'
codeEnd = 0x100 + firstStrOffset
# Windows NT6.0
elif firstStrOffset == 0x80:
expectedLoader = 'NT6.0 VBR'
codeEnd = 0x100 + firstStrOffset
# Windows NT6.1
elif firstStrOffset == 0x8c:
expectedLoader = 'NT6.1 VBR'
codeEnd = 0x100 + firstStrOffset
# Windows NT6.2+
else:
firstStrOffset = Int16ul.parse(self._raw[0x1f6:0x1f8])
if firstStrOffset == 0x18a:
expectedLoader = 'NT6.2+ VBR'
codeEnd = firstStrOffset
if codeEnd is None:
self._suspiciousBehaviour.append(
'Invalid string offset: {0:#x}'.format(firstStrOffset))
self._logger.debug(
'First localized string offset is wrong for a NTFS VBR: {0:#x}. '
'It should be 0x83, 0x80, 0x8c or 0x18a.'.format(
firstStrOffset))
codeEnd = 0
elif vbrType == 'bitlocker':
expectedLoader = 'NT6.1+ Bitlocker VBR'
# The first three bytes are a jump over the NTFS BPB to where the code really starts (0x5A) and a NOP
invariantCode += vbrStruct.JumpOverBPB
# First section of code (_BITLOCKER_VBR_STRUCT.Code1)
invariantCode += vbrStruct.Code1
# In the second section of code, there are localized strings which must be excluded from hash computation.
# Their offsets are stored in the last 3 bytes before the VBR signature (0x55aa).
# For Windows 8, 8.1 and 10, the first string offset seems to always be 0x100 (ie. FirstStrOffset = 0x00)
if vbrStruct.FirstStrOffset != 0:
self._suspiciousBehaviour.append(
'Invalid string offset: {0:#x}'.format(
vbrStruct.FirstStrOffset))
self._logger.debug(
'First localized string offset is wrong for a Bitlocker VBR. '
'It should be 0x00) : {0:#x}'.format(
vbrStruct.FirstStrOffset))
codeStart = 0xc8 # Offset of Code2
codeEnd = 0x100 + vbrStruct.FirstStrOffset
else:
raise NotImplementedError(
'VBR type "{0}" is not implemented yet'.format(vbrType))
self._logger.debug(
'Expecting {0}. Code starts at {1:#x} and ends at {2:#x}'.format(
expectedLoader, codeStart, codeEnd))
invariantCode += self._raw[codeStart:codeEnd]
return expectedLoader, invariantCode
def _checkCode(self, code):
md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_16)
md.detail = True
for i in md.disasm(code, 0):
# Check for unknown interrupt
if i.mnemonic == 'int' and i.bytes[1] not in (0x10, 0x13, 0x18,
0x1a):
self._suspiciousBehaviour.append(
'Unknown Interrupt : {0:#x}'.format(i.bytes[1]))
})
)
# Specialization for loading float data faster
TaggedFloat64 = ExprAdapter(
Struct(Const('tag'/Int16ul, 5), 'value'/Float64l),
encoder=lambda obj, ctx: Container(tag=obj.tag, value=obj.value),
decoder=lambda obj, ctx: obj.value)
DataList = Struct(
'size'/Int64ul,
OnDemand(Array(lambda ctx: ctx.size, TaggedFloat64)),
Const('\xc0\xff\xee\x01') # XXX: probably useful
)
# XXX: hacks
bad_strings = ('\xc0\xff\xee\x01\x00\x00', '\x01#Eg\x00\x00')
Property = Struct(
'peek'/Peek(String(6)),
Embedded(IfThenElse(
this.peek in bad_strings,
Padding(6),
Struct('label'/VBString, 'TaggedData'/TaggedData)))
)
Properties = GreedyRange(Property)
LabeledDataList = Struct(
'label'/VBString,
Padding(18),
'DataList'/Embedded(DataList)
)
DataSet = Struct(
'number'/Int64ul,
# XXX: may have more than two. Might use ctx.number to decide?
'LabeledDataList'/Array(2, LabeledDataList),
INDEX_RECORD_DATA = Struct('field_index',
Bytes('data', lambda ctx: ctx['data_size'] - 10),
ULInt32('record_number'))
RECORD_STRUCT = Struct(
'record', ULInt16('data_size'), Peek(Byte('first_byte')),
Embed(
IfThenElse(
'record_type', lambda ctx: ctx['first_byte'] == 0xFE,
Embed(
Struct(
'record',
RECORD_TYPE,
String('table_name',
lambda ctx: ctx['data_size'] - 5,
encoding=record_encoding),
UBInt32('table_number'),
)),
Embed(
Struct(
'record', UBInt32('table_number'), RECORD_TYPE,
Switch(
'record_type', lambda ctx: ctx.type, {
'DATA': Embed(DATA_RECORD_DATA),
'METADATA': Embed(METADATA_RECORD_DATA),
'TABLE_DEFINITION':
Embed(TABLE_DEFINITION_RECORD_DATA),
'INDEX': Embed(INDEX_RECORD_DATA)
}))))))
def __init__(self):
# TODO: replace this with construct.PascalString
vbs = Struct('length'/Int32ul,
'value'/String(this.length - 2),
Const(b'\x00\x00')) # There's always an ending null
Adapter.__init__(self, vbs)
from construct import Computed, ExprAdapter, FocusedSeq, Int8ul, Int24ul, Pointer, RepeatUntil, String, Switch, this
#PioString = Struct(
PioString = FocusedSeq(
1,
"padded_length" / RepeatUntil(lambda x, lst, ctx: x != 0, Int8ul),
"data" / Switch(
this.padded_length[-1],
{
# string longer than 127 bytes, prefixed with 3 bytes length
0x40:
FocusedSeq(
1, "actual_length" /
ExprAdapter(Int24ul, lambda o, c: o + 4, lambda o, c: o - 4),
"text" / String(this.actual_length, encoding="ascii")),
# iso-8859 text with \x00 between every character (like utf-16, but its iso-8859)
0x90:
FocusedSeq(
1, "actual_length" /
ExprAdapter(Int24ul, lambda o, c: o + 4, lambda o, c: o - 4),
"text" / ExprAdapter(
String(this.actual_length, encoding="iso-8859-1"),
lambda o, c: "".join(x + "\x00"
for x in o), lambda o, c: o[::2])),
},
default= # just ascii text
FocusedSeq(
1,
"actual_length" / Computed((this.padded_length[-1] - 1) // 2 - 1),
"text" / String(this.actual_length, encoding="ascii"))))
from construct import Struct, If, Array, PrefixedArray, Padding, \
SLInt8, ULInt16, SLInt16, ULInt32, SLInt32, LFloat32, LFloat64, String
def array(struct):
"""Standard prefixed arrays."""
return PrefixedArray(struct, ULInt32('count'))
BEAT = Struct('ebeats', LFloat32('time'), ULInt16('measure'), ULInt16('beat'),
ULInt32('phraseIteration'), ULInt32('mask'))
PHRASE = Struct('phrases', SLInt8('solo'), SLInt8('disparity'),
SLInt8('ignore'), Padding(1), ULInt32('maxDifficulty'),
ULInt32('phraseIterationLinks'),
String('name', 32, padchar='\x00'))
CHORD_TEMPLATE = Struct('chordTemplates', ULInt32('mask'), SLInt8('fret0'),
SLInt8('fret1'), SLInt8('fret2'), SLInt8('fret3'),
SLInt8('fret4'), SLInt8('fret5'), SLInt8('finger0'),
SLInt8('finger1'), SLInt8('finger2'),
SLInt8('finger3'), SLInt8('finger4'),
SLInt8('finger5'), Array(6, SLInt32('notes')),
String('chordName', 32, padchar='\x00'))
BEND_VALUE = Struct(
'bendValues',
LFloat32('time'),
LFloat32('step'),
Padding(3),
# Seens values: 0 1 3 31 32 36..49 51..54 56
Bytes(4),
FXP_PARAMS=b'FxCk',
FXP_OPAQUE_CHUNK=b'FPCh',
FXB_REGULAR=b'FxBk',
FXB_OPAQUE_CHUNK=b'FBCh',
),
"version" / Int32ub,
"fxID" / Int32ub,
"fxVersion" / Int32ub,
"count" / Int32ub,
"data" / Switch(
lambda ctx: ctx.fxMagic,
{
'FXP_PARAMS':
"data" / Struct(
"prgName" / String(28, padchar='\0'),
Array(lambda ctx: ctx['_']['count'], "params" / Float32b),
),
'FXP_OPAQUE_CHUNK':
"data" / Struct(
"prgName" / String(28, padchar='\0'),
"size" / Int32ub,
"chunk" / Bytes(lambda ctx: ctx['size']),
),
'FXB_REGULAR':
"data" / Struct(
"future" / Bytes(128), # zeros
# Array of FXP_PARAMS vst2preset
Array(lambda ctx: ctx['_']['count'],
"presets" / LazyBound(lambda: vst2preset)),
),
This section is a work in progress.
"""
from construct import (Array, Computed, Embedded, GreedyBytes, If, Int16ul,
Int32ul, Padding, Peek, String, Struct, Switch)
from mgz import subheader
# pylint: disable=invalid-name
# Embedded chat message
chat = Struct(
"subtype" / Computed("chat"), "data" / Struct(
"length" / Computed(lambda ctx: ctx._._._.op),
"text" / String(lambda ctx: ctx._._._.op,
padchar=b'\x00',
trimdir='right',
encoding='latin1'),
))
# Embedded header (aka saved chapter)
header = Struct(
"subtype" / Computed("savedchapter"), "data" / Struct(
"header_length" / Computed(lambda ctx: ctx._._._.op - ctx._._._.start),
Embedded(subheader)))
# Unknown embedded structure - looks like a partial action?
other = Struct(
"subtype" / Computed("unknown"), "data" / Struct(
Padding(4), "num_ints" / Int32ul,
If(lambda ctx: ctx.num_ints < 0xff,
Array(lambda ctx: ctx.num_ints, Int32ul)), Padding(12)))
Array(lambda ctx: ctx.selected, Int32ul)))
ai_waypoint = "ai_waypoint" / Struct(
"selected" / Byte, "waypoint_count" / Byte,
Array(lambda ctx: ctx.selected, "unit_ids" / Int32ul),
Array(lambda ctx: ctx.waypoint_count, "x_more" / Byte),
Array(lambda ctx: ctx.waypoint_count, "y_more" / Byte))
backtowork = "backtowork" / Struct(Padding(3), "towncenter_id" / Int32ul)
ai_command = "ai_command" / Struct(Padding(lambda ctx: ctx._._.length - 1))
postgame = "achievements" / Struct(
Padding(3),
"scenario_filename" /
String(32, padchar=b'\x00', trimdir='right', encoding='latin1'),
"player_num" / Byte,
"computer_num" / Byte,
Padding(2),
Peek("duration_int" / Int32ul),
TimeSecAdapter("duration" / Int32ul),
"cheats" / Flag,
"complete" / Flag,
Padding(14),
"map_size" / Byte,
MapAdapter("map_id" / Byte),
"population" / Byte,
Padding(1),
VictoryEnum("victory_type" / Byte),
StartingAgeEnum("starting_age" / Byte),
ResourceLevelEnum("resource_level" / Byte),
def test_parse_padded(self):
s = String("foo", 10, padchar="X", paddir="right")
self.assertEqual(s.parse("helloXXXXX"), "hello")
