def _struct(cls):
return construct.Struct(
"type" / construct.Enum(construct.Byte, OpeDataType),
"name" / construct.PascalString(construct.Int32ul, "cp932"),
"count" / construct.Int32ul,
"func" / construct.Switch(
construct.this.type,
{"Func": construct.Enum(construct.Byte, OpeFuncType)}),
"operands" /
construct.Array(construct.this.count, Param._struct()),
)
def _struct(cls):
return construct.Struct(
'type' / construct.Enum(construct.Byte, OpeDataType),
'name' / construct.PascalString(construct.Int32ul, 'cp932'),
'count' / construct.Int32ul,
'func' / construct.Switch(
construct.this.type, {
'Func': construct.Enum(construct.Byte, OpeFuncType),
}),
'operands' /
construct.Array(construct.this.count, Param._struct()),
)
def CommTypeEnum(subcon):
return c2.Enum(subcon,
INVALID=smc.INVALID,
TOPIC=smc.TOPIC,
SERVICE_REQUEST=smc.SERVICE_REQUEST,
SERVICE_REPLY=smc.SERVICE_REPLY,
_default_=Pass)
def _struct(cls):
return construct.Struct(
"type" / construct.Enum(construct.Byte, ParamType),
"value" / construct.Switch(
construct.this.type,
{
"Int":
construct.Int32sl,
"Float":
construct.ExprAdapter(
construct.Bytes(10),
lambda obj, ctx: numpy.frombuffer(
obj.rjust(16, b"\x00"), dtype=numpy.longdouble),
lambda obj, ctx: numpy.longdouble(obj).tobytes()[-10:],
),
"Flag":
construct.Byte,
"Str":
construct.PascalString(construct.Int32ul, "cp932"),
},
# else 'Var' variable name type
construct.Select(
construct.PascalString(construct.Int32ul, "cp932"), ),
),
)
def cmd_type():
kwargs = {
llenums.CmdType.NULL_CMD.name: 0,
llenums.CmdType.CIRC_CMD.name: 1,
llenums.CmdType.EXPERIMENT_CMD.name: 2,
llenums.CmdType.FLUSH_CMD.name: 3
}
return cstruct.Enum(cstruct.Int8ul, **kwargs)
def lut_source_t():
kwargs = {
llenums.LUTSourceType.ADC0.value: 0,
llenums.LUTSourceType.ADC1.value: 1,
llenums.LUTSourceType.EXTERN.value: 2,
llenums.LUTSourceType.CONTROLLER.value: 3
}
return cstruct.Enum(cstruct.Int8ul, **kwargs)
def range_t():
kwargs = {
llenums.RangeType.HIGH.value: 0,
llenums.RangeType.MED.value: 1,
llenums.RangeType.LOW.value: 2,
llenums.RangeType.UNKNOWN.value: 3
}
return cstruct.Enum(cstruct.Int8ul, **kwargs)
def dac_source_t():
kwargs = {
llenums.DACSourceType.MEM.value: 0,
llenums.DACSourceType.EXTERN.value: 1,
llenums.DACSourceType.LUT0.value: 2,
llenums.DACSourceType.LUT1.value: 3
}
return cstruct.Enum(cstruct.Int8ul, **kwargs)
def exp_cmd_type():
kwargs = {
llenums.ExpCmdType.RESET.name: 0,
llenums.ExpCmdType.USE_ANALOG_CHIP.name: 1,
llenums.ExpCmdType.SET_SIM_TIME.name: 2,
llenums.ExpCmdType.USE_OSC.name: 3,
llenums.ExpCmdType.RUN.name: 4
}
return cstruct.Enum(cstruct.Int8ul, **kwargs)
def profile_type_t():
kwargs = {
llenums.ProfileOpType.INPUT_OUTPUT.name: 0,
llenums.ProfileOpType.INTEG_INITIAL_COND.name: 1,
llenums.ProfileOpType.INTEG_DERIVATIVE_STABLE.name: 2,
llenums.ProfileOpType.INTEG_DERIVATIVE_BIAS.name: 3,
llenums.ProfileOpType.INTEG_DERIVATIVE_GAIN.name: 4,
}
return cstruct.Enum(cstruct.Int8ul, **kwargs)
def port_type_t():
kwargs = {
llenums.PortType.IN0.name: 0,
llenums.PortType.IN1.name: 1,
llenums.PortType.OUT0.name: 2,
llenums.PortType.OUT1.name: 3,
llenums.PortType.OUT2.name: 4,
llenums.PortType.NOPORT.name: 5
}
return cstruct.Enum(cstruct.Int8ul, **kwargs)
def MsgTypeEnum(subcon):
return c2.Enum(subcon,
PING=smt.PING,
JOINT_POSITION=smt.JOINT_POSITION,
JOINT=smt.JOINT,
READ_INPUT=smt.READ_INPUT,
WRITE_OUTPUT=smt.WRITE_OUTPUT,
JOINT_TRAJ_PT=smt.JOINT_TRAJ_PT,
JOINT_TRAJ=smt.JOINT_TRAJ,
STATUS=smt.STATUS,
JOINT_TRAJ_PT_FULL=smt.JOINT_TRAJ_PT_FULL,
JOINT_FEEDBACK=smt.JOINT_FEEDBACK,
_default_=Pass)
def block_type_t():
kwargs = {
llenums.BlockType.NOBLOCK.name: 0,
llenums.BlockType.DAC.name: 1,
llenums.BlockType.CHIP_INPUT.name: 2,
llenums.BlockType.CHIP_OUTPUT.name: 3,
llenums.BlockType.TILE_INPUT.name: 4,
llenums.BlockType.TILE_OUTPUT.name: 5,
llenums.BlockType.MULT.name: 6,
llenums.BlockType.INTEG.name: 7,
llenums.BlockType.FANOUT.name: 8,
llenums.BlockType.LUT.name: 9,
llenums.BlockType.ADC.name: 10
}
return cstruct.Enum(cstruct.Int8ul, **kwargs)
def circ_cmd_type():
kwargs = {
llenums.CircCmdType.NULLCMD.name: 0,
llenums.CircCmdType.DISABLE.name: 1,
llenums.CircCmdType.CONNECT.name: 2,
llenums.CircCmdType.BREAK.name: 3,
llenums.CircCmdType.GET_STATUS.name: 4,
llenums.CircCmdType.WRITE_LUT.name: 5,
llenums.CircCmdType.CALIBRATE.name: 6,
llenums.CircCmdType.GET_STATE.name: 7,
llenums.CircCmdType.SET_STATE.name: 8,
llenums.CircCmdType.DEFAULTS.name: 9,
llenums.CircCmdType.PROFILE.name: 10
}
return cstruct.Enum(cstruct.Int8ul, **kwargs)
def _struct(cls):
return construct.Struct(
'type' / construct.Enum(construct.Byte, ParamType),
'value' / construct.Switch(
construct.this.type,
{
'Int': construct.Int32ul,
'Float': construct.ExprAdapter(
construct.Bytes(10),
lambda obj, ctx: numpy.frombuffer(obj.rjust(16, b'\x00'),
dtype=numpy.longdouble),
lambda obj, ctx: numpy.longdouble(obj).tobytes()[-10:]
),
'Flag': construct.Byte,
'Str': construct.PascalString(construct.Int32ul, 'cp932'),
},
# else 'Var' variable name type
construct.Select(
construct.PascalString(construct.Int32ul, 'cp932'),
),
),
)
def convert_fields_to_struct_fields(fields, verbose=False):
total_bits = 0
struct_fields = []
for identifier, op, options in fields:
s_field = None
if op == "&":
bits = len(options)
description = f"{bits}-bit bitfield"
f_enum = enum.IntFlag(
identifier, [(op, 1 << i) for (i, op) in enumerate(options)]
)
s_field = identifier / construct.FlagsEnum(
construct.BitsInteger(bits), f_enum
)
elif op == "|":
bits = math.ceil(math.log2(len(options)))
description = f"{bits}-bit enum"
if set(options) == {True, False}:
s_field = identifier / construct.Flag
else:
f_enum = enum.IntEnum(
identifier, [(op, 1 << i) for (i, op) in enumerate(options)]
)
s_field = identifier / construct.Enum(
construct.BitsInteger(bits), f_enum
)
struct_fields.append(s_field)
if verbose:
print(f"{identifier:15}: {description} ({len(options)} options)")
assert bits == s_field.sizeof()
total_bits += bits
if verbose:
print(f"Total bits: {total_bits}")
return (struct_fields, total_bits)
con = cst.Container # alias
HEADER = cst.Struct('pc_header', cst.Magic('\x00'),
cst.Const(cst.UBInt8('lines'), 1), cst.UBInt8('address'),
cst.Magic('\x03'))
SER_STATUS = cst.BitStruct('serst', cst.Magic('\x01\x01\x00'),
cst.Flag('schedule_enabled'),
cst.Flag('ack_enabled'), cst.Flag('further_pages'),
cst.Flag('interrupt_mode'), cst.Magic('\x00'))
PAGE_IDX = cst.Bytes('page_num', 3)
TEMPO = cst.BitStruct(
'tempo', cst.Magic('\x01\x01'),
cst.Enum(cst.BitField('display_ctrl', 2), TIMED=0, FIXED_ON=1,
FIXED_OFF=2),
cst.Enum(cst.BitField('persist_time', 4),
S2=1,
S5=2,
S10=3,
S20=4,
S30=5,
S45=6,
S60=7,
S90=8,
S120=9))
# TODO: union persist_time with scroll_speed
PAGE_FUNC = cst.BitStruct(
'page_func', cst.Magic('\x01\x01'), cst.Flag('show_temp'),
class MasterBootRecord(BootRecord):
_MBR_STRUCT = construct.Struct(
"mbr",
construct.HexDumpAdapter(construct.Bytes("bootloader_code", 440)),
construct.Field('disk_signature', 4),
construct.Padding(2),
construct.Array(
4,
construct.Struct(
"partitions",
construct.SLInt8("state"),
construct.BitStruct(
"beginning",
construct.Octet("head"),
construct.Bits("sect", 6),
construct.Bits("cyl", 10),
),
construct.Enum(
construct.UBInt8("type"),
Nothing=0x00,
FAT12=0x01,
XENIX_ROOT=0x02,
XENIX_USR=0x03,
FAT16_old=0x04,
Extended_DOS=0x05,
FAT16=0x06,
FAT32=0x0b,
FAT32_LBA=0x0c,
NTFS=0x07,
LINUX_SWAP=0x82,
LINUX_NATIVE=0x83,
PROTECTIVE_MBR=0xee,
_default_=construct.Pass,
),
construct.BitStruct(
"ending",
construct.Octet("head"),
construct.Bits("sect", 6),
construct.Bits("cyl", 10),
),
construct.ULInt32(
"sector_offset"), # offset from MBR in sectors
construct.ULInt32("size"), # in sectors
)),
construct.Const(construct.Bytes("signature", 2), '55aa'.decode('hex')),
)
def __init__(self, filePath, size, offset=None, whitelist=()):
self._type = 'MBR'
super(MasterBootRecord, self).__init__(filePath, size, offset,
whitelist)
def _parse(self):
"""
Main method in charge of parsing the MBR.
It will try to parse the boot record according to documented known structure and extract the partition table
disk signature and code section.
It will then try to narrow down invariant code, hash it and match the hash against a whitelist.
If no match was found, it will try some simple heuristics to detect malicious behaviours.
Returns: nothing
"""
try:
mbr = self._MBR_STRUCT.parse(self._raw)
except construct.core.ConstructError as e:
raise InvalidMBRError('Invalid MBR structure: {0}\n{1}'.format(
e, hexdump(self._raw)))
self._parsePartTable(mbr.partitions)
# Windows stores the disk signature at 0x1B8, other MBRs seem to leave this area alone
self._diskSignature = mbr.disk_signature
# If code section is null, check for protective MBR signature (detected in partition table parsing). If found,
# then the machine is likely using UEFI instead of BIOS to boot. If not, it could mean that the sample being
# analyzed has been tampered by a bootkit
if mbr.bootloader_code.encode('hex') == 440 * '00':
if 'Protective MBR' in self._signature:
self._signature.append('UEFI (no legacy boot code)')
else:
self._suspiciousBehaviour.append('Code section is null')
else:
expectedLoader, invariantCode = self._getInvariantCode(
mbr.bootloader_code)
codeHash = hashlib.sha256(invariantCode)
self._matchHash(codeHash, expectedLoader)
if len(self._signature) == 0:
# No whitelisted signature matched, try some simple heuristics to flag this MBR as malicious
# Note that the self._checkCode method is only given the "invariant" code section to help with the
# disassembling. This will obviously leads to broken offsets, but it doesn't matter since the heuristics
# don't use them.
self._checkCode(invariantCode)
def _parsePartTable(self, partitions):
"""
Private method that parses the partition table of the MBR. Updates self._partTable list.
Args:
partitions: Construct.Container object of the partition table
Returns: nothing
"""
partNum = 0
for part in partitions:
partNum += 1
# Assume a partition entry without size (in LBA) or type is invalid, and do not include it in the listing.
if part.size != 0 and part.type != 'Nothing':
self._partTable.append((partNum, part.state < 0, part.type,
part.sector_offset, part.size))
else:
self._logger.debug('Ignoring invalid partition: %s', part)
# Early detection of protective MBR so that we don't try to make sense of the MBR partition table
if part.type == 'PROTECTIVE_MBR' and partNum == 1:
self._logger.debug(
'Protective MBR detected, MBR partition table should not be taken into account. '
'GPT partition table parser not implemented yet')
self._signature.append('Protective MBR')
def _getInvariantCode(self, rawCode):
"""
Helper method that tries to narrow down "invariant code" which can be hashed and compared to well known
signatures. Most MBRs have localized error strings which must be excluded from the hash computation because
they may vary from a country to another.
First, this method tries to detect what kind of MBR it is dealing with. Most of the time, it is enough to
to look for some known hardcoded strings that identify "well known" MBR (such as Truecrypt, GRUB2, etc...).
Then, this method finds where the strings are and "removes" them (as in "does not include them").
Finding these strings can be achieved by quickly studying the assembly code and looking for how these
strings are echoed on screen at boot time (using interrupt 0x10).
This research only needs to be done once for each type of MBR but requires an analyst to do it by static
analysis. This script cannot take care of this. This method merely implements the results of such work.
Currently supported MBR are:
- Truecrypt
- McAfee Endpoint Encryption (Safeboot)
- GRUB2
- Windows (XP to 10)
Args:
rawCode: str of the code section
Returns: 2-tuple (unicode string of expected loader, concatenated strings of invariant sections of code)
"""
# By default, assume all the MBR code section will be hashed. It is obviously wrong in most cases, but it allows
# for a "default case" which will automatically matches no known hash in case something goes wrong with the
# detection.
codeStart = 0
codeEnd = len(rawCode)
expectedLoader = None
invariantCode = str()
# TrueCrypt (detected with the hardcoded string following the first jump: " TrueCrypt Boot Loader")
if rawCode[0x5:0x1b].encode('hex').upper(
) == '2054727565437279707420426F6F74204C6F61646572':
# TrueCrypt uses hardcoded and not-localized error strings. Therefore every TrueCrypt MBR should have the
# same code from start to end
expectedLoader = 'TrueCrypt MBR'
# MacAfee SafeBoot (detected with the hardcoded string following the first jump: "Safeboot ")
elif rawCode[0x3:0xc].encode('hex').upper() == '53616665426F6F7420':
# Two versions have been seen but both start with a jump to the same offset (0x26).
# There are some strings at the of the code section but localization is unlikely so it will be assumed
# to be hardcoded (until a localized version is found...).
# Therefore, Safeboot code can be hashed from 0x26 to the end of code section
invariantCode += rawCode[:0x3] # Hash the first JMP
codeStart = 0x26
expectedLoader = 'Safeboot MBR'
# GRUB (detected with the hardcoded string "GRUB " located at 0x188)
elif rawCode[0x188:0x18d].encode('hex').upper() == '4752554220':
# GRUB has some error strings but they are hardcoded and not localized so they can be included in the hash
# computation. However GRUB can be installed on a disk (MBR) as well as on a partition (in a kind of VBR).
# But in both cases the code used is the same. Since a BPB is needed for the latter case it is also present
# in the MBR (but not needed). It therefore has to be excluded from the hash computation.
# GRUB is jumping over the BIOS Parameter Block located between 0x3 and 0x5a.
# It should be followed by the kernel address (word), kernel sector (dword), kernel sector high (dword) and
# boot drive (byte). Therefore the code really starts at 0x65.
# These values are hardcoded in boot.img and have little chance to change anytime soon.
codeStart = 0x65
invariantCode += rawCode[:0x3] # Hash the first JMP
expectedLoader = 'GRUB2 MBR'
# Windows MBR cannot be detected with hardcoded strings, so they fall in the default case and further checks
# are then made based on the hypothesis that this is indeed a Windows MBR.
else:
# Starting with NT5.0, the MBR contains localized strings which must be excluded from the hash computation.
# These strings are located after the code, at 3 different offsets which can be calculated by adding 0x100
# to the values respectively stored in bytes 0x1b5, 0x1b6 and 0x1b7 (last bytes of the code section).
# Eg: The first localized string is at : 0x100 + the value saved at offset 0x1B5
# Even though localized strings can be of different lengths, the offset of the first one does not vary
# given one Windows version. This can therefore be used to tell Windows versions apart.
firstStrOffset = construct.UBInt8('FirstStringOffset').parse(
rawCode[0x1b5])
# Windows NT5
if firstStrOffset == 0x2c:
expectedLoader = 'NT5.1/NT5.2 MBR'
codeEnd = 0x100 + firstStrOffset
# Windows NT6.0
elif firstStrOffset == 0x62:
expectedLoader = 'NT6.0 MBR'
codeEnd = 0x100 + firstStrOffset
# Windows NT6.1+
elif firstStrOffset == 0x63:
expectedLoader = 'NT6.1+ MBR'
codeEnd = 0x100 + firstStrOffset
else:
self._suspiciousBehaviour.append(
'Invalid string offset: {0:#x}'.format(firstStrOffset))
self._logger.debug(
'First localized string offset is wrong for a windows MBR.'
'It should be 0x2c, 0x62 or 0x63) : {0:#x}'.format(
firstStrOffset))
self._logger.debug(
'Expecting {0}. Code starts at {1:#x} and ends at {2:#x}'.format(
expectedLoader, codeStart, codeEnd))
invariantCode += rawCode[codeStart:codeEnd]
return expectedLoader, invariantCode
def _checkCode(self, rawCode):
md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_16)
md.detail = True
checkJmp = True
for i in md.disasm(rawCode, 0):
# Check for JUMPs and CALLs before the first PUSH/RET.
if checkJmp and len(i.groups) > 0:
# Group check if available
if hasattr(capstone.x86, 'X86_GRP_CALL') and hasattr(
capstone.x86, 'X86_GRP_RET'):
if capstone.x86.X86_GRP_CALL in i.groups or capstone.x86.X86_GRP_JUMP in i.groups:
self._suspiciousBehaviour.append(
'JMP or CALL before relocation')
checkJmp = False
elif capstone.x86.X86_GRP_RET in i.groups:
# Stop search after the first PUSH/RET
checkJmp = False
# Manual check in case capstone version doesn't support CALL and RET groups
else:
if i.mnemonic[0] == 'j' or i.mnemonic == 'call':
self._suspiciousBehaviour.append(
'JMP or CALL before relocation')
checkJmp = False
elif i.mnemonic[:3] == 'ret':
# Stop search after the first PUSH/RET
checkJmp = False
# Check for unknown interrupt
if i.mnemonic == 'int' and i.bytes[1] not in (0x10, 0x13, 0x18,
0x1a):
self._suspiciousBehaviour.append(
'Unknown Interrupt : {0:#x}'.format(i.bytes[1]))
class SystemdJournalParser(interface.FileObjectParser):
"""Parses Systemd Journal files."""
NAME = 'systemd_journal'
DESCRIPTION = 'Parser for Systemd Journal files.'
_OBJECT_COMPRESSED_FLAG = 0x00000001
# Unfortunately this doesn't help us knowing about the "dirtiness" or
# "corrupted" file state.
# A file can be in any of these states and still be corrupted, for example, by
# an unexpected shut down. Once journald detects one of these, it will
# "rotate" the corrupted journal file, an store it away, and change the status
# to STATE_OFFLINE.
# STATE_ONLINE means the file wasn't closed, but the journal can still be in a
# clean state.
_JOURNAL_STATE = construct.Enum(
construct.ULInt8('state'),
STATE_OFFLINE=0,
STATE_ONLINE=1,
STATE_ARCHIVED=2
)
_OBJECT_HEADER_TYPE = construct.Enum(
construct.ULInt8('type'),
UNUSED=0,
DATA=1,
FIELD=2,
ENTRY=3,
DATA_HASH_TABLE=4,
FIELD_HASH_TABLE=5,
ENTRY_ARRAY=6,
TAG=7
)
_ULInt64 = construct.ULInt64('int')
_OBJECT_HEADER = construct.Struct(
'object_header',
_OBJECT_HEADER_TYPE,
construct.ULInt8('flags'),
construct.Bytes('reserved', 6),
construct.ULInt64('size')
)
_OBJECT_HEADER_SIZE = _OBJECT_HEADER.sizeof()
_DATA_OBJECT = construct.Struct(
'data_object',
construct.ULInt64('hash'),
construct.ULInt64('next_hash_offset'),
construct.ULInt64('next_field_offset'),
construct.ULInt64('entry_offset'),
construct.ULInt64('entry_array_offset'),
construct.ULInt64('n_entries')
)
_DATA_OBJECT_SIZE = _DATA_OBJECT.sizeof()
_ENTRY_ITEM = construct.Struct(
'entry_item',
construct.ULInt64('object_offset'),
construct.ULInt64('hash')
)
_ENTRY_OBJECT = construct.Struct(
'entry_object',
construct.ULInt64('seqnum'),
construct.ULInt64('realtime'),
construct.ULInt64('monotonic'),
construct.Struct(
'boot_id',
construct.Bytes('bytes', 16),
construct.ULInt64('qword1'),
construct.ULInt64('qword2')),
construct.ULInt64('xor_hash'),
construct.Rename('object_items', construct.GreedyRange(_ENTRY_ITEM))
)
_JOURNAL_HEADER = construct.Struct(
'journal_header',
construct.Const(construct.String('signature', 8), b'LPKSHHRH'),
construct.ULInt32('compatible_flags'),
construct.ULInt32('incompatible_flags'),
_JOURNAL_STATE,
construct.Bytes('reserved', 7),
construct.Bytes('file_id', 16),
construct.Bytes('machine_id', 16),
construct.Bytes('boot_id', 16),
construct.Bytes('seqnum_id', 16),
construct.ULInt64('header_size'),
construct.ULInt64('arena_size'),
construct.ULInt64('data_hash_table_offset'),
construct.ULInt64('data_hash_table_size'),
construct.ULInt64('field_hash_table_offset'),
construct.ULInt64('field_hash_table_size'),
construct.ULInt64('tail_object_offset'),
construct.ULInt64('n_objects'),
construct.ULInt64('n_entries'),
construct.ULInt64('tail_entry_seqnum'),
construct.ULInt64('head_entry_seqnum'),
construct.ULInt64('entry_array_offset'),
construct.ULInt64('head_entry_realtime'),
construct.ULInt64('tail_entry_realtime'),
construct.ULInt64('tail_entry_monotonic'),
# Added in format version 187
construct.ULInt64('n_data'),
construct.ULInt64('n_fields'),
# Added in format version 189
construct.ULInt64('n_tags'),
construct.ULInt64('n_entry_arrays')
)
def __init__(self):
"""Initializes a parser object."""
super(SystemdJournalParser, self).__init__()
self._max_journal_file_offset = 0
def _ParseObjectHeader(self, file_object, offset):
"""Parses a Systemd journal object header structure.
Args:
file_object (dfvfs.FileIO): a file-like object.
offset (int): offset to the object header.
Returns:
tuple[construct.Struct, int]: parsed object header and size of the
object payload (data) that follows.
"""
file_object.seek(offset, os.SEEK_SET)
object_header_data = file_object.read(self._OBJECT_HEADER_SIZE)
object_header = self._OBJECT_HEADER.parse(object_header_data)
payload_size = object_header.size - self._OBJECT_HEADER_SIZE
return (object_header, payload_size)
def _ParseItem(self, file_object, offset):
"""Parses a Systemd journal DATA object.
This method will read, and decompress if needed, the content of a DATA
object.
Args:
file_object (dfvfs.FileIO): a file-like object.
offset (int): offset to the DATA object.
Returns:
tuple[str, str]: key and value of this item.
Raises:
ParseError: When an unexpected object type is parsed.
"""
object_header, payload_size = self._ParseObjectHeader(file_object, offset)
file_object.read(self._DATA_OBJECT_SIZE)
if object_header.type != 'DATA':
raise errors.ParseError(
'Expected an object of type DATA, but got {0:s}'.format(
object_header.type))
event_data = file_object.read(payload_size - self._DATA_OBJECT_SIZE)
if object_header.flags & self._OBJECT_COMPRESSED_FLAG:
event_data = lzma.decompress(event_data)
event_string = event_data.decode('utf-8')
event_key, event_value = event_string.split('=', 1)
return (event_key, event_value)
def _ParseJournalEntry(self, parser_mediator, file_object, offset):
"""Parses a Systemd journal ENTRY object.
This method will generate an event per ENTRY object.
Args:
parser_mediator (ParserMediator): parser mediator.
file_object (dfvfs.FileIO): a file-like object.
offset (int): offset of the ENTRY object.
Raises:
ParseError: When an unexpected object type is parsed.
"""
object_header, payload_size = self._ParseObjectHeader(file_object, offset)
entry_object_data = file_object.read(payload_size)
entry_object = self._ENTRY_OBJECT.parse(entry_object_data)
if object_header.type != 'ENTRY':
raise errors.ParseError(
'Expected an object of type ENTRY, but got {0:s}'.format(
object_header.type))
fields = {}
for item in entry_object.object_items:
if item.object_offset < self._max_journal_file_offset:
raise errors.ParseError(
'object offset should be after hash tables ({0:d} < {1:d})'.format(
offset, self._max_journal_file_offset))
key, value = self._ParseItem(file_object, item.object_offset)
fields[key] = value
reporter = fields.get('SYSLOG_IDENTIFIER', None)
if reporter and reporter != 'kernel':
pid = fields.get('_PID', fields.get('SYSLOG_PID', None))
else:
pid = None
event_data = SystemdJournalEventData()
event_data.body = fields['MESSAGE']
event_data.hostname = fields['_HOSTNAME']
event_data.pid = pid
event_data.reporter = reporter
date_time = dfdatetime_posix_time.PosixTimeInMicroseconds(
timestamp=entry_object.realtime)
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_WRITTEN)
parser_mediator.ProduceEventWithEventData(event, event_data)
def _ParseEntries(self, file_object, offset):
"""Parses Systemd journal ENTRY_ARRAY objects.
Args:
file_object (dfvfs.FileIO): a file-like object.
offset (int): offset of the ENTRY_ARRAY object.
Returns:
list[dict]: every ENTRY objects offsets.
Raises:
ParseError: When an unexpected object type is parsed.
"""
entry_offsets = []
object_header, payload_size = self._ParseObjectHeader(file_object, offset)
if object_header.type != 'ENTRY_ARRAY':
raise errors.ParseError(
'Expected an object of type ENTRY_ARRAY, but got {0:s}'.format(
object_header.type))
next_array_offset = self._ULInt64.parse_stream(file_object)
entry_offests_numbers, _ = divmod((payload_size - 8), 8)
for entry_offset in range(entry_offests_numbers):
entry_offset = self._ULInt64.parse_stream(file_object)
if entry_offset != 0:
entry_offsets.append(entry_offset)
if next_array_offset != 0:
next_entry_offsets = self._ParseEntries(file_object, next_array_offset)
entry_offsets.extend(next_entry_offsets)
return entry_offsets
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses a Systemd journal file-like object.
Args:
parser_mediator (ParserMediator): parser mediator.
file_object (dfvfs.FileIO): a file-like object.
Raises:
UnableToParseFile: when the header cannot be parsed.
"""
try:
journal_header = self._JOURNAL_HEADER.parse_stream(file_object)
except construct.ConstructError as exception:
raise errors.UnableToParseFile(
'Unable to parse journal header with error: {0!s}'.format(exception))
max_data_hash_table_offset = (
journal_header.data_hash_table_offset +
journal_header.data_hash_table_size)
max_field_hash_table_offset = (
journal_header.field_hash_table_offset +
journal_header.field_hash_table_size)
self._max_journal_file_offset = max(
max_data_hash_table_offset, max_field_hash_table_offset)
entries_offsets = self._ParseEntries(
file_object, journal_header.entry_array_offset)
for entry_offset in entries_offsets:
try:
self._ParseJournalEntry(parser_mediator, file_object, entry_offset)
except errors.ParseError as exception:
parser_mediator.ProduceExtractionError((
'Unable to complete parsing journal file: {0:s} at offset '
'0x{1:08x}').format(exception, entry_offset))
return
except construct.ConstructError as exception:
raise errors.UnableToParseFile((
'Unable to parse journal header at offset: 0x{0:08x} with '
'error: {1:s}').format(entry_offset, exception))
def ReplyTypeEnum(subcon):
return c2.Enum(subcon,
INVALID_UNUSED=smr.INVALID_UNUSED,
SUCCES=smr.SUCCES,
FAILURE=smr.FAILURE,
_default_=Pass)
#
#######################################
RPC_TYPE = con.Enum(
Byte,
RPC_OK=0xa,
RPC_FAIL=0xb,
RPC_NOTIFY=0xc,
RPC_HELO=0xd,
PULL_MD=0xe,
PULL_MD_RESULT=0xf,
PUSH_MD=0x10,
PUSH_MD_RESULT=0x11,
# below messages are not implemented or not used by Lumina. Enjoy yourselves ;)
GET_POP=0x12,
GET_POP_RESULT=0x13,
LIST_PEERS=0x14,
LIST_PEERS_RESULT=0x15,
KILL_SESSIONS=0x16,
KILL_SESSIONS_RESULT=0x17,
DEL_ENTRIES=0x18,
DEL_ENTRIES_RESULT=0x19,
SHOW_ENTRIES=0x1a,
SHOW_ENTRIES_RESULT=0x1b,
DUMP_MD=0x1c,
DUMP_MD_RESULT=0x1d,
CLEAN_DB=0x1e,
DEBUGCTL=0x1f)
RpcMessage_FAIL = con.Struct(
"status" / IdaVarInt32,
DeviceCapabilityFlagsFormat = con.FlagsEnum(
U64,
doubly_redundant_can_bus=1 << 0,
battery_eliminator_circuit=1 << 1,
)
MathRangeFormat = con.Struct(
min=F32,
max=F32,
)
TaskIDFormat = con.Enum(
U8,
idle=0,
fault=1,
beep=2,
run=3,
hardware_test=4,
motor_identification=5,
low_level_manipulation=6,
)
ControlModeFormat = con.Enum(
U8,
ratiometric_current=0,
ratiometric_angular_velocity=1,
ratiometric_voltage=2,
current=3,
mechanical_rpm=4,
voltage=5,
)
from glucometerutils import common
from glucometerutils import exceptions
from glucometerutils.support import lifescan
from glucometerutils.support import lifescan_binary_protocol
# This device uses SCSI blocks as registers.
_REGISTER_SIZE = 512
_PACKET = construct.Padded(
_REGISTER_SIZE, lifescan_binary_protocol.LifeScanPacket(0x03, False))
_QUERY_REQUEST = construct.Struct(
construct.Const(b'\xe6\x02'),
'selector' /
construct.Enum(construct.Byte, serial=0x00, model=0x01, software=0x02),
)
_QUERY_RESPONSE = construct.Struct(
lifescan_binary_protocol.COMMAND_SUCCESS,
'value' / construct.CString(encoding='utf-16-le'),
)
_READ_PARAMETER_REQUEST = construct.Struct(
'selector' / construct.Enum(construct.Byte, unit=0x04), )
_READ_UNIT_RESPONSE = construct.Struct(
lifescan_binary_protocol.COMMAND_SUCCESS,
'unit' / lifescan_binary_protocol.GLUCOSE_UNIT,
construct.Padding(3),
)
)
ConstructResourceRequirement = Struct(
type=Byte,
index=VarInt,
amount=Short,
negate=Flag,
)
requirement_type_map = {
"resource": ConstructResourceRequirement,
"template": CString("utf8"),
}
ConstructRequirement = Struct(
type=construct.Enum(Byte, resource=0, **{"and": 1, "or": 2}, template=3),
data=Switch(lambda this: this.type, requirement_type_map)
)
requirement_type_map["and"] = PrefixedArray(VarInt, ConstructRequirement)
requirement_type_map["or"] = PrefixedArray(VarInt, ConstructRequirement)
ConstructDockWeakness = Struct(
index=VarInt,
name=CString("utf8"),
is_blast_door=Flag,
requirement=ConstructRequirement,
)
ConstructResourceDatabase = Struct(
items=PrefixedArray(VarInt, ConstructItemResourceInfo),
energy_tank_item_index=VarInt,
import tty
import time
import serial
import math
import construct as c
import numpy
###############################################################################
# Enumerations
PacketTypes = dict(
CONTROL=0x00,
CHANNEL=0x01,
SPEECH=0x02,
)
PacketType = c.Enum(c.Byte, **PacketTypes)
# Table 31
ControlPacketFields = dict(PKT_CHANNEL0=0x40,
PKT_ECMODE=0x05,
PKT_DCMODE=0x06,
PKT_COMPAND=0x32,
PKT_RATET=0x09,
PKT_RATEP=0x0A,
PKT_INIT=0x0B,
PKT_LOWPOWER=0x10,
PKT_CODECCFG=0x38,
PKT_CODECSTART=0x2A,
PKT_CODECSTOP=0x2B,
PKT_CHANFMT=0x15,
PKT_SPCHFMT=0x16,
_VERSION_RESPONSE = construct.Struct(
_COMMAND_SUCCESS,
'version' / construct.PascalString(construct.Byte, encoding='ascii'),
)
_SERIAL_NUMBER_REQUEST = construct.Const(
b'\x05\x0B\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00')
_SERIAL_NUMBER_RESPONSE = construct.Struct(
_COMMAND_SUCCESS,
'serial_number' / construct.GreedyString(encoding='ascii'),
)
_DATETIME_REQUEST = construct.Struct(
construct.Const(b'\x05\x20'), # 0x20 is the datetime
'request_type' / construct.Enum(construct.Byte, write=0x01, read=0x02),
'timestamp' / construct.Default(
construct_extras.Timestamp(construct.Int32ul), # type: ignore
datetime.datetime(1970, 1, 1, 0, 0)),
)
_DATETIME_RESPONSE = construct.Struct(
_COMMAND_SUCCESS,
'timestamp' /
construct_extras.Timestamp(construct.Int32ul), # type: ignore
)
_GLUCOSE_UNIT_REQUEST = construct.Const(b'\x05\x09\x02\x09\x00\x00\x00\x00')
_GLUCOSE_UNIT_RESPONSE = construct.Struct(
_COMMAND_SUCCESS,
ConstructResourceRequirement = Struct(
type=Byte,
index=VarInt,
amount=Short,
negate=Flag,
)
requirement_type_map = {
"resource": ConstructResourceRequirement,
"template": CString("utf8"),
}
ConstructRequirement = Struct(type=construct.Enum(Byte,
resource=0,
**{
"and": 1,
"or": 2
},
template=3),
data=Switch(lambda this: this.type,
requirement_type_map))
requirement_type_map["and"] = PrefixedArray(VarInt, ConstructRequirement)
requirement_type_map["or"] = PrefixedArray(VarInt, ConstructRequirement)
ConstructDockWeakness = Struct(
index=VarInt,
name=CString("utf8"),
is_blast_door=Flag,
requirement=ConstructRequirement,
)
OPCODE_TO_MNEMONIC = {
Opcode.NOP: 'nop',
Opcode.INVERT: 'not',
Opcode.BITWISE_OR: 'or',
Opcode.BITWISE_AND: 'and',
Opcode.BITWISE_NOR: 'nor',
Opcode.RETURN: 'ret',
}
symbol_entry = construct.Struct(
"argument_count" / construct.Int8ub,
"arguments" /
construct.Array(construct.this.argument_count,
construct.Enum(construct.Int8ub, ArgumentType)),
"name" / construct.PascalString(construct.VarInt, "utf8"),
"start" / construct.Int16ub, # Offset into "instructions"
# I don't think Construct's Pointer can be used for this.
# "start" / Pointer(8, Bytes(1)),
# For debug purposes this could potentially contain argument names.
# Or have a separate 'debug' entry for details only needed for debugging
# or reverse-engineering.
)
instruction_definition = construct.Struct(
"opcode" / construct.Enum(construct.Byte, Opcode),
"operand_a" / construct.Int8ub,
"operand_b" / construct.Int8ub,
"reserved" / construct.Const(0xFF, construct.Int8ub),
)
"""Byte-swap and bit-invert the VendorTrust field.
Vendor trust is interpreted as a bitmask in a 16-bit little-endian integer,
with the added twist that 0 means set and 1 means unset.
We feed it to a `BitStruct` that expects a big-endian sequence where bits have
the traditional meaning. We must therefore do a bitwise negation of each byte,
and return them in reverse order. This is the same transformation both ways,
fortunately, so we don't need two separate functions.
"""
return bytes(~b & 0xFF for b in data)[::-1]
# fmt: off
Toif = c.Struct(
"magic" / c.Const(b"TOI"),
"format" / c.Enum(c.Byte, full_color=b"f", grayscale=b"g"),
"width" / c.Int16ul,
"height" / c.Int16ul,
"data" / c.Prefixed(c.Int32ul, c.GreedyBytes),
)
VendorTrust = c.Transformed(
c.BitStruct(
"reserved" / c.Default(c.BitsInteger(9), 0),
"show_vendor_string" / c.Flag,
"require_user_click" / c.Flag,
"red_background" / c.Flag,
"delay" / c.BitsInteger(4),
), _transform_vendor_trust, 2, _transform_vendor_trust, 2)
VendorHeader = c.Struct(
construct.Int16ul, "minor_operating_system_version" / construct.Int16ul,
"major_image_version" / construct.Int16ul, "minor_image_version" /
construct.Int16ul, "major_subsystem_version" / construct.Int16ul,
"minor_subsystem_version" / construct.Int16ul, "win32_version_value" /
construct.Int32ul, "image_size" / construct.Int32ul,
"headers_size" / construct.Int32ul, "checksum" / construct.Int32ul,
"subsystem" / construct.Int16ul, "dll_characteristics" / construct.Int16ul,
"stack_reserve_size" / construct.Int64ul, "stack_commit_size" /
construct.Int64ul, "heap_reserve_size" / construct.Int64ul,
"heap_commit_size" / construct.Int64ul, "loader_flags" / construct.Int32ul,
"rva_and_sizes_count" / construct.Int32ul, "data_directory" /
construct.Array(construct.this.rva_and_sizes_count, ImageDataDirectory))
# TODO: When building this struct, the optional_header_size in ImageFileHeader should be fixed
ImageOptionalHeader = construct.Struct(
"signature" / construct.Enum(construct.Int16ul, PE32=267, PE64=523),
"header" / construct.Switch(construct.this.signature, {
"PE32": ImageOptionalHeader32,
"PE64": ImageOptionalHeader64
}))
"""
typedef struct _IMAGE_SECTION_HEADER {
BYTE Name[IMAGE_SIZEOF_SHORT_NAME];
union {
DWORD PhysicalAddress;
DWORD VirtualSize;
} Misc;
DWORD VirtualAddress;
DWORD SizeOfRawData;
DWORD PointerToRawData;
DWORD PointerToRelocations;
