def test_gif_quantization(name):
"""Encode unquantized image and compare result to reference."""
# load un-quantized PNG
png_path = get_test_gif_path(name + '.png')
rgba = run('convert {} rgba:-', png_path)
rgba_tuples = [tuple(col) for col in construct.Array(
lambda ctx: len(rgba) / 4,
construct.Array(4, construct.ULInt8('col')),
).parse(rgba)]
# load expected quantized gif
ref = load_reference_gif(get_test_gif_path(name + '.gif'))
# convert PNG to quantized GIF
gif = GIF()
gif.size = ref['size']
gif.images = [Image(rgba_tuples, ref['size'], ref['images'][0]['delay'])]
gif.loop_count = ref['loop']
gif.comment = ref['comment']
with tempfile.NamedTemporaryFile() as encoded_file:
gif.save(encoded_file)
encoded_file.flush()
# load resulting gif and compare to reference
reencoded_ref = load_reference_gif(encoded_file.name)
assert ref == reencoded_ref
def __init__(self):
super(AudioHandler, self).__init__()
self.header_base = construct.BitStruct(
'ASTRMBaseHeader', construct.BitField('fmt', 3),
construct.Bit('channel'), construct.Flag('vibrate'),
construct.Bit('packet_type'), construct.BitField('seq_id', 10),
construct.BitField('payload_size', 16))
self.header_aud = construct.Struct('ASTRMAudioHeader',
construct.ULInt32('timestamp'))
self.header_msg = construct.Struct(
'ASTRMMsgHeader',
# This is kind of a hack, (there are two timestamp fields, which one is used
# depends on packet_type
construct.ULInt32('timestamp_audio'),
construct.ULInt32('timestamp'),
construct.Array(2, construct.ULInt32('freq_0')), # -> mc_video
construct.Array(2, construct.ULInt32('freq_1')), # -> mc_sync
construct.ULInt8('vid_format'),
construct.Padding(3))
self.header = construct.Struct(
'ASTRMHeader', construct.Embed(self.header_base),
construct.Switch('format_hdr',
lambda ctx: ctx.packet_type, {
0: construct.Embed(self.header_aud),
1: construct.Embed(self.header_msg),
},
default=construct.Pass))
def build_floats(x, byteorder=sys.byteorder):
if byteorder == 'little':
c = construct.Array(len(x), construct.LFloat32("x"))
else:
c = construct.Array(len(x), construct.BFloat32("x"))
return c.build(x)
def mult_state_t():
return cstruct.Struct(
"vga" / bool_t(),
"enable" / bool_t(),
"range" / cstruct.Array(3, range_t()),
"pmos" / cstruct.Int8ul,
"nmos" / cstruct.Int8ul,
"port_cal" / cstruct.Array(3, cstruct.Int8ul),
"gain_cal" / cstruct.Int8ul,
"gain_code" / cstruct.Int8ul,
)
def sendCMDreceive(self, function, data, plformat):
format = structs.Message.struct
checksumformat = structs.Message.checksumFormat
if data == []:
payload = data
data_length = 0
else:
payload = plformat.build(data)
payload = bytearray(bytes(payload))
data_length = len(bytearray(bytes(payload)))
parameters = checksumformat.build(
dict(function=function, payloadSize=data_length, payload=payload))
checksum = 0
for i in bytearray(bytes(parameters)):
checksum = checkSum(checksum, i)
bytes2 = format.build(
dict(function=function,
payloadSize=data_length,
payload=payload,
checksum=checksum))
try:
b = self.ser.write(bytes2)
while True:
header = self.ser.read()
if header != b"":
blist = header
if header == b'$':
while header != b"":
header = self.ser.read()
blist = blist + header
break
format = structs.Message.structRecv
if ((function == 106 and len(blist) != 27)
or (function == 108 and len(blist) != 15)
or (function == 109 and len(blist) != 19)
or (function == 106 and len(blist) != 27)):
return None
if function == 209:
#print("lala ",blist)
return blist
#print(blist)
#print(len(blist),function)
parsed_data = dict(format.parse(blist))
blist = parsed_data["payload"]
payload1 = construct.Array(parsed_data["payloadSize"],
Int8ub).build(blist)
parsed_data = dict(plformat.parse(payload1))
#print("end")
#print(parsed_data)
return parsed_data
except Exception as error:
print("\n\nError in sendCMDreceive.")
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print(error, exc_type, fname, exc_tb.tb_lineno)
sys.exc_clear()
return None
def integ_state_t():
return cstruct.Struct(
"cal_enable" / cstruct.Array(3, bool_t()),
"inv" / sign_t(),
"enable" / bool_t(),
"exception" / bool_t(),
# 7 bytes in
"range" / cstruct.Array(3, range_t()),
# 10 bytes in
"pmos" / cstruct.Int8ul,
"nmos" / cstruct.Int8ul,
"ic_cal" / cstruct.Int8ul,
"ic_code" / cstruct.Int8ul,
# 14 bytes in
"port_cal" / cstruct.Array(3, cstruct.Int8ul),
)
def _ReadFileHeader(self):
"""Reads the file header.
Raises:
IOError: if the file header cannot be read.
"""
if self._debug:
print(u'Seeking file header offset: 0x{0:08x}'.format(0))
self._file_object.seek(0, os.SEEK_SET)
file_header_data = self._file_object.read(self._FILE_HEADER.sizeof())
if self._debug:
print(u'File header data:')
print(hexdump.Hexdump(file_header_data))
try:
file_header_struct = self._FILE_HEADER.parse(file_header_data)
except construct.FieldError as exception:
raise IOError(
u'Unable to parse file header with error: {0:s}'.format(
exception))
if self._debug:
print(u'Signature\t\t\t\t\t\t\t: {0!s}'.format(
file_header_struct.signature))
print(u'Number of pages\t\t\t\t\t\t\t: {0:d}'.format(
file_header_struct.number_of_pages))
print(u'')
page_sizes_data_size = file_header_struct.number_of_pages * 4
page_sizes_data = self._file_object.read(page_sizes_data_size)
if self._debug:
print(u'Page sizes data:')
print(hexdump.Hexdump(page_sizes_data))
try:
page_sizes_array = construct.Array(
file_header_struct.number_of_pages,
construct.UBInt32(u'page_sizes')).parse(page_sizes_data)
except construct.FieldError as exception:
raise IOError(
u'Unable to parse page sizes array with error: {0:s}'.format(
exception))
self._page_sizes = []
for page_index in range(file_header_struct.number_of_pages):
self._page_sizes.append(page_sizes_array[page_index])
if self._debug:
print(u'Page: {0:d} size\t\t\t\t\t\t\t: {1:d}'.format(
page_index, page_sizes_array[page_index]))
if self._debug:
print(u'')
def load_reference_gif(filename):
"""Load a GIF that we can use as a reference for testing.
exiftool and ImageMagick are used here as reference decoders.
"""
# get comment, size, loop count from exiftool
exiftool_json = json.loads(run('exiftool -j {}', filename))[0]
comment = exiftool_json.get('Comment', None)
size = (exiftool_json['ImageWidth'], exiftool_json['ImageHeight'])
loop = exiftool_json.get('AnimationIterations', 0)
loop = 0 if loop == 'Infinite' else loop + 1
# get delay for each frame from ImageMagick
delays = run('identify -format %T, {}', filename)
delays = [int(d) * 10 for d in delays.split(',')[:-1]]
# Work around bugs in ImageMagick. Animations need to be coalesced so we
# can extract frames with disposal methods applied correctly. But for still
# images, do not coalesce because it can corrupt the image.
if exiftool_json.get('FrameCount', 1) > 1:
tmp_filename = '{}.coalesced.gif'.format(filename)
run('convert {} -coalesce {}', filename, tmp_filename)
else:
tmp_filename = filename
images = []
for i in xrange(len(delays)):
# get the coalesced frame data as RGBA using ImageMagick
rgba = run('convert {}[{}] rgba:-', tmp_filename, i)
rgba_tuples = [tuple(col) for col in construct.Array(
lambda ctx: len(rgba) / 4,
construct.Array(4, construct.ULInt8('col')),
).parse(rgba)]
images.append({
'data': rgba_tuples,
'delay': delays[i],
})
return {
'size': size,
'loop': loop,
'images': images,
'comment': comment,
}
def profile_spec_t():
return cstruct.Struct(
"inst" / block_loc_t(),
"method" / profile_type_t(),
"output" / port_type_t(),
cstruct.Padding(1),
"in_vals" / cstruct.Array(2, cstruct.Float32l),
"state" / state_t(),
)
def _struct(cls):
return construct.Struct(
"type" / construct.Enum(construct.Byte, OpeDataType),
"name" / construct.PascalString(construct.Int32ul, "cp932"),
"count" / construct.Int32ul,
"func" / construct.Switch(
construct.this.type,
{"Func": construct.Enum(construct.Byte, OpeFuncType)}),
"operands" /
construct.Array(construct.this.count, Param._struct()),
)
def _struct(cls):
return construct.Struct(
'type' / construct.Enum(construct.Byte, OpeDataType),
'name' / construct.PascalString(construct.Int32ul, 'cp932'),
'count' / construct.Int32ul,
'func' / construct.Switch(
construct.this.type, {
'Func': construct.Enum(construct.Byte, OpeFuncType),
}),
'operands' /
construct.Array(construct.this.count, Param._struct()),
)
def _struct(cls, subcon, prefixed=True):
"""Return a construct Struct for this class.
Args:
subcon: Can be a subcon or fixed size, depending on value of `prefixed`.
prefixed: True if this is a PrefixedArray.
"""
if prefixed:
return construct.PrefixedArray(subcon, LiveParser._struct())
else:
return construct.Array(subcon, LiveParser._struct())
def __init__(s):
super(ServiceASTRM, s).__init__()
s.header_base = construct.BitStruct('ASTRMBaseHeader',
construct.BitField('fmt', 3),
construct.Bit('channel'),
construct.Flag('vibrate'),
construct.Bit('packet_type'),
construct.BitField('seq_id', 10),
construct.BitField('payload_size', 16)
)
s.header_aud = construct.Struct('ASTRMAudioHeader',
construct.ULInt32('timestamp'),
# construct.Array(lambda ctx: ctx.payload_size, construct.UBInt8("data"))
)
s.header_msg = construct.Struct('ASTRMMsgHeader',
# This is kind of a hack, (there are two timestamp fields, which one is used depends on packet_type
construct.ULInt32('timestamp_audio'),
construct.ULInt32('timestamp'),
construct.Array(2, construct.ULInt32('freq_0')), # -> mc_video
construct.Array(2, construct.ULInt32('freq_1')), # -> mc_sync
construct.ULInt8('vid_format'),
construct.Padding(3)
)
s.header = construct.Struct('ASTRMHeader',
construct.Embed(s.header_base),
construct.Switch('format_hdr', lambda ctx: ctx.packet_type,
{
0 : construct.Embed(s.header_aud),
1 : construct.Embed(s.header_msg),
},
default = construct.Pass
)
)
s.is_streaming = False
s.p = pyaudio.PyAudio()
s.stream = None
s.pa_num_bufs = 15
s.pa_ring = [array.array('H', '\0' * 416 * 2)] * s.pa_num_bufs
s.pa_wpos = s.pa_rpos = 0
class JBSQ(ct.TContainerMixin):
magic: Optional[bytes] = ct.sfield(
c.Select(c.Const(b"IJBQ"), c.Const(b"IJSQ"), c.Const(b"JBSQ")))
num_events: int = ct.sfield(c.Int32ul)
combo: int = ct.sfield(c.Int32ul)
end_time: int = ct.sfield(c.Int32ul)
_1: None = ct.sfield(c.Padding(2))
starting_buttons: int = ct.sfield(c.Int16ul)
start_time: int = ct.sfield(c.Int32ul)
_2: None = ct.sfield(c.Padding(12))
density_graph: List[int] = ct.sfield(c.Byte[60])
events: List[Event] = ct.sfield(
c.Array(c.this.num_events, ct.TStruct(Event)))
def _struct(cls):
return construct.Struct(
'version' / _LsbVersionValidator(construct.Int32ul),
'flags' / construct.Byte,
'command_count' / construct.Int32ul,
'param_stream_size' / construct.Int32ul,
'command_params' / construct.Array(
construct.this.command_count,
_ParamStreamAdapter(
construct.Bytes(construct.this.param_stream_size)),
),
'commands' / construct.PrefixedArray(
construct.Int32ul, construct.Select(*_command_structs)),
)
def _struct(cls):
return construct.Struct(
"version" / LsbVersionValidator(construct.Int32ul),
"flags" / construct.Byte,
"command_count" / construct.Int32ul,
"param_stream_size" / construct.Int32ul,
"command_params" / construct.Array(
construct.this.command_count,
_ParamStreamAdapter(
construct.Bytes(construct.this.param_stream_size)),
),
"commands" / construct.PrefixedArray(
construct.Int32ul, construct.Select(*_command_structs)),
)
class STECSatElement(object):
"""STECSatElement.
STEC polynomial for the given satellite.
Parameters
----------
sv_id : SvId
Unique space vehicle identifier
stec_quality_indicator : int
Quality of the STEC data. Encoded following RTCM DF389 specifcation
but in units of TECU instead of m.
stec_coeff : array
Coefficents of the STEC polynomial in the order of C00, C01, C10, C11
"""
_parser = construct.Embedded(
construct.Struct(
'sv_id' / construct.Struct(SvId._parser),
'stec_quality_indicator' / construct.Int8ul,
'stec_coeff' / construct.Array(4, construct.Int16sl),
))
__slots__ = [
'sv_id',
'stec_quality_indicator',
'stec_coeff',
]
def __init__(self, payload=None, **kwargs):
if payload:
self.from_binary(payload)
else:
self.sv_id = kwargs.pop('sv_id')
self.stec_quality_indicator = kwargs.pop('stec_quality_indicator')
self.stec_coeff = kwargs.pop('stec_coeff')
def __repr__(self):
return fmt_repr(self)
def from_binary(self, d):
p = STECSatElement._parser.parse(d)
for n in self.__class__.__slots__:
setattr(self, n, getattr(p, n))
def to_binary(self):
d = dict([(k, getattr(obj, k)) for k in self.__slots__])
return STECSatElement.build(d)
def ReadVtableFixups(ClrHeader):
VTableFixup = construct.Struct('VTableFixup',
MakeRva('RVA'),
construct.ULInt16('Count'),
construct.FlagsEnum(construct.ULInt16('Type'),
COR_VTABLE_32BIT = 0x01, # V-table slots are 32-bits in size.
COR_VTABLE_64BIT = 0x02, # V-table slots are 64-bits in size.
COR_VTABLE_FROM_UNMANAGED = 0x04, # If set, transition from unmanaged.
COR_VTABLE_FROM_UNMANAGED_RETAIN_APPDOMAIN = 0x08, # If set, transition from unmanaged with keeping the current appdomain.
COR_VTABLE_CALL_MOST_DERIVED = 0x10, # Call most derived method described by
)
)
numFixups = ClrHeader.VTableFixups.Size / VTableFixup.sizeof()
VTableFixups = construct.Array(numFixups, VTableFixup)
if numFixups == 0: return []
return VTableFixups.parse(idc.GetManyBytes(clrHeader.VTableFixups.VA, VTableFixups.sizeof()))
def parse_buffer(byte_list, dtype, offset=0, count=1):
"""
Parse byte_list
`dtype` data type. It should have a method `parse`
`count` number of this type to parse
`offset` start of the parsing
`byte_list` python binary array or list like object
"""
if dtype == str:
return parse_str(byte_list, offset)
else:
if type(dtype) == construct.core.Array:
raise TypeError('dtype should be plain data type')
if count != 1:
dtype = construct.Array(count, dtype)
new_offset = offset + dtype.sizeof()
s = dtype.parse(byte_list[offset:new_offset])
if count != 1:
s = list(s)
return s, new_offset
def make_dataset_t(buf):
arr_t = construct.Array(len(buf),
construct.Float32l)
arr_d = arr_t.build(buf)
return arr_t,arr_d
class MsgSbasRaw(SBP):
"""SBP class for message MSG_SBAS_RAW (0x7777).
You can have MSG_SBAS_RAW inherit its fields directly
from an inherited SBP object, or construct it inline using a dict
of its fields.
This message is sent once per second per SBAS satellite. ME checks the
parity of the data block and sends only blocks that pass the check.
Parameters
----------
sbp : SBP
SBP parent object to inherit from.
sid : GnssSignal
GNSS signal identifier.
tow : int
GPS time-of-week at the start of the data block.
message_type : int
SBAS message type (0-63)
data : array
Raw SBAS data field of 212 bits (last byte padded with zeros).
sender : int
Optional sender ID, defaults to SENDER_ID (see sbp/msg.py).
"""
_parser = construct.Struct(
'sid' / construct.Struct(GnssSignal._parser),
'tow' / construct.Int32ul,
'message_type' / construct.Int8ul,
'data' / construct.Array(27, construct.Int8ul),
)
__slots__ = [
'sid',
'tow',
'message_type',
'data',
]
def __init__(self, sbp=None, **kwargs):
if sbp:
super(MsgSbasRaw, self).__init__(sbp.msg_type, sbp.sender,
sbp.length, sbp.payload, sbp.crc)
self.from_binary(sbp.payload)
else:
super(MsgSbasRaw, self).__init__()
self.msg_type = SBP_MSG_SBAS_RAW
self.sender = kwargs.pop('sender', SENDER_ID)
self.sid = kwargs.pop('sid')
self.tow = kwargs.pop('tow')
self.message_type = kwargs.pop('message_type')
self.data = kwargs.pop('data')
def __repr__(self):
return fmt_repr(self)
@staticmethod
def from_json(s):
"""Given a JSON-encoded string s, build a message object.
"""
d = json.loads(s)
return MsgSbasRaw.from_json_dict(d)
@staticmethod
def from_json_dict(d):
sbp = SBP.from_json_dict(d)
return MsgSbasRaw(sbp, **d)
def from_binary(self, d):
"""Given a binary payload d, update the appropriate payload fields of
the message.
"""
p = MsgSbasRaw._parser.parse(d)
for n in self.__class__.__slots__:
setattr(self, n, getattr(p, n))
def to_binary(self):
"""Produce a framed/packed SBP message.
"""
c = containerize(exclude_fields(self))
self.payload = MsgSbasRaw._parser.build(c)
return self.pack()
def to_json_dict(self):
self.to_binary()
d = super(MsgSbasRaw, self).to_json_dict()
j = walk_json_dict(exclude_fields(self))
d.update(j)
return d
class AppCompatCacheKeyParser(object):
"""Class that parses the Application Compatibility Cache data."""
FORMAT_TYPE_2000 = 1
FORMAT_TYPE_XP = 2
FORMAT_TYPE_2003 = 3
FORMAT_TYPE_VISTA = 4
FORMAT_TYPE_7 = 5
FORMAT_TYPE_8 = 6
# AppCompatCache format signature used in Windows XP.
_HEADER_SIGNATURE_XP = 0xdeadbeef
# AppCompatCache format used in Windows XP.
_HEADER_XP_32BIT_STRUCT = construct.Struct(
'appcompatcache_header_xp', construct.ULInt32('signature'),
construct.ULInt32('number_of_cached_entries'),
construct.ULInt32('unknown1'), construct.ULInt32('unknown2'),
construct.Padding(384))
_CACHED_ENTRY_XP_32BIT_STRUCT = construct.Struct(
'appcompatcache_cached_entry_xp_32bit',
construct.Array(528, construct.Byte('path')),
construct.ULInt64('last_modification_time'),
construct.ULInt64('file_size'), construct.ULInt64('last_update_time'))
# AppCompatCache format signature used in Windows 2003, Vista and 2008.
_HEADER_SIGNATURE_2003 = 0xbadc0ffe
# AppCompatCache format used in Windows 2003.
_HEADER_2003_STRUCT = construct.Struct(
'appcompatcache_header_2003', construct.ULInt32('signature'),
construct.ULInt32('number_of_cached_entries'))
_CACHED_ENTRY_2003_32BIT_STRUCT = construct.Struct(
'appcompatcache_cached_entry_2003_32bit',
construct.ULInt16('path_size'), construct.ULInt16('maximum_path_size'),
construct.ULInt32('path_offset'),
construct.ULInt64('last_modification_time'),
construct.ULInt64('file_size'))
_CACHED_ENTRY_2003_64BIT_STRUCT = construct.Struct(
'appcompatcache_cached_entry_2003_64bit',
construct.ULInt16('path_size'), construct.ULInt16('maximum_path_size'),
construct.ULInt32('unknown1'), construct.ULInt64('path_offset'),
construct.ULInt64('last_modification_time'),
construct.ULInt64('file_size'))
# AppCompatCache format used in Windows Vista and 2008.
_CACHED_ENTRY_VISTA_32BIT_STRUCT = construct.Struct(
'appcompatcache_cached_entry_vista_32bit',
construct.ULInt16('path_size'), construct.ULInt16('maximum_path_size'),
construct.ULInt32('path_offset'),
construct.ULInt64('last_modification_time'),
construct.ULInt32('insertion_flags'), construct.ULInt32('shim_flags'))
_CACHED_ENTRY_VISTA_64BIT_STRUCT = construct.Struct(
'appcompatcache_cached_entry_vista_64bit',
construct.ULInt16('path_size'), construct.ULInt16('maximum_path_size'),
construct.ULInt32('unknown1'), construct.ULInt64('path_offset'),
construct.ULInt64('last_modification_time'),
construct.ULInt32('insertion_flags'), construct.ULInt32('shim_flags'))
# AppCompatCache format signature used in Windows 7 and 2008 R2.
_HEADER_SIGNATURE_7 = 0xbadc0fee
# AppCompatCache format used in Windows 7 and 2008 R2.
_HEADER_7_STRUCT = construct.Struct(
'appcompatcache_header_7', construct.ULInt32('signature'),
construct.ULInt32('number_of_cached_entries'), construct.Padding(120))
_CACHED_ENTRY_7_32BIT_STRUCT = construct.Struct(
'appcompatcache_cached_entry_7_32bit', construct.ULInt16('path_size'),
construct.ULInt16('maximum_path_size'),
construct.ULInt32('path_offset'),
construct.ULInt64('last_modification_time'),
construct.ULInt32('insertion_flags'), construct.ULInt32('shim_flags'),
construct.ULInt32('data_size'), construct.ULInt32('data_offset'))
_CACHED_ENTRY_7_64BIT_STRUCT = construct.Struct(
'appcompatcache_cached_entry_7_64bit', construct.ULInt16('path_size'),
construct.ULInt16('maximum_path_size'), construct.ULInt32('unknown1'),
construct.ULInt64('path_offset'),
construct.ULInt64('last_modification_time'),
construct.ULInt32('insertion_flags'), construct.ULInt32('shim_flags'),
construct.ULInt64('data_size'), construct.ULInt64('data_offset'))
# AppCompatCache format used in Windows 8.0 and 8.1.
_HEADER_SIGNATURE_8 = 0x00000080
_HEADER_8_STRUCT = construct.Struct('appcompatcache_header_8',
construct.ULInt32('signature'),
construct.Padding(124))
_CACHED_ENTRY_HEADER_8_STRUCT = construct.Struct(
'appcompatcache_cached_entry_header_8', construct.ULInt32('signature'),
construct.ULInt32('unknown1'),
construct.ULInt32('cached_entry_data_size'),
construct.ULInt16('path_size'))
# AppCompatCache format used in Windows 8.0.
_CACHED_ENTRY_SIGNATURE_8_0 = '00ts'
# AppCompatCache format used in Windows 8.1.
_CACHED_ENTRY_SIGNATURE_8_1 = '10ts'
def CheckSignature(self, value_data):
"""Parses the signature.
Args:
value_data: a binary string containing the value data.
Returns:
The format type if successful or None otherwise.
"""
signature = construct.ULInt32('signature').parse(value_data)
if signature == self._HEADER_SIGNATURE_XP:
return self.FORMAT_TYPE_XP
elif signature == self._HEADER_SIGNATURE_2003:
# TODO: determine which format version is used (2003 or Vista).
return self.FORMAT_TYPE_2003
elif signature == self._HEADER_SIGNATURE_7:
return self.FORMAT_TYPE_7
elif signature == self._HEADER_SIGNATURE_8:
if value_data[signature:signature + 4] in [
self._CACHED_ENTRY_SIGNATURE_8_0,
self._CACHED_ENTRY_SIGNATURE_8_1
]:
return self.FORMAT_TYPE_8
def ParseHeader(self, format_type, value_data):
"""Parses the header.
Args:
format_type: integer value that contains the format type.
value_data: a binary string containing the value data.
Returns:
A header object (instance of AppCompatCacheHeader).
Raises:
RuntimeError: if the format type is not supported.
"""
if format_type not in [
self.FORMAT_TYPE_XP, self.FORMAT_TYPE_2003,
self.FORMAT_TYPE_VISTA, self.FORMAT_TYPE_7, self.FORMAT_TYPE_8
]:
raise RuntimeError(
u'Unsupported format type: {0:d}'.format(format_type))
# TODO: change to collections.namedtuple or use __slots__ if the overhead
# of a regular object becomes a problem.
header_object = AppCompatCacheHeader()
if format_type == self.FORMAT_TYPE_XP:
header_object.header_size = self._HEADER_XP_32BIT_STRUCT.sizeof()
header_struct = self._HEADER_XP_32BIT_STRUCT.parse(value_data)
elif format_type == self.FORMAT_TYPE_2003:
header_object.header_size = self._HEADER_2003_STRUCT.sizeof()
header_struct = self._HEADER_2003_STRUCT.parse(value_data)
elif format_type == self.FORMAT_TYPE_VISTA:
header_object.header_size = self._HEADER_VISTA_STRUCT.sizeof()
header_struct = self._HEADER_VISTA_STRUCT.parse(value_data)
elif format_type == self.FORMAT_TYPE_7:
header_object.header_size = self._HEADER_7_STRUCT.sizeof()
header_struct = self._HEADER_7_STRUCT.parse(value_data)
elif format_type == self.FORMAT_TYPE_8:
header_object.header_size = self._HEADER_8_STRUCT.sizeof()
header_struct = self._HEADER_8_STRUCT.parse(value_data)
if format_type in [
self.FORMAT_TYPE_XP, self.FORMAT_TYPE_2003,
self.FORMAT_TYPE_VISTA, self.FORMAT_TYPE_7
]:
header_object.number_of_cached_entries = header_struct.get(
'number_of_cached_entries')
return header_object
def DetermineCacheEntrySize(self, format_type, value_data,
cached_entry_offset):
"""Parses a cached entry.
Args:
format_type: integer value that contains the format type.
value_data: a binary string containing the value data.
cached_entry_offset: integer value that contains the offset of
the first cached entry data relative to the start of
the value data.
Returns:
The cached entry size if successful or None otherwise.
Raises:
RuntimeError: if the format type is not supported.
"""
if format_type not in [
self.FORMAT_TYPE_XP, self.FORMAT_TYPE_2003,
self.FORMAT_TYPE_VISTA, self.FORMAT_TYPE_7, self.FORMAT_TYPE_8
]:
raise RuntimeError(
u'Unsupported format type: {0:d}'.format(format_type))
cached_entry_data = value_data[cached_entry_offset:]
cached_entry_size = 0
if format_type == self.FORMAT_TYPE_XP:
cached_entry_size = self._CACHED_ENTRY_XP_32BIT_STRUCT.sizeof()
elif format_type in [
self.FORMAT_TYPE_2003, self.FORMAT_TYPE_VISTA,
self.FORMAT_TYPE_7
]:
path_size = construct.ULInt16('path_size').parse(
cached_entry_data[0:2])
maximum_path_size = construct.ULInt16('maximum_path_size').parse(
cached_entry_data[2:4])
path_offset_32bit = construct.ULInt32('path_offset').parse(
cached_entry_data[4:8])
path_offset_64bit = construct.ULInt32('path_offset').parse(
cached_entry_data[8:16])
if maximum_path_size < path_size:
logging.error(u'Path size value out of bounds.')
return
path_end_of_string_size = maximum_path_size - path_size
if path_size == 0 or path_end_of_string_size != 2:
logging.error(u'Unsupported path size values.')
return
# Assume the entry is 64-bit if the 32-bit path offset is 0 and
# the 64-bit path offset is set.
if path_offset_32bit == 0 and path_offset_64bit != 0:
if format_type == self.FORMAT_TYPE_2003:
cached_entry_size = self._CACHED_ENTRY_2003_64BIT_STRUCT.sizeof(
)
elif format_type == self.FORMAT_TYPE_VISTA:
cached_entry_size = self._CACHED_ENTRY_VISTA_64BIT_STRUCT.sizeof(
)
elif format_type == self.FORMAT_TYPE_7:
cached_entry_size = self._CACHED_ENTRY_7_64BIT_STRUCT.sizeof(
)
else:
if format_type == self.FORMAT_TYPE_2003:
cached_entry_size = self._CACHED_ENTRY_2003_32BIT_STRUCT.sizeof(
)
elif format_type == self.FORMAT_TYPE_VISTA:
cached_entry_size = self._CACHED_ENTRY_VISTA_32BIT_STRUCT.sizeof(
)
elif format_type == self.FORMAT_TYPE_7:
cached_entry_size = self._CACHED_ENTRY_7_32BIT_STRUCT.sizeof(
)
elif format_type == self.FORMAT_TYPE_8:
cached_entry_size = self._CACHED_ENTRY_HEADER_8_STRUCT.sizeof()
return cached_entry_size
def ParseCachedEntry(self, format_type, value_data, cached_entry_offset,
cached_entry_size):
"""Parses a cached entry.
Args:
format_type: integer value that contains the format type.
value_data: a binary string containing the value data.
cached_entry_offset: integer value that contains the offset of
the cached entry data relative to the start of
the value data.
cached_entry_size: integer value that contains the cached entry data size.
Returns:
A cached entry object (instance of AppCompatCacheCachedEntry).
Raises:
RuntimeError: if the format type is not supported.
"""
if format_type not in [
self.FORMAT_TYPE_XP, self.FORMAT_TYPE_2003,
self.FORMAT_TYPE_VISTA, self.FORMAT_TYPE_7, self.FORMAT_TYPE_8
]:
raise RuntimeError(
u'Unsupported format type: {0:d}'.format(format_type))
cached_entry_data = value_data[
cached_entry_offset:cached_entry_offset + cached_entry_size]
cached_entry_struct = None
if format_type == self.FORMAT_TYPE_XP:
if cached_entry_size == self._CACHED_ENTRY_XP_32BIT_STRUCT.sizeof(
):
cached_entry_struct = self._CACHED_ENTRY_XP_32BIT_STRUCT.parse(
cached_entry_data)
elif format_type == self.FORMAT_TYPE_2003:
if cached_entry_size == self._CACHED_ENTRY_2003_32BIT_STRUCT.sizeof(
):
cached_entry_struct = self._CACHED_ENTRY_2003_32BIT_STRUCT.parse(
cached_entry_data)
elif cached_entry_size == self._CACHED_ENTRY_2003_64BIT_STRUCT.sizeof(
):
cached_entry_struct = self._CACHED_ENTRY_2003_64BIT_STRUCT.parse(
cached_entry_data)
elif format_type == self.FORMAT_TYPE_VISTA:
if cached_entry_size == self._CACHED_ENTRY_VISTA_32BIT_STRUCT.sizeof(
):
cached_entry_struct = self._CACHED_ENTRY_VISTA_32BIT_STRUCT.parse(
cached_entry_data)
elif cached_entry_size == self._CACHED_ENTRY_VISTA_64BIT_STRUCT.sizeof(
):
cached_entry_struct = self._CACHED_ENTRY_VISTA_64BIT_STRUCT.parse(
cached_entry_data)
elif format_type == self.FORMAT_TYPE_7:
if cached_entry_size == self._CACHED_ENTRY_7_32BIT_STRUCT.sizeof():
cached_entry_struct = self._CACHED_ENTRY_7_32BIT_STRUCT.parse(
cached_entry_data)
elif cached_entry_size == self._CACHED_ENTRY_7_64BIT_STRUCT.sizeof(
):
cached_entry_struct = self._CACHED_ENTRY_7_64BIT_STRUCT.parse(
cached_entry_data)
elif format_type == self.FORMAT_TYPE_8:
if cached_entry_data[0:4] not in [
self._CACHED_ENTRY_SIGNATURE_8_0,
self._CACHED_ENTRY_SIGNATURE_8_1
]:
raise RuntimeError(u'Unsupported cache entry signature')
if cached_entry_size == self._CACHED_ENTRY_HEADER_8_STRUCT.sizeof(
):
cached_entry_struct = self._CACHED_ENTRY_HEADER_8_STRUCT.parse(
cached_entry_data)
cached_entry_data_size = cached_entry_struct.get(
'cached_entry_data_size')
cached_entry_size = 12 + cached_entry_data_size
cached_entry_data = value_data[
cached_entry_offset:cached_entry_offset +
cached_entry_size]
if not cached_entry_struct:
raise RuntimeError(u'Unsupported cache entry size: {0:d}'.format(
cached_entry_size))
cached_entry_object = AppCompatCacheCachedEntry()
cached_entry_object.cached_entry_size = cached_entry_size
path_offset = 0
data_size = 0
if format_type == self.FORMAT_TYPE_XP:
string_size = 0
for string_index in xrange(0, 528, 2):
if (ord(cached_entry_data[string_index]) == 0
and ord(cached_entry_data[string_index + 1]) == 0):
break
string_size += 2
cached_entry_object.path = binary.Ut16StreamCopyToString(
cached_entry_data[0:string_size])
elif format_type in [
self.FORMAT_TYPE_2003, self.FORMAT_TYPE_VISTA,
self.FORMAT_TYPE_7
]:
path_size = cached_entry_struct.get('path_size')
path_offset = cached_entry_struct.get('path_offset')
elif format_type == self.FORMAT_TYPE_8:
path_size = cached_entry_struct.get('path_size')
cached_entry_data_offset = 14 + path_size
cached_entry_object.path = binary.Ut16StreamCopyToString(
cached_entry_data[14:cached_entry_data_offset])
remaining_data = cached_entry_data[cached_entry_data_offset:]
cached_entry_object.insertion_flags = construct.ULInt32(
'insertion_flags').parse(remaining_data[0:4])
cached_entry_object.shim_flags = construct.ULInt32(
'shim_flags').parse(remaining_data[4:8])
if cached_entry_data[0:4] == self._CACHED_ENTRY_SIGNATURE_8_0:
cached_entry_data_offset += 8
elif cached_entry_data[0:4] == self._CACHED_ENTRY_SIGNATURE_8_1:
cached_entry_data_offset += 10
remaining_data = cached_entry_data[cached_entry_data_offset:]
if format_type in [
self.FORMAT_TYPE_XP, self.FORMAT_TYPE_2003,
self.FORMAT_TYPE_VISTA, self.FORMAT_TYPE_7
]:
cached_entry_object.last_modification_time = cached_entry_struct.get(
'last_modification_time')
elif format_type == self.FORMAT_TYPE_8:
cached_entry_object.last_modification_time = construct.ULInt64(
'last_modification_time').parse(remaining_data[0:8])
if format_type in [self.FORMAT_TYPE_XP, self.FORMAT_TYPE_2003]:
cached_entry_object.file_size = cached_entry_struct.get(
'file_size')
elif format_type in [self.FORMAT_TYPE_VISTA, self.FORMAT_TYPE_7]:
cached_entry_object.insertion_flags = cached_entry_struct.get(
'insertion_flags')
cached_entry_object.shim_flags = cached_entry_struct.get(
'shim_flags')
if format_type == self.FORMAT_TYPE_XP:
cached_entry_object.last_update_time = cached_entry_struct.get(
'last_update_time')
if format_type == self.FORMAT_TYPE_7:
data_offset = cached_entry_struct.get('data_offset')
data_size = cached_entry_struct.get('data_size')
elif format_type == self.FORMAT_TYPE_8:
data_offset = cached_entry_offset + cached_entry_data_offset + 12
data_size = construct.ULInt32('data_size').parse(
remaining_data[8:12])
if path_offset > 0 and path_size > 0:
path_size += path_offset
cached_entry_object.path = binary.Ut16StreamCopyToString(
value_data[path_offset:path_size])
if data_size > 0:
data_size += data_offset
cached_entry_object.data = value_data[data_offset:data_size]
return cached_entry_object
construct.ULInt32('size'),
construct.Bytes('raw_data', lambda ctx: ctx.size)
)
'''
CREDENTIAL_DEC_BLOCK_ENC = construct.Struct(
'CREDENTIAL_DEC_BLOCK_ENC', construct.ULInt32('empty'),
construct.Rename('block_name', UNICODE_STRING), construct.ULInt32('size'),
construct.Bytes('raw_data', lambda ctx: ctx.size))
CREDENTIAL_DECRYPTED = construct.Struct(
'CREDENTIAL_DECRYPTED', construct.Rename('header', CREDENTIAL_DEC_HEADER),
construct.Rename('main', CREDENTIAL_DEC_MAIN),
construct.If(
lambda ctx: ctx.header.unk_type == 2,
construct.Array(lambda ctx: ctx.header.unk_blocks,
CREDENTIAL_DEC_BLOCK_ENC)))
# VAULT file structs.
VAULT_ATTRIBUTE_ENCRYPTED = construct.Struct(
'VAULT_ATTRIBUTE_ENCRYPTED', construct.Byte('has_iv'),
construct.IfThenElse(
'', lambda ctx: ctx.has_iv,
construct.Embed(
construct.Struct(
'encrypted', construct.ULInt32('iv_size'),
construct.Bytes('iv', lambda ctx: ctx.iv_size),
construct.Bytes('data',
lambda ctx: ctx.size - 1 - 4 - ctx.iv_size))),
construct.Embed(
construct.Struct('encrypted',
con.Struct(
'failed_task_id' / TaskIDFormat,
'failed_task_exit_code' / U8,
),
'run':
con.Select(
# New format starting from firmware v0.2 - added a new field 'torque' and one reserved four bytes long field
con.Struct(
'stall_count' / U32,
'demand_factor' / F32,
# Mechanical parameters
'electrical_angular_velocity' / F32,
'mechanical_angular_velocity' / F32,
'torque' / F32,
# Rotating system parameters
'u_dq' / con.Array(2, F32),
'i_dq' / con.Array(2, F32),
# Control mode
'mode' / ControlModeFormat,
# State flags
'spinup_in_progress' / con.Flag,
'rotation_reversed' / con.Flag,
'controller_saturated' / con.Flag,
),
# An older format used in the firmware v0.1 - this one is shorter, hence it must be at the end of Select()
con.Struct(
'stall_count' / U32,
'demand_factor' / F32,
'electrical_angular_velocity' / F32,
'mechanical_angular_velocity' / F32,
'u_dq' / con.Array(2, F32),
class MsgLinuxSocketUsage(SBP):
"""SBP class for message MSG_LINUX_SOCKET_USAGE (0x7F05).
You can have MSG_LINUX_SOCKET_USAGE inherit its fields directly
from an inherited SBP object, or construct it inline using a dict
of its fields.
Summaries the socket usage across the system.
Parameters
----------
sbp : SBP
SBP parent object to inherit from.
avg_queue_depth : int
average socket queue depths across all sockets on the system
max_queue_depth : int
the max queue depth seen within the reporting period
socket_state_counts : array
A count for each socket type reported in the `socket_types_reported` field,
the first entry corresponds to the first enabled bit in `types_reported`.
socket_type_counts : array
A count for each socket type reported in the `socket_types_reported` field,
the first entry corresponds to the first enabled bit in `types_reported`.
sender : int
Optional sender ID, defaults to SENDER_ID (see sbp/msg.py).
"""
_parser = construct.Struct(
'avg_queue_depth' / construct.Int32ul,
'max_queue_depth' / construct.Int32ul,
'socket_state_counts' / construct.Array(16, construct.Int16ul),
'socket_type_counts' / construct.Array(16, construct.Int16ul),
)
__slots__ = [
'avg_queue_depth',
'max_queue_depth',
'socket_state_counts',
'socket_type_counts',
]
def __init__(self, sbp=None, **kwargs):
if sbp:
super(MsgLinuxSocketUsage,
self).__init__(sbp.msg_type, sbp.sender, sbp.length,
sbp.payload, sbp.crc)
self.from_binary(sbp.payload)
else:
super(MsgLinuxSocketUsage, self).__init__()
self.msg_type = SBP_MSG_LINUX_SOCKET_USAGE
self.sender = kwargs.pop('sender', SENDER_ID)
self.avg_queue_depth = kwargs.pop('avg_queue_depth')
self.max_queue_depth = kwargs.pop('max_queue_depth')
self.socket_state_counts = kwargs.pop('socket_state_counts')
self.socket_type_counts = kwargs.pop('socket_type_counts')
def __repr__(self):
return fmt_repr(self)
@staticmethod
def from_json(s):
"""Given a JSON-encoded string s, build a message object.
"""
d = json.loads(s)
return MsgLinuxSocketUsage.from_json_dict(d)
@staticmethod
def from_json_dict(d):
sbp = SBP.from_json_dict(d)
return MsgLinuxSocketUsage(sbp, **d)
def from_binary(self, d):
"""Given a binary payload d, update the appropriate payload fields of
the message.
"""
p = MsgLinuxSocketUsage._parser.parse(d)
for n in self.__class__.__slots__:
setattr(self, n, getattr(p, n))
def to_binary(self):
"""Produce a framed/packed SBP message.
"""
c = containerize(exclude_fields(self))
self.payload = MsgLinuxSocketUsage._parser.build(c)
return self.pack()
def to_json_dict(self):
self.to_binary()
d = super(MsgLinuxSocketUsage, self).to_json_dict()
j = walk_json_dict(exclude_fields(self))
d.update(j)
return d
class BSMParser(interface.FileObjectParser):
"""Parser for BSM files."""
NAME = 'bsm_log'
DESCRIPTION = 'Parser for BSM log files.'
# BSM supported version (0x0b = 11).
AUDIT_HEADER_VERSION = 11
# Magic Trail Header.
BSM_TOKEN_TRAILER_MAGIC = b'b105'
# IP Version constants.
AU_IPv4 = 4
AU_IPv6 = 16
IPV4_STRUCT = construct.UBInt32('ipv4')
IPV6_STRUCT = construct.Struct(
'ipv6',
construct.UBInt64('high'),
construct.UBInt64('low'))
# Tested structures.
# INFO: I have ommited the ID in the structures declaration.
# I used the BSM_TYPE first to read the ID, and then, the structure.
# Tokens always start with an ID value that identifies their token
# type and subsequent structure.
_BSM_TOKEN = construct.UBInt8('token_id')
# Data type structures.
BSM_TOKEN_DATA_CHAR = construct.String('value', 1)
BSM_TOKEN_DATA_SHORT = construct.UBInt16('value')
BSM_TOKEN_DATA_INTEGER = construct.UBInt32('value')
# Common structure used by other structures.
# audit_uid: integer, uid that generates the entry.
# effective_uid: integer, the permission user used.
# effective_gid: integer, the permission group used.
# real_uid: integer, user id of the user that execute the process.
# real_gid: integer, group id of the group that execute the process.
# pid: integer, identification number of the process.
# session_id: unknown, need research.
BSM_TOKEN_SUBJECT_SHORT = construct.Struct(
'subject_data',
construct.UBInt32('audit_uid'),
construct.UBInt32('effective_uid'),
construct.UBInt32('effective_gid'),
construct.UBInt32('real_uid'),
construct.UBInt32('real_gid'),
construct.UBInt32('pid'),
construct.UBInt32('session_id'))
# Common structure used by other structures.
# Identify the kind of inet (IPv4 or IPv6)
# TODO: instead of 16, AU_IPv6 must be used.
BSM_IP_TYPE_SHORT = construct.Struct(
'bsm_ip_type_short',
construct.UBInt32('net_type'),
construct.Switch(
'ip_addr',
_BSMTokenGetNetType,
{16: IPV6_STRUCT},
default=IPV4_STRUCT))
# Initial fields structure used by header structures.
# length: integer, the length of the entry, equal to trailer (doc: length).
# version: integer, version of BSM (AUDIT_HEADER_VERSION).
# event_type: integer, the type of event (/etc/security/audit_event).
# modifier: integer, unknown, need research (It is always 0).
BSM_HEADER = construct.Struct(
'bsm_header',
construct.UBInt32('length'),
construct.UBInt8('version'),
construct.UBInt16('event_type'),
construct.UBInt16('modifier'))
# First token of one entry.
# timestamp: unsigned integer, number of seconds since
# January 1, 1970 00:00:00 UTC.
# microseconds: unsigned integer, number of micro seconds.
BSM_HEADER32 = construct.Struct(
'bsm_header32',
BSM_HEADER,
construct.UBInt32('timestamp'),
construct.UBInt32('microseconds'))
BSM_HEADER64 = construct.Struct(
'bsm_header64',
BSM_HEADER,
construct.UBInt64('timestamp'),
construct.UBInt64('microseconds'))
BSM_HEADER32_EX = construct.Struct(
'bsm_header32_ex',
BSM_HEADER,
BSM_IP_TYPE_SHORT,
construct.UBInt32('timestamp'),
construct.UBInt32('microseconds'))
# Token TEXT, provides extra information.
BSM_TOKEN_TEXT = construct.Struct(
'bsm_token_text',
construct.UBInt16('length'),
construct.Array(_BSMTokenGetLength, construct.UBInt8('text')))
# Path of the executable.
BSM_TOKEN_PATH = BSM_TOKEN_TEXT
# Identified the end of the record (follow by TRAILER).
# status: integer that identifies the status of the exit (BSM_ERRORS).
# return: returned value from the operation.
BSM_TOKEN_RETURN32 = construct.Struct(
'bsm_token_return32',
construct.UBInt8('status'),
construct.UBInt32('return_value'))
BSM_TOKEN_RETURN64 = construct.Struct(
'bsm_token_return64',
construct.UBInt8('status'),
construct.UBInt64('return_value'))
# Identified the number of bytes that was written.
# magic: 2 bytes that identifies the TRAILER (BSM_TOKEN_TRAILER_MAGIC).
# length: integer that has the number of bytes from the entry size.
BSM_TOKEN_TRAILER = construct.Struct(
'bsm_token_trailer',
construct.UBInt16('magic'),
construct.UBInt32('record_length'))
# A 32-bits argument.
# num_arg: the number of the argument.
# name_arg: the argument's name.
# text: the string value of the argument.
BSM_TOKEN_ARGUMENT32 = construct.Struct(
'bsm_token_argument32',
construct.UBInt8('num_arg'),
construct.UBInt32('name_arg'),
construct.UBInt16('length'),
construct.Array(_BSMTokenGetLength, construct.UBInt8('text')))
# A 64-bits argument.
# num_arg: integer, the number of the argument.
# name_arg: text, the argument's name.
# text: the string value of the argument.
BSM_TOKEN_ARGUMENT64 = construct.Struct(
'bsm_token_argument64',
construct.UBInt8('num_arg'),
construct.UBInt64('name_arg'),
construct.UBInt16('length'),
construct.Array(_BSMTokenGetLength, construct.UBInt8('text')))
# Identify an user.
# terminal_id: unknown, research needed.
# terminal_addr: unknown, research needed.
BSM_TOKEN_SUBJECT32 = construct.Struct(
'bsm_token_subject32',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt32('terminal_port'),
IPV4_STRUCT)
# Identify an user using a extended Token.
# terminal_port: unknown, need research.
# net_type: unknown, need research.
BSM_TOKEN_SUBJECT32_EX = construct.Struct(
'bsm_token_subject32_ex',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt32('terminal_port'),
BSM_IP_TYPE_SHORT)
# au_to_opaque // AUT_OPAQUE
BSM_TOKEN_OPAQUE = BSM_TOKEN_TEXT
# au_to_seq // AUT_SEQ
BSM_TOKEN_SEQUENCE = BSM_TOKEN_DATA_INTEGER
# Program execution with options.
# For each argument we are going to have a string+ "\x00".
# Example: [00 00 00 02][41 42 43 00 42 42 00]
# 2 Arguments, Arg1: [414243] Arg2: [4242].
BSM_TOKEN_EXEC_ARGUMENTS = construct.UBInt32('number_arguments')
BSM_TOKEN_EXEC_ARGUMENT = construct.Struct(
'bsm_token_exec_argument',
construct.RepeatUntil(
_BSMTokenIsEndOfString, construct.StaticField("text", 1)))
# au_to_in_addr // AUT_IN_ADDR:
BSM_TOKEN_ADDR = IPV4_STRUCT
# au_to_in_addr_ext // AUT_IN_ADDR_EX:
BSM_TOKEN_ADDR_EXT = construct.Struct(
'bsm_token_addr_ext',
construct.UBInt32('net_type'),
IPV6_STRUCT)
# au_to_ip // AUT_IP:
# TODO: parse this header in the correct way.
BSM_TOKEN_IP = construct.String('binary_ipv4_add', 20)
# au_to_ipc // AUT_IPC:
BSM_TOKEN_IPC = construct.Struct(
'bsm_token_ipc',
construct.UBInt8('object_type'),
construct.UBInt32('object_id'))
# au_to_ipc_perm // au_to_ipc_perm
BSM_TOKEN_IPC_PERM = construct.Struct(
'bsm_token_ipc_perm',
construct.UBInt32('user_id'),
construct.UBInt32('group_id'),
construct.UBInt32('creator_user_id'),
construct.UBInt32('creator_group_id'),
construct.UBInt32('access_mode'),
construct.UBInt32('slot_seq'),
construct.UBInt32('key'))
# au_to_iport // AUT_IPORT:
BSM_TOKEN_PORT = construct.UBInt16('port_number')
# au_to_file // AUT_OTHER_FILE32:
BSM_TOKEN_FILE = construct.Struct(
'bsm_token_file',
construct.UBInt32('timestamp'),
construct.UBInt32('microseconds'),
construct.UBInt16('length'),
construct.Array(_BSMTokenGetLength, construct.UBInt8('text')))
# au_to_subject64 // AUT_SUBJECT64:
BSM_TOKEN_SUBJECT64 = construct.Struct(
'bsm_token_subject64',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt64('terminal_port'),
IPV4_STRUCT)
# au_to_subject64_ex // AU_IPv4:
BSM_TOKEN_SUBJECT64_EX = construct.Struct(
'bsm_token_subject64_ex',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt32('terminal_port'),
construct.UBInt32('terminal_type'),
BSM_IP_TYPE_SHORT)
# au_to_process32 // AUT_PROCESS32:
BSM_TOKEN_PROCESS32 = construct.Struct(
'bsm_token_process32',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt32('terminal_port'),
IPV4_STRUCT)
# au_to_process64 // AUT_PROCESS32:
BSM_TOKEN_PROCESS64 = construct.Struct(
'bsm_token_process64',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt64('terminal_port'),
IPV4_STRUCT)
# au_to_process32_ex // AUT_PROCESS32_EX:
BSM_TOKEN_PROCESS32_EX = construct.Struct(
'bsm_token_process32_ex',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt32('terminal_port'),
BSM_IP_TYPE_SHORT)
# au_to_process64_ex // AUT_PROCESS64_EX:
BSM_TOKEN_PROCESS64_EX = construct.Struct(
'bsm_token_process64_ex',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt64('terminal_port'),
BSM_IP_TYPE_SHORT)
# au_to_sock_inet32 // AUT_SOCKINET32:
BSM_TOKEN_AUT_SOCKINET32 = construct.Struct(
'bsm_token_aut_sockinet32',
construct.UBInt16('net_type'),
construct.UBInt16('port_number'),
IPV4_STRUCT)
# Info: checked against the source code of XNU, but not against
# real BSM file.
BSM_TOKEN_AUT_SOCKINET128 = construct.Struct(
'bsm_token_aut_sockinet128',
construct.UBInt16('net_type'),
construct.UBInt16('port_number'),
IPV6_STRUCT)
INET6_ADDR_TYPE = construct.Struct(
'addr_type',
construct.UBInt16('ip_type'),
construct.UBInt16('source_port'),
construct.UBInt64('saddr_high'),
construct.UBInt64('saddr_low'),
construct.UBInt16('destination_port'),
construct.UBInt64('daddr_high'),
construct.UBInt64('daddr_low'))
INET4_ADDR_TYPE = construct.Struct(
'addr_type',
construct.UBInt16('ip_type'),
construct.UBInt16('source_port'),
construct.UBInt32('source_address'),
construct.UBInt16('destination_port'),
construct.UBInt32('destination_address'))
# au_to_socket_ex // AUT_SOCKET_EX
# TODO: Change the 26 for unixbsm.BSM_PROTOCOLS.INET6.
BSM_TOKEN_AUT_SOCKINET32_EX = construct.Struct(
'bsm_token_aut_sockinet32_ex',
construct.UBInt16('socket_domain'),
construct.UBInt16('socket_type'),
construct.Switch(
'structure_addr_port',
_BSMTokenGetSocketDomain,
{26: INET6_ADDR_TYPE},
default=INET4_ADDR_TYPE))
# au_to_sock_unix // AUT_SOCKUNIX
BSM_TOKEN_SOCKET_UNIX = construct.Struct(
'bsm_token_au_to_sock_unix',
construct.UBInt16('family'),
construct.RepeatUntil(
_BSMTokenIsEndOfString,
construct.StaticField("path", 1)))
# au_to_data // au_to_data
# how to print: bsmtoken.BSM_TOKEN_DATA_PRINT.
# type: bsmtoken.BSM_TOKEN_DATA_TYPE.
# unit_count: number of type values.
# BSM_TOKEN_DATA has a end field = type * unit_count
BSM_TOKEN_DATA = construct.Struct(
'bsm_token_data',
construct.UBInt8('how_to_print'),
construct.UBInt8('data_type'),
construct.UBInt8('unit_count'))
# au_to_attr32 // AUT_ATTR32
BSM_TOKEN_ATTR32 = construct.Struct(
'bsm_token_attr32',
construct.UBInt32('file_mode'),
construct.UBInt32('uid'),
construct.UBInt32('gid'),
construct.UBInt32('file_system_id'),
construct.UBInt64('file_system_node_id'),
construct.UBInt32('device'))
# au_to_attr64 // AUT_ATTR64
BSM_TOKEN_ATTR64 = construct.Struct(
'bsm_token_attr64',
construct.UBInt32('file_mode'),
construct.UBInt32('uid'),
construct.UBInt32('gid'),
construct.UBInt32('file_system_id'),
construct.UBInt64('file_system_node_id'),
construct.UBInt64('device'))
# au_to_exit // AUT_EXIT
BSM_TOKEN_EXIT = construct.Struct(
'bsm_token_exit',
construct.UBInt32('status'),
construct.UBInt32('return_value'))
# au_to_newgroups // AUT_NEWGROUPS
# INFO: we must read BSM_TOKEN_DATA_INTEGER for each group.
BSM_TOKEN_GROUPS = construct.UBInt16('group_number')
# au_to_exec_env == au_to_exec_args
BSM_TOKEN_EXEC_ENV = BSM_TOKEN_EXEC_ARGUMENTS
# au_to_zonename //AUT_ZONENAME
BSM_TOKEN_ZONENAME = BSM_TOKEN_TEXT
# Token ID.
# List of valid Token_ID.
# Token_ID -> (NAME_STRUCTURE, STRUCTURE)
# Only the checked structures are been added to the valid structures lists.
_BSM_TOKEN_TYPES = {
17: ('BSM_TOKEN_FILE', BSM_TOKEN_FILE),
19: ('BSM_TOKEN_TRAILER', BSM_TOKEN_TRAILER),
20: ('BSM_HEADER32', BSM_HEADER32),
21: ('BSM_HEADER64', BSM_HEADER64),
33: ('BSM_TOKEN_DATA', BSM_TOKEN_DATA),
34: ('BSM_TOKEN_IPC', BSM_TOKEN_IPC),
35: ('BSM_TOKEN_PATH', BSM_TOKEN_PATH),
36: ('BSM_TOKEN_SUBJECT32', BSM_TOKEN_SUBJECT32),
38: ('BSM_TOKEN_PROCESS32', BSM_TOKEN_PROCESS32),
39: ('BSM_TOKEN_RETURN32', BSM_TOKEN_RETURN32),
40: ('BSM_TOKEN_TEXT', BSM_TOKEN_TEXT),
41: ('BSM_TOKEN_OPAQUE', BSM_TOKEN_OPAQUE),
42: ('BSM_TOKEN_ADDR', BSM_TOKEN_ADDR),
43: ('BSM_TOKEN_IP', BSM_TOKEN_IP),
44: ('BSM_TOKEN_PORT', BSM_TOKEN_PORT),
45: ('BSM_TOKEN_ARGUMENT32', BSM_TOKEN_ARGUMENT32),
47: ('BSM_TOKEN_SEQUENCE', BSM_TOKEN_SEQUENCE),
96: ('BSM_TOKEN_ZONENAME', BSM_TOKEN_ZONENAME),
113: ('BSM_TOKEN_ARGUMENT64', BSM_TOKEN_ARGUMENT64),
114: ('BSM_TOKEN_RETURN64', BSM_TOKEN_RETURN64),
116: ('BSM_HEADER32_EX', BSM_HEADER32_EX),
119: ('BSM_TOKEN_PROCESS64', BSM_TOKEN_PROCESS64),
122: ('BSM_TOKEN_SUBJECT32_EX', BSM_TOKEN_SUBJECT32_EX),
127: ('BSM_TOKEN_AUT_SOCKINET32_EX', BSM_TOKEN_AUT_SOCKINET32_EX),
128: ('BSM_TOKEN_AUT_SOCKINET32', BSM_TOKEN_AUT_SOCKINET32)}
# Untested structures.
# When not tested structure is found, we try to parse using also
# these structures.
BSM_TYPE_LIST_NOT_TESTED = {
49: ('BSM_TOKEN_ATTR', BSM_TOKEN_ATTR32),
50: ('BSM_TOKEN_IPC_PERM', BSM_TOKEN_IPC_PERM),
52: ('BSM_TOKEN_GROUPS', BSM_TOKEN_GROUPS),
59: ('BSM_TOKEN_GROUPS', BSM_TOKEN_GROUPS),
60: ('BSM_TOKEN_EXEC_ARGUMENTS', BSM_TOKEN_EXEC_ARGUMENTS),
61: ('BSM_TOKEN_EXEC_ENV', BSM_TOKEN_EXEC_ENV),
62: ('BSM_TOKEN_ATTR32', BSM_TOKEN_ATTR32),
82: ('BSM_TOKEN_EXIT', BSM_TOKEN_EXIT),
115: ('BSM_TOKEN_ATTR64', BSM_TOKEN_ATTR64),
117: ('BSM_TOKEN_SUBJECT64', BSM_TOKEN_SUBJECT64),
123: ('BSM_TOKEN_PROCESS32_EX', BSM_TOKEN_PROCESS32_EX),
124: ('BSM_TOKEN_PROCESS64_EX', BSM_TOKEN_PROCESS64_EX),
125: ('BSM_TOKEN_SUBJECT64_EX', BSM_TOKEN_SUBJECT64_EX),
126: ('BSM_TOKEN_ADDR_EXT', BSM_TOKEN_ADDR_EXT),
129: ('BSM_TOKEN_AUT_SOCKINET128', BSM_TOKEN_AUT_SOCKINET128),
130: ('BSM_TOKEN_SOCKET_UNIX', BSM_TOKEN_SOCKET_UNIX)}
MESSAGE_CAN_NOT_SAVE = (
'Plaso: some tokens from this entry can not be saved. Entry at 0x{0:X} '
'with unknown token id "0x{1:X}".')
# BSM token types:
# https://github.com/openbsm/openbsm/blob/master/sys/bsm/audit_record.h
_BSM_TOKEN_TYPE_ARGUMENT32 = 45
_BSM_TOKEN_TYPE_ARGUMENT64 = 113
_BSM_TOKEN_TYPE_ATTR = 49
_BSM_TOKEN_TYPE_ATTR32 = 62
_BSM_TOKEN_TYPE_ATTR64 = 115
_BSM_TOKEN_TYPE_EXEC_ARGUMENTS = 60
_BSM_TOKEN_TYPE_EXEC_ENV = 61
_BSM_TOKEN_TYPE_EXIT = 82
_BSM_TOKEN_TYPE_HEADER32 = 20
_BSM_TOKEN_TYPE_HEADER32_EX = 116
_BSM_TOKEN_TYPE_HEADER64 = 21
_BSM_TOKEN_TYPE_PATH = 35
_BSM_TOKEN_TYPE_PROCESS32 = 38
_BSM_TOKEN_TYPE_PROCESS32_EX = 123
_BSM_TOKEN_TYPE_PROCESS64 = 119
_BSM_TOKEN_TYPE_PROCESS64_EX = 124
_BSM_TOKEN_TYPE_RETURN32 = 39
_BSM_TOKEN_TYPE_RETURN64 = 114
_BSM_TOKEN_TYPE_SUBJECT32 = 36
_BSM_TOKEN_TYPE_SUBJECT32_EX = 122
_BSM_TOKEN_TYPE_SUBJECT64 = 117
_BSM_TOKEN_TYPE_SUBJECT64_EX = 125
_BSM_TOKEN_TYPE_TEXT = 40
_BSM_TOKEN_TYPE_ZONENAME = 96
_BSM_ARGUMENT_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_ARGUMENT32,
_BSM_TOKEN_TYPE_ARGUMENT64)
_BSM_ATTR_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_ATTR,
_BSM_TOKEN_TYPE_ATTR32,
_BSM_TOKEN_TYPE_ATTR64)
_BSM_EXEV_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_EXEC_ARGUMENTS,
_BSM_TOKEN_TYPE_EXEC_ENV)
_BSM_HEADER_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_HEADER32,
_BSM_TOKEN_TYPE_HEADER32_EX,
_BSM_TOKEN_TYPE_HEADER64)
_BSM_PROCESS_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_PROCESS32,
_BSM_TOKEN_TYPE_PROCESS64)
_BSM_PROCESS_EX_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_PROCESS32_EX,
_BSM_TOKEN_TYPE_PROCESS64_EX)
_BSM_RETURN_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_EXIT,
_BSM_TOKEN_TYPE_RETURN32,
_BSM_TOKEN_TYPE_RETURN64)
_BSM_SUBJECT_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_SUBJECT32,
_BSM_TOKEN_TYPE_SUBJECT64)
_BSM_SUBJECT_EX_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_SUBJECT32_EX,
_BSM_TOKEN_TYPE_SUBJECT64_EX)
_BSM_UTF8_BYTE_ARRAY_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_PATH,
_BSM_TOKEN_TYPE_TEXT,
_BSM_TOKEN_TYPE_ZONENAME)
def __init__(self):
"""Initializes a parser object."""
super(BSMParser, self).__init__()
# Create the dictionary with all token IDs: tested and untested.
self._bsm_type_list_all = self._BSM_TOKEN_TYPES.copy()
self._bsm_type_list_all.update(self.BSM_TYPE_LIST_NOT_TESTED)
def _CopyByteArrayToBase16String(self, byte_array):
"""Copies a byte array into a base-16 encoded Unicode string.
Args:
byte_array (bytes): A byte array.
Returns:
str: a base-16 encoded Unicode string.
"""
return ''.join(['{0:02x}'.format(byte) for byte in byte_array])
def _CopyUtf8ByteArrayToString(self, byte_array):
"""Copies a UTF-8 encoded byte array into a Unicode string.
Args:
byte_array (bytes): A byte array containing an UTF-8 encoded string.
Returns:
str: A Unicode string.
"""
byte_stream = b''.join(map(chr, byte_array))
try:
string = byte_stream.decode('utf-8')
except UnicodeDecodeError:
logging.warning('Unable to decode UTF-8 formatted byte array.')
string = byte_stream.decode('utf-8', errors='ignore')
string, _, _ = string.partition(b'\x00')
return string
def _IPv4Format(self, address):
"""Formats an IPv4 address as a human readable string.
Args:
address (int): IPv4 address.
Returns:
str: human readable string of IPv4 address in 4 octet representation:
"1.2.3.4".
"""
ipv4_string = self.IPV4_STRUCT.build(address)
return socket.inet_ntoa(ipv4_string)
def _IPv6Format(self, high, low):
"""Formats an IPv6 address as a human readable string.
Args:
high (int): upper 64-bit part of the IPv6 address.
low (int): lower 64-bit part of the IPv6 address.
Returns:
str: human readable string of IPv6 address.
"""
ipv6_string = self.IPV6_STRUCT.build(
construct.Container(high=high, low=low))
# socket.inet_ntop not supported in Windows.
if hasattr(socket, 'inet_ntop'):
return socket.inet_ntop(socket.AF_INET6, ipv6_string)
# TODO: this approach returns double "::", illegal IPv6 addr.
str_address = binascii.hexlify(ipv6_string)
address = []
blank = False
for pos in range(0, len(str_address), 4):
if str_address[pos:pos + 4] == '0000':
if not blank:
address.append('')
blank = True
else:
blank = False
address.append(str_address[pos:pos + 4].lstrip('0'))
return ':'.join(address)
def _ParseBSMEvent(self, parser_mediator, file_object):
"""Parses a BSM entry (BSMEvent) from the file-like object.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): a file-like object.
Returns:
bool: True if the BSM entry was parsed.
"""
record_start_offset = file_object.tell()
try:
token_type = self._BSM_TOKEN.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
parser_mediator.ProduceExtractionError((
'unable to parse BSM token type at offset: 0x{0:08x} with error: '
'{1:s}.').format(record_start_offset, exception))
return False
if token_type not in self._BSM_HEADER_TOKEN_TYPES:
parser_mediator.ProduceExtractionError(
'unsupported token type: {0:d} at offset: 0x{1:08x}.'.format(
token_type, record_start_offset))
# TODO: if it is a Mac OS X, search for the trailer magic value
# as a end of the entry can be a possibility to continue.
return False
_, record_structure = self._BSM_TOKEN_TYPES.get(token_type, ('', None))
try:
token = record_structure.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
parser_mediator.ProduceExtractionError((
'unable to parse BSM record at offset: 0x{0:08x} with error: '
'{1:s}.').format(record_start_offset, exception))
return False
event_type = bsmtoken.BSM_AUDIT_EVENT.get(
token.bsm_header.event_type, 'UNKNOWN')
event_type = '{0:s} ({1:d})'.format(
event_type, token.bsm_header.event_type)
timestamp = (token.timestamp * 1000000) + token.microseconds
date_time = dfdatetime_posix_time.PosixTimeInMicroseconds(
timestamp=timestamp)
record_length = token.bsm_header.length
record_end_offset = record_start_offset + record_length
# A dict of tokens that has the entry.
extra_tokens = {}
# Read until we reach the end of the record.
while file_object.tell() < record_end_offset:
# Check if it is a known token.
try:
token_type = self._BSM_TOKEN.parse_stream(file_object)
except (IOError, construct.FieldError):
logging.warning(
'Unable to parse the Token ID at position: {0:d}'.format(
file_object.tell()))
return False
_, record_structure = self._BSM_TOKEN_TYPES.get(token_type, ('', None))
if not record_structure:
pending = record_end_offset - file_object.tell()
new_extra_tokens = self.TryWithUntestedStructures(
file_object, token_type, pending)
extra_tokens.update(new_extra_tokens)
else:
token = record_structure.parse_stream(file_object)
new_extra_tokens = self.FormatToken(token_type, token, file_object)
extra_tokens.update(new_extra_tokens)
if file_object.tell() > record_end_offset:
logging.warning(
'Token ID {0:d} not expected at position 0x{1:08x}.'
'Jumping for the next entry.'.format(
token_type, file_object.tell()))
try:
file_object.seek(
record_end_offset - file_object.tell(), os.SEEK_CUR)
except (IOError, construct.FieldError) as exception:
logging.warning(
'Unable to jump to next entry with error: {0:s}'.format(exception))
return False
# BSM can be in more than one OS: BSD, Solaris and Mac OS X.
if parser_mediator.platform != 'MacOSX':
event_data = BSMEventData()
else:
event_data = MacBSMEventData()
# In Mac OS X the last two tokens are the return status and the trailer.
return_value = extra_tokens.get('BSM_TOKEN_RETURN32')
if not return_value:
return_value = extra_tokens.get('BSM_TOKEN_RETURN64')
if not return_value:
return_value = 'UNKNOWN'
event_data.return_value = return_value
event_data.event_type = event_type
event_data.extra_tokens = extra_tokens
event_data.offset = record_start_offset
event_data.record_length = record_length
# TODO: check why trailer was passed to event in original while
# event was expecting record length.
# if extra_tokens:
# trailer = extra_tokens.get('BSM_TOKEN_TRAILER', 'unknown')
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_CREATION)
parser_mediator.ProduceEventWithEventData(event, event_data)
return True
def _RawToUTF8(self, byte_stream):
"""Copies a UTF-8 byte stream into a Unicode string.
Args:
byte_stream (bytes): byte stream containing an UTF-8 encoded string.
Returns:
str: A Unicode string.
"""
try:
string = byte_stream.decode('utf-8')
except UnicodeDecodeError:
logging.warning(
'Decode UTF8 failed, the message string may be cut short.')
string = byte_stream.decode('utf-8', errors='ignore')
return string.partition(b'\x00')[0]
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses a BSM file-like object.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): a file-like object.
Raises:
UnableToParseFile: when the file cannot be parsed.
"""
try:
is_bsm = self.VerifyFile(parser_mediator, file_object)
except (IOError, construct.FieldError) as exception:
raise errors.UnableToParseFile(
'Unable to parse BSM file with error: {0:s}'.format(exception))
if not is_bsm:
raise errors.UnableToParseFile('Not a BSM File, unable to parse.')
file_object.seek(0, os.SEEK_SET)
while self._ParseBSMEvent(parser_mediator, file_object):
pass
def VerifyFile(self, parser_mediator, file_object):
"""Check if the file is a BSM file.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): a file-like object.
Returns:
bool: True if this is a valid BSM file, False otherwise.
"""
# First part of the entry is always a Header.
try:
token_type = self._BSM_TOKEN.parse_stream(file_object)
except (IOError, construct.FieldError):
return False
if token_type not in self._BSM_HEADER_TOKEN_TYPES:
return False
_, record_structure = self._BSM_TOKEN_TYPES.get(token_type, ('', None))
try:
header = record_structure.parse_stream(file_object)
except (IOError, construct.FieldError):
return False
if header.bsm_header.version != self.AUDIT_HEADER_VERSION:
return False
try:
token_identifier = self._BSM_TOKEN.parse_stream(file_object)
except (IOError, construct.FieldError):
return False
# If is Mac OS X BSM file, next entry is a text token indicating
# if it is a normal start or it is a recovery track.
if parser_mediator.platform == 'MacOSX':
token_type, record_structure = self._BSM_TOKEN_TYPES.get(
token_identifier, ('', None))
if not record_structure:
return False
if token_type != 'BSM_TOKEN_TEXT':
logging.warning('It is not a valid first entry for Mac OS X BSM.')
return False
try:
token = record_structure.parse_stream(file_object)
except (IOError, construct.FieldError):
return
text = self._CopyUtf8ByteArrayToString(token.text)
if (text != 'launchctl::Audit startup' and
text != 'launchctl::Audit recovery'):
logging.warning('It is not a valid first entry for Mac OS X BSM.')
return False
return True
def TryWithUntestedStructures(self, file_object, token_id, pending):
"""Try to parse the pending part of the entry using untested structures.
Args:
file_object: BSM file.
token_id: integer with the id that comes from the unknown token.
pending: pending length of the entry.
Returns:
A list of extra tokens data that can be parsed using non-tested
structures. A message indicating that a structure cannot be parsed
is added for unparsed structures.
"""
# Data from the unknown structure.
start_position = file_object.tell()
start_token_id = token_id
extra_tokens = {}
# Read all the "pending" bytes.
try:
if token_id in self._bsm_type_list_all:
token = self._bsm_type_list_all[token_id][1].parse_stream(file_object)
new_extra_tokens = self.FormatToken(token_id, token, file_object)
extra_tokens.update(new_extra_tokens)
while file_object.tell() < (start_position + pending):
# Check if it is a known token.
try:
token_id = self._BSM_TOKEN.parse_stream(file_object)
except (IOError, construct.FieldError):
logging.warning(
'Unable to parse the Token ID at position: {0:d}'.format(
file_object.tell()))
return
if token_id not in self._bsm_type_list_all:
break
token = self._bsm_type_list_all[token_id][1].parse_stream(file_object)
new_extra_tokens = self.FormatToken(token_id, token, file_object)
extra_tokens.update(new_extra_tokens)
except (IOError, construct.FieldError):
token_id = 255
next_entry = (start_position + pending)
if file_object.tell() != next_entry:
# Unknown Structure.
logging.warning('Unknown Token at "0x{0:X}", ID: {1} (0x{2:X})'.format(
start_position - 1, token_id, token_id))
# TODO: another way to save this information must be found.
extra_tokens.update(
{'message': self.MESSAGE_CAN_NOT_SAVE.format(
start_position - 1, start_token_id)})
# Move to next entry.
file_object.seek(next_entry - file_object.tell(), os.SEEK_CUR)
# It returns null list because it doesn't know witch structure was
# the incorrect structure that makes that it can arrive to the spected
# end of the entry.
return {}
return extra_tokens
def FormatToken(self, token_id, token, file_object):
"""Parse the Token depending of the type of the structure.
Args:
token_id (int): identification of the token_type.
token (structure): token struct to parse.
file_object: BSM file.
Returns:
(dict): parsed Token values.
Keys for returned dictionary are token name like BSM_TOKEN_SUBJECT32.
Values of this dictionary are key-value pairs like terminal_ip:127.0.0.1.
"""
if token_id not in self._bsm_type_list_all:
return {}
bsm_type, _ = self._bsm_type_list_all.get(token_id, ['', ''])
if token_id in self._BSM_UTF8_BYTE_ARRAY_TOKEN_TYPES:
try:
string = self._CopyUtf8ByteArrayToString(token.text)
except TypeError:
string = 'Unknown'
return {bsm_type: string}
elif token_id in self._BSM_RETURN_TOKEN_TYPES:
return {bsm_type: {
'error': bsmtoken.BSM_ERRORS.get(token.status, 'Unknown'),
'token_status': token.status,
'call_status': token.return_value
}}
elif token_id in self._BSM_SUBJECT_TOKEN_TYPES:
return {bsm_type: {
'aid': token.subject_data.audit_uid,
'euid': token.subject_data.effective_uid,
'egid': token.subject_data.effective_gid,
'uid': token.subject_data.real_uid,
'gid': token.subject_data.real_gid,
'pid': token.subject_data.pid,
'session_id': token.subject_data.session_id,
'terminal_port': token.terminal_port,
'terminal_ip': self._IPv4Format(token.ipv4)
}}
elif token_id in self._BSM_SUBJECT_EX_TOKEN_TYPES:
if token.bsm_ip_type_short.net_type == self.AU_IPv6:
ip = self._IPv6Format(
token.bsm_ip_type_short.ip_addr.high,
token.bsm_ip_type_short.ip_addr.low)
elif token.bsm_ip_type_short.net_type == self.AU_IPv4:
ip = self._IPv4Format(token.bsm_ip_type_short.ip_addr)
else:
ip = 'unknown'
return {bsm_type: {
'aid': token.subject_data.audit_uid,
'euid': token.subject_data.effective_uid,
'egid': token.subject_data.effective_gid,
'uid': token.subject_data.real_uid,
'gid': token.subject_data.real_gid,
'pid': token.subject_data.pid,
'session_id': token.subject_data.session_id,
'terminal_port': token.terminal_port,
'terminal_ip': ip
}}
elif token_id in self._BSM_ARGUMENT_TOKEN_TYPES:
string = self._CopyUtf8ByteArrayToString(token.text)
return {bsm_type: {
'string': string,
'num_arg': token.num_arg,
'is': token.name_arg}}
elif token_id in self._BSM_EXEV_TOKEN_TYPES:
arguments = []
for _ in range(0, token):
sub_token = self.BSM_TOKEN_EXEC_ARGUMENT.parse_stream(file_object)
string = self._CopyUtf8ByteArrayToString(sub_token.text)
arguments.append(string)
return {bsm_type: ' '.join(arguments)}
elif bsm_type == 'BSM_TOKEN_AUT_SOCKINET32':
return {bsm_type: {
'protocols':
bsmtoken.BSM_PROTOCOLS.get(token.net_type, 'UNKNOWN'),
'net_type': token.net_type,
'port': token.port_number,
'address': self._IPv4Format(token.ipv4)
}}
elif bsm_type == 'BSM_TOKEN_AUT_SOCKINET128':
return {bsm_type: {
'protocols':
bsmtoken.BSM_PROTOCOLS.get(token.net_type, 'UNKNOWN'),
'net_type': token.net_type,
'port': token.port_number,
'address': self._IPv6Format(token.ipv6.high, token.ipv6.low)
}}
elif bsm_type == 'BSM_TOKEN_ADDR':
return {bsm_type: self._IPv4Format(token)}
elif bsm_type == 'BSM_TOKEN_IP':
return {'IPv4_Header': '0x{0:s}]'.format(token.encode('hex'))}
elif bsm_type == 'BSM_TOKEN_ADDR_EXT':
return {bsm_type: {
'protocols':
bsmtoken.BSM_PROTOCOLS.get(token.net_type, 'UNKNOWN'),
'net_type': token.net_type,
'address': self._IPv6Format(token.ipv6.high, token.ipv6.low)
}}
elif bsm_type == 'BSM_TOKEN_PORT':
return {bsm_type: token}
elif bsm_type == 'BSM_TOKEN_TRAILER':
return {bsm_type: token.record_length}
elif bsm_type == 'BSM_TOKEN_FILE':
# TODO: if this timestamp is usefull, it must be extracted as a separate
# event object.
timestamp = timelib.Timestamp.FromPosixTimeWithMicrosecond(
token.timestamp, token.microseconds)
date_time = timelib.Timestamp.CopyToDatetime(timestamp, pytz.UTC)
date_time_string = date_time.strftime('%Y-%m-%d %H:%M:%S')
string = self._CopyUtf8ByteArrayToString(token.text)
return {bsm_type: {'string': string, 'timestamp': date_time_string}}
elif bsm_type == 'BSM_TOKEN_IPC':
return {bsm_type: {
'object_type': token.object_type,
'object_id': token.object_id
}}
elif token_id in self._BSM_PROCESS_TOKEN_TYPES:
return {bsm_type: {
'aid': token.subject_data.audit_uid,
'euid': token.subject_data.effective_uid,
'egid': token.subject_data.effective_gid,
'uid': token.subject_data.real_uid,
'gid': token.subject_data.real_gid,
'pid': token.subject_data.pid,
'session_id': token.subject_data.session_id,
'terminal_port': token.terminal_port,
'terminal_ip': self._IPv4Format(token.ipv4)
}}
elif token_id in self._BSM_PROCESS_EX_TOKEN_TYPES:
if token.bsm_ip_type_short.net_type == self.AU_IPv6:
ip = self._IPv6Format(
token.bsm_ip_type_short.ip_addr.high,
token.bsm_ip_type_short.ip_addr.low)
elif token.bsm_ip_type_short.net_type == self.AU_IPv4:
ip = self._IPv4Format(token.bsm_ip_type_short.ip_addr)
else:
ip = 'unknown'
return {bsm_type: {
'aid': token.subject_data.audit_uid,
'euid': token.subject_data.effective_uid,
'egid': token.subject_data.effective_gid,
'uid': token.subject_data.real_uid,
'gid': token.subject_data.real_gid,
'pid': token.subject_data.pid,
'session_id': token.subject_data.session_id,
'terminal_port': token.terminal_port,
'terminal_ip': ip
}}
elif bsm_type == 'BSM_TOKEN_DATA':
data = []
data_type = bsmtoken.BSM_TOKEN_DATA_TYPE.get(token.data_type, '')
if data_type == 'AUR_CHAR':
for _ in range(token.unit_count):
data.append(self.BSM_TOKEN_DATA_CHAR.parse_stream(file_object))
elif data_type == 'AUR_SHORT':
for _ in range(token.unit_count):
data.append(self.BSM_TOKEN_DATA_SHORT.parse_stream(file_object))
elif data_type == 'AUR_INT32':
for _ in range(token.unit_count):
data.append(self.BSM_TOKEN_DATA_INTEGER.parse_stream(file_object))
else:
data.append('Unknown type data')
# TODO: the data when it is string ends with ".", HW a space is return
# after uses the UTF-8 conversion.
return {bsm_type: {
'format': bsmtoken.BSM_TOKEN_DATA_PRINT[token.how_to_print],
'data':
'{0}'.format(self._RawToUTF8(''.join(map(str, data))))
}}
elif token_id in self._BSM_ATTR_TOKEN_TYPES:
return {bsm_type: {
'mode': token.file_mode,
'uid': token.uid,
'gid': token.gid,
'system_id': token.file_system_id,
'node_id': token.file_system_node_id,
'device': token.device}}
elif bsm_type == 'BSM_TOKEN_GROUPS':
arguments = []
for _ in range(token):
arguments.append(
self._RawToUTF8(
self.BSM_TOKEN_DATA_INTEGER.parse_stream(file_object)))
return {bsm_type: ','.join(arguments)}
elif bsm_type == 'BSM_TOKEN_AUT_SOCKINET32_EX':
if bsmtoken.BSM_PROTOCOLS.get(token.socket_domain, '') == 'INET6':
saddr = self._IPv6Format(
token.structure_addr_port.saddr_high,
token.structure_addr_port.saddr_low)
daddr = self._IPv6Format(
token.structure_addr_port.daddr_high,
token.structure_addr_port.daddr_low)
else:
saddr = self._IPv4Format(token.structure_addr_port.source_address)
daddr = self._IPv4Format(token.structure_addr_port.destination_address)
return {bsm_type:{
'from': saddr,
'from_port': token.structure_addr_port.source_port,
'to': daddr,
'to_port': token.structure_addr_port.destination_port}}
elif bsm_type == 'BSM_TOKEN_IPC_PERM':
return {bsm_type: {
'user_id': token.user_id,
'group_id': token.group_id,
'creator_user_id': token.creator_user_id,
'creator_group_id': token.creator_group_id,
'access': token.access_mode}}
elif bsm_type == 'BSM_TOKEN_SOCKET_UNIX':
string = self._CopyUtf8ByteArrayToString(token.path)
return {bsm_type: {'family': token.family, 'path': string}}
elif bsm_type == 'BSM_TOKEN_OPAQUE':
string = self._CopyByteArrayToBase16String(token.text)
return {bsm_type: string}
elif bsm_type == 'BSM_TOKEN_SEQUENCE':
return {bsm_type: token}
else:
print hexdump(buf[pos:pos + 0x0A])
pos = pos + 0x0A
i = i + 1
print "i : %X" % i
obj_unk_00 = construct.Struct(
"unk_field_00" / construct.Int16ul, # + 0x00
"unk_field_02" / construct.Int16ul, # + 0x02
"unk_field_04" / construct.Int16ul, # + 0x04
"unk_field_06" / construct.Int16ul, # + 0x06
"entries" / construct.Array(
lambda ctx: ctx.unk_field_06,
construct.Struct(
"unk_field_00" / construct.Int16ul, # + 0x00
"unk_field_02" / construct.Int16ul, # + 0x02
"unk_field_04" / construct.Int16ul, # + 0x04
"unk_field_06" / construct.Int16ul, # + 0x06
"unk_field_08" / construct.Int16ul, # + 0x08
"unk_field_0A" / construct.Int16ul, # + 0x0A
)))
rawf.stream.seek(rawf.raw_file.field_C, 0x00)
o = obj_unk_00.parse_stream(rawf.stream)
print o
# unk_word_00 : 0x18C4
# field_12 : 0x024C
# field_14 : 0x0614
# field_16 : 0x026A
# field_18 : 0x0008
# field_1A : 0x04BC
c.FocusedSeq(
"keyvalue",
"terminator" / c.Peek(c.Byte),
c.StopIf(c.this.terminator == 0),
"keyvalue" / KeyValue,
)
),
c.Const(b'\0'),
)
PSBT = c.Struct(
"magic" / c.Const(b'psbt'),
"sep" / c.Const(b'\xff'),
"general" / Sequence,
"transaction" / c.RestreamData(c.this.general[0].value, Transaction),
"inputs" / c.Array(c.len_(c.this.transaction.inputs), Sequence),
"outputs" / c.Array(c.len_(c.this.transaction.outputs), Sequence),
c.Terminated,
)
# key-less format:
ValueOnly = c.Prefixed(CompactUint, c.Struct(
"type" / c.Byte,
"value" / c.GreedyBytes,
))
ValueSequence = c.FocusedSeq("content",
"content" / c.GreedyRange(
c.FocusedSeq("valueonly",
"terminator" / c.Peek(c.Byte),
"VERSTRING" / c.CString("utf8"))
RateTCmd = c.Struct(
"FIELD_ID" / c.Const(ControlPacketField.build("PKT_RATET")),
"RATE_IDX" / c.Byte)
RateTResp = c.Struct(
"FIELD_ID" / c.Const(ControlPacketField.build("PKT_RATET")),
"RESULT" / c.Byte)
###############################################################################
# Speech Messages
SpeechPCM = c.Struct(
"FIELD_ID" / c.Const(b'\x00'),
"NUM_SAMPLES" / c.Byte, # 156 - 164
"DATA" / c.Array(c.this.NUM_SAMPLES, c.Int16ub))
SpeechCMODE = c.Struct("FIELD_ID" / c.Const(b'\x02'), "CMODE_IN" / ECMODE_IN)
SpeechDCMODE = c.Struct("FIELD_ID" / c.Const(b'\x02'),
"DCMODE_OUT" / DCMODE_OUT)
SpeechTONE = c.Struct(
"FIELD_ID" / c.Const(b'\x08'),
"TONE_IDX" / c.Byte,
"TONE_AMPLITUDE" / c.Byte,
)
SpeechPCMPacket = c.Struct(
"FIELD_ID" / c.Const(b'\x40'),
"SPEECHD" / c.Optional(SpeechPCM),
class BinaryCookieParser(interface.FileObjectParser):
"""Parser for Safari Binary Cookie files."""
NAME = u'binary_cookies'
DESCRIPTION = u'Parser for Safari Binary Cookie files.'
COOKIE_HEADER = construct.Struct(
u'binary_cookie_header', construct.UBInt32(u'pages'),
construct.Array(lambda ctx: ctx.pages,
construct.UBInt32(u'page_sizes')))
COOKIE_DATA = construct.Struct(u'binary_cookie_data',
construct.ULInt32(u'size'),
construct.Bytes(u'unknown_1', 4),
construct.ULInt32(u'flags'),
construct.Bytes(u'unknown_2', 4),
construct.ULInt32(u'url_offset'),
construct.ULInt32(u'name_offset'),
construct.ULInt32(u'path_offset'),
construct.ULInt32(u'value_offset'),
construct.Bytes(u'end_of_cookie', 8),
construct.LFloat64(u'expiration_date'),
construct.LFloat64(u'creation_date'))
PAGE_DATA = construct.Struct(
u'page_data', construct.Bytes(u'header', 4),
construct.ULInt32(u'number_of_cookies'),
construct.Array(lambda ctx: ctx.number_of_cookies,
construct.ULInt32(u'offsets')))
# Cookie flags.
COOKIE_FLAG_NONE = 0
COOKIE_FLAG_SECURE = 1
COOKIE_FLAG_UNKNOWN = 2
COOKIE_FLAG_HTTP_ONLY = 4
def __init__(self):
"""Initializes a parser object."""
super(BinaryCookieParser, self).__init__()
self._cookie_plugins = (
cookie_plugins_manager.CookiePluginsManager.GetPlugins())
def _ParsePage(self, page_data, parser_mediator):
"""Extract events from a page and produce events.
Args:
page_data: Raw bytes of the page.
file_entry: The file entry (instance of dfvfs.FileEntry).
parser_mediator: A parser mediator object (instance of ParserMediator).
"""
try:
page = self.PAGE_DATA.parse(page_data)
except construct.FieldError:
parser_mediator.ProduceParseError(u'Unable to parse page')
return
for page_offset in page.offsets:
try:
cookie = self.COOKIE_DATA.parse(page_data[page_offset:])
except construct.FieldError:
message = u'Unable to parse cookie data from offset: {0:d}'.format(
page_offset)
parser_mediator.ProduceParseError(message)
continue
# The offset is determine by the range between the start of the current
# offset until the start of the next offset. Thus we need to determine
# the proper ordering of the offsets, since they are not always in the
# same ordering.
offset_dict = {
cookie.url_offset: u'url',
cookie.name_offset: u'name',
cookie.value_offset: u'value',
cookie.path_offset: u'path'
}
offsets = sorted(offset_dict.keys())
offsets.append(cookie.size + page_offset)
# TODO: Find a better approach to parsing the data than this.
data_dict = {}
for current_offset in range(0, len(offsets) - 1):
# Get the current offset and the offset for the next entry.
start, end = offsets[current_offset:current_offset + 2]
value = offset_dict.get(offsets[current_offset])
# Read the data.
data_all = page_data[start + page_offset:end + page_offset]
data, _, _ = data_all.partition(b'\x00')
data_dict[value] = data
url = data_dict.get(u'url')
cookie_name = data_dict.get(u'name')
cookie_value = data_dict.get(u'value')
path = data_dict.get(u'path')
flags = []
flag_int = cookie.flags
if flag_int & self.COOKIE_FLAG_HTTP_ONLY:
flags.append(u'HttpOnly')
if flag_int & self.COOKIE_FLAG_UNKNOWN:
flags.append(u'Unknown')
if flag_int & self.COOKIE_FLAG_SECURE:
flags.append(u'Secure')
cookie_flags = u'|'.join(flags)
if cookie.creation_date:
event_object = BinaryCookieEvent(
cookie.creation_date,
eventdata.EventTimestamp.CREATION_TIME, cookie_flags, url,
cookie_value, cookie_name, path)
parser_mediator.ProduceEvent(event_object)
if cookie.expiration_date:
event_object = BinaryCookieEvent(
cookie.expiration_date,
eventdata.EventTimestamp.EXPIRATION_TIME, cookie_flags,
url, cookie_value, cookie_name, path)
parser_mediator.ProduceEvent(event_object)
for cookie_plugin in self._cookie_plugins:
try:
cookie_plugin.UpdateChainAndProcess(
parser_mediator,
cookie_name=data_dict.get(u'name'),
cookie_data=data_dict.get(u'value'),
url=data_dict.get(u'url'))
except errors.WrongPlugin:
pass
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses a Safari binary cookie file-like object.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
file_object: A file-like object.
Raises:
UnableToParseFile: when the file cannot be parsed.
"""
# Start by verifying magic value.
# We do this here instead of in the header parsing routine due to the
# fact that we read an integer there and create an array, which is part
# of the header. For false hits this could end up with reading large chunks
# of data, which we don't want for false hits.
magic = file_object.read(4)
if magic != b'cook':
raise errors.UnableToParseFile(
u'The file is not a Binary Cookie file. Unsupported file signature.'
)
try:
header = self.COOKIE_HEADER.parse_stream(file_object)
except (IOError, construct.ArrayError, construct.FieldError):
raise errors.UnableToParseFile(
u'The file is not a Binary Cookie file (bad header).')
for page_size in header.page_sizes:
page = file_object.read(page_size)
if len(page) != page_size:
parser_mediator.ProduceParseError(
u'Unable to continue parsing Binary Cookie file')
break
self._ParsePage(page, parser_mediator)