class BSMParser(interface.FileObjectParser):
"""Parser for BSM files."""
NAME = 'bsm_log'
DESCRIPTION = 'Parser for BSM log files.'
# BSM supported version (0x0b = 11).
AUDIT_HEADER_VERSION = 11
# Magic Trail Header.
BSM_TOKEN_TRAILER_MAGIC = b'b105'
# IP Version constants.
AU_IPv4 = 4
AU_IPv6 = 16
IPV4_STRUCT = construct.UBInt32('ipv4')
IPV6_STRUCT = construct.Struct(
'ipv6',
construct.UBInt64('high'),
construct.UBInt64('low'))
# Tested structures.
# INFO: I have ommited the ID in the structures declaration.
# I used the BSM_TYPE first to read the ID, and then, the structure.
# Tokens always start with an ID value that identifies their token
# type and subsequent structure.
_BSM_TOKEN = construct.UBInt8('token_id')
# Data type structures.
BSM_TOKEN_DATA_CHAR = construct.String('value', 1)
BSM_TOKEN_DATA_SHORT = construct.UBInt16('value')
BSM_TOKEN_DATA_INTEGER = construct.UBInt32('value')
# Common structure used by other structures.
# audit_uid: integer, uid that generates the entry.
# effective_uid: integer, the permission user used.
# effective_gid: integer, the permission group used.
# real_uid: integer, user id of the user that execute the process.
# real_gid: integer, group id of the group that execute the process.
# pid: integer, identification number of the process.
# session_id: unknown, need research.
BSM_TOKEN_SUBJECT_SHORT = construct.Struct(
'subject_data',
construct.UBInt32('audit_uid'),
construct.UBInt32('effective_uid'),
construct.UBInt32('effective_gid'),
construct.UBInt32('real_uid'),
construct.UBInt32('real_gid'),
construct.UBInt32('pid'),
construct.UBInt32('session_id'))
# Common structure used by other structures.
# Identify the kind of inet (IPv4 or IPv6)
# TODO: instead of 16, AU_IPv6 must be used.
BSM_IP_TYPE_SHORT = construct.Struct(
'bsm_ip_type_short',
construct.UBInt32('net_type'),
construct.Switch(
'ip_addr',
_BSMTokenGetNetType,
{16: IPV6_STRUCT},
default=IPV4_STRUCT))
# Initial fields structure used by header structures.
# length: integer, the length of the entry, equal to trailer (doc: length).
# version: integer, version of BSM (AUDIT_HEADER_VERSION).
# event_type: integer, the type of event (/etc/security/audit_event).
# modifier: integer, unknown, need research (It is always 0).
BSM_HEADER = construct.Struct(
'bsm_header',
construct.UBInt32('length'),
construct.UBInt8('version'),
construct.UBInt16('event_type'),
construct.UBInt16('modifier'))
# First token of one entry.
# timestamp: unsigned integer, number of seconds since
# January 1, 1970 00:00:00 UTC.
# microseconds: unsigned integer, number of micro seconds.
BSM_HEADER32 = construct.Struct(
'bsm_header32',
BSM_HEADER,
construct.UBInt32('timestamp'),
construct.UBInt32('microseconds'))
BSM_HEADER64 = construct.Struct(
'bsm_header64',
BSM_HEADER,
construct.UBInt64('timestamp'),
construct.UBInt64('microseconds'))
BSM_HEADER32_EX = construct.Struct(
'bsm_header32_ex',
BSM_HEADER,
BSM_IP_TYPE_SHORT,
construct.UBInt32('timestamp'),
construct.UBInt32('microseconds'))
# Token TEXT, provides extra information.
BSM_TOKEN_TEXT = construct.Struct(
'bsm_token_text',
construct.UBInt16('length'),
construct.Array(_BSMTokenGetLength, construct.UBInt8('text')))
# Path of the executable.
BSM_TOKEN_PATH = BSM_TOKEN_TEXT
# Identified the end of the record (follow by TRAILER).
# status: integer that identifies the status of the exit (BSM_ERRORS).
# return: returned value from the operation.
BSM_TOKEN_RETURN32 = construct.Struct(
'bsm_token_return32',
construct.UBInt8('status'),
construct.UBInt32('return_value'))
BSM_TOKEN_RETURN64 = construct.Struct(
'bsm_token_return64',
construct.UBInt8('status'),
construct.UBInt64('return_value'))
# Identified the number of bytes that was written.
# magic: 2 bytes that identifies the TRAILER (BSM_TOKEN_TRAILER_MAGIC).
# length: integer that has the number of bytes from the entry size.
BSM_TOKEN_TRAILER = construct.Struct(
'bsm_token_trailer',
construct.UBInt16('magic'),
construct.UBInt32('record_length'))
# A 32-bits argument.
# num_arg: the number of the argument.
# name_arg: the argument's name.
# text: the string value of the argument.
BSM_TOKEN_ARGUMENT32 = construct.Struct(
'bsm_token_argument32',
construct.UBInt8('num_arg'),
construct.UBInt32('name_arg'),
construct.UBInt16('length'),
construct.Array(_BSMTokenGetLength, construct.UBInt8('text')))
# A 64-bits argument.
# num_arg: integer, the number of the argument.
# name_arg: text, the argument's name.
# text: the string value of the argument.
BSM_TOKEN_ARGUMENT64 = construct.Struct(
'bsm_token_argument64',
construct.UBInt8('num_arg'),
construct.UBInt64('name_arg'),
construct.UBInt16('length'),
construct.Array(_BSMTokenGetLength, construct.UBInt8('text')))
# Identify an user.
# terminal_id: unknown, research needed.
# terminal_addr: unknown, research needed.
BSM_TOKEN_SUBJECT32 = construct.Struct(
'bsm_token_subject32',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt32('terminal_port'),
IPV4_STRUCT)
# Identify an user using a extended Token.
# terminal_port: unknown, need research.
# net_type: unknown, need research.
BSM_TOKEN_SUBJECT32_EX = construct.Struct(
'bsm_token_subject32_ex',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt32('terminal_port'),
BSM_IP_TYPE_SHORT)
# au_to_opaque // AUT_OPAQUE
BSM_TOKEN_OPAQUE = BSM_TOKEN_TEXT
# au_to_seq // AUT_SEQ
BSM_TOKEN_SEQUENCE = BSM_TOKEN_DATA_INTEGER
# Program execution with options.
# For each argument we are going to have a string+ "\x00".
# Example: [00 00 00 02][41 42 43 00 42 42 00]
# 2 Arguments, Arg1: [414243] Arg2: [4242].
BSM_TOKEN_EXEC_ARGUMENTS = construct.UBInt32('number_arguments')
BSM_TOKEN_EXEC_ARGUMENT = construct.Struct(
'bsm_token_exec_argument',
construct.RepeatUntil(
_BSMTokenIsEndOfString, construct.StaticField("text", 1)))
# au_to_in_addr // AUT_IN_ADDR:
BSM_TOKEN_ADDR = IPV4_STRUCT
# au_to_in_addr_ext // AUT_IN_ADDR_EX:
BSM_TOKEN_ADDR_EXT = construct.Struct(
'bsm_token_addr_ext',
construct.UBInt32('net_type'),
IPV6_STRUCT)
# au_to_ip // AUT_IP:
# TODO: parse this header in the correct way.
BSM_TOKEN_IP = construct.String('binary_ipv4_add', 20)
# au_to_ipc // AUT_IPC:
BSM_TOKEN_IPC = construct.Struct(
'bsm_token_ipc',
construct.UBInt8('object_type'),
construct.UBInt32('object_id'))
# au_to_ipc_perm // au_to_ipc_perm
BSM_TOKEN_IPC_PERM = construct.Struct(
'bsm_token_ipc_perm',
construct.UBInt32('user_id'),
construct.UBInt32('group_id'),
construct.UBInt32('creator_user_id'),
construct.UBInt32('creator_group_id'),
construct.UBInt32('access_mode'),
construct.UBInt32('slot_seq'),
construct.UBInt32('key'))
# au_to_iport // AUT_IPORT:
BSM_TOKEN_PORT = construct.UBInt16('port_number')
# au_to_file // AUT_OTHER_FILE32:
BSM_TOKEN_FILE = construct.Struct(
'bsm_token_file',
construct.UBInt32('timestamp'),
construct.UBInt32('microseconds'),
construct.UBInt16('length'),
construct.Array(_BSMTokenGetLength, construct.UBInt8('text')))
# au_to_subject64 // AUT_SUBJECT64:
BSM_TOKEN_SUBJECT64 = construct.Struct(
'bsm_token_subject64',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt64('terminal_port'),
IPV4_STRUCT)
# au_to_subject64_ex // AU_IPv4:
BSM_TOKEN_SUBJECT64_EX = construct.Struct(
'bsm_token_subject64_ex',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt32('terminal_port'),
construct.UBInt32('terminal_type'),
BSM_IP_TYPE_SHORT)
# au_to_process32 // AUT_PROCESS32:
BSM_TOKEN_PROCESS32 = construct.Struct(
'bsm_token_process32',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt32('terminal_port'),
IPV4_STRUCT)
# au_to_process64 // AUT_PROCESS32:
BSM_TOKEN_PROCESS64 = construct.Struct(
'bsm_token_process64',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt64('terminal_port'),
IPV4_STRUCT)
# au_to_process32_ex // AUT_PROCESS32_EX:
BSM_TOKEN_PROCESS32_EX = construct.Struct(
'bsm_token_process32_ex',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt32('terminal_port'),
BSM_IP_TYPE_SHORT)
# au_to_process64_ex // AUT_PROCESS64_EX:
BSM_TOKEN_PROCESS64_EX = construct.Struct(
'bsm_token_process64_ex',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt64('terminal_port'),
BSM_IP_TYPE_SHORT)
# au_to_sock_inet32 // AUT_SOCKINET32:
BSM_TOKEN_AUT_SOCKINET32 = construct.Struct(
'bsm_token_aut_sockinet32',
construct.UBInt16('net_type'),
construct.UBInt16('port_number'),
IPV4_STRUCT)
# Info: checked against the source code of XNU, but not against
# real BSM file.
BSM_TOKEN_AUT_SOCKINET128 = construct.Struct(
'bsm_token_aut_sockinet128',
construct.UBInt16('net_type'),
construct.UBInt16('port_number'),
IPV6_STRUCT)
INET6_ADDR_TYPE = construct.Struct(
'addr_type',
construct.UBInt16('ip_type'),
construct.UBInt16('source_port'),
construct.UBInt64('saddr_high'),
construct.UBInt64('saddr_low'),
construct.UBInt16('destination_port'),
construct.UBInt64('daddr_high'),
construct.UBInt64('daddr_low'))
INET4_ADDR_TYPE = construct.Struct(
'addr_type',
construct.UBInt16('ip_type'),
construct.UBInt16('source_port'),
construct.UBInt32('source_address'),
construct.UBInt16('destination_port'),
construct.UBInt32('destination_address'))
# au_to_socket_ex // AUT_SOCKET_EX
# TODO: Change the 26 for unixbsm.BSM_PROTOCOLS.INET6.
BSM_TOKEN_AUT_SOCKINET32_EX = construct.Struct(
'bsm_token_aut_sockinet32_ex',
construct.UBInt16('socket_domain'),
construct.UBInt16('socket_type'),
construct.Switch(
'structure_addr_port',
_BSMTokenGetSocketDomain,
{26: INET6_ADDR_TYPE},
default=INET4_ADDR_TYPE))
# au_to_sock_unix // AUT_SOCKUNIX
BSM_TOKEN_SOCKET_UNIX = construct.Struct(
'bsm_token_au_to_sock_unix',
construct.UBInt16('family'),
construct.RepeatUntil(
_BSMTokenIsEndOfString,
construct.StaticField("path", 1)))
# au_to_data // au_to_data
# how to print: bsmtoken.BSM_TOKEN_DATA_PRINT.
# type: bsmtoken.BSM_TOKEN_DATA_TYPE.
# unit_count: number of type values.
# BSM_TOKEN_DATA has a end field = type * unit_count
BSM_TOKEN_DATA = construct.Struct(
'bsm_token_data',
construct.UBInt8('how_to_print'),
construct.UBInt8('data_type'),
construct.UBInt8('unit_count'))
# au_to_attr32 // AUT_ATTR32
BSM_TOKEN_ATTR32 = construct.Struct(
'bsm_token_attr32',
construct.UBInt32('file_mode'),
construct.UBInt32('uid'),
construct.UBInt32('gid'),
construct.UBInt32('file_system_id'),
construct.UBInt64('file_system_node_id'),
construct.UBInt32('device'))
# au_to_attr64 // AUT_ATTR64
BSM_TOKEN_ATTR64 = construct.Struct(
'bsm_token_attr64',
construct.UBInt32('file_mode'),
construct.UBInt32('uid'),
construct.UBInt32('gid'),
construct.UBInt32('file_system_id'),
construct.UBInt64('file_system_node_id'),
construct.UBInt64('device'))
# au_to_exit // AUT_EXIT
BSM_TOKEN_EXIT = construct.Struct(
'bsm_token_exit',
construct.UBInt32('status'),
construct.UBInt32('return_value'))
# au_to_newgroups // AUT_NEWGROUPS
# INFO: we must read BSM_TOKEN_DATA_INTEGER for each group.
BSM_TOKEN_GROUPS = construct.UBInt16('group_number')
# au_to_exec_env == au_to_exec_args
BSM_TOKEN_EXEC_ENV = BSM_TOKEN_EXEC_ARGUMENTS
# au_to_zonename //AUT_ZONENAME
BSM_TOKEN_ZONENAME = BSM_TOKEN_TEXT
# Token ID.
# List of valid Token_ID.
# Token_ID -> (NAME_STRUCTURE, STRUCTURE)
# Only the checked structures are been added to the valid structures lists.
_BSM_TOKEN_TYPES = {
17: ('BSM_TOKEN_FILE', BSM_TOKEN_FILE),
19: ('BSM_TOKEN_TRAILER', BSM_TOKEN_TRAILER),
20: ('BSM_HEADER32', BSM_HEADER32),
21: ('BSM_HEADER64', BSM_HEADER64),
33: ('BSM_TOKEN_DATA', BSM_TOKEN_DATA),
34: ('BSM_TOKEN_IPC', BSM_TOKEN_IPC),
35: ('BSM_TOKEN_PATH', BSM_TOKEN_PATH),
36: ('BSM_TOKEN_SUBJECT32', BSM_TOKEN_SUBJECT32),
38: ('BSM_TOKEN_PROCESS32', BSM_TOKEN_PROCESS32),
39: ('BSM_TOKEN_RETURN32', BSM_TOKEN_RETURN32),
40: ('BSM_TOKEN_TEXT', BSM_TOKEN_TEXT),
41: ('BSM_TOKEN_OPAQUE', BSM_TOKEN_OPAQUE),
42: ('BSM_TOKEN_ADDR', BSM_TOKEN_ADDR),
43: ('BSM_TOKEN_IP', BSM_TOKEN_IP),
44: ('BSM_TOKEN_PORT', BSM_TOKEN_PORT),
45: ('BSM_TOKEN_ARGUMENT32', BSM_TOKEN_ARGUMENT32),
47: ('BSM_TOKEN_SEQUENCE', BSM_TOKEN_SEQUENCE),
96: ('BSM_TOKEN_ZONENAME', BSM_TOKEN_ZONENAME),
113: ('BSM_TOKEN_ARGUMENT64', BSM_TOKEN_ARGUMENT64),
114: ('BSM_TOKEN_RETURN64', BSM_TOKEN_RETURN64),
116: ('BSM_HEADER32_EX', BSM_HEADER32_EX),
119: ('BSM_TOKEN_PROCESS64', BSM_TOKEN_PROCESS64),
122: ('BSM_TOKEN_SUBJECT32_EX', BSM_TOKEN_SUBJECT32_EX),
127: ('BSM_TOKEN_AUT_SOCKINET32_EX', BSM_TOKEN_AUT_SOCKINET32_EX),
128: ('BSM_TOKEN_AUT_SOCKINET32', BSM_TOKEN_AUT_SOCKINET32)}
# Untested structures.
# When not tested structure is found, we try to parse using also
# these structures.
BSM_TYPE_LIST_NOT_TESTED = {
49: ('BSM_TOKEN_ATTR', BSM_TOKEN_ATTR32),
50: ('BSM_TOKEN_IPC_PERM', BSM_TOKEN_IPC_PERM),
52: ('BSM_TOKEN_GROUPS', BSM_TOKEN_GROUPS),
59: ('BSM_TOKEN_GROUPS', BSM_TOKEN_GROUPS),
60: ('BSM_TOKEN_EXEC_ARGUMENTS', BSM_TOKEN_EXEC_ARGUMENTS),
61: ('BSM_TOKEN_EXEC_ENV', BSM_TOKEN_EXEC_ENV),
62: ('BSM_TOKEN_ATTR32', BSM_TOKEN_ATTR32),
82: ('BSM_TOKEN_EXIT', BSM_TOKEN_EXIT),
115: ('BSM_TOKEN_ATTR64', BSM_TOKEN_ATTR64),
117: ('BSM_TOKEN_SUBJECT64', BSM_TOKEN_SUBJECT64),
123: ('BSM_TOKEN_PROCESS32_EX', BSM_TOKEN_PROCESS32_EX),
124: ('BSM_TOKEN_PROCESS64_EX', BSM_TOKEN_PROCESS64_EX),
125: ('BSM_TOKEN_SUBJECT64_EX', BSM_TOKEN_SUBJECT64_EX),
126: ('BSM_TOKEN_ADDR_EXT', BSM_TOKEN_ADDR_EXT),
129: ('BSM_TOKEN_AUT_SOCKINET128', BSM_TOKEN_AUT_SOCKINET128),
130: ('BSM_TOKEN_SOCKET_UNIX', BSM_TOKEN_SOCKET_UNIX)}
MESSAGE_CAN_NOT_SAVE = (
'Plaso: some tokens from this entry can not be saved. Entry at 0x{0:X} '
'with unknown token id "0x{1:X}".')
# BSM token types:
# https://github.com/openbsm/openbsm/blob/master/sys/bsm/audit_record.h
_BSM_TOKEN_TYPE_ARGUMENT32 = 45
_BSM_TOKEN_TYPE_ARGUMENT64 = 113
_BSM_TOKEN_TYPE_ATTR = 49
_BSM_TOKEN_TYPE_ATTR32 = 62
_BSM_TOKEN_TYPE_ATTR64 = 115
_BSM_TOKEN_TYPE_EXEC_ARGUMENTS = 60
_BSM_TOKEN_TYPE_EXEC_ENV = 61
_BSM_TOKEN_TYPE_EXIT = 82
_BSM_TOKEN_TYPE_HEADER32 = 20
_BSM_TOKEN_TYPE_HEADER32_EX = 116
_BSM_TOKEN_TYPE_HEADER64 = 21
_BSM_TOKEN_TYPE_PATH = 35
_BSM_TOKEN_TYPE_PROCESS32 = 38
_BSM_TOKEN_TYPE_PROCESS32_EX = 123
_BSM_TOKEN_TYPE_PROCESS64 = 119
_BSM_TOKEN_TYPE_PROCESS64_EX = 124
_BSM_TOKEN_TYPE_RETURN32 = 39
_BSM_TOKEN_TYPE_RETURN64 = 114
_BSM_TOKEN_TYPE_SUBJECT32 = 36
_BSM_TOKEN_TYPE_SUBJECT32_EX = 122
_BSM_TOKEN_TYPE_SUBJECT64 = 117
_BSM_TOKEN_TYPE_SUBJECT64_EX = 125
_BSM_TOKEN_TYPE_TEXT = 40
_BSM_TOKEN_TYPE_ZONENAME = 96
_BSM_ARGUMENT_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_ARGUMENT32,
_BSM_TOKEN_TYPE_ARGUMENT64)
_BSM_ATTR_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_ATTR,
_BSM_TOKEN_TYPE_ATTR32,
_BSM_TOKEN_TYPE_ATTR64)
_BSM_EXEV_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_EXEC_ARGUMENTS,
_BSM_TOKEN_TYPE_EXEC_ENV)
_BSM_HEADER_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_HEADER32,
_BSM_TOKEN_TYPE_HEADER32_EX,
_BSM_TOKEN_TYPE_HEADER64)
_BSM_PROCESS_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_PROCESS32,
_BSM_TOKEN_TYPE_PROCESS64)
_BSM_PROCESS_EX_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_PROCESS32_EX,
_BSM_TOKEN_TYPE_PROCESS64_EX)
_BSM_RETURN_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_EXIT,
_BSM_TOKEN_TYPE_RETURN32,
_BSM_TOKEN_TYPE_RETURN64)
_BSM_SUBJECT_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_SUBJECT32,
_BSM_TOKEN_TYPE_SUBJECT64)
_BSM_SUBJECT_EX_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_SUBJECT32_EX,
_BSM_TOKEN_TYPE_SUBJECT64_EX)
_BSM_UTF8_BYTE_ARRAY_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_PATH,
_BSM_TOKEN_TYPE_TEXT,
_BSM_TOKEN_TYPE_ZONENAME)
def __init__(self):
"""Initializes a parser object."""
super(BSMParser, self).__init__()
# Create the dictionary with all token IDs: tested and untested.
self._bsm_type_list_all = self._BSM_TOKEN_TYPES.copy()
self._bsm_type_list_all.update(self.BSM_TYPE_LIST_NOT_TESTED)
def _CopyByteArrayToBase16String(self, byte_array):
"""Copies a byte array into a base-16 encoded Unicode string.
Args:
byte_array (bytes): A byte array.
Returns:
str: a base-16 encoded Unicode string.
"""
return ''.join(['{0:02x}'.format(byte) for byte in byte_array])
def _CopyUtf8ByteArrayToString(self, byte_array):
"""Copies a UTF-8 encoded byte array into a Unicode string.
Args:
byte_array (bytes): A byte array containing an UTF-8 encoded string.
Returns:
str: A Unicode string.
"""
byte_stream = b''.join(map(chr, byte_array))
try:
string = byte_stream.decode('utf-8')
except UnicodeDecodeError:
logging.warning('Unable to decode UTF-8 formatted byte array.')
string = byte_stream.decode('utf-8', errors='ignore')
string, _, _ = string.partition(b'\x00')
return string
def _IPv4Format(self, address):
"""Formats an IPv4 address as a human readable string.
Args:
address (int): IPv4 address.
Returns:
str: human readable string of IPv4 address in 4 octet representation:
"1.2.3.4".
"""
ipv4_string = self.IPV4_STRUCT.build(address)
return socket.inet_ntoa(ipv4_string)
def _IPv6Format(self, high, low):
"""Formats an IPv6 address as a human readable string.
Args:
high (int): upper 64-bit part of the IPv6 address.
low (int): lower 64-bit part of the IPv6 address.
Returns:
str: human readable string of IPv6 address.
"""
ipv6_string = self.IPV6_STRUCT.build(
construct.Container(high=high, low=low))
# socket.inet_ntop not supported in Windows.
if hasattr(socket, 'inet_ntop'):
return socket.inet_ntop(socket.AF_INET6, ipv6_string)
# TODO: this approach returns double "::", illegal IPv6 addr.
str_address = binascii.hexlify(ipv6_string)
address = []
blank = False
for pos in range(0, len(str_address), 4):
if str_address[pos:pos + 4] == '0000':
if not blank:
address.append('')
blank = True
else:
blank = False
address.append(str_address[pos:pos + 4].lstrip('0'))
return ':'.join(address)
def _ParseBSMEvent(self, parser_mediator, file_object):
"""Parses a BSM entry (BSMEvent) from the file-like object.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): a file-like object.
Returns:
bool: True if the BSM entry was parsed.
"""
record_start_offset = file_object.tell()
try:
token_type = self._BSM_TOKEN.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
parser_mediator.ProduceExtractionError((
'unable to parse BSM token type at offset: 0x{0:08x} with error: '
'{1:s}.').format(record_start_offset, exception))
return False
if token_type not in self._BSM_HEADER_TOKEN_TYPES:
parser_mediator.ProduceExtractionError(
'unsupported token type: {0:d} at offset: 0x{1:08x}.'.format(
token_type, record_start_offset))
# TODO: if it is a Mac OS X, search for the trailer magic value
# as a end of the entry can be a possibility to continue.
return False
_, record_structure = self._BSM_TOKEN_TYPES.get(token_type, ('', None))
try:
token = record_structure.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
parser_mediator.ProduceExtractionError((
'unable to parse BSM record at offset: 0x{0:08x} with error: '
'{1:s}.').format(record_start_offset, exception))
return False
event_type = bsmtoken.BSM_AUDIT_EVENT.get(
token.bsm_header.event_type, 'UNKNOWN')
event_type = '{0:s} ({1:d})'.format(
event_type, token.bsm_header.event_type)
timestamp = (token.timestamp * 1000000) + token.microseconds
date_time = dfdatetime_posix_time.PosixTimeInMicroseconds(
timestamp=timestamp)
record_length = token.bsm_header.length
record_end_offset = record_start_offset + record_length
# A dict of tokens that has the entry.
extra_tokens = {}
# Read until we reach the end of the record.
while file_object.tell() < record_end_offset:
# Check if it is a known token.
try:
token_type = self._BSM_TOKEN.parse_stream(file_object)
except (IOError, construct.FieldError):
logging.warning(
'Unable to parse the Token ID at position: {0:d}'.format(
file_object.tell()))
return False
_, record_structure = self._BSM_TOKEN_TYPES.get(token_type, ('', None))
if not record_structure:
pending = record_end_offset - file_object.tell()
new_extra_tokens = self.TryWithUntestedStructures(
file_object, token_type, pending)
extra_tokens.update(new_extra_tokens)
else:
token = record_structure.parse_stream(file_object)
new_extra_tokens = self.FormatToken(token_type, token, file_object)
extra_tokens.update(new_extra_tokens)
if file_object.tell() > record_end_offset:
logging.warning(
'Token ID {0:d} not expected at position 0x{1:08x}.'
'Jumping for the next entry.'.format(
token_type, file_object.tell()))
try:
file_object.seek(
record_end_offset - file_object.tell(), os.SEEK_CUR)
except (IOError, construct.FieldError) as exception:
logging.warning(
'Unable to jump to next entry with error: {0:s}'.format(exception))
return False
# BSM can be in more than one OS: BSD, Solaris and Mac OS X.
if parser_mediator.platform != 'MacOSX':
event_data = BSMEventData()
else:
event_data = MacBSMEventData()
# In Mac OS X the last two tokens are the return status and the trailer.
return_value = extra_tokens.get('BSM_TOKEN_RETURN32')
if not return_value:
return_value = extra_tokens.get('BSM_TOKEN_RETURN64')
if not return_value:
return_value = 'UNKNOWN'
event_data.return_value = return_value
event_data.event_type = event_type
event_data.extra_tokens = extra_tokens
event_data.offset = record_start_offset
event_data.record_length = record_length
# TODO: check why trailer was passed to event in original while
# event was expecting record length.
# if extra_tokens:
# trailer = extra_tokens.get('BSM_TOKEN_TRAILER', 'unknown')
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_CREATION)
parser_mediator.ProduceEventWithEventData(event, event_data)
return True
def _RawToUTF8(self, byte_stream):
"""Copies a UTF-8 byte stream into a Unicode string.
Args:
byte_stream (bytes): byte stream containing an UTF-8 encoded string.
Returns:
str: A Unicode string.
"""
try:
string = byte_stream.decode('utf-8')
except UnicodeDecodeError:
logging.warning(
'Decode UTF8 failed, the message string may be cut short.')
string = byte_stream.decode('utf-8', errors='ignore')
return string.partition(b'\x00')[0]
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses a BSM file-like object.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): a file-like object.
Raises:
UnableToParseFile: when the file cannot be parsed.
"""
try:
is_bsm = self.VerifyFile(parser_mediator, file_object)
except (IOError, construct.FieldError) as exception:
raise errors.UnableToParseFile(
'Unable to parse BSM file with error: {0:s}'.format(exception))
if not is_bsm:
raise errors.UnableToParseFile('Not a BSM File, unable to parse.')
file_object.seek(0, os.SEEK_SET)
while self._ParseBSMEvent(parser_mediator, file_object):
pass
def VerifyFile(self, parser_mediator, file_object):
"""Check if the file is a BSM file.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): a file-like object.
Returns:
bool: True if this is a valid BSM file, False otherwise.
"""
# First part of the entry is always a Header.
try:
token_type = self._BSM_TOKEN.parse_stream(file_object)
except (IOError, construct.FieldError):
return False
if token_type not in self._BSM_HEADER_TOKEN_TYPES:
return False
_, record_structure = self._BSM_TOKEN_TYPES.get(token_type, ('', None))
try:
header = record_structure.parse_stream(file_object)
except (IOError, construct.FieldError):
return False
if header.bsm_header.version != self.AUDIT_HEADER_VERSION:
return False
try:
token_identifier = self._BSM_TOKEN.parse_stream(file_object)
except (IOError, construct.FieldError):
return False
# If is Mac OS X BSM file, next entry is a text token indicating
# if it is a normal start or it is a recovery track.
if parser_mediator.platform == 'MacOSX':
token_type, record_structure = self._BSM_TOKEN_TYPES.get(
token_identifier, ('', None))
if not record_structure:
return False
if token_type != 'BSM_TOKEN_TEXT':
logging.warning('It is not a valid first entry for Mac OS X BSM.')
return False
try:
token = record_structure.parse_stream(file_object)
except (IOError, construct.FieldError):
return
text = self._CopyUtf8ByteArrayToString(token.text)
if (text != 'launchctl::Audit startup' and
text != 'launchctl::Audit recovery'):
logging.warning('It is not a valid first entry for Mac OS X BSM.')
return False
return True
def TryWithUntestedStructures(self, file_object, token_id, pending):
"""Try to parse the pending part of the entry using untested structures.
Args:
file_object: BSM file.
token_id: integer with the id that comes from the unknown token.
pending: pending length of the entry.
Returns:
A list of extra tokens data that can be parsed using non-tested
structures. A message indicating that a structure cannot be parsed
is added for unparsed structures.
"""
# Data from the unknown structure.
start_position = file_object.tell()
start_token_id = token_id
extra_tokens = {}
# Read all the "pending" bytes.
try:
if token_id in self._bsm_type_list_all:
token = self._bsm_type_list_all[token_id][1].parse_stream(file_object)
new_extra_tokens = self.FormatToken(token_id, token, file_object)
extra_tokens.update(new_extra_tokens)
while file_object.tell() < (start_position + pending):
# Check if it is a known token.
try:
token_id = self._BSM_TOKEN.parse_stream(file_object)
except (IOError, construct.FieldError):
logging.warning(
'Unable to parse the Token ID at position: {0:d}'.format(
file_object.tell()))
return
if token_id not in self._bsm_type_list_all:
break
token = self._bsm_type_list_all[token_id][1].parse_stream(file_object)
new_extra_tokens = self.FormatToken(token_id, token, file_object)
extra_tokens.update(new_extra_tokens)
except (IOError, construct.FieldError):
token_id = 255
next_entry = (start_position + pending)
if file_object.tell() != next_entry:
# Unknown Structure.
logging.warning('Unknown Token at "0x{0:X}", ID: {1} (0x{2:X})'.format(
start_position - 1, token_id, token_id))
# TODO: another way to save this information must be found.
extra_tokens.update(
{'message': self.MESSAGE_CAN_NOT_SAVE.format(
start_position - 1, start_token_id)})
# Move to next entry.
file_object.seek(next_entry - file_object.tell(), os.SEEK_CUR)
# It returns null list because it doesn't know witch structure was
# the incorrect structure that makes that it can arrive to the spected
# end of the entry.
return {}
return extra_tokens
def FormatToken(self, token_id, token, file_object):
"""Parse the Token depending of the type of the structure.
Args:
token_id (int): identification of the token_type.
token (structure): token struct to parse.
file_object: BSM file.
Returns:
(dict): parsed Token values.
Keys for returned dictionary are token name like BSM_TOKEN_SUBJECT32.
Values of this dictionary are key-value pairs like terminal_ip:127.0.0.1.
"""
if token_id not in self._bsm_type_list_all:
return {}
bsm_type, _ = self._bsm_type_list_all.get(token_id, ['', ''])
if token_id in self._BSM_UTF8_BYTE_ARRAY_TOKEN_TYPES:
try:
string = self._CopyUtf8ByteArrayToString(token.text)
except TypeError:
string = 'Unknown'
return {bsm_type: string}
elif token_id in self._BSM_RETURN_TOKEN_TYPES:
return {bsm_type: {
'error': bsmtoken.BSM_ERRORS.get(token.status, 'Unknown'),
'token_status': token.status,
'call_status': token.return_value
}}
elif token_id in self._BSM_SUBJECT_TOKEN_TYPES:
return {bsm_type: {
'aid': token.subject_data.audit_uid,
'euid': token.subject_data.effective_uid,
'egid': token.subject_data.effective_gid,
'uid': token.subject_data.real_uid,
'gid': token.subject_data.real_gid,
'pid': token.subject_data.pid,
'session_id': token.subject_data.session_id,
'terminal_port': token.terminal_port,
'terminal_ip': self._IPv4Format(token.ipv4)
}}
elif token_id in self._BSM_SUBJECT_EX_TOKEN_TYPES:
if token.bsm_ip_type_short.net_type == self.AU_IPv6:
ip = self._IPv6Format(
token.bsm_ip_type_short.ip_addr.high,
token.bsm_ip_type_short.ip_addr.low)
elif token.bsm_ip_type_short.net_type == self.AU_IPv4:
ip = self._IPv4Format(token.bsm_ip_type_short.ip_addr)
else:
ip = 'unknown'
return {bsm_type: {
'aid': token.subject_data.audit_uid,
'euid': token.subject_data.effective_uid,
'egid': token.subject_data.effective_gid,
'uid': token.subject_data.real_uid,
'gid': token.subject_data.real_gid,
'pid': token.subject_data.pid,
'session_id': token.subject_data.session_id,
'terminal_port': token.terminal_port,
'terminal_ip': ip
}}
elif token_id in self._BSM_ARGUMENT_TOKEN_TYPES:
string = self._CopyUtf8ByteArrayToString(token.text)
return {bsm_type: {
'string': string,
'num_arg': token.num_arg,
'is': token.name_arg}}
elif token_id in self._BSM_EXEV_TOKEN_TYPES:
arguments = []
for _ in range(0, token):
sub_token = self.BSM_TOKEN_EXEC_ARGUMENT.parse_stream(file_object)
string = self._CopyUtf8ByteArrayToString(sub_token.text)
arguments.append(string)
return {bsm_type: ' '.join(arguments)}
elif bsm_type == 'BSM_TOKEN_AUT_SOCKINET32':
return {bsm_type: {
'protocols':
bsmtoken.BSM_PROTOCOLS.get(token.net_type, 'UNKNOWN'),
'net_type': token.net_type,
'port': token.port_number,
'address': self._IPv4Format(token.ipv4)
}}
elif bsm_type == 'BSM_TOKEN_AUT_SOCKINET128':
return {bsm_type: {
'protocols':
bsmtoken.BSM_PROTOCOLS.get(token.net_type, 'UNKNOWN'),
'net_type': token.net_type,
'port': token.port_number,
'address': self._IPv6Format(token.ipv6.high, token.ipv6.low)
}}
elif bsm_type == 'BSM_TOKEN_ADDR':
return {bsm_type: self._IPv4Format(token)}
elif bsm_type == 'BSM_TOKEN_IP':
return {'IPv4_Header': '0x{0:s}]'.format(token.encode('hex'))}
elif bsm_type == 'BSM_TOKEN_ADDR_EXT':
return {bsm_type: {
'protocols':
bsmtoken.BSM_PROTOCOLS.get(token.net_type, 'UNKNOWN'),
'net_type': token.net_type,
'address': self._IPv6Format(token.ipv6.high, token.ipv6.low)
}}
elif bsm_type == 'BSM_TOKEN_PORT':
return {bsm_type: token}
elif bsm_type == 'BSM_TOKEN_TRAILER':
return {bsm_type: token.record_length}
elif bsm_type == 'BSM_TOKEN_FILE':
# TODO: if this timestamp is usefull, it must be extracted as a separate
# event object.
timestamp = timelib.Timestamp.FromPosixTimeWithMicrosecond(
token.timestamp, token.microseconds)
date_time = timelib.Timestamp.CopyToDatetime(timestamp, pytz.UTC)
date_time_string = date_time.strftime('%Y-%m-%d %H:%M:%S')
string = self._CopyUtf8ByteArrayToString(token.text)
return {bsm_type: {'string': string, 'timestamp': date_time_string}}
elif bsm_type == 'BSM_TOKEN_IPC':
return {bsm_type: {
'object_type': token.object_type,
'object_id': token.object_id
}}
elif token_id in self._BSM_PROCESS_TOKEN_TYPES:
return {bsm_type: {
'aid': token.subject_data.audit_uid,
'euid': token.subject_data.effective_uid,
'egid': token.subject_data.effective_gid,
'uid': token.subject_data.real_uid,
'gid': token.subject_data.real_gid,
'pid': token.subject_data.pid,
'session_id': token.subject_data.session_id,
'terminal_port': token.terminal_port,
'terminal_ip': self._IPv4Format(token.ipv4)
}}
elif token_id in self._BSM_PROCESS_EX_TOKEN_TYPES:
if token.bsm_ip_type_short.net_type == self.AU_IPv6:
ip = self._IPv6Format(
token.bsm_ip_type_short.ip_addr.high,
token.bsm_ip_type_short.ip_addr.low)
elif token.bsm_ip_type_short.net_type == self.AU_IPv4:
ip = self._IPv4Format(token.bsm_ip_type_short.ip_addr)
else:
ip = 'unknown'
return {bsm_type: {
'aid': token.subject_data.audit_uid,
'euid': token.subject_data.effective_uid,
'egid': token.subject_data.effective_gid,
'uid': token.subject_data.real_uid,
'gid': token.subject_data.real_gid,
'pid': token.subject_data.pid,
'session_id': token.subject_data.session_id,
'terminal_port': token.terminal_port,
'terminal_ip': ip
}}
elif bsm_type == 'BSM_TOKEN_DATA':
data = []
data_type = bsmtoken.BSM_TOKEN_DATA_TYPE.get(token.data_type, '')
if data_type == 'AUR_CHAR':
for _ in range(token.unit_count):
data.append(self.BSM_TOKEN_DATA_CHAR.parse_stream(file_object))
elif data_type == 'AUR_SHORT':
for _ in range(token.unit_count):
data.append(self.BSM_TOKEN_DATA_SHORT.parse_stream(file_object))
elif data_type == 'AUR_INT32':
for _ in range(token.unit_count):
data.append(self.BSM_TOKEN_DATA_INTEGER.parse_stream(file_object))
else:
data.append('Unknown type data')
# TODO: the data when it is string ends with ".", HW a space is return
# after uses the UTF-8 conversion.
return {bsm_type: {
'format': bsmtoken.BSM_TOKEN_DATA_PRINT[token.how_to_print],
'data':
'{0}'.format(self._RawToUTF8(''.join(map(str, data))))
}}
elif token_id in self._BSM_ATTR_TOKEN_TYPES:
return {bsm_type: {
'mode': token.file_mode,
'uid': token.uid,
'gid': token.gid,
'system_id': token.file_system_id,
'node_id': token.file_system_node_id,
'device': token.device}}
elif bsm_type == 'BSM_TOKEN_GROUPS':
arguments = []
for _ in range(token):
arguments.append(
self._RawToUTF8(
self.BSM_TOKEN_DATA_INTEGER.parse_stream(file_object)))
return {bsm_type: ','.join(arguments)}
elif bsm_type == 'BSM_TOKEN_AUT_SOCKINET32_EX':
if bsmtoken.BSM_PROTOCOLS.get(token.socket_domain, '') == 'INET6':
saddr = self._IPv6Format(
token.structure_addr_port.saddr_high,
token.structure_addr_port.saddr_low)
daddr = self._IPv6Format(
token.structure_addr_port.daddr_high,
token.structure_addr_port.daddr_low)
else:
saddr = self._IPv4Format(token.structure_addr_port.source_address)
daddr = self._IPv4Format(token.structure_addr_port.destination_address)
return {bsm_type:{
'from': saddr,
'from_port': token.structure_addr_port.source_port,
'to': daddr,
'to_port': token.structure_addr_port.destination_port}}
elif bsm_type == 'BSM_TOKEN_IPC_PERM':
return {bsm_type: {
'user_id': token.user_id,
'group_id': token.group_id,
'creator_user_id': token.creator_user_id,
'creator_group_id': token.creator_group_id,
'access': token.access_mode}}
elif bsm_type == 'BSM_TOKEN_SOCKET_UNIX':
string = self._CopyUtf8ByteArrayToString(token.path)
return {bsm_type: {'family': token.family, 'path': string}}
elif bsm_type == 'BSM_TOKEN_OPAQUE':
string = self._CopyByteArrayToBase16String(token.text)
return {bsm_type: string}
elif bsm_type == 'BSM_TOKEN_SEQUENCE':
return {bsm_type: token}
class JavaIDXParser(interface.BaseParser):
"""Parse Java IDX files for download events.
There are five structures defined. 6.02 files had one generic section
that retained all data. From 6.03, the file went to a multi-section
format where later sections were optional and had variable-lengths.
6.03, 6.04, and 6.05 files all have their main data section (#2)
begin at offset 128. The short structure is because 6.05 files
deviate after the 8th byte. So, grab the first 8 bytes to ensure it's
valid, get the file version, then continue on with the correct
structures.
"""
NAME = 'java_idx'
DESCRIPTION = u'Parser for Java IDX files.'
IDX_SHORT_STRUCT = construct.Struct('magic', construct.UBInt8('busy'),
construct.UBInt8('incomplete'),
construct.UBInt32('idx_version'))
IDX_602_STRUCT = construct.Struct(
'IDX_602_Full', construct.UBInt16('null_space'),
construct.UBInt8('shortcut'), construct.UBInt32('content_length'),
construct.UBInt64('last_modified_date'),
construct.UBInt64('expiration_date'),
construct.PascalString('version_string',
length_field=construct.UBInt16('length')),
construct.PascalString('url',
length_field=construct.UBInt16('length')),
construct.PascalString('namespace',
length_field=construct.UBInt16('length')),
construct.UBInt32('FieldCount'))
IDX_605_SECTION_ONE_STRUCT = construct.Struct(
'IDX_605_Section1', construct.UBInt8('shortcut'),
construct.UBInt32('content_length'),
construct.UBInt64('last_modified_date'),
construct.UBInt64('expiration_date'),
construct.UBInt64('validation_date'), construct.UBInt8('signed'),
construct.UBInt32('sec2len'), construct.UBInt32('sec3len'),
construct.UBInt32('sec4len'))
IDX_605_SECTION_TWO_STRUCT = construct.Struct(
'IDX_605_Section2',
construct.PascalString('version',
length_field=construct.UBInt16('length')),
construct.PascalString('url',
length_field=construct.UBInt16('length')),
construct.PascalString('namespec',
length_field=construct.UBInt16('length')),
construct.PascalString('ip_address',
length_field=construct.UBInt16('length')),
construct.UBInt32('FieldCount'))
# Java uses Pascal-style strings, but with a 2-byte length field.
JAVA_READUTF_STRING = construct.Struct(
'Java.ReadUTF',
construct.PascalString('string',
length_field=construct.UBInt16('length')))
def Parse(self, parser_context, file_entry):
"""Extract data from a Java cache IDX file.
This is the main parsing engine for the parser. It determines if
the selected file is a proper IDX file. It then checks the file
version to determine the correct structure to apply to extract
data.
Args:
parser_context: A parser context object (instance of ParserContext).
file_entry: A file entry object (instance of dfvfs.FileEntry).
"""
file_object = file_entry.GetFileObject()
try:
magic = self.IDX_SHORT_STRUCT.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
raise errors.UnableToParseFile(
u'Unable to parse Java IDX file with error: {0:s}.'.format(
exception))
# Fields magic.busy and magic.incomplete are normally 0x00. They
# are set to 0x01 if the file is currently being downloaded. Logic
# checks for > 1 to avoid a race condition and still reject any
# file with other data.
# Field magic.idx_version is the file version, of which only
# certain versions are supported.
if magic.busy > 1 or magic.incomplete > 1:
raise errors.UnableToParseFile(u'Not a valid Java IDX file')
if not magic.idx_version in [602, 603, 604, 605]:
raise errors.UnableToParseFile(u'Not a valid Java IDX file')
# Obtain the relevant values from the file. The last modified date
# denotes when the file was last modified on the HOST. For example,
# when the file was uploaded to a web server.
if magic.idx_version == 602:
section_one = self.IDX_602_STRUCT.parse_stream(file_object)
last_modified_date = section_one.last_modified_date
url = section_one.url
ip_address = 'Unknown'
http_header_count = section_one.FieldCount
elif magic.idx_version in [603, 604, 605]:
# IDX 6.03 and 6.04 have two unused bytes before the structure.
if magic.idx_version in [603, 604]:
file_object.read(2)
# IDX 6.03, 6.04, and 6.05 files use the same structures for the
# remaining data.
section_one = self.IDX_605_SECTION_ONE_STRUCT.parse_stream(
file_object)
last_modified_date = section_one.last_modified_date
if file_object.get_size() > 128:
file_object.seek(128) # Static offset for section 2.
section_two = self.IDX_605_SECTION_TWO_STRUCT.parse_stream(
file_object)
url = section_two.url
ip_address = section_two.ip_address
http_header_count = section_two.FieldCount
else:
url = 'Unknown'
ip_address = 'Unknown'
http_header_count = 0
# File offset is now just prior to HTTP headers. Make sure there
# are headers, and then parse them to retrieve the download date.
download_date = None
for field in range(0, http_header_count):
field = self.JAVA_READUTF_STRING.parse_stream(file_object)
value = self.JAVA_READUTF_STRING.parse_stream(file_object)
if field.string == 'date':
# Time string "should" be in UTC or have an associated time zone
# information in the string itself. If that is not the case then
# there is no reliable method for plaso to determine the proper
# timezone, so the assumption is that it is UTC.
download_date = timelib.Timestamp.FromTimeString(
value.string, gmt_as_timezone=False)
if not url or not ip_address:
raise errors.UnableToParseFile(
u'Unexpected Error: URL or IP address not found in file.')
last_modified_timestamp = timelib.Timestamp.FromJavaTime(
last_modified_date)
# TODO: Move the timestamp description fields into eventdata.
event_object = JavaIDXEvent(last_modified_timestamp,
'File Hosted Date', magic.idx_version, url,
ip_address)
parser_context.ProduceEvent(event_object,
parser_name=self.NAME,
file_entry=file_entry)
if section_one:
expiration_date = section_one.get('expiration_date', None)
if expiration_date:
expiration_timestamp = timelib.Timestamp.FromJavaTime(
expiration_date)
event_object = JavaIDXEvent(expiration_timestamp,
'File Expiration Date',
magic.idx_version, url, ip_address)
parser_context.ProduceEvent(event_object,
parser_name=self.NAME,
file_entry=file_entry)
if download_date:
event_object = JavaIDXEvent(
download_date, eventdata.EventTimestamp.FILE_DOWNLOADED,
magic.idx_version, url, ip_address)
parser_context.ProduceEvent(event_object,
parser_name=self.NAME,
file_entry=file_entry)
class ESEDBPlugin(plugins.BasePlugin):
"""The ESE database plugin interface."""
NAME = 'esedb'
BINARY_DATA_COLUMN_TYPES = frozenset([
pyesedb.column_types.BINARY_DATA,
pyesedb.column_types.LARGE_BINARY_DATA])
FLOATING_POINT_COLUMN_TYPES = frozenset([
pyesedb.column_types.FLOAT_32BIT,
pyesedb.column_types.DOUBLE_64BIT])
INTEGER_COLUMN_TYPES = frozenset([
pyesedb.column_types.CURRENCY,
pyesedb.column_types.DATE_TIME,
pyesedb.column_types.INTEGER_8BIT_UNSIGNED,
pyesedb.column_types.INTEGER_16BIT_SIGNED,
pyesedb.column_types.INTEGER_16BIT_UNSIGNED,
pyesedb.column_types.INTEGER_32BIT_SIGNED,
pyesedb.column_types.INTEGER_32BIT_UNSIGNED,
pyesedb.column_types.INTEGER_64BIT_SIGNED])
STRING_COLUMN_TYPES = frozenset([
pyesedb.column_types.TEXT,
pyesedb.column_types.LARGE_TEXT])
_UINT64_BIG_ENDIAN = construct.UBInt64('value')
_UINT64_LITTLE_ENDIAN = construct.ULInt64('value')
# Dictionary containing a callback method per table name.
# E.g. 'SystemIndex_0A': 'ParseSystemIndex_0A'
REQUIRED_TABLES = {}
OPTIONAL_TABLES = {}
def __init__(self):
"""Initializes the ESE database plugin."""
super(ESEDBPlugin, self).__init__()
self._tables = {}
self._tables.update(self.REQUIRED_TABLES)
self._tables.update(self.OPTIONAL_TABLES)
@property
def required_tables(self):
"""set[str]: required table names."""
return frozenset(self.REQUIRED_TABLES.keys())
def _ConvertValueBinaryDataToStringAscii(self, value):
"""Converts a binary data value into a string.
Args:
value (bytes): binary data value containing an ASCII string or None.
Returns:
str: string representation of binary data value or None.
"""
if value:
return value.decode('ascii')
return None
def _ConvertValueBinaryDataToStringBase16(self, value):
"""Converts a binary data value into a base-16 (hexadecimal) string.
Args:
value (bytes): binary data value or None.
Returns:
str: string representation of binary data value or None.
"""
if value:
return value.encode('hex')
return None
def _ConvertValueBinaryDataToUBInt64(self, value):
"""Converts a binary data value into an integer.
Args:
value (bytes): binary data value containing an unsigned 64-bit big-endian
integer.
Returns:
int: integer representation of binary data value or None.
"""
if value:
return self._UINT64_BIG_ENDIAN.parse(value)
return None
def _ConvertValueBinaryDataToULInt64(self, value):
"""Converts a binary data value into an integer.
Args:
value (int): binary data value containing an unsigned 64-bit little-endian
integer.
Returns:
int: integer representation of binary data value or None.
"""
if value:
return self._UINT64_LITTLE_ENDIAN.parse(value)
return None
def _GetRecordValue(self, record, value_entry):
"""Retrieves a specific value from the record.
Args:
record (pyesedb.record): ESE record.
value_entry (int): value entry.
Returns:
object: value.
Raises:
ValueError: if the value is not supported.
"""
column_type = record.get_column_type(value_entry)
long_value = None
if record.is_long_value(value_entry):
long_value = record.get_value_data_as_long_value(value_entry)
if record.is_multi_value(value_entry):
# TODO: implement
raise ValueError('Multi value support not implemented yet.')
if column_type == pyesedb.column_types.NULL:
return None
elif column_type == pyesedb.column_types.BOOLEAN:
# TODO: implement
raise ValueError('Boolean value support not implemented yet.')
elif column_type in self.INTEGER_COLUMN_TYPES:
if long_value:
raise ValueError('Long integer value not supported.')
return record.get_value_data_as_integer(value_entry)
elif column_type in self.FLOATING_POINT_COLUMN_TYPES:
if long_value:
raise ValueError('Long floating point value not supported.')
return record.get_value_data_as_floating_point(value_entry)
elif column_type in self.STRING_COLUMN_TYPES:
if long_value:
return long_value.get_data_as_string()
return record.get_value_data_as_string(value_entry)
elif column_type == pyesedb.column_types.GUID:
# TODO: implement
raise ValueError('GUID value support not implemented yet.')
if long_value:
return long_value.get_data()
return record.get_value_data(value_entry)
def _GetRecordValues(
self, parser_mediator, table_name, record, value_mappings=None):
"""Retrieves the values from the record.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
table_name (str): name of the table.
record (pyesedb.record): ESE record.
value_mappings (Optional[dict[str, str]): value mappings, which map
the column name to a callback method.
Returns:
dict[str,object]: values per column name.
"""
record_values = {}
for value_entry in range(0, record.number_of_values):
if parser_mediator.abort:
break
column_name = record.get_column_name(value_entry)
if column_name in record_values:
logger.warning(
'[{0:s}] duplicate column: {1:s} in table: {2:s}'.format(
self.NAME, column_name, table_name))
continue
value_callback = None
if value_mappings and column_name in value_mappings:
value_callback_method = value_mappings.get(column_name)
if value_callback_method:
value_callback = getattr(self, value_callback_method, None)
if value_callback is None:
logger.warning((
'[{0:s}] missing value callback method: {1:s} for column: '
'{2:s} in table: {3:s}').format(
self.NAME, value_callback_method, column_name, table_name))
if value_callback:
try:
value_data = record.get_value_data(value_entry)
value = value_callback(value_data)
except Exception as exception: # pylint: disable=broad-except
logger.error(exception)
value = None
parser_mediator.ProduceExtractionError((
'unable to parse value: {0:s} with callback: {1:s} with error: '
'{2!s}').format(column_name, value_callback_method, exception))
else:
try:
value = self._GetRecordValue(record, value_entry)
except ValueError as exception:
value = None
parser_mediator.ProduceExtractionError(
'unable to parse value: {0:s} with error: {1!s}'.format(
column_name, exception))
record_values[column_name] = value
return record_values
# pylint 1.9.3 wants a docstring for kwargs, but this is not useful to add.
# pylint: disable=missing-param-doc
def GetEntries(self, parser_mediator, cache=None, database=None, **kwargs):
"""Extracts event objects from the database.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
cache (Optional[ESEDBCache]): cache.
database (Optional[pyesedb.file]): ESE database.
Raises:
ValueError: If the database attribute is not valid.
"""
if database is None:
raise ValueError('Invalid database.')
for table_name, callback_method in iter(self._tables.items()):
if parser_mediator.abort:
break
if not callback_method:
# Table names without a callback method are allowed to improve
# the detection of a database based on its table names.
continue
callback = getattr(self, callback_method, None)
if callback is None:
logger.warning(
'[{0:s}] missing callback method: {1:s} for table: {2:s}'.format(
self.NAME, callback_method, table_name))
continue
esedb_table = database.get_table_by_name(table_name)
if not esedb_table:
logger.warning('[{0:s}] missing table: {1:s}'.format(
self.NAME, table_name))
continue
# The database is passed in case the database contains table names
# that are assigned dynamically and cannot be defined by
# the table name-callback mechanism.
callback(
parser_mediator, cache=cache, database=database, table=esedb_table,
**kwargs)
# pylint 1.9.3 wants a docstring for kwargs, but this is not useful to add.
# pylint: disable=missing-param-doc,arguments-differ
def Process(self, parser_mediator, cache=None, database=None, **kwargs):
"""Determines if this is the appropriate plugin for the database.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
cache (Optional[ESEDBCache]): cache.
database (Optional[pyesedb.file]): ESE database.
Raises:
ValueError: If the database attribute is not valid.
"""
if database is None:
raise ValueError('Invalid database.')
# This will raise if unhandled keyword arguments are passed.
super(ESEDBPlugin, self).Process(parser_mediator)
self.GetEntries(
parser_mediator, cache=cache, database=database, **kwargs)
class EseDbPlugin(plugins.BasePlugin):
"""The ESE database plugin interface."""
NAME = u'esedb'
BINARY_DATA_COLUMN_TYPES = frozenset([
pyesedb.column_types.BINARY_DATA,
pyesedb.column_types.LARGE_BINARY_DATA
])
FLOATING_POINT_COLUMN_TYPES = frozenset(
[pyesedb.column_types.FLOAT_32BIT, pyesedb.column_types.DOUBLE_64BIT])
INTEGER_COLUMN_TYPES = frozenset([
pyesedb.column_types.CURRENCY, pyesedb.column_types.DATE_TIME,
pyesedb.column_types.INTEGER_8BIT_UNSIGNED,
pyesedb.column_types.INTEGER_16BIT_SIGNED,
pyesedb.column_types.INTEGER_16BIT_UNSIGNED,
pyesedb.column_types.INTEGER_32BIT_SIGNED,
pyesedb.column_types.INTEGER_32BIT_UNSIGNED,
pyesedb.column_types.INTEGER_64BIT_SIGNED
])
STRING_COLUMN_TYPES = frozenset(
[pyesedb.column_types.TEXT, pyesedb.column_types.LARGE_TEXT])
_UINT64_BIG_ENDIAN = construct.UBInt64(u'value')
_UINT64_LITTLE_ENDIAN = construct.ULInt64(u'value')
# Dictionary containing a callback method per table name.
# E.g. 'SystemIndex_0A': 'ParseSystemIndex_0A'
REQUIRED_TABLES = {}
OPTIONAL_TABLES = {}
def __init__(self):
"""Initializes the ESE database plugin."""
super(EseDbPlugin, self).__init__()
self._required_tables = frozenset(self.REQUIRED_TABLES.keys())
self._tables = {}
self._tables.update(self.REQUIRED_TABLES)
self._tables.update(self.OPTIONAL_TABLES)
def _ConvertValueBinaryDataToStringAscii(self, value):
"""Converts a binary data value into a string.
Args:
value: The binary data value containing an ASCII string or None.
Returns:
A string or None if value is None.
"""
if value:
return value.decode(u'ascii')
def _ConvertValueBinaryDataToStringBase16(self, value):
"""Converts a binary data value into a base-16 (hexadecimal) string.
Args:
value: The binary data value or None.
Returns:
A string or None if value is None.
"""
if value:
return value.encode(u'hex')
def _ConvertValueBinaryDataToUBInt64(self, value):
"""Converts a binary data value into an integer.
Args:
value: The binary data value containing an unsigned 64-bit big-endian
integer.
Returns:
An integer or None if value is None.
"""
if value:
return self._UINT64_BIG_ENDIAN.parse(value)
def _ConvertValueBinaryDataToULInt64(self, value):
"""Converts a binary data value into an integer.
Args:
value: The binary data value containing an unsigned 64-bit little-endian
integer.
Returns:
An integer or None if value is None.
"""
if value:
return self._UINT64_LITTLE_ENDIAN.parse(value)
def _GetRecordValue(self, record, value_entry):
"""Retrieves a specific value from the record.
Args:
record: The ESE record object (instance of pyesedb.record).
value_entry: The value entry.
Returns:
An object containing the value.
Raises:
ValueError: if the value is not supported.
"""
column_type = record.get_column_type(value_entry)
long_value = None
if record.is_long_value(value_entry):
long_value = record.get_value_data_as_long_value(value_entry)
if record.is_multi_value(value_entry):
# TODO: implement
raise ValueError(u'Multi value support not implemented yet.')
if column_type == pyesedb.column_types.NULL:
return
elif column_type == pyesedb.column_types.BOOLEAN:
# TODO: implement
raise ValueError(u'Boolean value support not implemented yet.')
elif column_type in self.INTEGER_COLUMN_TYPES:
if long_value:
raise ValueError(u'Long integer value not supported.')
return record.get_value_data_as_integer(value_entry)
elif column_type in self.FLOATING_POINT_COLUMN_TYPES:
if long_value:
raise ValueError(u'Long floating point value not supported.')
return record.get_value_data_as_floating_point(value_entry)
elif column_type in self.STRING_COLUMN_TYPES:
if long_value:
return long_value.get_data_as_string()
return record.get_value_data_as_string(value_entry)
elif column_type == pyesedb.column_types.GUID:
# TODO: implement
raise ValueError(u'GUID value support not implemented yet.')
if long_value:
return long_value.get_data()
return record.get_value_data(value_entry)
def _GetRecordValues(self, table_name, record, value_mappings=None):
"""Retrieves the values from the record.
Args:
table_name: The name of the table.
record: The ESE record object (instance of pyesedb.record).
value_mappings: Optional dict of value mappings, which map the column
name to a callback method. The default is None.
Returns:
An dict containing the values.
"""
record_values = {}
for value_entry in range(0, record.number_of_values):
column_name = record.get_column_name(value_entry)
if column_name in record_values:
logging.warning(
u'[{0:s}] duplicate column: {1:s} in table: {2:s}'.format(
self.NAME, column_name, table_name))
continue
value_callback = None
if value_mappings and column_name in value_mappings:
value_callback_method = value_mappings.get(column_name)
if value_callback_method:
value_callback = getattr(self, value_callback_method, None)
if value_callback is None:
logging.warning((
u'[{0:s}] missing value callback method: {1:s} for column: '
u'{2:s} in table: {3:s}').format(
self.NAME, value_callback_method, column_name,
table_name))
try:
value = self._GetRecordValue(record, value_entry)
except ValueError as exception:
logging.warning(exception)
if value_callback:
value = value_callback(value)
record_values[column_name] = value
return record_values
def _GetTableNames(self, database):
"""Retrieves the table names in a database.
Args:
database: The ESE database object (instance of pyesedb.file).
Returns:
A list of the table names.
"""
table_names = []
for esedb_table in database.tables:
table_names.append(esedb_table.name)
return table_names
def GetEntries(self, parser_mediator, database=None, cache=None, **kwargs):
"""Extracts event objects from the database.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
database: Optional ESE database object (instance of pyesedb.file).
The default is None.
cache: Optional cache object (instance of EseDbCache). The default is
None.
Raises:
ValueError: If the database attribute is not valid.
"""
if database is None:
raise ValueError(u'Invalid database.')
for table_name, callback_method in self._tables.iteritems():
if not callback_method:
# Table names without a callback method are allowed to improve
# the detection of a database based on its table names.
continue
callback = getattr(self, callback_method, None)
if callback is None:
logging.warning(
u'[{0:s}] missing callback method: {1:s} for table: {2:s}'.
format(self.NAME, callback_method, table_name))
continue
esedb_table = database.get_table_by_name(table_name)
if not esedb_table:
logging.warning(u'[{0:s}] missing table: {1:s}'.format(
self.NAME, table_name))
continue
# The database is passed in case the database contains table names
# that are assigned dynamically and cannot be defined by
# the table name-callback mechanism.
callback(parser_mediator,
database=database,
table=esedb_table,
cache=cache,
**kwargs)
def Process(self, parser_mediator, database=None, cache=None, **kwargs):
"""Determines if this is the appropriate plugin for the database.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
database: Optional ESE database object (instance of pyesedb.file).
The default is None.
cache: Optional cache object (instance of EseDbCache). The default is
None.
Raises:
errors.WrongPlugin: If the database does not contain all the tables
defined in the required_tables set.
ValueError: If the database attribute is not valid.
"""
if database is None:
raise ValueError(u'Invalid database.')
table_names = frozenset(self._GetTableNames(database))
if self._required_tables.difference(table_names):
raise errors.WrongPlugin(
u'[{0:s}] required tables not found.'.format(self.NAME))
# This will raise if unhandled keyword arguments are passed.
super(EseDbPlugin, self).Process(parser_mediator)
self.GetEntries(parser_mediator,
database=database,
cache=cache,
**kwargs)
1: 'ALERT',
2: 'CRITICAL',
3: 'ERROR',
4: 'WARNING',
5: 'NOTICE',
6: 'INFO',
7: 'DEBUG'
}
# ASL Required Structures.
# ASL header file structure
ASL_HEADER_STRUCT = construct.Struct('asl_header_struct',
construct.String('magic', 12),
construct.UBInt32('version'),
construct.UBInt64('offset'),
construct.UBInt64('timestamp'),
construct.UBInt32('cache_size'),
construct.UBInt64('last_offset'),
construct.Padding(36))
# Record = [Heap][Record_Struct][Values]
# Heap = [Group of Dyn_Value]*
# Values = [ADDR_TXT][ADDR_TXT][ADDR_TXT][ADDR_TXT](2x[ADDR_TXT])*
# (Host) (Sender) (Facility) (message)
# Record Struct
ASL_RECORD_STRUCT = construct.Struct('asl_record_struct', construct.Padding(2),
construct.UBInt32('tam_entry'),
construct.UBInt64('next_offset'),
construct.UBInt64('ASLMessageID'),
class AslParser(interface.BaseParser):
"""Parser for ASL log files."""
NAME = 'asl_log'
DESCRIPTION = u'Parser for ASL log files.'
ASL_MAGIC = 'ASL DB\x00\x00\x00\x00\x00\x00'
# If not right assigned, the value is "-1".
ASL_NO_RIGHTS = 'ffffffff'
# Priority level (criticity)
ASL_MESSAGE_PRIORITY = {
0: 'EMERGENCY',
1: 'ALERT',
2: 'CRITICAL',
3: 'ERROR',
4: 'WARNING',
5: 'NOTICE',
6: 'INFO',
7: 'DEBUG'
}
# ASL File header.
# magic: magic number that identify ASL files.
# version: version of the file.
# offset: first record in the file.
# timestamp: epoch time when the first entry was written.
# last_offset: last record in the file.
ASL_HEADER_STRUCT = construct.Struct('asl_header_struct',
construct.String('magic', 12),
construct.UBInt32('version'),
construct.UBInt64('offset'),
construct.UBInt64('timestamp'),
construct.UBInt32('cache_size'),
construct.UBInt64('last_offset'),
construct.Padding(36))
# The record structure is:
# [HEAP][STRUCTURE][4xExtraField][2xExtraField]*[PreviousEntry]
# Record static structure.
# tam_entry: it contains the number of bytes from this file position
# until the end of the record, without counts itself.
# next_offset: next record. If is equal to 0x00, it is the last record.
# asl_message_id: integer that has the numeric identification of the event.
# timestamp: Epoch integer that has the time when the entry was created.
# nanosecond: nanosecond to add to the timestamp.
# level: level of priority.
# pid: process identification that ask to save the record.
# uid: user identification that has lunched the process.
# gid: group identification that has lunched the process.
# read_uid: identification id of a user. Only applied if is not -1 (all FF).
# Only root and this user can read the entry.
# read_gid: the same than read_uid, but for the group.
ASL_RECORD_STRUCT = construct.Struct('asl_record_struct',
construct.Padding(2),
construct.UBInt32('tam_entry'),
construct.UBInt64('next_offset'),
construct.UBInt64('asl_message_id'),
construct.UBInt64('timestamp'),
construct.UBInt32('nanosec'),
construct.UBInt16('level'),
construct.UBInt16('flags'),
construct.UBInt32('pid'),
construct.UBInt32('uid'),
construct.UBInt32('gid'),
construct.UBInt32('read_uid'),
construct.UBInt32('read_gid'),
construct.UBInt64('ref_pid'))
ASL_RECORD_STRUCT_SIZE = ASL_RECORD_STRUCT.sizeof()
# 8-byte fields, they can be:
# - String: [Nibble = 1000 (8)][Nibble = Length][7 Bytes = String].
# - Integer: integer that has the byte position in the file that points
# to an ASL_RECORD_DYN_VALUE struct. If the value of the integer
# is equal to 0, it means that it has not data (skip).
# If the field is a String, we use this structure to decode each
# integer byte in the corresponding character (ASCII Char).
ASL_OCTET_STRING = construct.ExprAdapter(construct.Octet('string'),
encoder=lambda obj, ctx: ord(obj),
decoder=lambda obj, ctx: chr(obj))
# Field string structure. If the first bit is 1, it means that it
# is a String (1000) = 8, then the next nibble has the number of
# characters. The last 7 bytes are the number of bytes.
ASL_STRING = construct.BitStruct(
'string', construct.Flag('type'), construct.Bits('filler', 3),
construct.If(lambda ctx: ctx.type, construct.Nibble('string_length')),
construct.If(lambda ctx: ctx.type,
construct.Array(7, ASL_OCTET_STRING)))
# 8-byte pointer to a byte position in the file.
ASL_POINTER = construct.UBInt64('pointer')
# Dynamic data structure pointed by a pointer that contains a String:
# [2 bytes padding][4 bytes lenght of String][String].
ASL_RECORD_DYN_VALUE = construct.Struct(
'asl_record_dyn_value', construct.Padding(2),
construct.PascalString('value',
length_field=construct.UBInt32('length')))
def Parse(self, parser_context, file_entry):
"""Extract entries from an ASL file.
Args:
parser_context: A parser context object (instance of ParserContext).
file_entry: A file entry object (instance of dfvfs.FileEntry).
"""
file_object = file_entry.GetFileObject()
file_object.seek(0, os.SEEK_SET)
try:
header = self.ASL_HEADER_STRUCT.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
file_object.close()
raise errors.UnableToParseFile(
u'Unable to parse ASL Header with error: {0:s}.'.format(
exception))
if header.magic != self.ASL_MAGIC:
file_object.close()
raise errors.UnableToParseFile(
u'Not an ASL Header, unable to parse.')
# Get the first and the last entry.
offset = header.offset
old_offset = header.offset
last_offset_header = header.last_offset
# If the ASL file has entries.
if offset:
event_object, offset = self.ReadAslEvent(file_object, offset)
while event_object:
parser_context.ProduceEvent(event_object,
parser_name=self.NAME,
file_entry=file_entry)
# TODO: an anomaly object must be emitted once that is implemented.
# Sanity check, the last read element must be the same as
# indicated by the header.
if offset == 0 and old_offset != last_offset_header:
logging.warning(u'Parsing ended before the header ends.')
old_offset = offset
event_object, offset = self.ReadAslEvent(file_object, offset)
file_object.close()
def ReadAslEvent(self, file_object, offset):
"""Returns an AslEvent from a single ASL entry.
Args:
file_object: a file-like object that points to an ASL file.
offset: offset where the static part of the entry starts.
Returns:
An event object constructed from a single ASL record.
"""
# The heap of the entry is saved to try to avoid seek (performance issue).
# It has the real start position of the entry.
dynamic_start = file_object.tell()
dynamic_part = file_object.read(offset - file_object.tell())
if not offset:
return None, None
try:
record_header = self.ASL_RECORD_STRUCT.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
logging.warning(
u'Unable to parse ASL event with error: {0:s}'.format(
exception))
return None, None
# Variable tam_fields = is the real length of the dynamic fields.
# We have this: [Record_Struct] + [Dynamic_Fields] + [Pointer_Entry_Before]
# In Record_Struct we have a field called tam_entry, where it has the number
# of bytes until the end of the entry from the position that the field is.
# The tam_entry is between the 2th and the 6th byte in the [Record_Struct].
# tam_entry = ([Record_Struct]-6)+[Dynamic_Fields]+[Pointer_Entry_Before]
# Also, we do not need [Point_Entry_Before] and then we delete the size of
# [Point_Entry_Before] that it is 8 bytes (8):
# tam_entry = ([Record_Struct]-6)+[Dynamic_Fields]+[Pointer_Entry_Before]
# [Dynamic_Fields] = tam_entry - [Record_Struct] + 6 - 8
# [Dynamic_Fields] = tam_entry - [Record_Struct] - 2
tam_fields = record_header.tam_entry - self.ASL_RECORD_STRUCT_SIZE - 2
# Dynamic part of the entry that contains minimal four fields of 8 bytes
# plus 2x[8bytes] fields for each extra ASL_Field.
# The four first fields are always the Host, Sender, Facility and Message.
# After the four first fields, the entry might have extra ASL_Fields.
# For each extra ASL_field, it has a pair of 8-byte fields where the first
# 8 bytes contains the name of the extra ASL_field and the second 8 bytes
# contains the text of the exta field.
# All of this 8-byte field can be saved using one of these three differents
# types:
# - Null value ('0000000000000000'): nothing to do.
# - String: It is string if first bit = 1 or first nibble = 8 (1000).
# Second nibble has the length of string.
# The next 7 bytes have the text characters of the string
# padding the end with null characters: '0x00'.
# Example: [8468 6964 6400 0000]
# [8] String, [4] length, value: [68 69 64 64] = hidd.
# - Pointer: static position in the file to a special struct
# implemented as an ASL_RECORD_DYN_VALUE.
# Example: [0000 0000 0000 0077]
# It points to the file position 0x077 that has a
# ASL_RECORD_DYN_VALUE structure.
values = []
while tam_fields > 0:
try:
raw_field = file_object.read(8)
except (IOError, construct.FieldError) as exception:
logging.warning(
u'Unable to parse ASL event with error: {0:d}'.format(
exception))
return None, None
try:
# Try to read as a String.
field = self.ASL_STRING.parse(raw_field)
values.append(''.join(field.string[0:field.string_length]))
# Go to parse the next extra field.
tam_fields -= 8
continue
except ValueError:
pass
# If it is not a string, it must be a pointer.
try:
field = self.ASL_POINTER.parse(raw_field)
except ValueError as exception:
logging.warning(
u'Unable to parse ASL event with error: {0:s}'.format(
exception))
return None, None
if field != 0:
# The next IF ELSE is only for performance issues, avoiding seek.
# If the pointer points a lower position than where the actual entry
# starts, it means that it points to a previuos entry.
pos = field - dynamic_start
# Bigger or equal 0 means that the data is in the actual entry.
if pos >= 0:
try:
values.append((self.ASL_RECORD_DYN_VALUE.parse(
dynamic_part[pos:])).value.partition('\x00')[0])
except (IOError, construct.FieldError) as exception:
logging.warning(
u'Unable to parse ASL event with error: {0:s}'.
format(exception))
return None, None
else:
# Only if it is a pointer that points to the
# heap from another entry we use the seek method.
main_position = file_object.tell()
# If the pointer is in a previous entry.
if main_position > field:
file_object.seek(field - main_position, os.SEEK_CUR)
try:
values.append(
(self.ASL_RECORD_DYN_VALUE.parse_stream(
file_object)).value.partition('\x00')[0])
except (IOError, construct.FieldError):
logging.warning((
u'The pointer at {0:d} (0x{0:x}) points to invalid '
u'information.'
).format(main_position -
self.ASL_POINTER.sizeof()))
# Come back to the position in the entry.
_ = file_object.read(main_position -
file_object.tell())
else:
_ = file_object.read(field - main_position)
values.append((self.ASL_RECORD_DYN_VALUE.parse_stream(
file_object)).value.partition('\x00')[0])
# Come back to the position in the entry.
file_object.seek(main_position - file_object.tell(),
os.SEEK_CUR)
# Next extra field: 8 bytes more.
tam_fields -= 8
# Read the last 8 bytes of the record that points to the previous entry.
_ = file_object.read(8)
# Parsed section, we translate the read data to an appropriate format.
microsecond = record_header.nanosec // 1000
timestamp = timelib.Timestamp.FromPosixTimeWithMicrosecond(
record_header.timestamp, microsecond)
record_position = offset
message_id = record_header.asl_message_id
level = u'{0} ({1})'.format(
self.ASL_MESSAGE_PRIORITY[record_header.level],
record_header.level)
# If the value is -1 (FFFFFFFF), it can be read by everyone.
if record_header.read_uid != int(self.ASL_NO_RIGHTS, 16):
read_uid = record_header.read_uid
else:
read_uid = 'ALL'
if record_header.read_gid != int(self.ASL_NO_RIGHTS, 16):
read_gid = record_header.read_gid
else:
read_gid = 'ALL'
# Parsing the dynamic values (text or pointers to position with text).
# The first four are always the host, sender, facility, and message.
computer_name = values[0]
sender = values[1]
facility = values[2]
message = values[3]
# If the entry has an extra fields, they works as a pairs:
# The first is the name of the field and the second the value.
extra_information = ''
if len(values) > 4:
values = values[4:]
for index in xrange(0, len(values) // 2):
extra_information += (u'[{0}: {1}]'.format(
values[index * 2], values[(index * 2) + 1]))
# Return the event and the offset for the next entry.
return AslEvent(timestamp, record_position, message_id, level,
record_header, read_uid, read_gid, computer_name,
sender, facility, message,
extra_information), record_header.next_offset
class ASLParser(interface.FileObjectParser):
"""Parser for ASL log files."""
_INITIAL_FILE_OFFSET = None
NAME = u'asl_log'
DESCRIPTION = u'Parser for ASL log files.'
_ASL_MAGIC = b'ASL DB\x00\x00\x00\x00\x00\x00'
# ASL File header.
# magic: magic number that identify ASL files.
# version: version of the file.
# offset: first record in the file.
# timestamp: time when the first entry was written.
# Contains the number of seconds since January 1, 1970 00:00:00 UTC.
# last_offset: last record in the file.
_ASL_HEADER_STRUCT = construct.Struct(u'asl_header_struct',
construct.String(u'magic', 12),
construct.UBInt32(u'version'),
construct.UBInt64(u'offset'),
construct.UBInt64(u'timestamp'),
construct.UBInt32(u'cache_size'),
construct.UBInt64(u'last_offset'),
construct.Padding(36))
# The record structure is:
# [HEAP][STRUCTURE][4xExtraField][2xExtraField]*[PreviousEntry]
# Record static structure.
# tam_entry: it contains the number of bytes from this file position
# until the end of the record, without counts itself.
# next_offset: next record. If is equal to 0x00, it is the last record.
# asl_message_id: integer that has the numeric identification of the event.
# timestamp: the entry creation date and time.
# Contains the number of seconds since January 1, 1970 00:00:00 UTC.
# nanosecond: nanosecond to add to the timestamp.
# level: level of priority.
# pid: process identification that ask to save the record.
# uid: user identification that has lunched the process.
# gid: group identification that has lunched the process.
# read_uid: identification id of a user. Only applied if is not -1 (all FF).
# Only root and this user can read the entry.
# read_gid: the same than read_uid, but for the group.
_ASL_RECORD_STRUCT = construct.Struct(u'asl_record_struct',
construct.Padding(2),
construct.UBInt32(u'tam_entry'),
construct.UBInt64(u'next_offset'),
construct.UBInt64(u'asl_message_id'),
construct.UBInt64(u'timestamp'),
construct.UBInt32(u'nanosec'),
construct.UBInt16(u'level'),
construct.UBInt16(u'flags'),
construct.UBInt32(u'pid'),
construct.UBInt32(u'uid'),
construct.UBInt32(u'gid'),
construct.UBInt32(u'read_uid'),
construct.UBInt32(u'read_gid'),
construct.UBInt64(u'ref_pid'))
_ASL_RECORD_STRUCT_SIZE = _ASL_RECORD_STRUCT.sizeof()
# 8-byte fields, they can be:
# - String: [Nibble = 1000 (8)][Nibble = Length][7 Bytes = String].
# - Integer: integer that has the byte position in the file that points
# to an ASL_RECORD_DYN_VALUE struct. If the value of the integer
# is equal to 0, it means that it has not data (skip).
# If the field is a String, we use this structure to decode each
# integer byte in the corresponding character (ASCII Char).
_ASL_OCTET_STRING = construct.ExprAdapter(
construct.Octet(u'string'),
encoder=lambda obj, ctx: ord(obj),
decoder=lambda obj, ctx: chr(obj))
# Field string structure. If the first bit is 1, it means that it
# is a String (1000) = 8, then the next nibble has the number of
# characters. The last 7 bytes are the number of bytes.
_ASL_STRING = construct.BitStruct(
u'string', construct.Flag(u'type'), construct.Bits(u'filler', 3),
construct.If(lambda ctx: ctx.type, construct.Nibble(u'string_length')),
construct.If(lambda ctx: ctx.type,
construct.Array(7, _ASL_OCTET_STRING)))
# 8-byte pointer to a byte position in the file.
_ASL_POINTER = construct.UBInt64(u'pointer')
# Dynamic data structure pointed by a pointer that contains a String:
# [2 bytes padding][4 bytes size of String][String].
_ASL_RECORD_DYN_VALUE = construct.Struct(
u'asl_record_dyn_value', construct.Padding(2),
construct.UBInt32(u'size'),
construct.Bytes(u'value', lambda ctx: ctx.size))
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses an ALS file-like object.
Args:
parser_mediator: a parser mediator object (instance of ParserMediator).
file_object: a file-like object.
Raises:
UnableToParseFile: when the file cannot be parsed.
"""
file_object.seek(0, os.SEEK_SET)
try:
header = self._ASL_HEADER_STRUCT.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
raise errors.UnableToParseFile(
u'Unable to parse ASL Header with error: {0:s}.'.format(
exception))
if header.magic != self._ASL_MAGIC:
raise errors.UnableToParseFile(
u'Not an ASL Header, unable to parse.')
offset = header.offset
if not offset:
return
header_last_offset = header.last_offset
previous_offset = offset
event_object, offset = self.ReadASLEvent(parser_mediator, file_object,
offset)
while event_object:
# Sanity check, the last read element must be the same as
# indicated by the header.
if offset == 0 and previous_offset != header_last_offset:
parser_mediator.ProduceParseError(
u'Unable to parse header. Last element header does not match '
u'header offset.')
previous_offset = offset
event_object, offset = self.ReadASLEvent(parser_mediator,
file_object, offset)
def ReadASLEvent(self, parser_mediator, file_object, offset):
"""Reads an ASL record at a specific offset.
Args:
parser_mediator: a parser mediator object (instance of ParserMediator).
file_object: a file-like object that points to an ASL file.
offset: an integer containing the offset of the ASL record.
Returns:
A tuple of an event object extracted from the ASL record,
and the offset to the next ASL record in the file.
"""
# The heap of the entry is saved to try to avoid seek (performance issue).
# It has the real start position of the entry.
dynamic_data_offset = file_object.tell()
try:
dynamic_data = file_object.read(offset - dynamic_data_offset)
except IOError as exception:
parser_mediator.ProduceParseError(
u'unable to read ASL record dynamic data with error: {0:s}'.
format(exception))
return None, None
if not offset:
return None, None
try:
record_struct = self._ASL_RECORD_STRUCT.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
parser_mediator.ProduceParseError(
u'unable to parse ASL record with error: {0:s}'.format(
exception))
return None, None
# Variable tam_fields = is the real length of the dynamic fields.
# We have this: [Record_Struct] + [Dynamic_Fields] + [Pointer_Entry_Before]
# In Record_Struct we have a field called tam_entry, where it has the number
# of bytes until the end of the entry from the position that the field is.
# The tam_entry is between the 2th and the 6th byte in the [Record_Struct].
# tam_entry = ([Record_Struct]-6)+[Dynamic_Fields]+[Pointer_Entry_Before]
# Also, we do not need [Point_Entry_Before] and then we delete the size of
# [Point_Entry_Before] that it is 8 bytes (8):
# tam_entry = ([Record_Struct]-6)+[Dynamic_Fields]+[Pointer_Entry_Before]
# [Dynamic_Fields] = tam_entry - [Record_Struct] + 6 - 8
# [Dynamic_Fields] = tam_entry - [Record_Struct] - 2
tam_fields = record_struct.tam_entry - self._ASL_RECORD_STRUCT_SIZE - 2
# Dynamic part of the entry that contains minimal four fields of 8 bytes
# plus 2 x [8 bytes] fields for each extra ASL_Field.
# The four first fields are always the Host, Sender, Facility and Message.
# After the four first fields, the entry might have extra ASL_Fields.
# For each extra ASL_field, it has a pair of 8-byte fields where the first
# 8 bytes contains the name of the extra ASL_field and the second 8 bytes
# contains the text of the extra field.
# All of this 8-byte field can be saved using one of these three different
# types:
# - Null value ('0000000000000000'): nothing to do.
# - String: It is string if first bit = 1 or first nibble = 8 (1000).
# Second nibble has the length of string.
# The next 7 bytes have the text characters of the string
# padding the end with null characters: '0x00'.
# Example: [8468 6964 6400 0000]
# [8] String, [4] length, value: [68 69 64 64] = hidd.
# - Pointer: static position in the file to a special struct
# implemented as an ASL_RECORD_DYN_VALUE.
# Example: [0000 0000 0000 0077]
# It points to the file position 0x077 that has a
# ASL_RECORD_DYN_VALUE structure.
values = []
while tam_fields > 0:
try:
field_data = file_object.read(8)
except IOError as exception:
parser_mediator.ProduceParseError(
u'unable to read ASL field with error: {0:s}'.format(
exception))
return None, None
# Try to read the field data as a string.
try:
asl_string_struct = self._ASL_STRING.parse(field_data)
string_data = b''.join(
asl_string_struct.string[0:asl_string_struct.
string_length])
values.append(string_data)
# Go to parse the next extra field.
tam_fields -= 8
continue
except ValueError:
pass
# If the field is not a string it must be a pointer.
try:
pointer_value = self._ASL_POINTER.parse(field_data)
except ValueError as exception:
parser_mediator.ProduceParseError(
u'unable to parse ASL field with error: {0:s}'.format(
exception))
return None, None
if not pointer_value:
# Next extra field: 8 bytes more.
tam_fields -= 8
continue
# The next IF ELSE is only for performance issues, avoiding seek.
# If the pointer points a lower position than where the actual entry
# starts, it means that it points to a previous entry.
pos = pointer_value - dynamic_data_offset
# Greater or equal 0 means that the data is in the actual entry.
if pos >= 0:
try:
dyn_value_struct = self._ASL_RECORD_DYN_VALUE.parse(
dynamic_data[pos:])
dyn_value = dyn_value_struct.value.partition(b'\x00')[0]
values.append(dyn_value)
except (IOError, construct.FieldError) as exception:
parser_mediator.ProduceParseError((
u'unable to parse ASL record dynamic value with error: '
u'{0:s}').format(exception))
return None, None
else:
# Only if it is a pointer that points to the
# heap from another entry we use the seek method.
main_position = file_object.tell()
# If the pointer is in a previous entry.
if main_position > pointer_value:
file_object.seek(pointer_value - main_position,
os.SEEK_CUR)
try:
dyn_value_struct = self._ASL_RECORD_DYN_VALUE.parse_stream(
file_object)
dyn_value = dyn_value_struct.value.partition(
b'\x00')[0]
values.append(dyn_value)
except (IOError, construct.FieldError):
parser_mediator.ProduceParseError((
u'the pointer at {0:d} (0x{0:08x}) points to invalid '
u'information.'
).format(main_position - self._ASL_POINTER.sizeof()))
# Come back to the position in the entry.
_ = file_object.read(main_position - file_object.tell())
else:
_ = file_object.read(pointer_value - main_position)
dyn_value_struct = self._ASL_RECORD_DYN_VALUE.parse_stream(
file_object)
dyn_value = dyn_value_struct.value.partition(b'\x00')[0]
values.append(dyn_value)
# Come back to the position in the entry.
file_object.seek(main_position - file_object.tell(),
os.SEEK_CUR)
# Next extra field: 8 bytes more.
tam_fields -= 8
# Read the last 8 bytes of the record that points to the previous entry.
_ = file_object.read(8)
# Parsed section, we translate the read data to an appropriate format.
micro_seconds, _ = divmod(record_struct.nanosec, 1000)
# Parsing the dynamic values (text or pointers to position with text).
# The first four are always the host, sender, facility, and message.
number_of_values = len(values)
if number_of_values < 4:
parser_mediator.ProduceParseError(
u'less than four values read from an ASL event.')
computer_name = u'N/A'
sender = u'N/A'
facility = u'N/A'
message = u'N/A'
if number_of_values >= 1:
computer_name = values[0].decode(u'utf-8')
if number_of_values >= 2:
sender = values[1].decode(u'utf-8')
if number_of_values >= 3:
facility = values[2].decode(u'utf-8')
if number_of_values >= 4:
message = values[3].decode(u'utf-8')
# If the entry has an extra fields, they works as a pairs:
# The first is the name of the field and the second the value.
extra_information = u''
if number_of_values > 4 and number_of_values % 2 == 0:
# Taking all the extra attributes and merging them together,
# eg: a = [1, 2, 3, 4] will look like "1: 2, 3: 4".
try:
extra_values = map(py2to3.UNICODE_TYPE, values[4:])
extra_information = u', '.join(
map(u': '.join, zip(extra_values[0::2],
extra_values[1::2])))
except UnicodeDecodeError as exception:
parser_mediator.ProduceParseError(
u'Unable to decode all ASL values in the extra information fields.'
)
event_object = ASLEvent(record_struct.timestamp,
offset,
record_struct.asl_message_id,
record_struct.level,
record_struct.pid,
record_struct.uid,
record_struct.gid,
record_struct.read_uid,
record_struct.read_gid,
computer_name,
sender,
facility,
message,
extra_information,
micro_seconds=micro_seconds)
parser_mediator.ProduceEvent(event_object)
return (event_object, record_struct.next_offset)
assert False
self.offset = min(self.offset, self.size())
self.offset = max(self.offset, 0)
def tell(self):
return self.offset
FileVaultV2Header = construct.Struct(
"FileVaultV2Header", construct.Magic("encrcdsa"),
construct.UBInt32("version"), construct.UBInt32("encIVSize"),
construct.UBInt32("_unk1"), construct.UBInt32("_unk2"),
construct.UBInt32("keyBits"), construct.UBInt32("_unk4"),
construct.UBInt32("_unk5"), construct.Array(4,
construct.UBInt32("UDIFID")),
construct.UBInt32("blockSize"), construct.UBInt64("dataSize"),
construct.UBInt64("dataOffset"), construct.Padding(0x260),
construct.UBInt32("kdfAlgorithm"), construct.UBInt32("kdfPRNGAlgorithm"),
construct.UBInt32("kdfIterationCount"), construct.UBInt32("kdfSaltLen"),
construct.String("kdfSalt", 0x20), construct.UBInt32("blobEncIVSize"),
construct.String("blobEncIV", 0x20), construct.UBInt32("blobEncKeyBits"),
construct.UBInt32("blobEncAlgorithm"), construct.UBInt32("blobEncPadding"),
construct.UBInt32("blobEncMode"),
construct.UBInt32("encryptedKeyblobSize"),
construct.String("encryptedKeyblob", 0x30))
class FileVaultFile(BaseFile):
def __init__(self, f, key):
BaseFile.__init__(self)
class BsmParser(interface.FileObjectParser):
"""Parser for BSM files."""
_INITIAL_FILE_OFFSET = None
NAME = u'bsm_log'
DESCRIPTION = u'Parser for BSM log files.'
# BSM supported version (0x0b = 11).
AUDIT_HEADER_VERSION = 11
# Magic Trail Header.
BSM_TOKEN_TRAILER_MAGIC = b'b105'
# IP Version constants.
AU_IPv4 = 4
AU_IPv6 = 16
IPV4_STRUCT = construct.UBInt32(u'ipv4')
IPV6_STRUCT = construct.Struct(u'ipv6', construct.UBInt64(u'high'),
construct.UBInt64(u'low'))
# Tested structures.
# INFO: I have ommited the ID in the structures declaration.
# I used the BSM_TYPE first to read the ID, and then, the structure.
# Tokens always start with an ID value that identifies their token
# type and subsequent structure.
BSM_TYPE = construct.UBInt8(u'token_id')
# Data type structures.
BSM_TOKEN_DATA_CHAR = construct.String(u'value', 1)
BSM_TOKEN_DATA_SHORT = construct.UBInt16(u'value')
BSM_TOKEN_DATA_INTEGER = construct.UBInt32(u'value')
# Common structure used by other structures.
# audit_uid: integer, uid that generates the entry.
# effective_uid: integer, the permission user used.
# effective_gid: integer, the permission group used.
# real_uid: integer, user id of the user that execute the process.
# real_gid: integer, group id of the group that execute the process.
# pid: integer, identification number of the process.
# session_id: unknown, need research.
BSM_TOKEN_SUBJECT_SHORT = construct.Struct(
u'subject_data', construct.UBInt32(u'audit_uid'),
construct.UBInt32(u'effective_uid'),
construct.UBInt32(u'effective_gid'), construct.UBInt32(u'real_uid'),
construct.UBInt32(u'real_gid'), construct.UBInt32(u'pid'),
construct.UBInt32(u'session_id'))
# Common structure used by other structures.
# Identify the kind of inet (IPv4 or IPv6)
# TODO: instead of 16, AU_IPv6 must be used.
BSM_IP_TYPE_SHORT = construct.Struct(
u'bsm_ip_type_short', construct.UBInt32(u'net_type'),
construct.Switch(u'ip_addr',
_BsmTokenGetNetType, {16: IPV6_STRUCT},
default=IPV4_STRUCT))
# Initial fields structure used by header structures.
# length: integer, the length of the entry, equal to trailer (doc: length).
# version: integer, version of BSM (AUDIT_HEADER_VERSION).
# event_type: integer, the type of event (/etc/security/audit_event).
# modifier: integer, unknown, need research (It is always 0).
BSM_HEADER = construct.Struct(u'bsm_header', construct.UBInt32(u'length'),
construct.UBInt8(u'version'),
construct.UBInt16(u'event_type'),
construct.UBInt16(u'modifier'))
# First token of one entry.
# timestamp: unsigned integer, number of seconds since
# January 1, 1970 00:00:00 UTC.
# microsecond: unsigned integer, number of micro seconds.
BSM_HEADER32 = construct.Struct(u'bsm_header32', BSM_HEADER,
construct.UBInt32(u'timestamp'),
construct.UBInt32(u'microsecond'))
BSM_HEADER64 = construct.Struct(u'bsm_header64', BSM_HEADER,
construct.UBInt64(u'timestamp'),
construct.UBInt64(u'microsecond'))
BSM_HEADER32_EX = construct.Struct(u'bsm_header32_ex', BSM_HEADER,
BSM_IP_TYPE_SHORT,
construct.UBInt32(u'timestamp'),
construct.UBInt32(u'microsecond'))
# Token TEXT, provides extra information.
BSM_TOKEN_TEXT = construct.Struct(
u'bsm_token_text', construct.UBInt16(u'length'),
construct.Array(_BsmTokenGetLength, construct.UBInt8(u'text')))
# Path of the executable.
BSM_TOKEN_PATH = BSM_TOKEN_TEXT
# Identified the end of the record (follow by TRAILER).
# status: integer that identifies the status of the exit (BSM_ERRORS).
# return: returned value from the operation.
BSM_TOKEN_RETURN32 = construct.Struct(u'bsm_token_return32',
construct.UBInt8(u'status'),
construct.UBInt32(u'return_value'))
BSM_TOKEN_RETURN64 = construct.Struct(u'bsm_token_return64',
construct.UBInt8(u'status'),
construct.UBInt64(u'return_value'))
# Identified the number of bytes that was written.
# magic: 2 bytes that identifies the TRAILER (BSM_TOKEN_TRAILER_MAGIC).
# length: integer that has the number of bytes from the entry size.
BSM_TOKEN_TRAILER = construct.Struct(u'bsm_token_trailer',
construct.UBInt16(u'magic'),
construct.UBInt32(u'record_length'))
# A 32-bits argument.
# num_arg: the number of the argument.
# name_arg: the argument's name.
# text: the string value of the argument.
BSM_TOKEN_ARGUMENT32 = construct.Struct(
u'bsm_token_argument32', construct.UBInt8(u'num_arg'),
construct.UBInt32(u'name_arg'), construct.UBInt16(u'length'),
construct.Array(_BsmTokenGetLength, construct.UBInt8(u'text')))
# A 64-bits argument.
# num_arg: integer, the number of the argument.
# name_arg: text, the argument's name.
# text: the string value of the argument.
BSM_TOKEN_ARGUMENT64 = construct.Struct(
u'bsm_token_argument64', construct.UBInt8(u'num_arg'),
construct.UBInt64(u'name_arg'), construct.UBInt16(u'length'),
construct.Array(_BsmTokenGetLength, construct.UBInt8(u'text')))
# Identify an user.
# terminal_id: unknown, research needed.
# terminal_addr: unknown, research needed.
BSM_TOKEN_SUBJECT32 = construct.Struct(u'bsm_token_subject32',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt32(u'terminal_port'),
IPV4_STRUCT)
# Identify an user using a extended Token.
# terminal_port: unknown, need research.
# net_type: unknown, need research.
BSM_TOKEN_SUBJECT32_EX = construct.Struct(
u'bsm_token_subject32_ex', BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt32(u'terminal_port'), BSM_IP_TYPE_SHORT)
# au_to_opaque // AUT_OPAQUE
BSM_TOKEN_OPAQUE = BSM_TOKEN_TEXT
# au_to_seq // AUT_SEQ
BSM_TOKEN_SEQUENCE = BSM_TOKEN_DATA_INTEGER
# Program execution with options.
# For each argument we are going to have a string+ "\x00".
# Example: [00 00 00 02][41 42 43 00 42 42 00]
# 2 Arguments, Arg1: [414243] Arg2: [4242].
BSM_TOKEN_EXEC_ARGUMENTS = construct.UBInt32(u'number_arguments')
BSM_TOKEN_EXEC_ARGUMENT = construct.Struct(
u'bsm_token_exec_argument',
construct.RepeatUntil(_BsmTokenIsEndOfString,
construct.StaticField("text", 1)))
# au_to_in_addr // AUT_IN_ADDR:
BSM_TOKEN_ADDR = IPV4_STRUCT
# au_to_in_addr_ext // AUT_IN_ADDR_EX:
BSM_TOKEN_ADDR_EXT = construct.Struct(u'bsm_token_addr_ext',
construct.UBInt32(u'net_type'),
IPV6_STRUCT)
# au_to_ip // AUT_IP:
# TODO: parse this header in the correct way.
BSM_TOKEN_IP = construct.String(u'binary_ipv4_add', 20)
# au_to_ipc // AUT_IPC:
BSM_TOKEN_IPC = construct.Struct(u'bsm_token_ipc',
construct.UBInt8(u'object_type'),
construct.UBInt32(u'object_id'))
# au_to_ipc_perm // au_to_ipc_perm
BSM_TOKEN_IPC_PERM = construct.Struct(
u'bsm_token_ipc_perm', construct.UBInt32(u'user_id'),
construct.UBInt32(u'group_id'), construct.UBInt32(u'creator_user_id'),
construct.UBInt32(u'creator_group_id'),
construct.UBInt32(u'access_mode'), construct.UBInt32(u'slot_seq'),
construct.UBInt32(u'key'))
# au_to_iport // AUT_IPORT:
BSM_TOKEN_PORT = construct.UBInt16(u'port_number')
# au_to_file // AUT_OTHER_FILE32:
BSM_TOKEN_FILE = construct.Struct(
u'bsm_token_file', construct.UBInt32(u'timestamp'),
construct.UBInt32(u'microsecond'), construct.UBInt16(u'length'),
construct.Array(_BsmTokenGetLength, construct.UBInt8(u'text')))
# au_to_subject64 // AUT_SUBJECT64:
BSM_TOKEN_SUBJECT64 = construct.Struct(u'bsm_token_subject64',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt64(u'terminal_port'),
IPV4_STRUCT)
# au_to_subject64_ex // AU_IPv4:
BSM_TOKEN_SUBJECT64_EX = construct.Struct(
u'bsm_token_subject64_ex', BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt32(u'terminal_port'),
construct.UBInt32(u'terminal_type'), BSM_IP_TYPE_SHORT)
# au_to_process32 // AUT_PROCESS32:
BSM_TOKEN_PROCESS32 = construct.Struct(u'bsm_token_process32',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt32(u'terminal_port'),
IPV4_STRUCT)
# au_to_process64 // AUT_PROCESS32:
BSM_TOKEN_PROCESS64 = construct.Struct(u'bsm_token_process64',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt64(u'terminal_port'),
IPV4_STRUCT)
# au_to_process32_ex // AUT_PROCESS32_EX:
BSM_TOKEN_PROCESS32_EX = construct.Struct(
u'bsm_token_process32_ex', BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt32(u'terminal_port'), BSM_IP_TYPE_SHORT)
# au_to_process64_ex // AUT_PROCESS64_EX:
BSM_TOKEN_PROCESS64_EX = construct.Struct(
u'bsm_token_process64_ex', BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt64(u'terminal_port'), BSM_IP_TYPE_SHORT)
# au_to_sock_inet32 // AUT_SOCKINET32:
BSM_TOKEN_AUT_SOCKINET32 = construct.Struct(
u'bsm_token_aut_sockinet32', construct.UBInt16(u'net_type'),
construct.UBInt16(u'port_number'), IPV4_STRUCT)
# Info: checked against the source code of XNU, but not against
# real BSM file.
BSM_TOKEN_AUT_SOCKINET128 = construct.Struct(
u'bsm_token_aut_sockinet128', construct.UBInt16(u'net_type'),
construct.UBInt16(u'port_number'), IPV6_STRUCT)
INET6_ADDR_TYPE = construct.Struct(u'addr_type',
construct.UBInt16(u'ip_type'),
construct.UBInt16(u'source_port'),
construct.UBInt64(u'saddr_high'),
construct.UBInt64(u'saddr_low'),
construct.UBInt16(u'destination_port'),
construct.UBInt64(u'daddr_high'),
construct.UBInt64(u'daddr_low'))
INET4_ADDR_TYPE = construct.Struct(
u'addr_type', construct.UBInt16(u'ip_type'),
construct.UBInt16(u'source_port'),
construct.UBInt32(u'source_address'),
construct.UBInt16(u'destination_port'),
construct.UBInt32(u'destination_address'))
# au_to_socket_ex // AUT_SOCKET_EX
# TODO: Change the 26 for unixbsm.BSM_PROTOCOLS.INET6.
BSM_TOKEN_AUT_SOCKINET32_EX = construct.Struct(
u'bsm_token_aut_sockinet32_ex', construct.UBInt16(u'socket_domain'),
construct.UBInt16(u'socket_type'),
construct.Switch(u'structure_addr_port',
_BsmTokenGetSocketDomain, {26: INET6_ADDR_TYPE},
default=INET4_ADDR_TYPE))
# au_to_sock_unix // AUT_SOCKUNIX
BSM_TOKEN_SOCKET_UNIX = construct.Struct(
u'bsm_token_au_to_sock_unix', construct.UBInt16(u'family'),
construct.RepeatUntil(_BsmTokenIsEndOfString,
construct.StaticField("path", 1)))
# au_to_data // au_to_data
# how to print: bsmtoken.BSM_TOKEN_DATA_PRINT.
# type: bsmtoken.BSM_TOKEN_DATA_TYPE.
# unit_count: number of type values.
# BSM_TOKEN_DATA has a end field = type * unit_count
BSM_TOKEN_DATA = construct.Struct(u'bsm_token_data',
construct.UBInt8(u'how_to_print'),
construct.UBInt8(u'data_type'),
construct.UBInt8(u'unit_count'))
# au_to_attr32 // AUT_ATTR32
BSM_TOKEN_ATTR32 = construct.Struct(
u'bsm_token_attr32', construct.UBInt32(u'file_mode'),
construct.UBInt32(u'uid'), construct.UBInt32(u'gid'),
construct.UBInt32(u'file_system_id'),
construct.UBInt64(u'file_system_node_id'),
construct.UBInt32(u'device'))
# au_to_attr64 // AUT_ATTR64
BSM_TOKEN_ATTR64 = construct.Struct(
u'bsm_token_attr64', construct.UBInt32(u'file_mode'),
construct.UBInt32(u'uid'), construct.UBInt32(u'gid'),
construct.UBInt32(u'file_system_id'),
construct.UBInt64(u'file_system_node_id'),
construct.UBInt64(u'device'))
# au_to_exit // AUT_EXIT
BSM_TOKEN_EXIT = construct.Struct(u'bsm_token_exit',
construct.UBInt32(u'status'),
construct.UBInt32(u'return_value'))
# au_to_newgroups // AUT_NEWGROUPS
# INFO: we must read BSM_TOKEN_DATA_INTEGER for each group.
BSM_TOKEN_GROUPS = construct.UBInt16(u'group_number')
# au_to_exec_env == au_to_exec_args
BSM_TOKEN_EXEC_ENV = BSM_TOKEN_EXEC_ARGUMENTS
# au_to_zonename //AUT_ZONENAME
BSM_TOKEN_ZONENAME = BSM_TOKEN_TEXT
# Token ID.
# List of valid Token_ID.
# Token_ID -> [NAME_STRUCTURE, STRUCTURE]
# Only the checked structures are been added to the valid structures lists.
BSM_TYPE_LIST = {
17: [u'BSM_TOKEN_FILE', BSM_TOKEN_FILE],
19: [u'BSM_TOKEN_TRAILER', BSM_TOKEN_TRAILER],
20: [u'BSM_HEADER32', BSM_HEADER32],
21: [u'BSM_HEADER64', BSM_HEADER64],
33: [u'BSM_TOKEN_DATA', BSM_TOKEN_DATA],
34: [u'BSM_TOKEN_IPC', BSM_TOKEN_IPC],
35: [u'BSM_TOKEN_PATH', BSM_TOKEN_PATH],
36: [u'BSM_TOKEN_SUBJECT32', BSM_TOKEN_SUBJECT32],
38: [u'BSM_TOKEN_PROCESS32', BSM_TOKEN_PROCESS32],
39: [u'BSM_TOKEN_RETURN32', BSM_TOKEN_RETURN32],
40: [u'BSM_TOKEN_TEXT', BSM_TOKEN_TEXT],
41: [u'BSM_TOKEN_OPAQUE', BSM_TOKEN_OPAQUE],
42: [u'BSM_TOKEN_ADDR', BSM_TOKEN_ADDR],
43: [u'BSM_TOKEN_IP', BSM_TOKEN_IP],
44: [u'BSM_TOKEN_PORT', BSM_TOKEN_PORT],
45: [u'BSM_TOKEN_ARGUMENT32', BSM_TOKEN_ARGUMENT32],
47: [u'BSM_TOKEN_SEQUENCE', BSM_TOKEN_SEQUENCE],
96: [u'BSM_TOKEN_ZONENAME', BSM_TOKEN_ZONENAME],
113: [u'BSM_TOKEN_ARGUMENT64', BSM_TOKEN_ARGUMENT64],
114: [u'BSM_TOKEN_RETURN64', BSM_TOKEN_RETURN64],
116: [u'BSM_HEADER32_EX', BSM_HEADER32_EX],
119: [u'BSM_TOKEN_PROCESS64', BSM_TOKEN_PROCESS64],
122: [u'BSM_TOKEN_SUBJECT32_EX', BSM_TOKEN_SUBJECT32_EX],
127: [u'BSM_TOKEN_AUT_SOCKINET32_EX', BSM_TOKEN_AUT_SOCKINET32_EX],
128: [u'BSM_TOKEN_AUT_SOCKINET32', BSM_TOKEN_AUT_SOCKINET32]
}
# Untested structures.
# When not tested structure is found, we try to parse using also
# these structures.
BSM_TYPE_LIST_NOT_TESTED = {
49: [u'BSM_TOKEN_ATTR32', BSM_TOKEN_ATTR32],
50: [u'BSM_TOKEN_IPC_PERM', BSM_TOKEN_IPC_PERM],
52: [u'BSM_TOKEN_GROUPS', BSM_TOKEN_GROUPS],
59: [u'BSM_TOKEN_GROUPS', BSM_TOKEN_GROUPS],
60: [u'BSM_TOKEN_EXEC_ARGUMENTS', BSM_TOKEN_EXEC_ARGUMENTS],
61: [u'BSM_TOKEN_EXEC_ENV', BSM_TOKEN_EXEC_ENV],
62: [u'BSM_TOKEN_ATTR32', BSM_TOKEN_ATTR32],
82: [u'BSM_TOKEN_EXIT', BSM_TOKEN_EXIT],
115: [u'BSM_TOKEN_ATTR64', BSM_TOKEN_ATTR64],
117: [u'BSM_TOKEN_SUBJECT64', BSM_TOKEN_SUBJECT64],
123: [u'BSM_TOKEN_PROCESS32_EX', BSM_TOKEN_PROCESS32_EX],
124: [u'BSM_TOKEN_PROCESS64_EX', BSM_TOKEN_PROCESS64_EX],
125: [u'BSM_TOKEN_SUBJECT64_EX', BSM_TOKEN_SUBJECT64_EX],
126: [u'BSM_TOKEN_ADDR_EXT', BSM_TOKEN_ADDR_EXT],
129: [u'BSM_TOKEN_AUT_SOCKINET128', BSM_TOKEN_AUT_SOCKINET128],
130: [u'BSM_TOKEN_SOCKET_UNIX', BSM_TOKEN_SOCKET_UNIX]
}
def __init__(self):
"""Initializes a parser object."""
super(BsmParser, self).__init__()
# Create the dictionary with all token IDs: tested and untested.
self.bsm_type_list_all = self.BSM_TYPE_LIST.copy()
self.bsm_type_list_all.update(self.BSM_TYPE_LIST_NOT_TESTED)
def _CopyByteArrayToBase16String(self, byte_array):
"""Copies a byte array into a base-16 encoded Unicode string.
Args:
byte_array: A byte array.
Returns:
A base-16 encoded Unicode string.
"""
return u''.join([u'{0:02x}'.format(byte) for byte in byte_array])
def _CopyUtf8ByteArrayToString(self, byte_array):
"""Copies a UTF-8 encoded byte array into a Unicode string.
Args:
byte_array: A byte array containing an UTF-8 encoded string.
Returns:
A Unicode string.
"""
byte_stream = b''.join(map(chr, byte_array))
try:
string = byte_stream.decode(u'utf-8')
except UnicodeDecodeError:
logging.warning(u'Unable to decode UTF-8 formatted byte array.')
string = byte_stream.decode(u'utf-8', errors=u'ignore')
string, _, _ = string.partition(b'\x00')
return string
def _IPv4Format(self, address):
"""Change an integer IPv4 address value for its 4 octets representation.
Args:
address: integer with the IPv4 address.
Returns:
IPv4 address in 4 octet representation (class A, B, C, D).
"""
ipv4_string = self.IPV4_STRUCT.build(address)
return socket.inet_ntoa(ipv4_string)
def _IPv6Format(self, high, low):
"""Provide a readable IPv6 IP having the high and low part in 2 integers.
Args:
high: 64 bits integers number with the high part of the IPv6.
low: 64 bits integers number with the low part of the IPv6.
Returns:
String with a well represented IPv6.
"""
ipv6_string = self.IPV6_STRUCT.build(
construct.Container(high=high, low=low))
# socket.inet_ntop not supported in Windows.
if hasattr(socket, u'inet_ntop'):
return socket.inet_ntop(socket.AF_INET6, ipv6_string)
# TODO: this approach returns double "::", illegal IPv6 addr.
str_address = binascii.hexlify(ipv6_string)
address = []
blank = False
for pos in range(0, len(str_address), 4):
if str_address[pos:pos + 4] == u'0000':
if not blank:
address.append(u'')
blank = True
else:
blank = False
address.append(str_address[pos:pos + 4].lstrip(u'0'))
return u':'.join(address)
def _RawToUTF8(self, byte_stream):
"""Copies a UTF-8 byte stream into a Unicode string.
Args:
byte_stream: A byte stream containing an UTF-8 encoded string.
Returns:
A Unicode string.
"""
try:
string = byte_stream.decode(u'utf-8')
except UnicodeDecodeError:
logging.warning(
u'Decode UTF8 failed, the message string may be cut short.')
string = byte_stream.decode(u'utf-8', errors=u'ignore')
return string.partition(b'\x00')[0]
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses a BSM file-like object.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
file_object: A file-like object.
Raises:
UnableToParseFile: when the file cannot be parsed.
"""
file_object.seek(0, os.SEEK_SET)
try:
is_bsm = self.VerifyFile(parser_mediator, file_object)
except (IOError, construct.FieldError) as exception:
raise errors.UnableToParseFile(
u'Unable to parse BSM file with error: {0:s}'.format(
exception))
if not is_bsm:
raise errors.UnableToParseFile(u'Not a BSM File, unable to parse.')
event_object = self.ReadBSMEvent(parser_mediator, file_object)
while event_object:
parser_mediator.ProduceEvent(event_object)
event_object = self.ReadBSMEvent(parser_mediator, file_object)
def ReadBSMEvent(self, parser_mediator, file_object):
"""Returns a BsmEvent from a single BSM entry.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
file_object: A file-like object.
Returns:
An event object.
"""
# A list of tokens that has the entry.
extra_tokens = []
offset = file_object.tell()
# Token header, first token for each entry.
try:
token_id = self.BSM_TYPE.parse_stream(file_object)
except (IOError, construct.FieldError):
return
bsm_type, structure = self.BSM_TYPE_LIST.get(token_id, [u'', u''])
if bsm_type == u'BSM_HEADER32':
token = structure.parse_stream(file_object)
elif bsm_type == u'BSM_HEADER64':
token = structure.parse_stream(file_object)
elif bsm_type == u'BSM_HEADER32_EX':
token = structure.parse_stream(file_object)
else:
logging.warning(
u'Token ID Header {0} not expected at position 0x{1:X}.'
u'The parsing of the file cannot be continued'.format(
token_id, file_object.tell()))
# TODO: if it is a Mac OS X, search for the trailer magic value
# as a end of the entry can be a possibility to continue.
return
length = token.bsm_header.length
event_type = u'{0} ({1})'.format(
bsmtoken.BSM_AUDIT_EVENT.get(token.bsm_header.event_type,
u'UNKNOWN'),
token.bsm_header.event_type)
timestamp = timelib.Timestamp.FromPosixTimeWithMicrosecond(
token.timestamp, token.microsecond)
# Read until we reach the end of the record.
while file_object.tell() < (offset + length):
# Check if it is a known token.
try:
token_id = self.BSM_TYPE.parse_stream(file_object)
except (IOError, construct.FieldError):
logging.warning(
u'Unable to parse the Token ID at position: {0:d}'.format(
file_object.tell()))
return
if not token_id in self.BSM_TYPE_LIST:
pending = (offset + length) - file_object.tell()
extra_tokens.extend(
self.TryWithUntestedStructures(file_object, token_id,
pending))
else:
token = self.BSM_TYPE_LIST[token_id][1].parse_stream(
file_object)
extra_tokens.append(
self.FormatToken(token_id, token, file_object))
if file_object.tell() > (offset + length):
logging.warning(u'Token ID {0} not expected at position 0x{1:X}.'
u'Jumping for the next entry.'.format(
token_id, file_object.tell()))
try:
file_object.seek((offset + length) - file_object.tell(),
os.SEEK_CUR)
except (IOError, construct.FieldError) as exception:
logging.warning(
u'Unable to jump to next entry with error: {0:s}'.format(
exception))
return
# BSM can be in more than one OS: BSD, Solaris and Mac OS X.
if parser_mediator.platform == u'MacOSX':
# In Mac OS X the last two tokens are the return status and the trailer.
if len(extra_tokens) >= 2:
return_value = extra_tokens[-2:-1][0]
if (return_value.startswith(u'[BSM_TOKEN_RETURN32')
or return_value.startswith(u'[BSM_TOKEN_RETURN64')):
_ = extra_tokens.pop(len(extra_tokens) - 2)
else:
return_value = u'Return unknown'
else:
return_value = u'Return unknown'
if extra_tokens:
trailer = extra_tokens[-1]
if trailer.startswith(u'[BSM_TOKEN_TRAILER'):
_ = extra_tokens.pop(len(extra_tokens) - 1)
else:
trailer = u'Trailer unknown'
else:
trailer = u'Trailer unknown'
return MacBsmEvent(event_type, timestamp, u'. '.join(extra_tokens),
return_value, trailer, offset)
else:
# Generic BSM format.
if extra_tokens:
trailer = extra_tokens[-1]
if trailer.startswith(u'[BSM_TOKEN_TRAILER'):
_ = extra_tokens.pop(len(extra_tokens) - 1)
else:
trailer = u'Trailer unknown'
else:
trailer = u'Trailer unknown'
return BsmEvent(event_type, timestamp, u'. '.join(extra_tokens),
trailer, offset)
def VerifyFile(self, parser_mediator, file_object):
"""Check if the file is a BSM file.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
file_event: file that we want to check.
Returns:
True if this is a valid BSM file, otherwise False.
"""
if file_object.tell() != 0:
file_object.seek(0)
# First part of the entry is always a Header.
try:
token_id = self.BSM_TYPE.parse_stream(file_object)
except (IOError, construct.FieldError):
return False
if token_id not in self.BSM_TYPE_LIST:
return False
bsm_type, structure = self.BSM_TYPE_LIST.get(token_id, [u'', u''])
try:
if bsm_type == u'BSM_HEADER32':
header = structure.parse_stream(file_object)
elif bsm_type == u'BSM_HEADER64':
header = structure.parse_stream(file_object)
elif bsm_type == u'BSM_HEADER32_EX':
header = structure.parse_stream(file_object)
else:
return False
except (IOError, construct.FieldError):
return False
if header.bsm_header.version != self.AUDIT_HEADER_VERSION:
return False
try:
token_id = self.BSM_TYPE.parse_stream(file_object)
except (IOError, construct.FieldError):
return False
# If is Mac OS X BSM file, next entry is a text token indicating
# if it is a normal start or it is a recovery track.
if parser_mediator.platform == u'MacOSX':
bsm_type_list = self.BSM_TYPE_LIST.get(token_id)
if not bsm_type_list:
return False
if bsm_type_list[0] != u'BSM_TOKEN_TEXT':
logging.warning(
u'It is not a valid first entry for Mac OS X BSM.')
return False
try:
token = self.BSM_TOKEN_TEXT.parse_stream(file_object)
except (IOError, construct.FieldError):
return
text = self._CopyUtf8ByteArrayToString(token.text)
if (text != u'launchctl::Audit startup'
and text != u'launchctl::Audit recovery'):
logging.warning(
u'It is not a valid first entry for Mac OS X BSM.')
return False
file_object.seek(0)
return True
def TryWithUntestedStructures(self, file_object, token_id, pending):
"""Try to parse the pending part of the entry using untested structures.
Args:
file_object: BSM file.
token_id: integer with the id that comes from the unknown token.
pending: pending length of the entry.
Returns:
A list of extra tokens data that can be parsed using non-tested
structures. A message indicating that a structure cannot be parsed
is added for unparsed structures.
"""
# Data from the unknown structure.
start_position = file_object.tell()
start_token_id = token_id
extra_tokens = []
# Read all the "pending" bytes.
try:
if token_id in self.bsm_type_list_all:
token = self.bsm_type_list_all[token_id][1].parse_stream(
file_object)
extra_tokens.append(
self.FormatToken(token_id, token, file_object))
while file_object.tell() < (start_position + pending):
# Check if it is a known token.
try:
token_id = self.BSM_TYPE.parse_stream(file_object)
except (IOError, construct.FieldError):
logging.warning(
u'Unable to parse the Token ID at position: {0:d}'.
format(file_object.tell()))
return
if token_id not in self.bsm_type_list_all:
break
token = self.bsm_type_list_all[token_id][1].parse_stream(
file_object)
extra_tokens.append(
self.FormatToken(token_id, token, file_object))
except (IOError, construct.FieldError):
token_id = 255
next_entry = (start_position + pending)
if file_object.tell() != next_entry:
# Unknown Structure.
logging.warning(
u'Unknown Token at "0x{0:X}", ID: {1} (0x{2:X})'.format(
start_position - 1, token_id, token_id))
# TODO: another way to save this information must be found.
extra_tokens.append(u'Plaso: some tokens from this entry can '
u'not be saved. Entry at 0x{0:X} with unknown '
u'token id "0x{1:X}".'.format(
start_position - 1, start_token_id))
# Move to next entry.
file_object.seek(next_entry - file_object.tell(), os.SEEK_CUR)
# It returns null list because it doesn't know witch structure was
# the incorrect structure that makes that it can arrive to the spected
# end of the entry.
return []
return extra_tokens
# TODO: instead of compare the text to know what structure was parsed
# is better to compare directly the numeric number (token_id),
# less readable, but better performance.
def FormatToken(self, token_id, token, file_object):
"""Parse the Token depending of the type of the structure.
Args:
token_id: Identification integer of the token_type.
token: Token struct to parse.
file_object: BSM file.
Returns:
String with the parsed Token values.
"""
if token_id not in self.bsm_type_list_all:
return u'Type Unknown: {0:d} (0x{0:X})'.format(token_id)
bsm_type, _ = self.bsm_type_list_all.get(token_id, [u'', u''])
if bsm_type in [
u'BSM_TOKEN_TEXT', u'BSM_TOKEN_PATH', u'BSM_TOKEN_ZONENAME'
]:
try:
string = self._CopyUtf8ByteArrayToString(token.text)
except TypeError:
string = u'Unknown'
return u'[{0}: {1:s}]'.format(bsm_type, string)
elif bsm_type in [
u'BSM_TOKEN_RETURN32', u'BSM_TOKEN_RETURN64', u'BSM_TOKEN_EXIT'
]:
return u'[{0}: {1} ({2}), System call status: {3}]'.format(
bsm_type, bsmtoken.BSM_ERRORS.get(token.status, u'Unknown'),
token.status, token.return_value)
elif bsm_type in [u'BSM_TOKEN_SUBJECT32', u'BSM_TOKEN_SUBJECT64']:
return (
u'[{0}: aid({1}), euid({2}), egid({3}), uid({4}), gid({5}), '
u'pid({6}), session_id({7}), terminal_port({8}), '
u'terminal_ip({9})]').format(
bsm_type, token.subject_data.audit_uid,
token.subject_data.effective_uid,
token.subject_data.effective_gid,
token.subject_data.real_uid, token.subject_data.real_gid,
token.subject_data.pid, token.subject_data.session_id,
token.terminal_port, self._IPv4Format(token.ipv4))
elif bsm_type in [
u'BSM_TOKEN_SUBJECT32_EX', u'BSM_TOKEN_SUBJECT64_EX'
]:
if token.bsm_ip_type_short.net_type == self.AU_IPv6:
ip = self._IPv6Format(token.bsm_ip_type_short.ip_addr.high,
token.bsm_ip_type_short.ip_addr.low)
elif token.bsm_ip_type_short.net_type == self.AU_IPv4:
ip = self._IPv4Format(token.bsm_ip_type_short.ip_addr)
else:
ip = u'unknown'
return (
u'[{0}: aid({1}), euid({2}), egid({3}), uid({4}), gid({5}), '
u'pid({6}), session_id({7}), terminal_port({8}), '
u'terminal_ip({9})]').format(
bsm_type, token.subject_data.audit_uid,
token.subject_data.effective_uid,
token.subject_data.effective_gid,
token.subject_data.real_uid, token.subject_data.real_gid,
token.subject_data.pid, token.subject_data.session_id,
token.terminal_port, ip)
elif bsm_type in [u'BSM_TOKEN_ARGUMENT32', u'BSM_TOKEN_ARGUMENT64']:
string = self._CopyUtf8ByteArrayToString(token.text)
return u'[{0}: {1:s}({2}) is 0x{3:X}]'.format(
bsm_type, string, token.num_arg, token.name_arg)
elif bsm_type in [u'BSM_TOKEN_EXEC_ARGUMENTS', u'BSM_TOKEN_EXEC_ENV']:
arguments = []
for _ in range(0, token):
sub_token = self.BSM_TOKEN_EXEC_ARGUMENT.parse_stream(
file_object)
string = self._CopyUtf8ByteArrayToString(sub_token.text)
arguments.append(string)
return u'[{0}: {1:s}]'.format(bsm_type, u' '.join(arguments))
elif bsm_type == u'BSM_TOKEN_AUT_SOCKINET32':
return (u'[{0}: {1} ({2}) open in port {3}. Address {4}]'.format(
bsm_type, bsmtoken.BSM_PROTOCOLS.get(token.net_type,
u'UNKNOWN'),
token.net_type, token.port_number,
self._IPv4Format(token.ipv4)))
elif bsm_type == u'BSM_TOKEN_AUT_SOCKINET128':
return u'[{0}: {1} ({2}) open in port {3}. Address {4}]'.format(
bsm_type, bsmtoken.BSM_PROTOCOLS.get(token.net_type,
u'UNKNOWN'),
token.net_type, token.port_number,
self._IPv6Format(token.ipv6.high, token.ipv6.low))
elif bsm_type == u'BSM_TOKEN_ADDR':
return u'[{0}: {1}]'.format(bsm_type, self._IPv4Format(token))
elif bsm_type == u'BSM_TOKEN_IP':
return u'[IPv4_Header: 0x{0:s}]'.format(token.encode(u'hex'))
elif bsm_type == u'BSM_TOKEN_ADDR_EXT':
return u'[{0}: {1} ({2}). Address {3}]'.format(
bsm_type, bsmtoken.BSM_PROTOCOLS.get(token.net_type,
u'UNKNOWN'),
token.net_type,
self._IPv6Format(token.ipv6.high, token.ipv6.low))
elif bsm_type == u'BSM_TOKEN_PORT':
return u'[{0}: {1}]'.format(bsm_type, token)
elif bsm_type == u'BSM_TOKEN_TRAILER':
return u'[{0}: {1}]'.format(bsm_type, token.record_length)
elif bsm_type == u'BSM_TOKEN_FILE':
# TODO: if this timestamp is usefull, it must be extracted as a separate
# event object.
timestamp = timelib.Timestamp.FromPosixTimeWithMicrosecond(
token.timestamp, token.microsecond)
date_time = timelib.Timestamp.CopyToDatetime(timestamp, pytz.UTC)
date_time_string = date_time.strftime(u'%Y-%m-%d %H:%M:%S')
string = self._CopyUtf8ByteArrayToString(token.text)
return u'[{0}: {1:s}, timestamp: {2:s}]'.format(
bsm_type, string, date_time_string)
elif bsm_type == u'BSM_TOKEN_IPC':
return u'[{0}: object type {1}, object id {2}]'.format(
bsm_type, token.object_type, token.object_id)
elif bsm_type in [u'BSM_TOKEN_PROCESS32', u'BSM_TOKEN_PROCESS64']:
return (
u'[{0}: aid({1}), euid({2}), egid({3}), uid({4}), gid({5}), '
u'pid({6}), session_id({7}), terminal_port({8}), '
u'terminal_ip({9})]').format(
bsm_type, token.subject_data.audit_uid,
token.subject_data.effective_uid,
token.subject_data.effective_gid,
token.subject_data.real_uid, token.subject_data.real_gid,
token.subject_data.pid, token.subject_data.session_id,
token.terminal_port, self._IPv4Format(token.ipv4))
elif bsm_type in [
u'BSM_TOKEN_PROCESS32_EX', u'BSM_TOKEN_PROCESS64_EX'
]:
if token.bsm_ip_type_short.net_type == self.AU_IPv6:
ip = self._IPv6Format(token.bsm_ip_type_short.ip_addr.high,
token.bsm_ip_type_short.ip_addr.low)
elif token.bsm_ip_type_short.net_type == self.AU_IPv4:
ip = self._IPv4Format(token.bsm_ip_type_short.ip_addr)
else:
ip = u'unknown'
return (
u'[{0}: aid({1}), euid({2}), egid({3}), uid({4}), gid({5}), '
u'pid({6}), session_id({7}), terminal_port({8}), '
u'terminal_ip({9})]').format(
bsm_type, token.subject_data.audit_uid,
token.subject_data.effective_uid,
token.subject_data.effective_gid,
token.subject_data.real_uid, token.subject_data.real_gid,
token.subject_data.pid, token.subject_data.session_id,
token.terminal_port, ip)
elif bsm_type == u'BSM_TOKEN_DATA':
data = []
data_type = bsmtoken.BSM_TOKEN_DATA_TYPE.get(token.data_type, u'')
if data_type == u'AUR_CHAR':
for _ in range(token.unit_count):
data.append(
self.BSM_TOKEN_DATA_CHAR.parse_stream(file_object))
elif data_type == u'AUR_SHORT':
for _ in range(token.unit_count):
data.append(
self.BSM_TOKEN_DAT_SHORT.parse_stream(file_object))
elif data_type == u'AUR_INT32':
for _ in range(token.unit_count):
data.append(
self.BSM_TOKEN_DATA_INTEGER.parse_stream(file_object))
else:
data.append(u'Unknown type data')
# TODO: the data when it is string ends with ".", HW a space is return
# after uses the UTF-8 conversion.
return u'[{0}: Format data: {1}, Data: {2}]'.format(
bsm_type, bsmtoken.BSM_TOKEN_DATA_PRINT[token.how_to_print],
self._RawToUTF8(u''.join(data)))
elif bsm_type in [u'BSM_TOKEN_ATTR32', u'BSM_TOKEN_ATTR64']:
return (u'[{0}: Mode: {1}, UID: {2}, GID: {3}, '
u'File system ID: {4}, Node ID: {5}, Device: {6}]').format(
bsm_type, token.file_mode, token.uid, token.gid,
token.file_system_id, token.file_system_node_id,
token.device)
elif bsm_type == u'BSM_TOKEN_GROUPS':
arguments = []
for _ in range(token):
arguments.append(
self._RawToUTF8(
self.BSM_TOKEN_DATA_INTEGER.parse_stream(file_object)))
return u'[{0}: {1:s}]'.format(bsm_type, u','.join(arguments))
elif bsm_type == u'BSM_TOKEN_AUT_SOCKINET32_EX':
if bsmtoken.BSM_PROTOCOLS.get(token.socket_domain,
u'') == u'INET6':
saddr = self._IPv6Format(token.structure_addr_port.saddr_high,
token.structure_addr_port.saddr_low)
daddr = self._IPv6Format(token.structure_addr_port.daddr_high,
token.structure_addr_port.daddr_low)
else:
saddr = self._IPv4Format(
token.structure_addr_port.source_address)
daddr = self._IPv4Format(
token.structure_addr_port.destination_address)
return u'[{0}: from {1} port {2} to {3} port {4}]'.format(
bsm_type, saddr, token.structure_addr_port.source_port, daddr,
token.structure_addr_port.destination_port)
elif bsm_type == u'BSM_TOKEN_IPC_PERM':
return (u'[{0}: user id {1}, group id {2}, create user id {3}, '
u'create group id {4}, access {5}]').format(
bsm_type, token.user_id, token.group_id,
token.creator_user_id, token.creator_group_id,
token.access_mode)
elif bsm_type == u'BSM_TOKEN_SOCKET_UNIX':
string = self._CopyUtf8ByteArrayToString(token.path)
return u'[{0}: Family {1}, Path {2:s}]'.format(
bsm_type, token.family, string)
elif bsm_type == u'BSM_TOKEN_OPAQUE':
string = self._CopyByteArrayToBase16String(token.text)
return u'[{0}: {1:s}]'.format(bsm_type, string)
elif bsm_type == u'BSM_TOKEN_SEQUENCE':
return u'[{0}: {1}]'.format(bsm_type, token)
import bencode
import construct as c
from gevent import socket
class Error(Exception):
pass
MAX_PACKET_SIZE = 2**16
log = logging.getLogger('tracker')
INIT_ID = 0x41727101980
connect_req = c.Struct('request',
c.UBInt64('connection_id'),
c.UBInt32('action'),
c.UBInt32('transaction_id')
)
connect_resp = c.Struct('response',
c.UBInt32('action'),
c.UBInt32('transaction_id'),
c.UBInt64('connection_id'),
)
announce_req = c.Struct('request',
c.UBInt64('connection_id'),
c.UBInt32('action'),
c.UBInt32('transaction_id'),
c.Bytes('info_hash', 20),
class JavaIDXParser(interface.FileObjectParser):
"""Parse Java WebStart Cache IDX files for download events.
There are five structures defined. 6.02 files had one generic section
that retained all data. From 6.03, the file went to a multi-section
format where later sections were optional and had variable-lengths.
6.03, 6.04, and 6.05 files all have their main data section (#2)
begin at offset 128. The short structure is because 6.05 files
deviate after the 8th byte. So, grab the first 8 bytes to ensure it's
valid, get the file version, then continue on with the correct
structures.
"""
_INITIAL_FILE_OFFSET = None
NAME = u'java_idx'
DESCRIPTION = u'Parser for Java WebStart Cache IDX files.'
IDX_SHORT_STRUCT = construct.Struct(u'magic', construct.UBInt8(u'busy'),
construct.UBInt8(u'incomplete'),
construct.UBInt32(u'idx_version'))
IDX_602_STRUCT = construct.Struct(
u'IDX_602_Full', construct.UBInt16(u'null_space'),
construct.UBInt8(u'shortcut'), construct.UBInt32(u'content_length'),
construct.UBInt64(u'last_modified_date'),
construct.UBInt64(u'expiration_date'),
construct.PascalString(u'version_string',
length_field=construct.UBInt16(u'length')),
construct.PascalString(u'url',
length_field=construct.UBInt16(u'length')),
construct.PascalString(u'namespace',
length_field=construct.UBInt16(u'length')),
construct.UBInt32(u'FieldCount'))
IDX_605_SECTION_ONE_STRUCT = construct.Struct(
u'IDX_605_Section1', construct.UBInt8(u'shortcut'),
construct.UBInt32(u'content_length'),
construct.UBInt64(u'last_modified_date'),
construct.UBInt64(u'expiration_date'),
construct.UBInt64(u'validation_date'), construct.UBInt8(u'signed'),
construct.UBInt32(u'sec2len'), construct.UBInt32(u'sec3len'),
construct.UBInt32(u'sec4len'))
IDX_605_SECTION_TWO_STRUCT = construct.Struct(
u'IDX_605_Section2',
construct.PascalString(u'version',
length_field=construct.UBInt16(u'length')),
construct.PascalString(u'url',
length_field=construct.UBInt16(u'length')),
construct.PascalString(u'namespec',
length_field=construct.UBInt16(u'length')),
construct.PascalString(u'ip_address',
length_field=construct.UBInt16(u'length')),
construct.UBInt32(u'FieldCount'))
# Java uses Pascal-style strings, but with a 2-byte length field.
JAVA_READUTF_STRING = construct.Struct(
u'Java.ReadUTF',
construct.PascalString(u'string',
length_field=construct.UBInt16(u'length')))
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses a Java WebStart Cache IDX file-like object.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
file_object: A file-like object.
Raises:
UnableToParseFile: when the file cannot be parsed.
"""
file_object.seek(0, os.SEEK_SET)
try:
magic = self.IDX_SHORT_STRUCT.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
raise errors.UnableToParseFile(
u'Unable to parse Java IDX file with error: {0:s}.'.format(
exception))
# Fields magic.busy and magic.incomplete are normally 0x00. They
# are set to 0x01 if the file is currently being downloaded. Logic
# checks for > 1 to avoid a race condition and still reject any
# file with other data.
# Field magic.idx_version is the file version, of which only
# certain versions are supported.
if magic.busy > 1 or magic.incomplete > 1:
raise errors.UnableToParseFile(u'Not a valid Java IDX file')
if not magic.idx_version in [602, 603, 604, 605]:
raise errors.UnableToParseFile(u'Not a valid Java IDX file')
# Obtain the relevant values from the file. The last modified date
# denotes when the file was last modified on the HOST. For example,
# when the file was uploaded to a web server.
if magic.idx_version == 602:
section_one = self.IDX_602_STRUCT.parse_stream(file_object)
last_modified_date = section_one.last_modified_date
url = section_one.url
ip_address = u'Unknown'
http_header_count = section_one.FieldCount
elif magic.idx_version in [603, 604, 605]:
# IDX 6.03 and 6.04 have two unused bytes before the structure.
if magic.idx_version in [603, 604]:
file_object.read(2)
# IDX 6.03, 6.04, and 6.05 files use the same structures for the
# remaining data.
section_one = self.IDX_605_SECTION_ONE_STRUCT.parse_stream(
file_object)
last_modified_date = section_one.last_modified_date
if file_object.get_size() > 128:
file_object.seek(128,
os.SEEK_SET) # Static offset for section 2.
section_two = self.IDX_605_SECTION_TWO_STRUCT.parse_stream(
file_object)
url = section_two.url
ip_address = section_two.ip_address
http_header_count = section_two.FieldCount
else:
url = u'Unknown'
ip_address = u'Unknown'
http_header_count = 0
# File offset is now just prior to HTTP headers. Make sure there
# are headers, and then parse them to retrieve the download date.
download_date = None
for field in range(0, http_header_count):
field = self.JAVA_READUTF_STRING.parse_stream(file_object)
value = self.JAVA_READUTF_STRING.parse_stream(file_object)
if field.string == u'date':
# Time string "should" be in UTC or have an associated time zone
# information in the string itself. If that is not the case then
# there is no reliable method for plaso to determine the proper
# timezone, so the assumption is that it is UTC.
try:
download_date = timelib.Timestamp.FromTimeString(
value.string, gmt_as_timezone=False)
except errors.TimestampError:
download_date = None
parser_mediator.ProduceExtractionError(
u'Unable to parse time value: {0:s}'.format(
value.string))
if not url or not ip_address:
raise errors.UnableToParseFile(
u'Unexpected Error: URL or IP address not found in file.')
event_data = JavaIDXEventData()
event_data.idx_version = magic.idx_version
event_data.ip_address = ip_address
event_data.url = url
date_time = dfdatetime_java_time.JavaTime(timestamp=last_modified_date)
# TODO: Move the timestamp description into eventdata.
event = time_events.DateTimeValuesEvent(date_time, u'File Hosted Date')
parser_mediator.ProduceEventWithEventData(event, event_data)
if section_one:
expiration_date = section_one.get(u'expiration_date', None)
if expiration_date:
date_time = dfdatetime_java_time.JavaTime(
timestamp=expiration_date)
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_EXPIRATION)
parser_mediator.ProduceEventWithEventData(event, event_data)
if download_date:
event = time_events.TimestampEvent(
download_date, definitions.TIME_DESCRIPTION_FILE_DOWNLOADED)
parser_mediator.ProduceEventWithEventData(event, event_data)
