class SoundBuffer(FixedObjectByteArray):
classID = 12
_construct = Struct("",
UBInt32("length"),
construct.String("items", lambda ctx: ctx.length * 2),
)
@classmethod
def from_value(cls, obj):
return cls(obj.items)
def to_value(self):
value = self.value
length = (len(value) + 3) / 2
value += "\x00" * (length * 2 - len(value)) # padding
return Container(items=value, length=length)
def interpret(data_record, bookmark):
count = data_record.length // 4
if data_record.length % 4 > 0:
count += 1
if data_record.type == 1537: # offsets
offsets = []
data = data_record.data
for offset in range(count):
offsets.append(bookmark.offset +
_INT.parse(data)) # base offset + relative offset
data = data[4:]
return offsets
if data_record.type == 257 or 513 or 2305:
string_data = construct.String('string', data_record.length).parse(
data_record.data)
return string_data
class InitialProgramLoader(BootRecord):
_IPL_HEADER = construct.Struct(
'IPL_HEADER', construct.ULInt16('sig_len'),
construct.String('signature',
length=lambda ctx: ctx.sig_len * 2,
encoding='utf16'.encode('utf8')))
def __init__(self, filePath, size, offset=None, whitelist=()):
self._type = 'IPL'
super(InitialProgramLoader, self).__init__(filePath, size, offset,
whitelist)
def _parse(self):
try:
header = self._IPL_HEADER.parse(self._raw)
except (construct.ConstructError, UnicodeDecodeError) as e:
raise InvalidIPLError('Invalid IPL structure: {0}\n{1}'.format(
e, hexdump(self._raw[:0x200])))
try:
# IPL's code section is usually contained is the first 9 sectors. The remaining sectors are filled with
# padding but it appears that the last (15th) sector can sometimes hold data not related to the boot process
# and we need to exclude that from hash calculation.
invariantCode = self._raw[:14 * 512]
except IndexError:
raise InvalidIPLError(
'Invalid sample size for IPL: {0} (should be 15 * 512-bytes sectors)'
.format(len(self._raw)))
expectedLoader = None
# Starting with NT 6.2, IPL has a localized string that must be excluded from hash computation.
# The difference between these two kinds of IPL can be told from the instruction located at 0x56 :
# a Jump Short (EB) in case of IPL<6.2 or a Jump Near (E9) otherwise
if header.signature == 'BOOTMGR' and self._raw[0x56].encode(
'hex').upper() == 'E9':
# The offset of the localized string seems to be stored in a DWORD at 0x117 (just before the beginning
# of the assembly code). But the value seems to be an offset relative to the start of the whole
# boot record (including the VBR) and not just the IPL.
# Therefore we need to substract 0x200 to get the offset inside the IPL.
strOffset = construct.ULInt16('offset').parse(
self._raw[0x117:]) - 0x200
# Exclude from hash calculation everything between the string offset and the beginning of code
invariantCode = invariantCode[:strOffset] + invariantCode[0x119:]
expectedLoader = 'NT6.2+ IPL'
codeHash = hashlib.sha256(invariantCode)
self._matchHash(codeHash, expectedLoader)
# If no whitelisted signature matched, try some simple heuristics to flag this IPL as malicious
# Note that the self._checkCode method is only given the "stripped" code section to help the disassembling.
# This will obviously leads to broken offsets, but it doesn't matter since the heuristics don't use them.
if len(self._signature) == 0:
self._checkCode(invariantCode)
def _checkCode(self, code):
md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_16)
md.detail = True
for i in md.disasm(code, 0):
# Check for unknown interrupt
if i.mnemonic == 'int' and i.bytes[1] not in (0x10, 0x13, 0x18,
0x1a):
self._suspiciousBehaviour.append(
'Unknown Interrupt : {0:#x}'.format(i.bytes[1]))
class Bitmap(FixedObjectByteArray):
classID = 13
_construct = Struct(
"",
UBInt32("length"),
construct.String("items", lambda ctx: ctx.length * 4),
# Identically named "String" class -_-
)
@classmethod
def from_value(cls, obj):
return cls(obj.items)
def to_value(self):
value = self.value
length = (len(value) + 3) / 4
value += "\x00" * (length * 4 - len(value)) # padding
return Container(items=value, length=length)
_int = Struct(
"int",
UBInt8("_value"),
If(
lambda ctx: ctx._value > 223,
IfThenElse(
"", lambda ctx: ctx._value <= 254,
Embed(
Struct(
"",
UBInt8("_second_byte"),
Value(
"_value", lambda ctx:
(ctx._value - 224) * 256 + ctx._second_byte),
)), Embed(Struct(
"",
UBInt32("_value"),
)))),
)
_length_run_coding = Struct(
"",
Embed(_int), #ERROR?
Value("length", lambda ctx: ctx._value),
OptionalGreedyRepeater(
Struct(
"data",
Embed(_int),
Value("data_code", lambda ctx: ctx._value % 4),
Value("run_length", lambda ctx:
(ctx._value - ctx.data_code) / 4),
Switch(
"", lambda ctx: ctx.data_code, {
0:
Embed(
Struct(
"",
StrictRepeater(
get_run_length,
Value("pixels",
lambda ctx: "\x00\x00\x00\x00")),
)),
1:
Embed(
Struct(
"",
Bytes("_b", 1),
StrictRepeater(
get_run_length,
Value("pixels", lambda ctx: ctx._b * 4),
),
)),
2:
Embed(
Struct(
"",
Bytes("_pixel", 4),
StrictRepeater(
get_run_length,
Value("pixels", lambda ctx: ctx._pixel),
),
)),
3:
Embed(
Struct(
"",
StrictRepeater(
get_run_length,
Bytes("pixels", 4),
),
)),
}),
)))
@classmethod
def from_byte_array(cls, bytes_):
"""Decodes a run-length encoded ByteArray and returns a Bitmap.
The ByteArray decompresses to a sequence of 32-bit values, which are
stored as a byte string. (The specific encoding depends on Form.depth.)
"""
runs = cls._length_run_coding.parse(bytes_)
pixels = (run.pixels for run in runs.data)
data = "".join(itertools.chain.from_iterable(pixels))
return cls(data)
def compress(self):
"""Compress to a ByteArray"""
raise NotImplementedError
class NTFSUsnJrnlParser(interface.FileObjectParser):
"""Parses a NTFS USN change journal."""
_INITIAL_FILE_OFFSET = None
NAME = u'usnjrnl'
DESCRIPTION = u'Parser for NTFS USN change journal ($UsnJrnl).'
_USN_RECORD_V2 = construct.Struct(
u'usn_record_v2',
construct.ULInt32(u'size'),
construct.ULInt16(u'major_version'),
construct.ULInt16(u'minor_version'),
construct.ULInt64(u'file_reference'),
construct.ULInt64(u'parent_file_reference'),
construct.ULInt64(u'update_sequence_number'),
construct.ULInt64(u'update_date_time'),
construct.ULInt32(u'update_reason_flags'),
construct.ULInt32(u'update_source_flags'),
construct.ULInt32(u'security_descriptor_identifier'),
construct.ULInt32(u'file_attribute_flags'),
construct.ULInt16(u'name_size'),
construct.ULInt16(u'name_offset'),
construct.String(u'name', lambda ctx: ctx.size - 60))
# TODO: add support for USN_RECORD_V3 when actually seen to be used.
def _ParseUSNChangeJournal(self, parser_mediator, usn_change_journal):
"""Parses an USN change journal.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
usn_change_journal: An USN change journal object (instance of
pyfsntsfs.usn_change_journal).
"""
if not usn_change_journal:
return
usn_record_data = usn_change_journal.read_usn_record()
while usn_record_data:
current_offset = usn_change_journal.get_offset()
try:
usn_record_struct = self._USN_RECORD_V2.parse(usn_record_data)
except (IOError, construct.FieldError) as exception:
parser_mediator.ProduceParseError((
u'unable to parse USN record at offset: 0x{0:08x} '
u'with error: {1:s}').format(current_offset, exception))
continue
name_offset = usn_record_struct.name_offset - 60
utf16_stream = usn_record_struct.name[
name_offset:usn_record_struct.name_size]
try:
name_string = utf16_stream.decode(u'utf-16-le')
except (UnicodeDecodeError, UnicodeEncodeError) as exception:
name_string = utf16_stream.decode(u'utf-16-le', errors=u'replace')
parser_mediator.ProduceParseError((
u'unable to decode USN record name string with error: '
u'{0:s}. Characters that cannot be decoded will be replaced '
u'with "?" or "\\ufffd".').format(exception))
event_object = file_system_events.NTFSUSNChangeEvent(
usn_record_struct.update_date_time, current_offset,
name_string, usn_record_struct.file_reference,
usn_record_struct.update_sequence_number,
usn_record_struct.update_source_flags,
usn_record_struct.update_reason_flags,
file_attribute_flags=usn_record_struct.file_attribute_flags,
parent_file_reference=usn_record_struct.parent_file_reference)
parser_mediator.ProduceEvent(event_object)
usn_record_data = usn_change_journal.read_usn_record()
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses a NTFS $UsnJrnl metadata file-like object.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
file_object: A file-like object.
"""
volume = pyfsntfs.volume()
try:
volume.open_file_object(file_object)
except IOError as exception:
parser_mediator.ProduceParseError(
u'unable to open NTFS volume with error: {0:s}'.format(exception))
try:
usn_change_journal = volume.get_usn_change_journal()
self._ParseUSNChangeJournal(parser_mediator, usn_change_journal)
finally:
volume.close()
dt = datetime.datetime.utcfromtimestamp(unix_time)
# dt = dt.replace(tzinfo=pytz.UTC)
return dt.isoformat()
def FILETIME(name):
return FileTimeAdapter(construct.ULInt64(name))
# Common structs.
UNICODE_STRING = construct.Struct(
'UNICODE_STRING', construct.ULInt32('length'),
construct.String('data', lambda ctx: ctx.length, encoding='utf16'))
SIZED_DATA = construct.Struct('SIZED_DATA', construct.ULInt32('size'),
construct.Bytes('data', lambda ctx: ctx.size))
# DPAPI structs.
DPAPI_BLOB = construct.Struct(
'BLOB', construct.ULInt32('version'), GUID('provider'),
construct.ULInt32('mk_version'),
GUID('mk_guid'), construct.ULInt32('flags'),
construct.Rename('description', UNICODE_STRING),
construct.ULInt32('crypt_alg_id'), construct.ULInt32('crypt_alg_len'),
construct.ULInt32('salt_len'),
construct.Bytes('salt', lambda ctx: ctx.salt_len),
construct.ULInt32('unknown1'), construct.ULInt32('hash_alg_id'),
class KeychainParser(interface.BaseParser):
"""Parser for Keychain files."""
NAME = 'mac_keychain'
DESCRIPTION = u'Parser for Mac OS X Keychain files.'
KEYCHAIN_MAGIC_HEADER = 'kych'
KEYCHAIN_MAJOR_VERSION = 1
KEYCHAIN_MINOR_VERSION = 0
RECORD_TYPE_APPLICATION = 0x80000000
RECORD_TYPE_INTERNET = 0x80000001
# DB HEADER.
KEYCHAIN_DB_HEADER = construct.Struct(
'db_header',
construct.String('magic', 4),
construct.UBInt16('major_version'),
construct.UBInt16('minor_version'),
construct.UBInt32('header_size'),
construct.UBInt32('schema_offset'),
construct.Padding(4))
# DB SCHEMA.
KEYCHAIN_DB_SCHEMA = construct.Struct(
'db_schema',
construct.UBInt32('size'),
construct.UBInt32('number_of_tables'))
# For each number_of_tables, the schema has a TABLE_OFFSET with the
# offset starting in the DB_SCHEMA.
TABLE_OFFSET = construct.UBInt32('table_offset')
TABLE_HEADER = construct.Struct(
'table_header',
construct.UBInt32('table_size'),
construct.UBInt32('record_type'),
construct.UBInt32('number_of_records'),
construct.UBInt32('first_record'),
construct.UBInt32('index_offset'),
construct.Padding(4),
construct.UBInt32('recordnumbercount'))
RECORD_HEADER = construct.Struct(
'record_header',
construct.UBInt32('entry_length'),
construct.Padding(12),
construct.UBInt32('ssgp_length'),
construct.Padding(4),
construct.UBInt32('creation_time'),
construct.UBInt32('last_mod_time'),
construct.UBInt32('text_description'),
construct.Padding(4),
construct.UBInt32('comments'),
construct.Padding(8),
construct.UBInt32('entry_name'),
construct.Padding(20),
construct.UBInt32('account_name'),
construct.Padding(4))
RECORD_HEADER_APP = construct.Struct(
'record_entry_app',
RECORD_HEADER,
construct.Padding(4))
RECORD_HEADER_INET = construct.Struct(
'record_entry_inet',
RECORD_HEADER,
construct.UBInt32('where'),
construct.UBInt32('protocol'),
construct.UBInt32('type'),
construct.Padding(4),
construct.UBInt32('url'))
TEXT = construct.PascalString(
'text', length_field = construct.UBInt32('length'))
TIME = construct.Struct(
'timestamp',
construct.String('year', 4),
construct.String('month', 2),
construct.String('day', 2),
construct.String('hour', 2),
construct.String('minute', 2),
construct.String('second', 2),
construct.Padding(2))
TYPE_TEXT = construct.String('type', 4)
# TODO: add more protocols.
_PROTOCOL_TRANSLATION_DICT = {
u'htps': u'https',
u'smtp': u'smtp',
u'imap': u'imap',
u'http': u'http'}
def _GetTimestampFromEntry(self, parser_context, file_entry, structure):
"""Parse a time entry structure into a microseconds since Epoch in UTC.
Args:
parser_context: A parser context object (instance of ParserContext).
file_entry: A file entry object (instance of dfvfs.FileEntry).
structure: TIME entry structure:
year: String with the number of the year.
month: String with the number of the month.
day: String with the number of the day.
hour: String with the number of the month.
minute: String with the number of the minute.
second: String with the number of the second.
Returns:
Microseconds since Epoch in UTC.
"""
try:
return timelib.Timestamp.FromTimeParts(
int(structure.year, 10), int(structure.month, 10),
int(structure.day, 10), int(structure.hour, 10),
int(structure.minute, 10), int(structure.second, 10))
except ValueError:
logging.warning(
u'[{0:s}] Invalid keychain time {1!s} in file: {2:s}'.format(
self.NAME, parser_context.GetDisplayName(file_entry), structure))
return 0
def _ReadEntryApplication(self, parser_context, file_object, file_entry=None):
"""Extracts the information from an application password entry.
Args:
parser_context: A parser context object (instance of ParserContext).
file_object: A file-like object that points to an Keychain file.
file_entry: Optional file entry object (instance of dfvfs.FileEntry).
The default is None.
"""
offset = file_object.tell()
try:
record = self.RECORD_HEADER_APP.parse_stream(file_object)
except (IOError, construct.FieldError):
logging.warning((
u'[{0:s}] Unsupported record header at 0x{1:08x} in file: '
u'{2:s}').format(
self.NAME, offset, parser_context.GetDisplayName(file_entry)))
return
(ssgp_hash, creation_time, last_mod_time, text_description,
comments, entry_name, account_name) = self._ReadEntryHeader(
parser_context, file_entry, file_object, record.record_header, offset)
# Move to the end of the record, and then, prepared for the next record.
file_object.seek(
record.record_header.entry_length + offset - file_object.tell(),
os.SEEK_CUR)
event_object = KeychainApplicationRecordEvent(
creation_time, eventdata.EventTimestamp.CREATION_TIME,
entry_name, account_name, text_description, comments, ssgp_hash)
parser_context.ProduceEvent(
event_object, parser_name=self.NAME, file_entry=file_entry)
if creation_time != last_mod_time:
event_object = KeychainApplicationRecordEvent(
last_mod_time, eventdata.EventTimestamp.MODIFICATION_TIME,
entry_name, account_name, text_description, comments, ssgp_hash)
parser_context.ProduceEvent(
event_object, parser_name=self.NAME, file_entry=file_entry)
def _ReadEntryHeader(
self, parser_context, file_entry, file_object, record, offset):
"""Read the common record attributes.
Args:
parser_context: A parser context object (instance of ParserContext).
file_entry: A file entry object (instance of dfvfs.FileEntry).
file_object: A file-like object that points to an Keychain file.
record: Structure with the header of the record.
offset: First byte of the record.
Returns:
A list of:
ssgp_hash: Hash of the encrypted data (passwd, cert, note).
creation_time: When the entry was created.
last_mod_time: Last time the entry was updated.
text_description: A brief description of the entry.
entry_name: Name of the entry
account_name: Name of the account.
"""
# Info: The hash header always start with the string ssgp follow by
# the hash. Furthermore The fields are always a multiple of four.
# Then if it is not multiple the value is padded by 0x00.
ssgp_hash = binascii.hexlify(file_object.read(record.ssgp_length)[4:])
file_object.seek(
record.creation_time - file_object.tell() + offset - 1, os.SEEK_CUR)
creation_time = self._GetTimestampFromEntry(
parser_context, file_entry, self.TIME.parse_stream(file_object))
file_object.seek(
record.last_mod_time - file_object.tell() + offset - 1, os.SEEK_CUR)
last_mod_time = self._GetTimestampFromEntry(
parser_context, file_entry, self.TIME.parse_stream(file_object))
# The comment field does not always contain data.
if record.text_description:
file_object.seek(
record.text_description - file_object.tell() + offset -1,
os.SEEK_CUR)
text_description = self.TEXT.parse_stream(file_object)
else:
text_description = u'N/A'
# The comment field does not always contain data.
if record.comments:
file_object.seek(
record.text_description - file_object.tell() + offset -1,
os.SEEK_CUR)
comments = self.TEXT.parse_stream(file_object)
else:
comments = u'N/A'
file_object.seek(
record.entry_name - file_object.tell() + offset - 1, os.SEEK_CUR)
entry_name = self.TEXT.parse_stream(file_object)
file_object.seek(
record.account_name - file_object.tell() + offset - 1, os.SEEK_CUR)
account_name = self.TEXT.parse_stream(file_object)
return (
ssgp_hash, creation_time, last_mod_time,
text_description, comments, entry_name, account_name)
def _ReadEntryInternet(self, parser_context, file_object, file_entry=None):
"""Extracts the information from an Internet password entry.
Args:
parser_context: A parser context object (instance of ParserContext).
file_object: A file-like object that points to an Keychain file.
file_entry: Optional file entry object (instance of dfvfs.FileEntry).
The default is None.
"""
offset = file_object.tell()
try:
record = self.RECORD_HEADER_INET.parse_stream(file_object)
except (IOError, construct.FieldError):
logging.warning((
u'[{0:s}] Unsupported record header at 0x{1:08x} in file: '
u'{2:s}').format(
self.NAME, offset, parser_context.GetDisplayName(file_entry)))
return
(ssgp_hash, creation_time, last_mod_time, text_description,
comments, entry_name, account_name) = self._ReadEntryHeader(
parser_context, file_entry, file_object, record.record_header, offset)
if not record.where:
where = u'N/A'
protocol = u'N/A'
type_protocol = u'N/A'
else:
file_object.seek(
record.where - file_object.tell() + offset - 1, os.SEEK_CUR)
where = self.TEXT.parse_stream(file_object)
file_object.seek(
record.protocol - file_object.tell() + offset - 1, os.SEEK_CUR)
protocol = self.TYPE_TEXT.parse_stream(file_object)
file_object.seek(
record.type - file_object.tell() + offset - 1, os.SEEK_CUR)
type_protocol = self.TEXT.parse_stream(file_object)
type_protocol = self._PROTOCOL_TRANSLATION_DICT.get(
type_protocol, type_protocol)
if record.url:
file_object.seek(
record.url - file_object.tell() + offset - 1, os.SEEK_CUR)
url = self.TEXT.parse_stream(file_object)
where = u'{0:s}{1:s}'.format(where, url)
# Move to the end of the record, and then, prepared for the next record.
file_object.seek(
record.record_header.entry_length + offset - file_object.tell(),
os.SEEK_CUR)
event_object = KeychainInternetRecordEvent(
creation_time, eventdata.EventTimestamp.CREATION_TIME,
entry_name, account_name, text_description,
comments, where, protocol, type_protocol, ssgp_hash)
parser_context.ProduceEvent(
event_object, parser_name=self.NAME, file_entry=file_entry)
if creation_time != last_mod_time:
event_object = KeychainInternetRecordEvent(
last_mod_time, eventdata.EventTimestamp.MODIFICATION_TIME,
entry_name, account_name, text_description,
comments, where, protocol, type_protocol)
parser_context.ProduceEvent(
event_object, parser_name=self.NAME, file_entry=file_entry)
def _VerifyStructure(self, file_object):
"""Verify that we are dealing with an Keychain entry.
Args:
file_object: A file-like object that points to an Keychain file.
Returns:
A list of table positions if it is a keychain, None otherwise.
"""
# INFO: The HEADER KEYCHAIN:
# [DBHEADER] + [DBSCHEMA] + [OFFSET TABLE A] + ... + [OFFSET TABLE Z]
# Where the table offset is relative to the first byte of the DB Schema,
# then we must add to this offset the size of the [DBHEADER].
try:
db_header = self.KEYCHAIN_DB_HEADER.parse_stream(file_object)
except (IOError, construct.FieldError):
return
if (db_header.minor_version != self.KEYCHAIN_MINOR_VERSION or
db_header.major_version != self.KEYCHAIN_MAJOR_VERSION or
db_header.magic != self.KEYCHAIN_MAGIC_HEADER):
return
# Read the database schema and extract the offset for all the tables.
# They are ordered by file position from the top to the bottom of the file.
try:
db_schema = self.KEYCHAIN_DB_SCHEMA.parse_stream(file_object)
except (IOError, construct.FieldError):
return
table_offsets = []
for _ in range(db_schema.number_of_tables):
try:
table_offset = self.TABLE_OFFSET.parse_stream(file_object)
except (IOError, construct.FieldError):
return
table_offsets.append(table_offset + self.KEYCHAIN_DB_HEADER.sizeof())
return table_offsets
def Parse(self, parser_context, file_entry):
"""Extract data from a Keychain file.
Args:
parser_context: A parser context object (instance of ParserContext).
file_entry: A file entry object (instance of dfvfs.FileEntry).
"""
file_object = file_entry.GetFileObject()
table_offsets = self._VerifyStructure(file_object)
if not table_offsets:
file_object.close()
raise errors.UnableToParseFile(u'The file is not a Keychain file.')
for table_offset in table_offsets:
# Skipping X bytes, unknown data at this point.
file_object.seek(table_offset - file_object.tell(), os.SEEK_CUR)
try:
table = self.TABLE_HEADER.parse_stream(file_object)
except construct.FieldError as exception:
logging.warning((
u'[{0:s}] Unable to parse table header in file: {1:s} '
u'with error: {2:s}.').format(
self.NAME, parser_context.GetDisplayName(file_entry),
exception))
continue
# Table_offset: absolute byte in the file where the table starts.
# table.first_record: first record in the table, relative to the
# first byte of the table.
file_object.seek(
table_offset + table.first_record - file_object.tell(), os.SEEK_CUR)
if table.record_type == self.RECORD_TYPE_INTERNET:
for _ in range(table.number_of_records):
self._ReadEntryInternet(
parser_context, file_object, file_entry=file_entry)
elif table.record_type == self.RECORD_TYPE_APPLICATION:
for _ in range(table.number_of_records):
self._ReadEntryApplication(
parser_context, file_object, file_entry=file_entry)
file_object.close()
"unk_offset_11" / construct.Int16ul, # + 0x30
"unk_nb_11" / construct.Int16ul, # + 0x32
"unk_offset_12" / construct.Int16ul, # + 0x34
"unk_nb_12" / construct.Int16ul, # + 0x36
"unk_offset_13" / construct.Int16ul, # + 0x38
"unk_nb_13" / construct.Int16ul, # + 0x3A
"unk_offset_14" / construct.Int16ul, # + 0x3C
"unk_nb_14" / construct.Int16ul, # + 0x3E
"entries" / construct.OnDemandPointer(
lambda ctx: rawf.raw_file.unk_word_00 + rawf.raw_file.field_14 +
ctx.unk_offset_00,
construct.Array(
lambda ctx: ctx.unk_nb_00,
construct.Struct(
"length" / construct.Int16ul, "data" /
construct.String(lambda ctx: ctx.length - 0x02)))))
o = obj_unk_01.parse_stream(rawf.stream)
print o
import struct
for entry in o.entries():
#print hexdump(struct.pack("<H", entry.length) + entry.data, 0x20)
print hexdump(entry.data, 0x20)
# /!\ SIZE BEFORE
# + 0x00 : ????
# + 0x01 : TYPE
#
# Type
# - 0x1B : FIELD_08 => WORD
class WinRecycleBinParser(interface.FileObjectParser):
"""Parses the Windows $Recycle.Bin $I files."""
NAME = u'recycle_bin'
DESCRIPTION = u'Parser for Windows $Recycle.Bin $I files.'
_FILE_HEADER_STRUCT = construct.Struct(
u'file_header', construct.ULInt64(u'format_version'),
construct.ULInt64(u'file_size'), construct.ULInt64(u'deletion_time'))
_FILENAME_V2_STRUCT = construct.Struct(
u'filename_v2', construct.ULInt32(u'number_of_characters'),
construct.String(u'string', lambda ctx: ctx.number_of_characters * 2))
def _ReadFilename(self, parser_mediator, file_object, format_version):
"""Reads the filename.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (FileIO): file-like object.
format_version (int): format version.
Returns:
str: filename
"""
if format_version == 1:
return binary.ReadUTF16Stream(file_object)
try:
filename_struct = self._FILENAME_V2_STRUCT.parse_stream(
file_object)
except (IOError, construct.FieldError) as exception:
parser_mediator.ProduceExtractionError(
u'unable to parse filename with error: {0:s}'.format(
exception))
return
return binary.ReadUTF16(filename_struct.string)
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses a Windows RecycleBin $Ixx file-like object.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): file-like object.
"""
# We may have to rely on filenames since this header is very generic.
# TODO: Rethink this and potentially make a better test.
filename = parser_mediator.GetFilename()
if not filename.startswith(u'$I'):
return
try:
header_struct = self._FILE_HEADER_STRUCT.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
parser_mediator.ProduceExtractionError(
u'unable to parse file header with error: {0:s}'.format(
exception))
return
if header_struct.format_version not in (1, 2):
parser_mediator.ProduceExtractionError(
u'unsupported format version: {0:d}.'.format(
header_struct.format_version))
return
if header_struct.deletion_time == 0:
date_time = dfdatetime_semantic_time.SemanticTime(u'Not set')
else:
date_time = dfdatetime_filetime.Filetime(
timestamp=header_struct.deletion_time)
event_data = WinRecycleBinEventData()
event_data.original_filename = self._ReadFilename(
parser_mediator, file_object, header_struct.format_version)
event_data.file_size = header_struct.file_size
event = time_events.DateTimeValuesEvent(
date_time, eventdata.EventTimestamp.DELETED_TIME)
parser_mediator.ProduceEventWithEventData(event, event_data)
class AslParser(interface.BaseParser):
"""Parser for ASL log files."""
NAME = 'asl_log'
DESCRIPTION = u'Parser for ASL log files.'
ASL_MAGIC = 'ASL DB\x00\x00\x00\x00\x00\x00'
# If not right assigned, the value is "-1".
ASL_NO_RIGHTS = 'ffffffff'
# Priority level (criticity)
ASL_MESSAGE_PRIORITY = {
0: 'EMERGENCY',
1: 'ALERT',
2: 'CRITICAL',
3: 'ERROR',
4: 'WARNING',
5: 'NOTICE',
6: 'INFO',
7: 'DEBUG'
}
# ASL File header.
# magic: magic number that identify ASL files.
# version: version of the file.
# offset: first record in the file.
# timestamp: epoch time when the first entry was written.
# last_offset: last record in the file.
ASL_HEADER_STRUCT = construct.Struct('asl_header_struct',
construct.String('magic', 12),
construct.UBInt32('version'),
construct.UBInt64('offset'),
construct.UBInt64('timestamp'),
construct.UBInt32('cache_size'),
construct.UBInt64('last_offset'),
construct.Padding(36))
# The record structure is:
# [HEAP][STRUCTURE][4xExtraField][2xExtraField]*[PreviousEntry]
# Record static structure.
# tam_entry: it contains the number of bytes from this file position
# until the end of the record, without counts itself.
# next_offset: next record. If is equal to 0x00, it is the last record.
# asl_message_id: integer that has the numeric identification of the event.
# timestamp: Epoch integer that has the time when the entry was created.
# nanosecond: nanosecond to add to the timestamp.
# level: level of priority.
# pid: process identification that ask to save the record.
# uid: user identification that has lunched the process.
# gid: group identification that has lunched the process.
# read_uid: identification id of a user. Only applied if is not -1 (all FF).
# Only root and this user can read the entry.
# read_gid: the same than read_uid, but for the group.
ASL_RECORD_STRUCT = construct.Struct('asl_record_struct',
construct.Padding(2),
construct.UBInt32('tam_entry'),
construct.UBInt64('next_offset'),
construct.UBInt64('asl_message_id'),
construct.UBInt64('timestamp'),
construct.UBInt32('nanosec'),
construct.UBInt16('level'),
construct.UBInt16('flags'),
construct.UBInt32('pid'),
construct.UBInt32('uid'),
construct.UBInt32('gid'),
construct.UBInt32('read_uid'),
construct.UBInt32('read_gid'),
construct.UBInt64('ref_pid'))
ASL_RECORD_STRUCT_SIZE = ASL_RECORD_STRUCT.sizeof()
# 8-byte fields, they can be:
# - String: [Nibble = 1000 (8)][Nibble = Length][7 Bytes = String].
# - Integer: integer that has the byte position in the file that points
# to an ASL_RECORD_DYN_VALUE struct. If the value of the integer
# is equal to 0, it means that it has not data (skip).
# If the field is a String, we use this structure to decode each
# integer byte in the corresponding character (ASCII Char).
ASL_OCTET_STRING = construct.ExprAdapter(construct.Octet('string'),
encoder=lambda obj, ctx: ord(obj),
decoder=lambda obj, ctx: chr(obj))
# Field string structure. If the first bit is 1, it means that it
# is a String (1000) = 8, then the next nibble has the number of
# characters. The last 7 bytes are the number of bytes.
ASL_STRING = construct.BitStruct(
'string', construct.Flag('type'), construct.Bits('filler', 3),
construct.If(lambda ctx: ctx.type, construct.Nibble('string_length')),
construct.If(lambda ctx: ctx.type,
construct.Array(7, ASL_OCTET_STRING)))
# 8-byte pointer to a byte position in the file.
ASL_POINTER = construct.UBInt64('pointer')
# Dynamic data structure pointed by a pointer that contains a String:
# [2 bytes padding][4 bytes lenght of String][String].
ASL_RECORD_DYN_VALUE = construct.Struct(
'asl_record_dyn_value', construct.Padding(2),
construct.PascalString('value',
length_field=construct.UBInt32('length')))
def Parse(self, parser_context, file_entry):
"""Extract entries from an ASL file.
Args:
parser_context: A parser context object (instance of ParserContext).
file_entry: A file entry object (instance of dfvfs.FileEntry).
"""
file_object = file_entry.GetFileObject()
file_object.seek(0, os.SEEK_SET)
try:
header = self.ASL_HEADER_STRUCT.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
file_object.close()
raise errors.UnableToParseFile(
u'Unable to parse ASL Header with error: {0:s}.'.format(
exception))
if header.magic != self.ASL_MAGIC:
file_object.close()
raise errors.UnableToParseFile(
u'Not an ASL Header, unable to parse.')
# Get the first and the last entry.
offset = header.offset
old_offset = header.offset
last_offset_header = header.last_offset
# If the ASL file has entries.
if offset:
event_object, offset = self.ReadAslEvent(file_object, offset)
while event_object:
parser_context.ProduceEvent(event_object,
parser_name=self.NAME,
file_entry=file_entry)
# TODO: an anomaly object must be emitted once that is implemented.
# Sanity check, the last read element must be the same as
# indicated by the header.
if offset == 0 and old_offset != last_offset_header:
logging.warning(u'Parsing ended before the header ends.')
old_offset = offset
event_object, offset = self.ReadAslEvent(file_object, offset)
file_object.close()
def ReadAslEvent(self, file_object, offset):
"""Returns an AslEvent from a single ASL entry.
Args:
file_object: a file-like object that points to an ASL file.
offset: offset where the static part of the entry starts.
Returns:
An event object constructed from a single ASL record.
"""
# The heap of the entry is saved to try to avoid seek (performance issue).
# It has the real start position of the entry.
dynamic_start = file_object.tell()
dynamic_part = file_object.read(offset - file_object.tell())
if not offset:
return None, None
try:
record_header = self.ASL_RECORD_STRUCT.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
logging.warning(
u'Unable to parse ASL event with error: {0:s}'.format(
exception))
return None, None
# Variable tam_fields = is the real length of the dynamic fields.
# We have this: [Record_Struct] + [Dynamic_Fields] + [Pointer_Entry_Before]
# In Record_Struct we have a field called tam_entry, where it has the number
# of bytes until the end of the entry from the position that the field is.
# The tam_entry is between the 2th and the 6th byte in the [Record_Struct].
# tam_entry = ([Record_Struct]-6)+[Dynamic_Fields]+[Pointer_Entry_Before]
# Also, we do not need [Point_Entry_Before] and then we delete the size of
# [Point_Entry_Before] that it is 8 bytes (8):
# tam_entry = ([Record_Struct]-6)+[Dynamic_Fields]+[Pointer_Entry_Before]
# [Dynamic_Fields] = tam_entry - [Record_Struct] + 6 - 8
# [Dynamic_Fields] = tam_entry - [Record_Struct] - 2
tam_fields = record_header.tam_entry - self.ASL_RECORD_STRUCT_SIZE - 2
# Dynamic part of the entry that contains minimal four fields of 8 bytes
# plus 2x[8bytes] fields for each extra ASL_Field.
# The four first fields are always the Host, Sender, Facility and Message.
# After the four first fields, the entry might have extra ASL_Fields.
# For each extra ASL_field, it has a pair of 8-byte fields where the first
# 8 bytes contains the name of the extra ASL_field and the second 8 bytes
# contains the text of the exta field.
# All of this 8-byte field can be saved using one of these three differents
# types:
# - Null value ('0000000000000000'): nothing to do.
# - String: It is string if first bit = 1 or first nibble = 8 (1000).
# Second nibble has the length of string.
# The next 7 bytes have the text characters of the string
# padding the end with null characters: '0x00'.
# Example: [8468 6964 6400 0000]
# [8] String, [4] length, value: [68 69 64 64] = hidd.
# - Pointer: static position in the file to a special struct
# implemented as an ASL_RECORD_DYN_VALUE.
# Example: [0000 0000 0000 0077]
# It points to the file position 0x077 that has a
# ASL_RECORD_DYN_VALUE structure.
values = []
while tam_fields > 0:
try:
raw_field = file_object.read(8)
except (IOError, construct.FieldError) as exception:
logging.warning(
u'Unable to parse ASL event with error: {0:d}'.format(
exception))
return None, None
try:
# Try to read as a String.
field = self.ASL_STRING.parse(raw_field)
values.append(''.join(field.string[0:field.string_length]))
# Go to parse the next extra field.
tam_fields -= 8
continue
except ValueError:
pass
# If it is not a string, it must be a pointer.
try:
field = self.ASL_POINTER.parse(raw_field)
except ValueError as exception:
logging.warning(
u'Unable to parse ASL event with error: {0:s}'.format(
exception))
return None, None
if field != 0:
# The next IF ELSE is only for performance issues, avoiding seek.
# If the pointer points a lower position than where the actual entry
# starts, it means that it points to a previuos entry.
pos = field - dynamic_start
# Bigger or equal 0 means that the data is in the actual entry.
if pos >= 0:
try:
values.append((self.ASL_RECORD_DYN_VALUE.parse(
dynamic_part[pos:])).value.partition('\x00')[0])
except (IOError, construct.FieldError) as exception:
logging.warning(
u'Unable to parse ASL event with error: {0:s}'.
format(exception))
return None, None
else:
# Only if it is a pointer that points to the
# heap from another entry we use the seek method.
main_position = file_object.tell()
# If the pointer is in a previous entry.
if main_position > field:
file_object.seek(field - main_position, os.SEEK_CUR)
try:
values.append(
(self.ASL_RECORD_DYN_VALUE.parse_stream(
file_object)).value.partition('\x00')[0])
except (IOError, construct.FieldError):
logging.warning((
u'The pointer at {0:d} (0x{0:x}) points to invalid '
u'information.'
).format(main_position -
self.ASL_POINTER.sizeof()))
# Come back to the position in the entry.
_ = file_object.read(main_position -
file_object.tell())
else:
_ = file_object.read(field - main_position)
values.append((self.ASL_RECORD_DYN_VALUE.parse_stream(
file_object)).value.partition('\x00')[0])
# Come back to the position in the entry.
file_object.seek(main_position - file_object.tell(),
os.SEEK_CUR)
# Next extra field: 8 bytes more.
tam_fields -= 8
# Read the last 8 bytes of the record that points to the previous entry.
_ = file_object.read(8)
# Parsed section, we translate the read data to an appropriate format.
microsecond = record_header.nanosec // 1000
timestamp = timelib.Timestamp.FromPosixTimeWithMicrosecond(
record_header.timestamp, microsecond)
record_position = offset
message_id = record_header.asl_message_id
level = u'{0} ({1})'.format(
self.ASL_MESSAGE_PRIORITY[record_header.level],
record_header.level)
# If the value is -1 (FFFFFFFF), it can be read by everyone.
if record_header.read_uid != int(self.ASL_NO_RIGHTS, 16):
read_uid = record_header.read_uid
else:
read_uid = 'ALL'
if record_header.read_gid != int(self.ASL_NO_RIGHTS, 16):
read_gid = record_header.read_gid
else:
read_gid = 'ALL'
# Parsing the dynamic values (text or pointers to position with text).
# The first four are always the host, sender, facility, and message.
computer_name = values[0]
sender = values[1]
facility = values[2]
message = values[3]
# If the entry has an extra fields, they works as a pairs:
# The first is the name of the field and the second the value.
extra_information = ''
if len(values) > 4:
values = values[4:]
for index in xrange(0, len(values) // 2):
extra_information += (u'[{0}: {1}]'.format(
values[index * 2], values[(index * 2) + 1]))
# Return the event and the offset for the next entry.
return AslEvent(timestamp, record_position, message_id, level,
record_header, read_uid, read_gid, computer_name,
sender, facility, message,
extra_information), record_header.next_offset
An old version of yk_bmdata.py that makes use of construct library.
Limitation: Number of ToCs it can parse
"""
__author__ = 'yukai'
# to reduce code width
_ARRAY = construct.Array
_POINTER = construct.Pointer
_REPEAT = construct.RepeatUntil
_INT = construct.ULInt32('integer')
_INT64 = construct.ULInt64('integer64')
_BOOKMARK_DATA = construct.Struct(
'bookmark_data',
construct.String('magic', 4),
construct.ULInt32('length'),
construct.UBInt32('version'),
construct.ULInt32('offset'), # offset to "FirstToC Offset"
_POINTER(
lambda ctx: ctx.offset,
construct.Struct(
'ftoc_offset',
construct.Anchor('abs_offset'),
construct.ULInt32('offset'),
_POINTER(
lambda ctx: ctx.abs_offset + ctx.offset,
_REPEAT(
lambda obj, ctx: obj.offset == 0x00000000,
construct.Struct(
'toc',
class WinRecycleBinParser(interface.FileObjectParser):
"""Parses the Windows $Recycle.Bin $I files."""
NAME = u'recycle_bin'
DESCRIPTION = u'Parser for Windows $Recycle.Bin $I files.'
_FILE_HEADER_STRUCT = construct.Struct(
u'file_header', construct.ULInt64(u'format_version'),
construct.ULInt64(u'file_size'), construct.ULInt64(u'deletion_time'))
_FILENAME_V2_STRUCT = construct.Struct(
u'filename_v2', construct.ULInt32(u'number_of_characters'),
construct.String(u'string', lambda ctx: ctx.number_of_characters * 2))
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses a Windows RecycleBin $Ixx file-like object.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): file-like object.
"""
# We may have to rely on filenames since this header is very generic.
# TODO: Rethink this and potentially make a better test.
filename = parser_mediator.GetFilename()
if not filename.startswith(u'$I'):
return
try:
header_struct = self._FILE_HEADER_STRUCT.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
parser_mediator.ProduceExtractionError(
u'unable to parse file header with error: {0:s}'.format(
exception))
return
if header_struct.format_version not in (1, 2):
parser_mediator.ProduceExtractionError(
u'unsupported format version: {0:d}.'.format(
header_struct.format_version))
return
filename = None
if header_struct.format_version == 1:
filename = binary.ReadUTF16Stream(file_object)
else:
try:
filename_struct = self._FILENAME_V2_STRUCT.parse_stream(
file_object)
filename = binary.ReadUTF16(filename_struct.string)
except (IOError, construct.FieldError) as exception:
parser_mediator.ProduceExtractionError(
u'unable to parse filename with error: {0:s}'.format(
exception))
return
event = WinRecycleEvent(header_struct.deletion_time, filename,
header_struct.file_size)
parser_mediator.ProduceEvent(event)
class Game:
command_id = namedtuple("packet_type",
("register", "error", "update", "start", "state"))(
register=0,
error=1,
update=2,
start=3,
state=4)
players_struct = construct.Struct("x" / construct.Int16ul,
"y" / construct.Int16ul,
"score" / construct.Int8ul,
"active" / construct.Int8ul)
command_state = construct.Struct(
"puck_x" / construct.Int16ul, "puck_y" / construct.Int16ul,
"players" / construct.Array(4, players_struct))
command_update = construct.Struct("x" / construct.Int16ul,
"y" / construct.Int16ul)
command_register = construct.Struct("id" / construct.String(36))
command = construct.Struct(
"type" / construct.Int8ul,
construct.Embedded(
construct.Switch(lambda ctx: ctx.type, {
command_id.update: command_update,
command_id.register: command_register,
command_id.state: command_state,
},
default=construct.Pass)))
def __init__(self):
self.sockets = []
self.redis = rredis.from_url(os.environ.get("REDIS_URL"))
def add_state_socket(self, ws):
sock = ClientSocket(ws)
data = sock.receive()
if data:
data = self.command.parse(data)
if data and data.type == self.command_id.register:
sock.set_id(data.id)
self.sockets.append(sock)
else:
sock.close()
def update(self):
while True:
games = ast.literal_eval(self.redis.get("games"))
self.update_games(games)
# Send states
for ws in list(self.sockets):
# Clear inactive sockets
if ws.is_closed():
for game in list(games):
for player in range(0, len(game["players"])):
if game["players"][player]["id"] == ws.get_id():
game["players"][player]["id"] = None
game["players"][player]["active"] = 0
if self.active_players(game["players"]) == 0:
games.remove(game)
self.sockets.remove(ws)
continue
# Send state
for game in games:
for player in game["players"]:
if player["id"] == ws.get_id():
ws.send(self.build_state_packet(game), True)
self.redis.set("games", games)
gevent.sleep(0.05)
@staticmethod
def update_games(games):
if len(games):
games[0]["puck"]["x"] += 1
def start(self):
self.redis.set("games", [])
gevent.spawn(self.update)
def add_request_socket(self, ws):
sock = ClientSocket(ws)
while not sock.is_closed():
data = sock.receive()
if not data:
sock.close()
return
data = self.command.parse(data)
# Register
if data and data.type == self.command_id.register and sock.get_id(
) is None:
client_id = self.get_new_client_id()
sock.set_id(client_id)
if client_id:
sock.send(
self.command.build(
dict(type=self.command_id.register, id=client_id)),
True)
else:
sock.send(
self.command.build(dict(type=self.command_id.error)),
True)
# Position Update
elif data.type == self.command_id.update:
self.update_player_position(sock.client_id, data.x, data.y)
def get_new_client_id(self):
games = ast.literal_eval(self.redis.get("games"))
new_player_id = str(uuid.uuid4())
# Make sure the id is unique
for game in games:
for player in game["players"]:
if player["id"] == new_player_id:
return self.get_new_client_id()
# Find an open game
found_game = False
for game in games:
if self.active_players(game["players"]) < 4:
found_game = True
# Add player to game
for player in range(0, len(game["players"])):
if game["players"][player]["active"] == 0:
game["players"][player]["id"] = new_player_id
game["players"][player]["active"] = 1
break
# Check if game can start
if self.active_players(game["players"]) == 4:
for player in game["players"]:
for sock in self.sockets:
if sock.get_id() == player["id"]:
sock.send(
self.command.build(
dict(type=self.command_id.start)),
True)
# Create game if an open one was not found
if not found_game:
game = {"players": [], "puck": {"x": 0, "y": 0}}
for player_num in range(0, 4):
game["players"].append({
"id": None,
"x": 0,
"y": 0,
"score": 0,
"active": 0
})
game["players"][0]["id"] = new_player_id
game["players"][0]["active"] = 1
games.append(game)
self.redis.set("games", games)
return new_player_id
def build_state_packet(self, game):
state = dict(type=self.command_id.state,
puck_x=game["puck"]["x"],
puck_y=game["puck"]["y"],
players=game["players"])
return self.command.build(state)
def update_player_position(self, client_id, x, y):
games = ast.literal_eval(self.redis.get("games"))
for game in games:
for player in game["players"]:
if player["id"] == client_id:
player["x"] = x
player["y"] = y
self.redis.set("games", games)
@staticmethod
def active_players(players):
count = 0
for player in players:
if player["active"] == 1:
count += 1
return count
construct.If(
lambda ctx: ctx.image_descriptor.lct_flag,
construct.Array(
lambda ctx: pow(2, ctx.image_descriptor.lct_size + 1),
construct.Array(3, construct.ULInt8('lct')),
),
),
construct.ULInt8('lzw_min'),
_get_data_subblocks('compressed_indices'),
)
_application_extension = construct.Struct(
'application_extension',
construct.Value('block_type', lambda ctx: 'application'),
construct.Const(construct.ULInt8('block_size'), 11),
construct.String('app_id', 8),
construct.Bytes('app_auth_code', 3),
_get_data_subblocks('app_data'),
)
_comment_extension = construct.Struct(
'comment_extension',
construct.Value('block_type', lambda ctx: 'comment'),
_get_data_subblocks('comment'),
)
_gce_extension = construct.Struct(
'gce_extension',
construct.Value('block_type', lambda ctx: 'gce'),
construct.Const(construct.ULInt8('block_size'), 4),
construct.EmbeddedBitStruct(
class UtmpxParser(interface.BaseParser):
"""Parser for UTMPX files."""
NAME = 'utmpx'
DESCRIPTION = u'Parser for UTMPX files.'
# INFO: Type is suppose to be a short (2 bytes),
# however if we analyze the file it is always
# byte follow by 3 bytes with \x00 value.
MAC_UTMPX_ENTRY = construct.Struct('utmpx_mac',
construct.String('user', 256),
construct.ULInt32('id'),
construct.String('tty_name', 32),
construct.ULInt32('pid'),
construct.ULInt16('status_type'),
construct.ULInt16('unknown'),
construct.ULInt32('timestamp'),
construct.ULInt32('microsecond'),
construct.String('hostname', 256),
construct.Padding(64))
MAC_UTMPX_ENTRY_SIZE = MAC_UTMPX_ENTRY.sizeof()
# 9, 10 and 11 are only for Darwin and IOS.
MAC_STATUS_TYPE = {
0: 'EMPTY',
1: 'RUN_LVL',
2: 'BOOT_TIME',
3: 'OLD_TIME',
4: 'NEW_TIME',
5: 'INIT_PROCESS',
6: 'LOGIN_PROCESS',
7: 'USER_PROCESS',
8: 'DEAD_PROCESS',
9: 'ACCOUNTING',
10: 'SIGNATURE',
11: 'SHUTDOWN_TIME'
}
def _ReadEntry(self, file_object):
"""Reads an UTMPX entry.
Args:
file_object: a file-like object that points to an UTMPX file.
Returns:
An event object constructed from the UTMPX entry.
"""
data = file_object.read(self.MAC_UTMPX_ENTRY_SIZE)
if len(data) != self.MAC_UTMPX_ENTRY_SIZE:
return
try:
entry = self.MAC_UTMPX_ENTRY.parse(data)
except (IOError, construct.FieldError) as exception:
logging.warning(
u'Unable to parse Mac OS X UTMPX entry with error: {0:s}'.
format(exception))
return
user, _, _ = entry.user.partition('\x00')
if not user:
user = u'N/A'
terminal, _, _ = entry.tty_name.partition('\x00')
if not terminal:
terminal = u'N/A'
computer_name, _, _ = entry.hostname.partition('\x00')
if not computer_name:
computer_name = u'localhost'
value_status = self.MAC_STATUS_TYPE.get(entry.status_type, u'N/A')
status = u'{0}'.format(value_status)
timestamp = timelib.Timestamp.FromPosixTimeWithMicrosecond(
entry.timestamp, entry.microsecond)
return UtmpxMacOsXEvent(timestamp, user, terminal, status,
computer_name)
def _VerifyStructure(self, file_object):
"""Verify that we are dealing with an UTMPX entry.
Args:
file_object: a file-like object that points to an UTMPX file.
Returns:
True if it is a UTMPX entry or False otherwise.
"""
# First entry is a SIGNAL entry of the file ("header").
try:
header = self.MAC_UTMPX_ENTRY.parse_stream(file_object)
except (IOError, construct.FieldError):
return False
user, _, _ = header.user.partition('\x00')
# The UTMPX_ENTRY structure will often succesfully compile on various
# structures, such as binary plist files, and thus we need to do some
# additional validation. The first one is to check if the user name
# can be converted into a unicode string, otherwise we can assume
# we are dealing with non UTMPX data.
try:
_ = unicode(user)
except UnicodeDecodeError:
return False
if user != u'utmpx-1.00':
return False
if self.MAC_STATUS_TYPE[header.status_type] != 'SIGNATURE':
return False
if header.timestamp != 0 or header.microsecond != 0 or header.pid != 0:
return False
tty_name, _, _ = header.tty_name.partition('\x00')
hostname, _, _ = header.hostname.partition('\x00')
if tty_name or hostname:
return False
return True
def Parse(self, parser_context, file_entry):
"""Extract data from a UTMPX file.
Args:
parser_context: A parser context object (instance of ParserContext).
file_entry: A file entry object (instance of dfvfs.FileEntry).
"""
file_object = file_entry.GetFileObject()
if not self._VerifyStructure(file_object):
file_object.close()
raise errors.UnableToParseFile(u'The file is not an UTMPX file.')
event_object = self._ReadEntry(file_object)
while event_object:
event_object.offset = file_object.tell()
parser_context.ProduceEvent(event_object,
parser_name=self.NAME,
file_entry=file_entry)
event_object = self._ReadEntry(file_object)
file_object.close()
class PlsRecallParser(interface.SingleFileBaseParser):
"""Parse PL/SQL Recall files.
Parser is based on a::
TRecallRecord = packed record
Sequence: Integer;
TimeStamp: TDateTime;
Username: array[0..30] of Char;
Database: array[0..80] of Char;
Text: array[0..4000] of Char;
end;
Delphi TDateTime is a little endian 64-bit
floating point without any time zone information
"""
_INITIAL_FILE_OFFSET = None
_PLS_KEYWORD = frozenset([
u'begin', u'commit', u'create', u'declare', u'drop', u'end', u'exception',
u'execute', u'insert', u'replace', u'rollback', u'select', u'set',
u'update'])
# 6 * 365 * 24 * 60 * 60 * 1000000.
_SIX_YEARS_IN_MICRO_SECONDS = 189216000000000
NAME = u'pls_recall'
DESCRIPTION = u'Parser for PL/SQL Recall files.'
PLS_STRUCT = construct.Struct(
u'PL/SQL_Recall',
construct.ULInt32(u'Sequence'),
construct.LFloat64(u'TimeStamp'),
construct.String(u'Username', 31, None, b'\x00'),
construct.String(u'Database', 81, None, b'\x00'),
construct.String(u'Query', 4001, None, b'\x00'))
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses a PLSRecall.dat file-like object.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
file_object: A file-like object.
Raises:
UnableToParseFile: when the file cannot be parsed.
"""
try:
is_pls = self.VerifyFile(file_object)
except (IOError, construct.FieldError) as exception:
raise errors.UnableToParseFile((
u'Not a PLSrecall File, unable to parse.'
u'with error: {0:s}').format(exception))
if not is_pls:
raise errors.UnableToParseFile(
u'Not a PLSRecall File, unable to parse.')
file_object.seek(0, os.SEEK_SET)
pls_record = self.PLS_STRUCT.parse_stream(file_object)
while pls_record:
event_object = PlsRecallEvent(
timelib.Timestamp.FromDelphiTime(pls_record.TimeStamp),
pls_record.Sequence, pls_record.Username,
pls_record.Database, pls_record.Query)
parser_mediator.ProduceEvent(event_object)
try:
pls_record = self.PLS_STRUCT.parse_stream(file_object)
except construct.FieldError:
# The code has reached the end of file (EOF).
break
def VerifyFile(self, file_object):
"""Check if the file is a PLSRecall.dat file.
Args:
file_object: file that we want to check.
Returns:
True if this is a valid PLSRecall.dat file, otherwise False.
"""
file_object.seek(0, os.SEEK_SET)
# The file consists of PL/SQL structures that are equal
# size (4125 bytes) TRecallRecord records. It should be
# noted that the query value is free form.
try:
structure = self.PLS_STRUCT.parse_stream(file_object)
except (IOError, construct.FieldError):
return False
# Verify few entries inside the structure.
try:
timestamp = timelib.Timestamp.FromDelphiTime(structure.TimeStamp)
except ValueError:
return False
if timestamp <= 0:
return False
# Verify that the timestamp is no more than six years into the future.
# Six years is an arbitrary time length just to evaluate the timestamp
# against some value. There is no guarantee that this will catch everything.
# TODO: Add a check for similarly valid value back in time. Maybe if it the
# timestamp is before 1980 we are pretty sure it is invalid?
# TODO: This is a very flaky assumption. Find a better one.
current_timestamp = timelib.Timestamp.GetNow()
if timestamp > current_timestamp + self._SIX_YEARS_IN_MICRO_SECONDS:
return False
# TODO: Add other verification checks here. For instance make sure
# that the query actually looks like a SQL query. This structure produces a
# lot of false positives and thus we need to add additional verification to
# make sure we are not parsing non-PLSRecall files.
# Another check might be to make sure the username looks legitimate, or the
# sequence number, or the database name.
# For now we just check if all three fields pass our "is this a text" test.
if not utils.IsText(structure.Username):
return False
if not utils.IsText(structure.Query):
return False
if not utils.IsText(structure.Database):
return False
# Take the first word from the query field and attempt to match that against
# allowed queries.
first_word, _, _ = structure.Query.partition(b' ')
if first_word.lower() not in self._PLS_KEYWORD:
return False
return True
class CustomDestinationsParser(interface.SingleFileBaseParser):
"""Parses .customDestinations-ms files."""
_INITIAL_FILE_OFFSET = None
NAME = 'custom_destinations'
DESCRIPTION = u'Parser for *.customDestinations-ms files.'
# We cannot use the parser registry here since winlnk could be disabled.
# TODO: see if there is a more elegant solution for this.
_WINLNK_PARSER = winlnk.WinLnkParser()
_LNK_GUID = '\x01\x14\x02\x00\x00\x00\x00\x00\xc0\x00\x00\x00\x00\x00\x00\x46'
_FILE_HEADER = construct.Struct(u'file_header',
construct.ULInt32(u'unknown1'),
construct.ULInt32(u'unknown2'),
construct.ULInt32(u'unknown3'),
construct.ULInt32(u'header_values_type'))
_HEADER_VALUE_TYPE_0 = construct.Struct(
u'header_value_type_0', construct.ULInt32(u'number_of_characters'),
construct.String(u'string', lambda ctx: ctx.number_of_characters * 2),
construct.ULInt32(u'unknown1'))
_HEADER_VALUE_TYPE_1_OR_2 = construct.Struct(
u'header_value_type_1_or_2', construct.ULInt32(u'unknown1'))
_ENTRY_HEADER = construct.Struct(u'entry_header',
construct.String(u'guid', 16))
_FILE_FOOTER = construct.Struct(u'file_footer',
construct.ULInt32(u'signature'))
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses a *.customDestinations-ms file-like object.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
file_object: A file-like object.
Raises:
UnableToParseFile: when the file cannot be parsed.
"""
file_object.seek(0, os.SEEK_SET)
try:
file_header = self._FILE_HEADER.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
raise errors.UnableToParseFile(
(u'Unable to parse Custom Destination file header with error: '
u'{0:s}').format(exception))
if file_header.unknown1 != 2:
raise errors.UnableToParseFile(
(u'Unsupported Custom Destination file - invalid unknown1: '
u'{0:d}.').format(file_header.unknown1))
if file_header.header_values_type > 2:
raise errors.UnableToParseFile((
u'Unsupported Custom Destination file - invalid header value type: '
u'{0:d}.').format(file_header.header_values_type))
if file_header.header_values_type == 0:
data_structure = self._HEADER_VALUE_TYPE_0
else:
data_structure = self._HEADER_VALUE_TYPE_1_OR_2
try:
_ = data_structure.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
raise errors.UnableToParseFile((
u'Unable to parse Custom Destination file header value with error: '
u'{0:s}').format(exception))
file_size = file_object.get_size()
file_offset = file_object.get_offset()
remaining_file_size = file_size - file_offset
# The Custom Destination file does not have a unique signature in
# the file header that is why we use the first LNK class identifier (GUID)
# as a signature.
first_guid_checked = False
while remaining_file_size > 4:
try:
entry_header = self._ENTRY_HEADER.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
if not first_guid_checked:
raise errors.UnableToParseFile((
u'Unable to parse Custom Destination file entry header with '
u'error: {0:s}').format(exception))
else:
logging.warning((
u'Unable to parse Custom Destination file entry header with '
u'error: {0:s}').format(exception))
break
if entry_header.guid != self._LNK_GUID:
if not first_guid_checked:
raise errors.UnableToParseFile(
u'Unsupported Custom Destination file - invalid entry header.'
)
else:
logging.warning(
u'Unsupported Custom Destination file - invalid entry header.'
)
break
first_guid_checked = True
file_offset += 16
remaining_file_size -= 16
path_spec = path_spec_factory.Factory.NewPathSpec(
definitions.TYPE_INDICATOR_DATA_RANGE,
range_offset=file_offset,
range_size=remaining_file_size,
parent=parser_mediator.GetFileEntry().path_spec)
try:
lnk_file_object = resolver.Resolver.OpenFileObject(path_spec)
except RuntimeError as exception:
message = u'Unable to open LNK file with error'.format(
exception)
parser_mediator.ProduceParseError(message)
return
display_name = u'{0:s} # 0x{1:08x}'.format(
parser_mediator.GetFileEntry().name, file_offset)
self._WINLNK_PARSER.UpdateChainAndParseFileObject(
parser_mediator, lnk_file_object, display_name=display_name)
# We cannot trust the file size in the LNK data so we get the last offset
# that was read instead.
lnk_file_size = lnk_file_object.get_offset()
lnk_file_object.close()
file_offset += lnk_file_size
remaining_file_size -= lnk_file_size
file_object.seek(file_offset, os.SEEK_SET)
try:
file_footer = self._FILE_FOOTER.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
logging.warning(
(u'Unable to parse Custom Destination file footer with error: '
u'{0:s}').format(exception))
if file_footer.signature != 0xbabffbab:
logging.warning(
u'Unsupported Custom Destination file - invalid footer signature.'
)
class UtmpxParser(interface.FileObjectParser):
"""Parser for UTMPX files."""
NAME = u'utmpx'
DESCRIPTION = u'Parser for UTMPX files.'
# INFO: Type is suppose to be a short (2 bytes),
# however if we analyze the file it is always
# byte follow by 3 bytes with \x00 value.
_UTMPX_ENTRY = construct.Struct(u'utmpx_mac',
construct.String(u'user', 256),
construct.ULInt32(u'id'),
construct.String(u'tty_name', 32),
construct.ULInt32(u'pid'),
construct.ULInt16(u'status_type'),
construct.ULInt16(u'unknown'),
construct.ULInt32(u'timestamp'),
construct.ULInt32(u'microsecond'),
construct.String(u'hostname', 256),
construct.Padding(64))
_UTMPX_ENTRY_SIZE = _UTMPX_ENTRY.sizeof()
_STATUS_TYPE_SIGNATURE = 10
def _ReadEntry(self, file_object):
"""Reads an UTMPX entry.
Args:
file_object: a file-like object that points to an UTMPX file.
Returns:
An event object (instance of UtmpxMacOsXEvent) or None if
the UTMPX entry cannot be read.
"""
data = file_object.read(self._UTMPX_ENTRY_SIZE)
if len(data) != self._UTMPX_ENTRY_SIZE:
return
try:
entry_struct = self._UTMPX_ENTRY.parse(data)
except (IOError, construct.FieldError) as exception:
logging.warning(
u'Unable to parse Mac OS X UTMPX entry with error: {0:s}'.
format(exception))
return
user, _, _ = entry_struct.user.partition(b'\x00')
if not user:
user = u'N/A'
terminal, _, _ = entry_struct.tty_name.partition(b'\x00')
if not terminal:
terminal = u'N/A'
computer_name, _, _ = entry_struct.hostname.partition(b'\x00')
if not computer_name:
computer_name = u'localhost'
return UtmpxMacOsXEvent(entry_struct.timestamp,
user,
terminal,
entry_struct.status_type,
computer_name,
micro_seconds=entry_struct.microsecond)
def _VerifyStructure(self, file_object):
"""Verify that we are dealing with an UTMPX entry.
Args:
file_object: a file-like object that points to an UTMPX file.
Returns:
True if it is a UTMPX entry or False otherwise.
"""
# First entry is a SIGNAL entry of the file ("header").
try:
header_struct = self._UTMPX_ENTRY.parse_stream(file_object)
except (IOError, construct.FieldError):
return False
user, _, _ = header_struct.user.partition(b'\x00')
# The UTMPX_ENTRY structure will often successfully compile on various
# structures, such as binary plist files, and thus we need to do some
# additional validation. The first one is to check if the user name
# can be converted into a Unicode string, otherwise we can assume
# we are dealing with non UTMPX data.
try:
user.decode(u'utf-8')
except UnicodeDecodeError:
return False
if user != b'utmpx-1.00':
return False
if header_struct.status_type != self._STATUS_TYPE_SIGNATURE:
return False
if (header_struct.timestamp != 0 or header_struct.microsecond != 0
or header_struct.pid != 0):
return False
tty_name, _, _ = header_struct.tty_name.partition(b'\x00')
hostname, _, _ = header_struct.hostname.partition(b'\x00')
if tty_name or hostname:
return False
return True
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses an UTMPX file-like object.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
file_object: The file-like object to extract data from.
Raises:
UnableToParseFile: when the file cannot be parsed.
"""
if not self._VerifyStructure(file_object):
raise errors.UnableToParseFile(u'The file is not an UTMPX file.')
event_object = self._ReadEntry(file_object)
while event_object:
event_object.offset = file_object.tell()
parser_mediator.ProduceEvent(event_object)
event_object = self._ReadEntry(file_object)
class UtmpxParser(interface.FileObjectParser):
"""Parser for UTMPX files."""
NAME = 'utmpx'
DESCRIPTION = 'Parser for UTMPX files.'
# INFO: Type is suppose to be a short (2 bytes),
# however if we analyze the file it is always
# byte follow by 3 bytes with \x00 value.
_UTMPX_ENTRY = construct.Struct('utmpx_mac', construct.String('user', 256),
construct.ULInt32('id'),
construct.String('tty_name', 32),
construct.ULInt32('pid'),
construct.ULInt16('status_type'),
construct.ULInt16('unknown'),
construct.ULInt32('timestamp'),
construct.ULInt32('microseconds'),
construct.String('hostname', 256),
construct.Padding(64))
_UTMPX_ENTRY_SIZE = _UTMPX_ENTRY.sizeof()
_STATUS_TYPE_SIGNATURE = 10
def _ReadEntry(self, parser_mediator, file_object):
"""Reads an UTMPX entry.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): a file-like object.
Returns:
bool: True if the UTMPX entry was successfully read.
"""
data = file_object.read(self._UTMPX_ENTRY_SIZE)
if len(data) != self._UTMPX_ENTRY_SIZE:
return False
try:
entry_struct = self._UTMPX_ENTRY.parse(data)
except (IOError, construct.FieldError) as exception:
logging.warning(
'Unable to parse MacOS UTMPX entry with error: {0!s}'.format(
exception))
return False
user, _, _ = entry_struct.user.partition(b'\x00')
if not user:
user = '******'
terminal, _, _ = entry_struct.tty_name.partition(b'\x00')
if not terminal:
terminal = 'N/A'
computer_name, _, _ = entry_struct.hostname.partition(b'\x00')
if not computer_name:
computer_name = 'localhost'
event_data = UtmpxMacOSEventData()
event_data.computer_name = computer_name
event_data.offset = file_object.tell()
event_data.status_type = entry_struct.status_type
event_data.terminal = terminal
event_data.user = user
timestamp = (entry_struct.timestamp *
1000000) + entry_struct.microseconds
date_time = dfdatetime_posix_time.PosixTimeInMicroseconds(
timestamp=timestamp)
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_START)
parser_mediator.ProduceEventWithEventData(event, event_data)
return True
def _VerifyStructure(self, file_object):
"""Verify that we are dealing with an UTMPX entry.
Args:
file_object (dfvfs.FileIO): a file-like object.
Returns:
bool: True if it is a UTMPX entry or False otherwise.
"""
# First entry is a SIGNAL entry of the file ("header").
try:
header_struct = self._UTMPX_ENTRY.parse_stream(file_object)
except (IOError, construct.FieldError):
return False
user, _, _ = header_struct.user.partition(b'\x00')
# The UTMPX_ENTRY structure will often successfully compile on various
# structures, such as binary plist files, and thus we need to do some
# additional validation. The first one is to check if the user name
# can be converted into a Unicode string, otherwise we can assume
# we are dealing with non UTMPX data.
try:
user.decode('utf-8')
except UnicodeDecodeError:
return False
if user != b'utmpx-1.00':
return False
if header_struct.status_type != self._STATUS_TYPE_SIGNATURE:
return False
if (header_struct.timestamp != 0 or header_struct.microseconds != 0
or header_struct.pid != 0):
return False
tty_name, _, _ = header_struct.tty_name.partition(b'\x00')
hostname, _, _ = header_struct.hostname.partition(b'\x00')
if tty_name or hostname:
return False
return True
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses an UTMPX file-like object.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): a file-like object.
Raises:
UnableToParseFile: when the file cannot be parsed.
"""
if not self._VerifyStructure(file_object):
raise errors.UnableToParseFile('The file is not an UTMPX file.')
while self._ReadEntry(parser_mediator, file_object):
pass
class WinJobParser(interface.SingleFileBaseParser):
"""Parse Windows Scheduled Task files for job events."""
NAME = u'winjob'
DESCRIPTION = u'Parser for Windows Scheduled Task job (or At-job) files.'
PRODUCT_VERSIONS = {
0x0400: u'Windows NT 4.0',
0x0500: u'Windows 2000',
0x0501: u'Windows XP',
0x0600: u'Windows Vista',
0x0601: u'Windows 7',
0x0602: u'Windows 8',
0x0603: u'Windows 8.1'
}
TRIGGER_TYPES = {
0x0000: u'ONCE',
0x0001: u'DAILY',
0x0002: u'WEEKLY',
0x0003: u'MONTHLYDATE',
0x0004: u'MONTHLYDOW',
0x0005: u'EVENT_ON_IDLE',
0x0006: u'EVENT_AT_SYSTEMSTART',
0x0007: u'EVENT_AT_LOGON'
}
JOB_FIXED_STRUCT = construct.Struct(
u'job_fixed',
construct.ULInt16(u'product_version'),
construct.ULInt16(u'file_version'),
construct.Bytes(u'job_uuid', 16),
construct.ULInt16(u'app_name_len_offset'),
construct.ULInt16(u'trigger_offset'),
construct.ULInt16(u'error_retry_count'),
construct.ULInt16(u'error_retry_interval'),
construct.ULInt16(u'idle_deadline'),
construct.ULInt16(u'idle_wait'),
construct.ULInt32(u'priority'),
construct.ULInt32(u'max_run_time'),
construct.ULInt32(u'exit_code'),
construct.ULInt32(u'status'),
construct.ULInt32(u'flags'),
construct.ULInt16(u'ran_year'),
construct.ULInt16(u'ran_month'),
construct.ULInt16(u'ran_weekday'),
construct.ULInt16(u'ran_day'),
construct.ULInt16(u'ran_hour'),
construct.ULInt16(u'ran_minute'),
construct.ULInt16(u'ran_second'),
construct.ULInt16(u'ran_millisecond'),
)
# Using Construct's utf-16 encoding here will create strings with their
# null terminators exposed. Instead, we'll read these variables raw and
# convert them using Plaso's ReadUtf16() for proper formatting.
JOB_VARIABLE_STRUCT = construct.Struct(
u'job_variable',
construct.ULInt16(u'running_instance_count'),
construct.ULInt16(u'app_name_len'),
construct.String(
u'app_name',
lambda ctx: ctx.app_name_len * 2),
construct.ULInt16(u'parameter_len'),
construct.String(
u'parameter',
lambda ctx: ctx.parameter_len * 2),
construct.ULInt16(u'working_dir_len'),
construct.String(
u'working_dir',
lambda ctx: ctx.working_dir_len * 2),
construct.ULInt16(u'username_len'),
construct.String(
u'username',
lambda ctx: ctx.username_len * 2),
construct.ULInt16(u'comment_len'),
construct.String(
u'comment',
lambda ctx: ctx.comment_len * 2),
construct.ULInt16(u'userdata_len'),
construct.String(
u'userdata',
lambda ctx: ctx.userdata_len),
construct.ULInt16(u'reserved_len'),
construct.String(
u'reserved',
lambda ctx: ctx.reserved_len),
construct.ULInt16(u'test'),
construct.ULInt16(u'trigger_size'),
construct.ULInt16(u'trigger_reserved1'),
construct.ULInt16(u'sched_start_year'),
construct.ULInt16(u'sched_start_month'),
construct.ULInt16(u'sched_start_day'),
construct.ULInt16(u'sched_end_year'),
construct.ULInt16(u'sched_end_month'),
construct.ULInt16(u'sched_end_day'),
construct.ULInt16(u'sched_start_hour'),
construct.ULInt16(u'sched_start_minute'),
construct.ULInt32(u'sched_duration'),
construct.ULInt32(u'sched_interval'),
construct.ULInt32(u'trigger_flags'),
construct.ULInt32(u'trigger_type'),
construct.ULInt16(u'trigger_arg0'),
construct.ULInt16(u'trigger_arg1'),
construct.ULInt16(u'trigger_arg2'),
construct.ULInt16(u'trigger_padding'),
construct.ULInt16(u'trigger_reserved2'),
construct.ULInt16(u'trigger_reserved3'))
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses a Windows job file-like object.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
file_object: A file-like object.
Raises:
UnableToParseFile: when the file cannot be parsed.
"""
try:
header = self.JOB_FIXED_STRUCT.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
raise errors.UnableToParseFile(
u'Unable to parse Windows Task Job file with error: {0:s}'.format(
exception))
if not header.product_version in self.PRODUCT_VERSIONS:
raise errors.UnableToParseFile(u'Not a valid Scheduled Task file')
if not header.file_version == 1:
raise errors.UnableToParseFile(u'Not a valid Scheduled Task file')
# Obtain the relevant values from the file.
try:
data = self.JOB_VARIABLE_STRUCT.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
raise errors.UnableToParseFile(
u'Unable to parse Windows Task Job file with error: {0:s}'.format(
exception))
trigger_type = self.TRIGGER_TYPES.get(data.trigger_type, u'Unknown')
last_run_date = timelib.Timestamp.FromTimeParts(
header.ran_year,
header.ran_month,
header.ran_day,
header.ran_hour,
header.ran_minute,
header.ran_second,
microseconds=(header.ran_millisecond * 1000),
timezone=parser_mediator.timezone)
scheduled_date = timelib.Timestamp.FromTimeParts(
data.sched_start_year,
data.sched_start_month,
data.sched_start_day,
data.sched_start_hour,
data.sched_start_minute,
0, # Seconds are not stored.
timezone=parser_mediator.timezone)
# Create two timeline events, one for created date and the other for last
# run.
parser_mediator.ProduceEvents(
[WinJobEvent(
last_run_date, eventdata.EventTimestamp.LAST_RUNTIME, data.app_name,
data.parameter, data.working_dir, data.username, trigger_type,
data.comment),
WinJobEvent(
scheduled_date, u'Scheduled To Start', data.app_name,
data.parameter, data.working_dir, data.username, trigger_type,
data.comment)])
# A scheduled end date is optional.
if data.sched_end_year:
scheduled_end_date = timelib.Timestamp.FromTimeParts(
data.sched_end_year,
data.sched_end_month,
data.sched_end_day,
0, # Hours are not stored.
0, # Minutes are not stored.
0, # Seconds are not stored.
timezone=parser_mediator.timezone)
event_object = WinJobEvent(
scheduled_end_date, u'Scheduled To End', data.app_name,
data.parameter, data.working_dir, data.username, trigger_type,
data.comment)
parser_mediator.ProduceEvent(event_object)
class BSMParser(interface.FileObjectParser):
"""Parser for BSM files."""
NAME = 'bsm_log'
DESCRIPTION = 'Parser for BSM log files.'
# BSM supported version (0x0b = 11).
AUDIT_HEADER_VERSION = 11
# Magic Trail Header.
BSM_TOKEN_TRAILER_MAGIC = b'b105'
# IP Version constants.
AU_IPv4 = 4
AU_IPv6 = 16
IPV4_STRUCT = construct.UBInt32('ipv4')
IPV6_STRUCT = construct.Struct(
'ipv6',
construct.UBInt64('high'),
construct.UBInt64('low'))
# Tested structures.
# INFO: I have ommited the ID in the structures declaration.
# I used the BSM_TYPE first to read the ID, and then, the structure.
# Tokens always start with an ID value that identifies their token
# type and subsequent structure.
_BSM_TOKEN = construct.UBInt8('token_id')
# Data type structures.
BSM_TOKEN_DATA_CHAR = construct.String('value', 1)
BSM_TOKEN_DATA_SHORT = construct.UBInt16('value')
BSM_TOKEN_DATA_INTEGER = construct.UBInt32('value')
# Common structure used by other structures.
# audit_uid: integer, uid that generates the entry.
# effective_uid: integer, the permission user used.
# effective_gid: integer, the permission group used.
# real_uid: integer, user id of the user that execute the process.
# real_gid: integer, group id of the group that execute the process.
# pid: integer, identification number of the process.
# session_id: unknown, need research.
BSM_TOKEN_SUBJECT_SHORT = construct.Struct(
'subject_data',
construct.UBInt32('audit_uid'),
construct.UBInt32('effective_uid'),
construct.UBInt32('effective_gid'),
construct.UBInt32('real_uid'),
construct.UBInt32('real_gid'),
construct.UBInt32('pid'),
construct.UBInt32('session_id'))
# Common structure used by other structures.
# Identify the kind of inet (IPv4 or IPv6)
# TODO: instead of 16, AU_IPv6 must be used.
BSM_IP_TYPE_SHORT = construct.Struct(
'bsm_ip_type_short',
construct.UBInt32('net_type'),
construct.Switch(
'ip_addr',
_BSMTokenGetNetType,
{16: IPV6_STRUCT},
default=IPV4_STRUCT))
# Initial fields structure used by header structures.
# length: integer, the length of the entry, equal to trailer (doc: length).
# version: integer, version of BSM (AUDIT_HEADER_VERSION).
# event_type: integer, the type of event (/etc/security/audit_event).
# modifier: integer, unknown, need research (It is always 0).
BSM_HEADER = construct.Struct(
'bsm_header',
construct.UBInt32('length'),
construct.UBInt8('version'),
construct.UBInt16('event_type'),
construct.UBInt16('modifier'))
# First token of one entry.
# timestamp: unsigned integer, number of seconds since
# January 1, 1970 00:00:00 UTC.
# microseconds: unsigned integer, number of micro seconds.
BSM_HEADER32 = construct.Struct(
'bsm_header32',
BSM_HEADER,
construct.UBInt32('timestamp'),
construct.UBInt32('microseconds'))
BSM_HEADER64 = construct.Struct(
'bsm_header64',
BSM_HEADER,
construct.UBInt64('timestamp'),
construct.UBInt64('microseconds'))
BSM_HEADER32_EX = construct.Struct(
'bsm_header32_ex',
BSM_HEADER,
BSM_IP_TYPE_SHORT,
construct.UBInt32('timestamp'),
construct.UBInt32('microseconds'))
# Token TEXT, provides extra information.
BSM_TOKEN_TEXT = construct.Struct(
'bsm_token_text',
construct.UBInt16('length'),
construct.Array(_BSMTokenGetLength, construct.UBInt8('text')))
# Path of the executable.
BSM_TOKEN_PATH = BSM_TOKEN_TEXT
# Identified the end of the record (follow by TRAILER).
# status: integer that identifies the status of the exit (BSM_ERRORS).
# return: returned value from the operation.
BSM_TOKEN_RETURN32 = construct.Struct(
'bsm_token_return32',
construct.UBInt8('status'),
construct.UBInt32('return_value'))
BSM_TOKEN_RETURN64 = construct.Struct(
'bsm_token_return64',
construct.UBInt8('status'),
construct.UBInt64('return_value'))
# Identified the number of bytes that was written.
# magic: 2 bytes that identifies the TRAILER (BSM_TOKEN_TRAILER_MAGIC).
# length: integer that has the number of bytes from the entry size.
BSM_TOKEN_TRAILER = construct.Struct(
'bsm_token_trailer',
construct.UBInt16('magic'),
construct.UBInt32('record_length'))
# A 32-bits argument.
# num_arg: the number of the argument.
# name_arg: the argument's name.
# text: the string value of the argument.
BSM_TOKEN_ARGUMENT32 = construct.Struct(
'bsm_token_argument32',
construct.UBInt8('num_arg'),
construct.UBInt32('name_arg'),
construct.UBInt16('length'),
construct.Array(_BSMTokenGetLength, construct.UBInt8('text')))
# A 64-bits argument.
# num_arg: integer, the number of the argument.
# name_arg: text, the argument's name.
# text: the string value of the argument.
BSM_TOKEN_ARGUMENT64 = construct.Struct(
'bsm_token_argument64',
construct.UBInt8('num_arg'),
construct.UBInt64('name_arg'),
construct.UBInt16('length'),
construct.Array(_BSMTokenGetLength, construct.UBInt8('text')))
# Identify an user.
# terminal_id: unknown, research needed.
# terminal_addr: unknown, research needed.
BSM_TOKEN_SUBJECT32 = construct.Struct(
'bsm_token_subject32',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt32('terminal_port'),
IPV4_STRUCT)
# Identify an user using a extended Token.
# terminal_port: unknown, need research.
# net_type: unknown, need research.
BSM_TOKEN_SUBJECT32_EX = construct.Struct(
'bsm_token_subject32_ex',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt32('terminal_port'),
BSM_IP_TYPE_SHORT)
# au_to_opaque // AUT_OPAQUE
BSM_TOKEN_OPAQUE = BSM_TOKEN_TEXT
# au_to_seq // AUT_SEQ
BSM_TOKEN_SEQUENCE = BSM_TOKEN_DATA_INTEGER
# Program execution with options.
# For each argument we are going to have a string+ "\x00".
# Example: [00 00 00 02][41 42 43 00 42 42 00]
# 2 Arguments, Arg1: [414243] Arg2: [4242].
BSM_TOKEN_EXEC_ARGUMENTS = construct.UBInt32('number_arguments')
BSM_TOKEN_EXEC_ARGUMENT = construct.Struct(
'bsm_token_exec_argument',
construct.RepeatUntil(
_BSMTokenIsEndOfString, construct.StaticField("text", 1)))
# au_to_in_addr // AUT_IN_ADDR:
BSM_TOKEN_ADDR = IPV4_STRUCT
# au_to_in_addr_ext // AUT_IN_ADDR_EX:
BSM_TOKEN_ADDR_EXT = construct.Struct(
'bsm_token_addr_ext',
construct.UBInt32('net_type'),
IPV6_STRUCT)
# au_to_ip // AUT_IP:
# TODO: parse this header in the correct way.
BSM_TOKEN_IP = construct.String('binary_ipv4_add', 20)
# au_to_ipc // AUT_IPC:
BSM_TOKEN_IPC = construct.Struct(
'bsm_token_ipc',
construct.UBInt8('object_type'),
construct.UBInt32('object_id'))
# au_to_ipc_perm // au_to_ipc_perm
BSM_TOKEN_IPC_PERM = construct.Struct(
'bsm_token_ipc_perm',
construct.UBInt32('user_id'),
construct.UBInt32('group_id'),
construct.UBInt32('creator_user_id'),
construct.UBInt32('creator_group_id'),
construct.UBInt32('access_mode'),
construct.UBInt32('slot_seq'),
construct.UBInt32('key'))
# au_to_iport // AUT_IPORT:
BSM_TOKEN_PORT = construct.UBInt16('port_number')
# au_to_file // AUT_OTHER_FILE32:
BSM_TOKEN_FILE = construct.Struct(
'bsm_token_file',
construct.UBInt32('timestamp'),
construct.UBInt32('microseconds'),
construct.UBInt16('length'),
construct.Array(_BSMTokenGetLength, construct.UBInt8('text')))
# au_to_subject64 // AUT_SUBJECT64:
BSM_TOKEN_SUBJECT64 = construct.Struct(
'bsm_token_subject64',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt64('terminal_port'),
IPV4_STRUCT)
# au_to_subject64_ex // AU_IPv4:
BSM_TOKEN_SUBJECT64_EX = construct.Struct(
'bsm_token_subject64_ex',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt32('terminal_port'),
construct.UBInt32('terminal_type'),
BSM_IP_TYPE_SHORT)
# au_to_process32 // AUT_PROCESS32:
BSM_TOKEN_PROCESS32 = construct.Struct(
'bsm_token_process32',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt32('terminal_port'),
IPV4_STRUCT)
# au_to_process64 // AUT_PROCESS32:
BSM_TOKEN_PROCESS64 = construct.Struct(
'bsm_token_process64',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt64('terminal_port'),
IPV4_STRUCT)
# au_to_process32_ex // AUT_PROCESS32_EX:
BSM_TOKEN_PROCESS32_EX = construct.Struct(
'bsm_token_process32_ex',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt32('terminal_port'),
BSM_IP_TYPE_SHORT)
# au_to_process64_ex // AUT_PROCESS64_EX:
BSM_TOKEN_PROCESS64_EX = construct.Struct(
'bsm_token_process64_ex',
BSM_TOKEN_SUBJECT_SHORT,
construct.UBInt64('terminal_port'),
BSM_IP_TYPE_SHORT)
# au_to_sock_inet32 // AUT_SOCKINET32:
BSM_TOKEN_AUT_SOCKINET32 = construct.Struct(
'bsm_token_aut_sockinet32',
construct.UBInt16('net_type'),
construct.UBInt16('port_number'),
IPV4_STRUCT)
# Info: checked against the source code of XNU, but not against
# real BSM file.
BSM_TOKEN_AUT_SOCKINET128 = construct.Struct(
'bsm_token_aut_sockinet128',
construct.UBInt16('net_type'),
construct.UBInt16('port_number'),
IPV6_STRUCT)
INET6_ADDR_TYPE = construct.Struct(
'addr_type',
construct.UBInt16('ip_type'),
construct.UBInt16('source_port'),
construct.UBInt64('saddr_high'),
construct.UBInt64('saddr_low'),
construct.UBInt16('destination_port'),
construct.UBInt64('daddr_high'),
construct.UBInt64('daddr_low'))
INET4_ADDR_TYPE = construct.Struct(
'addr_type',
construct.UBInt16('ip_type'),
construct.UBInt16('source_port'),
construct.UBInt32('source_address'),
construct.UBInt16('destination_port'),
construct.UBInt32('destination_address'))
# au_to_socket_ex // AUT_SOCKET_EX
# TODO: Change the 26 for unixbsm.BSM_PROTOCOLS.INET6.
BSM_TOKEN_AUT_SOCKINET32_EX = construct.Struct(
'bsm_token_aut_sockinet32_ex',
construct.UBInt16('socket_domain'),
construct.UBInt16('socket_type'),
construct.Switch(
'structure_addr_port',
_BSMTokenGetSocketDomain,
{26: INET6_ADDR_TYPE},
default=INET4_ADDR_TYPE))
# au_to_sock_unix // AUT_SOCKUNIX
BSM_TOKEN_SOCKET_UNIX = construct.Struct(
'bsm_token_au_to_sock_unix',
construct.UBInt16('family'),
construct.RepeatUntil(
_BSMTokenIsEndOfString,
construct.StaticField("path", 1)))
# au_to_data // au_to_data
# how to print: bsmtoken.BSM_TOKEN_DATA_PRINT.
# type: bsmtoken.BSM_TOKEN_DATA_TYPE.
# unit_count: number of type values.
# BSM_TOKEN_DATA has a end field = type * unit_count
BSM_TOKEN_DATA = construct.Struct(
'bsm_token_data',
construct.UBInt8('how_to_print'),
construct.UBInt8('data_type'),
construct.UBInt8('unit_count'))
# au_to_attr32 // AUT_ATTR32
BSM_TOKEN_ATTR32 = construct.Struct(
'bsm_token_attr32',
construct.UBInt32('file_mode'),
construct.UBInt32('uid'),
construct.UBInt32('gid'),
construct.UBInt32('file_system_id'),
construct.UBInt64('file_system_node_id'),
construct.UBInt32('device'))
# au_to_attr64 // AUT_ATTR64
BSM_TOKEN_ATTR64 = construct.Struct(
'bsm_token_attr64',
construct.UBInt32('file_mode'),
construct.UBInt32('uid'),
construct.UBInt32('gid'),
construct.UBInt32('file_system_id'),
construct.UBInt64('file_system_node_id'),
construct.UBInt64('device'))
# au_to_exit // AUT_EXIT
BSM_TOKEN_EXIT = construct.Struct(
'bsm_token_exit',
construct.UBInt32('status'),
construct.UBInt32('return_value'))
# au_to_newgroups // AUT_NEWGROUPS
# INFO: we must read BSM_TOKEN_DATA_INTEGER for each group.
BSM_TOKEN_GROUPS = construct.UBInt16('group_number')
# au_to_exec_env == au_to_exec_args
BSM_TOKEN_EXEC_ENV = BSM_TOKEN_EXEC_ARGUMENTS
# au_to_zonename //AUT_ZONENAME
BSM_TOKEN_ZONENAME = BSM_TOKEN_TEXT
# Token ID.
# List of valid Token_ID.
# Token_ID -> (NAME_STRUCTURE, STRUCTURE)
# Only the checked structures are been added to the valid structures lists.
_BSM_TOKEN_TYPES = {
17: ('BSM_TOKEN_FILE', BSM_TOKEN_FILE),
19: ('BSM_TOKEN_TRAILER', BSM_TOKEN_TRAILER),
20: ('BSM_HEADER32', BSM_HEADER32),
21: ('BSM_HEADER64', BSM_HEADER64),
33: ('BSM_TOKEN_DATA', BSM_TOKEN_DATA),
34: ('BSM_TOKEN_IPC', BSM_TOKEN_IPC),
35: ('BSM_TOKEN_PATH', BSM_TOKEN_PATH),
36: ('BSM_TOKEN_SUBJECT32', BSM_TOKEN_SUBJECT32),
38: ('BSM_TOKEN_PROCESS32', BSM_TOKEN_PROCESS32),
39: ('BSM_TOKEN_RETURN32', BSM_TOKEN_RETURN32),
40: ('BSM_TOKEN_TEXT', BSM_TOKEN_TEXT),
41: ('BSM_TOKEN_OPAQUE', BSM_TOKEN_OPAQUE),
42: ('BSM_TOKEN_ADDR', BSM_TOKEN_ADDR),
43: ('BSM_TOKEN_IP', BSM_TOKEN_IP),
44: ('BSM_TOKEN_PORT', BSM_TOKEN_PORT),
45: ('BSM_TOKEN_ARGUMENT32', BSM_TOKEN_ARGUMENT32),
47: ('BSM_TOKEN_SEQUENCE', BSM_TOKEN_SEQUENCE),
96: ('BSM_TOKEN_ZONENAME', BSM_TOKEN_ZONENAME),
113: ('BSM_TOKEN_ARGUMENT64', BSM_TOKEN_ARGUMENT64),
114: ('BSM_TOKEN_RETURN64', BSM_TOKEN_RETURN64),
116: ('BSM_HEADER32_EX', BSM_HEADER32_EX),
119: ('BSM_TOKEN_PROCESS64', BSM_TOKEN_PROCESS64),
122: ('BSM_TOKEN_SUBJECT32_EX', BSM_TOKEN_SUBJECT32_EX),
127: ('BSM_TOKEN_AUT_SOCKINET32_EX', BSM_TOKEN_AUT_SOCKINET32_EX),
128: ('BSM_TOKEN_AUT_SOCKINET32', BSM_TOKEN_AUT_SOCKINET32)}
# Untested structures.
# When not tested structure is found, we try to parse using also
# these structures.
BSM_TYPE_LIST_NOT_TESTED = {
49: ('BSM_TOKEN_ATTR', BSM_TOKEN_ATTR32),
50: ('BSM_TOKEN_IPC_PERM', BSM_TOKEN_IPC_PERM),
52: ('BSM_TOKEN_GROUPS', BSM_TOKEN_GROUPS),
59: ('BSM_TOKEN_GROUPS', BSM_TOKEN_GROUPS),
60: ('BSM_TOKEN_EXEC_ARGUMENTS', BSM_TOKEN_EXEC_ARGUMENTS),
61: ('BSM_TOKEN_EXEC_ENV', BSM_TOKEN_EXEC_ENV),
62: ('BSM_TOKEN_ATTR32', BSM_TOKEN_ATTR32),
82: ('BSM_TOKEN_EXIT', BSM_TOKEN_EXIT),
115: ('BSM_TOKEN_ATTR64', BSM_TOKEN_ATTR64),
117: ('BSM_TOKEN_SUBJECT64', BSM_TOKEN_SUBJECT64),
123: ('BSM_TOKEN_PROCESS32_EX', BSM_TOKEN_PROCESS32_EX),
124: ('BSM_TOKEN_PROCESS64_EX', BSM_TOKEN_PROCESS64_EX),
125: ('BSM_TOKEN_SUBJECT64_EX', BSM_TOKEN_SUBJECT64_EX),
126: ('BSM_TOKEN_ADDR_EXT', BSM_TOKEN_ADDR_EXT),
129: ('BSM_TOKEN_AUT_SOCKINET128', BSM_TOKEN_AUT_SOCKINET128),
130: ('BSM_TOKEN_SOCKET_UNIX', BSM_TOKEN_SOCKET_UNIX)}
MESSAGE_CAN_NOT_SAVE = (
'Plaso: some tokens from this entry can not be saved. Entry at 0x{0:X} '
'with unknown token id "0x{1:X}".')
# BSM token types:
# https://github.com/openbsm/openbsm/blob/master/sys/bsm/audit_record.h
_BSM_TOKEN_TYPE_ARGUMENT32 = 45
_BSM_TOKEN_TYPE_ARGUMENT64 = 113
_BSM_TOKEN_TYPE_ATTR = 49
_BSM_TOKEN_TYPE_ATTR32 = 62
_BSM_TOKEN_TYPE_ATTR64 = 115
_BSM_TOKEN_TYPE_EXEC_ARGUMENTS = 60
_BSM_TOKEN_TYPE_EXEC_ENV = 61
_BSM_TOKEN_TYPE_EXIT = 82
_BSM_TOKEN_TYPE_HEADER32 = 20
_BSM_TOKEN_TYPE_HEADER32_EX = 116
_BSM_TOKEN_TYPE_HEADER64 = 21
_BSM_TOKEN_TYPE_PATH = 35
_BSM_TOKEN_TYPE_PROCESS32 = 38
_BSM_TOKEN_TYPE_PROCESS32_EX = 123
_BSM_TOKEN_TYPE_PROCESS64 = 119
_BSM_TOKEN_TYPE_PROCESS64_EX = 124
_BSM_TOKEN_TYPE_RETURN32 = 39
_BSM_TOKEN_TYPE_RETURN64 = 114
_BSM_TOKEN_TYPE_SUBJECT32 = 36
_BSM_TOKEN_TYPE_SUBJECT32_EX = 122
_BSM_TOKEN_TYPE_SUBJECT64 = 117
_BSM_TOKEN_TYPE_SUBJECT64_EX = 125
_BSM_TOKEN_TYPE_TEXT = 40
_BSM_TOKEN_TYPE_ZONENAME = 96
_BSM_ARGUMENT_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_ARGUMENT32,
_BSM_TOKEN_TYPE_ARGUMENT64)
_BSM_ATTR_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_ATTR,
_BSM_TOKEN_TYPE_ATTR32,
_BSM_TOKEN_TYPE_ATTR64)
_BSM_EXEV_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_EXEC_ARGUMENTS,
_BSM_TOKEN_TYPE_EXEC_ENV)
_BSM_HEADER_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_HEADER32,
_BSM_TOKEN_TYPE_HEADER32_EX,
_BSM_TOKEN_TYPE_HEADER64)
_BSM_PROCESS_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_PROCESS32,
_BSM_TOKEN_TYPE_PROCESS64)
_BSM_PROCESS_EX_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_PROCESS32_EX,
_BSM_TOKEN_TYPE_PROCESS64_EX)
_BSM_RETURN_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_EXIT,
_BSM_TOKEN_TYPE_RETURN32,
_BSM_TOKEN_TYPE_RETURN64)
_BSM_SUBJECT_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_SUBJECT32,
_BSM_TOKEN_TYPE_SUBJECT64)
_BSM_SUBJECT_EX_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_SUBJECT32_EX,
_BSM_TOKEN_TYPE_SUBJECT64_EX)
_BSM_UTF8_BYTE_ARRAY_TOKEN_TYPES = (
_BSM_TOKEN_TYPE_PATH,
_BSM_TOKEN_TYPE_TEXT,
_BSM_TOKEN_TYPE_ZONENAME)
def __init__(self):
"""Initializes a parser object."""
super(BSMParser, self).__init__()
# Create the dictionary with all token IDs: tested and untested.
self._bsm_type_list_all = self._BSM_TOKEN_TYPES.copy()
self._bsm_type_list_all.update(self.BSM_TYPE_LIST_NOT_TESTED)
def _CopyByteArrayToBase16String(self, byte_array):
"""Copies a byte array into a base-16 encoded Unicode string.
Args:
byte_array (bytes): A byte array.
Returns:
str: a base-16 encoded Unicode string.
"""
return ''.join(['{0:02x}'.format(byte) for byte in byte_array])
def _CopyUtf8ByteArrayToString(self, byte_array):
"""Copies a UTF-8 encoded byte array into a Unicode string.
Args:
byte_array (bytes): A byte array containing an UTF-8 encoded string.
Returns:
str: A Unicode string.
"""
byte_stream = b''.join(map(chr, byte_array))
try:
string = byte_stream.decode('utf-8')
except UnicodeDecodeError:
logging.warning('Unable to decode UTF-8 formatted byte array.')
string = byte_stream.decode('utf-8', errors='ignore')
string, _, _ = string.partition(b'\x00')
return string
def _IPv4Format(self, address):
"""Formats an IPv4 address as a human readable string.
Args:
address (int): IPv4 address.
Returns:
str: human readable string of IPv4 address in 4 octet representation:
"1.2.3.4".
"""
ipv4_string = self.IPV4_STRUCT.build(address)
return socket.inet_ntoa(ipv4_string)
def _IPv6Format(self, high, low):
"""Formats an IPv6 address as a human readable string.
Args:
high (int): upper 64-bit part of the IPv6 address.
low (int): lower 64-bit part of the IPv6 address.
Returns:
str: human readable string of IPv6 address.
"""
ipv6_string = self.IPV6_STRUCT.build(
construct.Container(high=high, low=low))
# socket.inet_ntop not supported in Windows.
if hasattr(socket, 'inet_ntop'):
return socket.inet_ntop(socket.AF_INET6, ipv6_string)
# TODO: this approach returns double "::", illegal IPv6 addr.
str_address = binascii.hexlify(ipv6_string)
address = []
blank = False
for pos in range(0, len(str_address), 4):
if str_address[pos:pos + 4] == '0000':
if not blank:
address.append('')
blank = True
else:
blank = False
address.append(str_address[pos:pos + 4].lstrip('0'))
return ':'.join(address)
def _ParseBSMEvent(self, parser_mediator, file_object):
"""Parses a BSM entry (BSMEvent) from the file-like object.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): a file-like object.
Returns:
bool: True if the BSM entry was parsed.
"""
record_start_offset = file_object.tell()
try:
token_type = self._BSM_TOKEN.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
parser_mediator.ProduceExtractionError((
'unable to parse BSM token type at offset: 0x{0:08x} with error: '
'{1:s}.').format(record_start_offset, exception))
return False
if token_type not in self._BSM_HEADER_TOKEN_TYPES:
parser_mediator.ProduceExtractionError(
'unsupported token type: {0:d} at offset: 0x{1:08x}.'.format(
token_type, record_start_offset))
# TODO: if it is a Mac OS X, search for the trailer magic value
# as a end of the entry can be a possibility to continue.
return False
_, record_structure = self._BSM_TOKEN_TYPES.get(token_type, ('', None))
try:
token = record_structure.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
parser_mediator.ProduceExtractionError((
'unable to parse BSM record at offset: 0x{0:08x} with error: '
'{1:s}.').format(record_start_offset, exception))
return False
event_type = bsmtoken.BSM_AUDIT_EVENT.get(
token.bsm_header.event_type, 'UNKNOWN')
event_type = '{0:s} ({1:d})'.format(
event_type, token.bsm_header.event_type)
timestamp = (token.timestamp * 1000000) + token.microseconds
date_time = dfdatetime_posix_time.PosixTimeInMicroseconds(
timestamp=timestamp)
record_length = token.bsm_header.length
record_end_offset = record_start_offset + record_length
# A dict of tokens that has the entry.
extra_tokens = {}
# Read until we reach the end of the record.
while file_object.tell() < record_end_offset:
# Check if it is a known token.
try:
token_type = self._BSM_TOKEN.parse_stream(file_object)
except (IOError, construct.FieldError):
logging.warning(
'Unable to parse the Token ID at position: {0:d}'.format(
file_object.tell()))
return False
_, record_structure = self._BSM_TOKEN_TYPES.get(token_type, ('', None))
if not record_structure:
pending = record_end_offset - file_object.tell()
new_extra_tokens = self.TryWithUntestedStructures(
file_object, token_type, pending)
extra_tokens.update(new_extra_tokens)
else:
token = record_structure.parse_stream(file_object)
new_extra_tokens = self.FormatToken(token_type, token, file_object)
extra_tokens.update(new_extra_tokens)
if file_object.tell() > record_end_offset:
logging.warning(
'Token ID {0:d} not expected at position 0x{1:08x}.'
'Jumping for the next entry.'.format(
token_type, file_object.tell()))
try:
file_object.seek(
record_end_offset - file_object.tell(), os.SEEK_CUR)
except (IOError, construct.FieldError) as exception:
logging.warning(
'Unable to jump to next entry with error: {0:s}'.format(exception))
return False
# BSM can be in more than one OS: BSD, Solaris and Mac OS X.
if parser_mediator.platform != 'MacOSX':
event_data = BSMEventData()
else:
event_data = MacBSMEventData()
# In Mac OS X the last two tokens are the return status and the trailer.
return_value = extra_tokens.get('BSM_TOKEN_RETURN32')
if not return_value:
return_value = extra_tokens.get('BSM_TOKEN_RETURN64')
if not return_value:
return_value = 'UNKNOWN'
event_data.return_value = return_value
event_data.event_type = event_type
event_data.extra_tokens = extra_tokens
event_data.offset = record_start_offset
event_data.record_length = record_length
# TODO: check why trailer was passed to event in original while
# event was expecting record length.
# if extra_tokens:
# trailer = extra_tokens.get('BSM_TOKEN_TRAILER', 'unknown')
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_CREATION)
parser_mediator.ProduceEventWithEventData(event, event_data)
return True
def _RawToUTF8(self, byte_stream):
"""Copies a UTF-8 byte stream into a Unicode string.
Args:
byte_stream (bytes): byte stream containing an UTF-8 encoded string.
Returns:
str: A Unicode string.
"""
try:
string = byte_stream.decode('utf-8')
except UnicodeDecodeError:
logging.warning(
'Decode UTF8 failed, the message string may be cut short.')
string = byte_stream.decode('utf-8', errors='ignore')
return string.partition(b'\x00')[0]
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses a BSM file-like object.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): a file-like object.
Raises:
UnableToParseFile: when the file cannot be parsed.
"""
try:
is_bsm = self.VerifyFile(parser_mediator, file_object)
except (IOError, construct.FieldError) as exception:
raise errors.UnableToParseFile(
'Unable to parse BSM file with error: {0:s}'.format(exception))
if not is_bsm:
raise errors.UnableToParseFile('Not a BSM File, unable to parse.')
file_object.seek(0, os.SEEK_SET)
while self._ParseBSMEvent(parser_mediator, file_object):
pass
def VerifyFile(self, parser_mediator, file_object):
"""Check if the file is a BSM file.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): a file-like object.
Returns:
bool: True if this is a valid BSM file, False otherwise.
"""
# First part of the entry is always a Header.
try:
token_type = self._BSM_TOKEN.parse_stream(file_object)
except (IOError, construct.FieldError):
return False
if token_type not in self._BSM_HEADER_TOKEN_TYPES:
return False
_, record_structure = self._BSM_TOKEN_TYPES.get(token_type, ('', None))
try:
header = record_structure.parse_stream(file_object)
except (IOError, construct.FieldError):
return False
if header.bsm_header.version != self.AUDIT_HEADER_VERSION:
return False
try:
token_identifier = self._BSM_TOKEN.parse_stream(file_object)
except (IOError, construct.FieldError):
return False
# If is Mac OS X BSM file, next entry is a text token indicating
# if it is a normal start or it is a recovery track.
if parser_mediator.platform == 'MacOSX':
token_type, record_structure = self._BSM_TOKEN_TYPES.get(
token_identifier, ('', None))
if not record_structure:
return False
if token_type != 'BSM_TOKEN_TEXT':
logging.warning('It is not a valid first entry for Mac OS X BSM.')
return False
try:
token = record_structure.parse_stream(file_object)
except (IOError, construct.FieldError):
return
text = self._CopyUtf8ByteArrayToString(token.text)
if (text != 'launchctl::Audit startup' and
text != 'launchctl::Audit recovery'):
logging.warning('It is not a valid first entry for Mac OS X BSM.')
return False
return True
def TryWithUntestedStructures(self, file_object, token_id, pending):
"""Try to parse the pending part of the entry using untested structures.
Args:
file_object: BSM file.
token_id: integer with the id that comes from the unknown token.
pending: pending length of the entry.
Returns:
A list of extra tokens data that can be parsed using non-tested
structures. A message indicating that a structure cannot be parsed
is added for unparsed structures.
"""
# Data from the unknown structure.
start_position = file_object.tell()
start_token_id = token_id
extra_tokens = {}
# Read all the "pending" bytes.
try:
if token_id in self._bsm_type_list_all:
token = self._bsm_type_list_all[token_id][1].parse_stream(file_object)
new_extra_tokens = self.FormatToken(token_id, token, file_object)
extra_tokens.update(new_extra_tokens)
while file_object.tell() < (start_position + pending):
# Check if it is a known token.
try:
token_id = self._BSM_TOKEN.parse_stream(file_object)
except (IOError, construct.FieldError):
logging.warning(
'Unable to parse the Token ID at position: {0:d}'.format(
file_object.tell()))
return
if token_id not in self._bsm_type_list_all:
break
token = self._bsm_type_list_all[token_id][1].parse_stream(file_object)
new_extra_tokens = self.FormatToken(token_id, token, file_object)
extra_tokens.update(new_extra_tokens)
except (IOError, construct.FieldError):
token_id = 255
next_entry = (start_position + pending)
if file_object.tell() != next_entry:
# Unknown Structure.
logging.warning('Unknown Token at "0x{0:X}", ID: {1} (0x{2:X})'.format(
start_position - 1, token_id, token_id))
# TODO: another way to save this information must be found.
extra_tokens.update(
{'message': self.MESSAGE_CAN_NOT_SAVE.format(
start_position - 1, start_token_id)})
# Move to next entry.
file_object.seek(next_entry - file_object.tell(), os.SEEK_CUR)
# It returns null list because it doesn't know witch structure was
# the incorrect structure that makes that it can arrive to the spected
# end of the entry.
return {}
return extra_tokens
def FormatToken(self, token_id, token, file_object):
"""Parse the Token depending of the type of the structure.
Args:
token_id (int): identification of the token_type.
token (structure): token struct to parse.
file_object: BSM file.
Returns:
(dict): parsed Token values.
Keys for returned dictionary are token name like BSM_TOKEN_SUBJECT32.
Values of this dictionary are key-value pairs like terminal_ip:127.0.0.1.
"""
if token_id not in self._bsm_type_list_all:
return {}
bsm_type, _ = self._bsm_type_list_all.get(token_id, ['', ''])
if token_id in self._BSM_UTF8_BYTE_ARRAY_TOKEN_TYPES:
try:
string = self._CopyUtf8ByteArrayToString(token.text)
except TypeError:
string = 'Unknown'
return {bsm_type: string}
elif token_id in self._BSM_RETURN_TOKEN_TYPES:
return {bsm_type: {
'error': bsmtoken.BSM_ERRORS.get(token.status, 'Unknown'),
'token_status': token.status,
'call_status': token.return_value
}}
elif token_id in self._BSM_SUBJECT_TOKEN_TYPES:
return {bsm_type: {
'aid': token.subject_data.audit_uid,
'euid': token.subject_data.effective_uid,
'egid': token.subject_data.effective_gid,
'uid': token.subject_data.real_uid,
'gid': token.subject_data.real_gid,
'pid': token.subject_data.pid,
'session_id': token.subject_data.session_id,
'terminal_port': token.terminal_port,
'terminal_ip': self._IPv4Format(token.ipv4)
}}
elif token_id in self._BSM_SUBJECT_EX_TOKEN_TYPES:
if token.bsm_ip_type_short.net_type == self.AU_IPv6:
ip = self._IPv6Format(
token.bsm_ip_type_short.ip_addr.high,
token.bsm_ip_type_short.ip_addr.low)
elif token.bsm_ip_type_short.net_type == self.AU_IPv4:
ip = self._IPv4Format(token.bsm_ip_type_short.ip_addr)
else:
ip = 'unknown'
return {bsm_type: {
'aid': token.subject_data.audit_uid,
'euid': token.subject_data.effective_uid,
'egid': token.subject_data.effective_gid,
'uid': token.subject_data.real_uid,
'gid': token.subject_data.real_gid,
'pid': token.subject_data.pid,
'session_id': token.subject_data.session_id,
'terminal_port': token.terminal_port,
'terminal_ip': ip
}}
elif token_id in self._BSM_ARGUMENT_TOKEN_TYPES:
string = self._CopyUtf8ByteArrayToString(token.text)
return {bsm_type: {
'string': string,
'num_arg': token.num_arg,
'is': token.name_arg}}
elif token_id in self._BSM_EXEV_TOKEN_TYPES:
arguments = []
for _ in range(0, token):
sub_token = self.BSM_TOKEN_EXEC_ARGUMENT.parse_stream(file_object)
string = self._CopyUtf8ByteArrayToString(sub_token.text)
arguments.append(string)
return {bsm_type: ' '.join(arguments)}
elif bsm_type == 'BSM_TOKEN_AUT_SOCKINET32':
return {bsm_type: {
'protocols':
bsmtoken.BSM_PROTOCOLS.get(token.net_type, 'UNKNOWN'),
'net_type': token.net_type,
'port': token.port_number,
'address': self._IPv4Format(token.ipv4)
}}
elif bsm_type == 'BSM_TOKEN_AUT_SOCKINET128':
return {bsm_type: {
'protocols':
bsmtoken.BSM_PROTOCOLS.get(token.net_type, 'UNKNOWN'),
'net_type': token.net_type,
'port': token.port_number,
'address': self._IPv6Format(token.ipv6.high, token.ipv6.low)
}}
elif bsm_type == 'BSM_TOKEN_ADDR':
return {bsm_type: self._IPv4Format(token)}
elif bsm_type == 'BSM_TOKEN_IP':
return {'IPv4_Header': '0x{0:s}]'.format(token.encode('hex'))}
elif bsm_type == 'BSM_TOKEN_ADDR_EXT':
return {bsm_type: {
'protocols':
bsmtoken.BSM_PROTOCOLS.get(token.net_type, 'UNKNOWN'),
'net_type': token.net_type,
'address': self._IPv6Format(token.ipv6.high, token.ipv6.low)
}}
elif bsm_type == 'BSM_TOKEN_PORT':
return {bsm_type: token}
elif bsm_type == 'BSM_TOKEN_TRAILER':
return {bsm_type: token.record_length}
elif bsm_type == 'BSM_TOKEN_FILE':
# TODO: if this timestamp is usefull, it must be extracted as a separate
# event object.
timestamp = timelib.Timestamp.FromPosixTimeWithMicrosecond(
token.timestamp, token.microseconds)
date_time = timelib.Timestamp.CopyToDatetime(timestamp, pytz.UTC)
date_time_string = date_time.strftime('%Y-%m-%d %H:%M:%S')
string = self._CopyUtf8ByteArrayToString(token.text)
return {bsm_type: {'string': string, 'timestamp': date_time_string}}
elif bsm_type == 'BSM_TOKEN_IPC':
return {bsm_type: {
'object_type': token.object_type,
'object_id': token.object_id
}}
elif token_id in self._BSM_PROCESS_TOKEN_TYPES:
return {bsm_type: {
'aid': token.subject_data.audit_uid,
'euid': token.subject_data.effective_uid,
'egid': token.subject_data.effective_gid,
'uid': token.subject_data.real_uid,
'gid': token.subject_data.real_gid,
'pid': token.subject_data.pid,
'session_id': token.subject_data.session_id,
'terminal_port': token.terminal_port,
'terminal_ip': self._IPv4Format(token.ipv4)
}}
elif token_id in self._BSM_PROCESS_EX_TOKEN_TYPES:
if token.bsm_ip_type_short.net_type == self.AU_IPv6:
ip = self._IPv6Format(
token.bsm_ip_type_short.ip_addr.high,
token.bsm_ip_type_short.ip_addr.low)
elif token.bsm_ip_type_short.net_type == self.AU_IPv4:
ip = self._IPv4Format(token.bsm_ip_type_short.ip_addr)
else:
ip = 'unknown'
return {bsm_type: {
'aid': token.subject_data.audit_uid,
'euid': token.subject_data.effective_uid,
'egid': token.subject_data.effective_gid,
'uid': token.subject_data.real_uid,
'gid': token.subject_data.real_gid,
'pid': token.subject_data.pid,
'session_id': token.subject_data.session_id,
'terminal_port': token.terminal_port,
'terminal_ip': ip
}}
elif bsm_type == 'BSM_TOKEN_DATA':
data = []
data_type = bsmtoken.BSM_TOKEN_DATA_TYPE.get(token.data_type, '')
if data_type == 'AUR_CHAR':
for _ in range(token.unit_count):
data.append(self.BSM_TOKEN_DATA_CHAR.parse_stream(file_object))
elif data_type == 'AUR_SHORT':
for _ in range(token.unit_count):
data.append(self.BSM_TOKEN_DATA_SHORT.parse_stream(file_object))
elif data_type == 'AUR_INT32':
for _ in range(token.unit_count):
data.append(self.BSM_TOKEN_DATA_INTEGER.parse_stream(file_object))
else:
data.append('Unknown type data')
# TODO: the data when it is string ends with ".", HW a space is return
# after uses the UTF-8 conversion.
return {bsm_type: {
'format': bsmtoken.BSM_TOKEN_DATA_PRINT[token.how_to_print],
'data':
'{0}'.format(self._RawToUTF8(''.join(map(str, data))))
}}
elif token_id in self._BSM_ATTR_TOKEN_TYPES:
return {bsm_type: {
'mode': token.file_mode,
'uid': token.uid,
'gid': token.gid,
'system_id': token.file_system_id,
'node_id': token.file_system_node_id,
'device': token.device}}
elif bsm_type == 'BSM_TOKEN_GROUPS':
arguments = []
for _ in range(token):
arguments.append(
self._RawToUTF8(
self.BSM_TOKEN_DATA_INTEGER.parse_stream(file_object)))
return {bsm_type: ','.join(arguments)}
elif bsm_type == 'BSM_TOKEN_AUT_SOCKINET32_EX':
if bsmtoken.BSM_PROTOCOLS.get(token.socket_domain, '') == 'INET6':
saddr = self._IPv6Format(
token.structure_addr_port.saddr_high,
token.structure_addr_port.saddr_low)
daddr = self._IPv6Format(
token.structure_addr_port.daddr_high,
token.structure_addr_port.daddr_low)
else:
saddr = self._IPv4Format(token.structure_addr_port.source_address)
daddr = self._IPv4Format(token.structure_addr_port.destination_address)
return {bsm_type:{
'from': saddr,
'from_port': token.structure_addr_port.source_port,
'to': daddr,
'to_port': token.structure_addr_port.destination_port}}
elif bsm_type == 'BSM_TOKEN_IPC_PERM':
return {bsm_type: {
'user_id': token.user_id,
'group_id': token.group_id,
'creator_user_id': token.creator_user_id,
'creator_group_id': token.creator_group_id,
'access': token.access_mode}}
elif bsm_type == 'BSM_TOKEN_SOCKET_UNIX':
string = self._CopyUtf8ByteArrayToString(token.path)
return {bsm_type: {'family': token.family, 'path': string}}
elif bsm_type == 'BSM_TOKEN_OPAQUE':
string = self._CopyByteArrayToBase16String(token.text)
return {bsm_type: string}
elif bsm_type == 'BSM_TOKEN_SEQUENCE':
return {bsm_type: token}
class NTFSUsnJrnlParser(interface.FileObjectParser):
"""Parses a NTFS USN change journal."""
_INITIAL_FILE_OFFSET = None
NAME = u'usnjrnl'
DESCRIPTION = u'Parser for NTFS USN change journal ($UsnJrnl).'
_USN_RECORD_V2 = construct.Struct(
u'usn_record_v2', construct.ULInt32(u'size'),
construct.ULInt16(u'major_version'),
construct.ULInt16(u'minor_version'),
construct.ULInt64(u'file_reference'),
construct.ULInt64(u'parent_file_reference'),
construct.ULInt64(u'update_sequence_number'),
construct.ULInt64(u'update_date_time'),
construct.ULInt32(u'update_reason_flags'),
construct.ULInt32(u'update_source_flags'),
construct.ULInt32(u'security_descriptor_identifier'),
construct.ULInt32(u'file_attribute_flags'),
construct.ULInt16(u'name_size'), construct.ULInt16(u'name_offset'),
construct.String(u'name', lambda ctx: ctx.size - 60))
# TODO: add support for USN_RECORD_V3 when actually seen to be used.
def _ParseUSNChangeJournal(self, parser_mediator, usn_change_journal):
"""Parses an USN change journal.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
usn_change_journal (pyfsntsfs.usn_change_journal): USN change journal.
"""
if not usn_change_journal:
return
usn_record_data = usn_change_journal.read_usn_record()
while usn_record_data:
current_offset = usn_change_journal.get_offset()
try:
usn_record_struct = self._USN_RECORD_V2.parse(usn_record_data)
except (IOError, construct.FieldError) as exception:
parser_mediator.ProduceExtractionError(
(u'unable to parse USN record at offset: 0x{0:08x} '
u'with error: {1:s}').format(current_offset, exception))
continue
name_offset = usn_record_struct.name_offset - 60
utf16_stream = usn_record_struct.name[name_offset:usn_record_struct
.name_size]
try:
name_string = utf16_stream.decode(u'utf-16-le')
except (UnicodeDecodeError, UnicodeEncodeError) as exception:
name_string = utf16_stream.decode(u'utf-16-le',
errors=u'replace')
parser_mediator.ProduceExtractionError((
u'unable to decode USN record name string with error: '
u'{0:s}. Characters that cannot be decoded will be replaced '
u'with "?" or "\\ufffd".').format(exception))
event_data = NTFSUSNChangeEventData()
event_data.file_attribute_flags = usn_record_struct.file_attribute_flags
event_data.file_reference = usn_record_struct.file_reference
event_data.filename = name_string
event_data.offset = current_offset
event_data.parent_file_reference = usn_record_struct.parent_file_reference
event_data.update_reason_flags = usn_record_struct.update_reason_flags
event_data.update_sequence_number = (
usn_record_struct.update_sequence_number)
event_data.update_source_flags = usn_record_struct.update_source_flags
if not usn_record_struct.update_date_time:
date_time = dfdatetime_semantic_time.SemanticTime(u'Not set')
else:
date_time = dfdatetime_filetime.Filetime(
timestamp=usn_record_struct.update_date_time)
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_ENTRY_MODIFICATION)
parser_mediator.ProduceEventWithEventData(event, event_data)
usn_record_data = usn_change_journal.read_usn_record()
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses a NTFS $UsnJrnl metadata file-like object.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): file-like object.
"""
volume = pyfsntfs.volume()
try:
volume.open_file_object(file_object)
except IOError as exception:
parser_mediator.ProduceExtractionError(
u'unable to open NTFS volume with error: {0:s}'.format(
exception))
try:
usn_change_journal = volume.get_usn_change_journal()
self._ParseUSNChangeJournal(parser_mediator, usn_change_journal)
finally:
volume.close()
class KeychainParser(interface.FileObjectParser):
"""Parser for Keychain files."""
NAME = 'mac_keychain'
DESCRIPTION = 'Parser for MacOS Keychain files.'
KEYCHAIN_SIGNATURE = b'kych'
KEYCHAIN_MAJOR_VERSION = 1
KEYCHAIN_MINOR_VERSION = 0
RECORD_TYPE_APPLICATION = 0x80000000
RECORD_TYPE_INTERNET = 0x80000001
# DB HEADER.
KEYCHAIN_DB_HEADER = construct.Struct(
'db_header',
construct.Bytes('signature', 4),
construct.UBInt16('major_version'),
construct.UBInt16('minor_version'),
construct.UBInt32('header_size'),
construct.UBInt32('schema_offset'),
construct.Padding(4))
# DB SCHEMA.
KEYCHAIN_DB_SCHEMA = construct.Struct(
'db_schema',
construct.UBInt32('size'),
construct.UBInt32('number_of_tables'))
# For each number_of_tables, the schema has a TABLE_OFFSET with the
# offset starting in the DB_SCHEMA.
TABLE_OFFSET = construct.UBInt32('table_offset')
TABLE_HEADER = construct.Struct(
'table_header',
construct.UBInt32('table_size'),
construct.UBInt32('record_type'),
construct.UBInt32('number_of_records'),
construct.UBInt32('first_record'),
construct.UBInt32('index_offset'),
construct.Padding(4),
construct.UBInt32('recordnumbercount'))
RECORD_HEADER = construct.Struct(
'record_header',
construct.UBInt32('entry_length'),
construct.Padding(12),
construct.UBInt32('ssgp_length'),
construct.Padding(4),
construct.UBInt32('creation_time'),
construct.UBInt32('last_modification_time'),
construct.UBInt32('text_description'),
construct.Padding(4),
construct.UBInt32('comments'),
construct.Padding(8),
construct.UBInt32('entry_name'),
construct.Padding(20),
construct.UBInt32('account_name'),
construct.Padding(4))
RECORD_HEADER_APP = construct.Struct(
'record_entry_app',
RECORD_HEADER,
construct.Padding(4))
RECORD_HEADER_INET = construct.Struct(
'record_entry_inet',
RECORD_HEADER,
construct.UBInt32('where'),
construct.UBInt32('protocol'),
construct.UBInt32('type'),
construct.Padding(4),
construct.UBInt32('url'))
TEXT = construct.PascalString(
'text', length_field=construct.UBInt32('length'))
TIME = construct.Struct(
'timestamp',
construct.String('year', 4),
construct.String('month', 2),
construct.String('day', 2),
construct.String('hour', 2),
construct.String('minute', 2),
construct.String('second', 2),
construct.Padding(2))
TYPE_TEXT = construct.String('type', 4)
# TODO: add more protocols.
_PROTOCOL_TRANSLATION_DICT = {
'htps': 'https',
'smtp': 'smtp',
'imap': 'imap',
'http': 'http'}
def _ReadEntryApplication(self, parser_mediator, file_object):
"""Extracts the information from an application password entry.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): a file-like object.
"""
record_offset = file_object.tell()
try:
record_struct = self.RECORD_HEADER_APP.parse_stream(file_object)
except (IOError, construct.FieldError):
parser_mediator.ProduceExtractionError(
'unable to parse record structure at offset: 0x{0:08x}'.format(
record_offset))
return
(ssgp_hash, creation_time, last_modification_time, text_description,
comments, entry_name, account_name) = self._ReadEntryHeader(
parser_mediator, file_object, record_struct.record_header,
record_offset)
# Move to the end of the record.
next_record_offset = (
record_offset + record_struct.record_header.entry_length)
file_object.seek(next_record_offset, os.SEEK_SET)
event_data = KeychainApplicationRecordEventData()
event_data.account_name = account_name
event_data.comments = comments
event_data.entry_name = entry_name
event_data.ssgp_hash = ssgp_hash
event_data.text_description = text_description
if creation_time:
event = time_events.DateTimeValuesEvent(
creation_time, definitions.TIME_DESCRIPTION_CREATION)
parser_mediator.ProduceEventWithEventData(event, event_data)
if last_modification_time:
event = time_events.DateTimeValuesEvent(
last_modification_time, definitions.TIME_DESCRIPTION_MODIFICATION)
parser_mediator.ProduceEventWithEventData(event, event_data)
def _ReadEntryHeader(
self, parser_mediator, file_object, record, record_offset):
"""Read the common record attributes.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_entry (dfvfs.FileEntry): a file entry object.
file_object (dfvfs.FileIO): a file-like object.
record (construct.Struct): record header structure.
record_offset (int): offset of the start of the record.
Returns:
A tuple containing:
ssgp_hash: Hash of the encrypted data (passwd, cert, note).
creation_time (dfdatetime.TimeElements): entry creation time or None.
last_modification_time ((dfdatetime.TimeElements): entry last
modification time or None.
text_description: A brief description of the entry.
entry_name: Name of the entry
account_name: Name of the account.
"""
# TODO: reduce number of seeks and/or offset calculations needed
# for parsing.
# Info: The hash header always start with the string ssgp follow by
# the hash. Furthermore The fields are always a multiple of four.
# Then if it is not multiple the value is padded by 0x00.
ssgp_hash = binascii.hexlify(file_object.read(record.ssgp_length)[4:])
creation_time = None
structure_offset = record_offset + record.creation_time - 1
file_object.seek(structure_offset, os.SEEK_SET)
try:
time_structure = self.TIME.parse_stream(file_object)
except construct.FieldError as exception:
time_structure = None
parser_mediator.ProduceExtractionError(
'unable to parse creation time with error: {0!s}'.format(exception))
if time_structure:
time_elements_tuple = (
time_structure.year, time_structure.month, time_structure.day,
time_structure.hour, time_structure.minute, time_structure.second)
creation_time = dfdatetime_time_elements.TimeElements()
try:
creation_time.CopyFromStringTuple(
time_elements_tuple=time_elements_tuple)
except ValueError:
creation_time = None
parser_mediator.ProduceExtractionError(
'invalid creation time value: {0!s}'.format(time_elements_tuple))
last_modification_time = None
structure_offset = record_offset + record.last_modification_time - 1
file_object.seek(structure_offset, os.SEEK_SET)
try:
time_structure = self.TIME.parse_stream(file_object)
except construct.FieldError as exception:
time_structure = None
parser_mediator.ProduceExtractionError(
'unable to parse last modification time with error: {0!s}'.format(
exception))
if time_structure:
time_elements_tuple = (
time_structure.year, time_structure.month, time_structure.day,
time_structure.hour, time_structure.minute, time_structure.second)
last_modification_time = dfdatetime_time_elements.TimeElements()
try:
last_modification_time.CopyFromStringTuple(
time_elements_tuple=time_elements_tuple)
except ValueError:
last_modification_time = None
parser_mediator.ProduceExtractionError(
'invalid last modification time value: {0!s}'.format(
time_elements_tuple))
text_description = 'N/A'
if record.text_description:
structure_offset = record_offset + record.text_description - 1
file_object.seek(structure_offset, os.SEEK_SET)
try:
text_description = self.TEXT.parse_stream(file_object)
except construct.FieldError as exception:
parser_mediator.ProduceExtractionError(
'unable to parse text description with error: {0!s}'.format(
exception))
comments = 'N/A'
if record.comments:
structure_offset = record_offset + record.comments - 1
file_object.seek(structure_offset, os.SEEK_SET)
try:
comments = self.TEXT.parse_stream(file_object)
except construct.FieldError as exception:
parser_mediator.ProduceExtractionError(
'unable to parse comments with error: {0!s}'.format(exception))
structure_offset = record_offset + record.entry_name - 1
file_object.seek(structure_offset, os.SEEK_SET)
try:
entry_name = self.TEXT.parse_stream(file_object)
except construct.FieldError as exception:
entry_name = 'N/A'
parser_mediator.ProduceExtractionError(
'unable to parse entry name with error: {0!s}'.format(exception))
structure_offset = record_offset + record.account_name - 1
file_object.seek(structure_offset, os.SEEK_SET)
try:
account_name = self.TEXT.parse_stream(file_object)
except construct.FieldError as exception:
account_name = 'N/A'
parser_mediator.ProduceExtractionError(
'unable to parse account name with error: {0!s}'.format(exception))
return (
ssgp_hash, creation_time, last_modification_time,
text_description, comments, entry_name, account_name)
def _ReadEntryInternet(self, parser_mediator, file_object):
"""Extracts the information from an Internet password entry.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): a file-like object.
"""
record_offset = file_object.tell()
try:
record_header_struct = self.RECORD_HEADER_INET.parse_stream(file_object)
except (IOError, construct.FieldError):
parser_mediator.ProduceExtractionError((
'unable to parse record header structure at offset: '
'0x{0:08x}').format(record_offset))
return
(ssgp_hash, creation_time, last_modification_time, text_description,
comments, entry_name, account_name) = self._ReadEntryHeader(
parser_mediator, file_object, record_header_struct.record_header,
record_offset)
if not record_header_struct.where:
where = 'N/A'
protocol = 'N/A'
type_protocol = 'N/A'
else:
offset = record_offset + record_header_struct.where - 1
file_object.seek(offset, os.SEEK_SET)
where = self.TEXT.parse_stream(file_object)
offset = record_offset + record_header_struct.protocol - 1
file_object.seek(offset, os.SEEK_SET)
protocol = self.TYPE_TEXT.parse_stream(file_object)
offset = record_offset + record_header_struct.type - 1
file_object.seek(offset, os.SEEK_SET)
type_protocol = self.TEXT.parse_stream(file_object)
type_protocol = self._PROTOCOL_TRANSLATION_DICT.get(
type_protocol, type_protocol)
if record_header_struct.url:
offset = record_offset + record_header_struct.url - 1
file_object.seek(offset, os.SEEK_SET)
url = self.TEXT.parse_stream(file_object)
where = '{0:s}{1:s}'.format(where, url)
# Move to the end of the record.
next_record_offset = (
record_offset + record_header_struct.record_header.entry_length)
file_object.seek(next_record_offset, os.SEEK_SET)
event_data = KeychainInternetRecordEventData()
event_data.account_name = account_name
event_data.comments = comments
event_data.entry_name = entry_name
event_data.protocol = protocol
event_data.ssgp_hash = ssgp_hash
event_data.text_description = text_description
event_data.type_protocol = type_protocol
event_data.where = where
if creation_time:
event = time_events.DateTimeValuesEvent(
creation_time, definitions.TIME_DESCRIPTION_CREATION)
parser_mediator.ProduceEventWithEventData(event, event_data)
if last_modification_time:
event = time_events.DateTimeValuesEvent(
last_modification_time, definitions.TIME_DESCRIPTION_MODIFICATION)
parser_mediator.ProduceEventWithEventData(event, event_data)
def _ReadTableOffsets(self, parser_mediator, file_object):
"""Reads the table offsets.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): a file-like object.
Returns:
list[int]: table offsets.
"""
# INFO: The HEADER KEYCHAIN:
# [DBHEADER] + [DBSCHEMA] + [OFFSET TABLE A] + ... + [OFFSET TABLE Z]
# Where the table offset is relative to the first byte of the DB Schema,
# then we must add to this offset the size of the [DBHEADER].
# Read the database schema and extract the offset for all the tables.
# They are ordered by file position from the top to the bottom of the file.
table_offsets = []
try:
db_schema_struct = self.KEYCHAIN_DB_SCHEMA.parse_stream(file_object)
except (IOError, construct.FieldError):
parser_mediator.ProduceExtractionError(
'unable to parse database schema structure')
return []
for index in range(db_schema_struct.number_of_tables):
try:
table_offset = self.TABLE_OFFSET.parse_stream(file_object)
except (IOError, construct.FieldError):
parser_mediator.ProduceExtractionError(
'unable to parse table offsets: {0:d}'.format(index))
return
table_offsets.append(table_offset + self.KEYCHAIN_DB_HEADER.sizeof())
return table_offsets
@classmethod
def GetFormatSpecification(cls):
"""Retrieves the format specification.
Returns:
FormatSpecification: format specification.
"""
format_specification = specification.FormatSpecification(cls.NAME)
format_specification.AddNewSignature(
cls.KEYCHAIN_SIGNATURE, offset=0)
return format_specification
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses a MacOS keychain file-like object.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): a file-like object.
Raises:
UnableToParseFile: when the file cannot be parsed.
"""
try:
db_header = self.KEYCHAIN_DB_HEADER.parse_stream(file_object)
except (IOError, construct.FieldError):
raise errors.UnableToParseFile('Unable to parse file header.')
if db_header.signature != self.KEYCHAIN_SIGNATURE:
raise errors.UnableToParseFile('Not a MacOS keychain file.')
if (db_header.major_version != self.KEYCHAIN_MAJOR_VERSION or
db_header.minor_version != self.KEYCHAIN_MINOR_VERSION):
parser_mediator.ProduceExtractionError(
'unsupported format version: {0:s}.{1:s}'.format(
db_header.major_version, db_header.minor_version))
return
# TODO: document format and determine if -1 offset correction is needed.
table_offsets = self._ReadTableOffsets(parser_mediator, file_object)
for table_offset in table_offsets:
# Skipping X bytes, unknown data at this point.
file_object.seek(table_offset, os.SEEK_SET)
try:
table = self.TABLE_HEADER.parse_stream(file_object)
except (IOError, construct.FieldError):
parser_mediator.ProduceExtractionError(
'unable to parse table structure at offset: 0x{0:08x}'.format(
table_offset))
continue
# Table_offset: absolute byte in the file where the table starts.
# table.first_record: first record in the table, relative to the
# first byte of the table.
file_object.seek(table_offset + table.first_record, os.SEEK_SET)
if table.record_type == self.RECORD_TYPE_INTERNET:
for _ in range(table.number_of_records):
self._ReadEntryInternet(parser_mediator, file_object)
elif table.record_type == self.RECORD_TYPE_APPLICATION:
for _ in range(table.number_of_records):
self._ReadEntryApplication(parser_mediator, file_object)
class WinJobParser(interface.FileObjectParser):
"""Parse Windows Scheduled Task files for job events."""
NAME = u'winjob'
DESCRIPTION = u'Parser for Windows Scheduled Task job (or At-job) files.'
_EMPTY_SYSTEM_TIME_TUPLE = (0, 0, 0, 0, 0, 0, 0, 0)
_PRODUCT_VERSIONS = {
0x0400: u'Windows NT 4.0',
0x0500: u'Windows 2000',
0x0501: u'Windows XP',
0x0600: u'Windows Vista',
0x0601: u'Windows 7',
0x0602: u'Windows 8',
0x0603: u'Windows 8.1',
0x0a00: u'Windows 10',
}
_JOB_FIXED_LENGTH_SECTION_STRUCT = construct.Struct(
u'job_fixed_length_section', construct.ULInt16(u'product_version'),
construct.ULInt16(u'format_version'), construct.Bytes(u'job_uuid', 16),
construct.ULInt16(u'application_length_offset'),
construct.ULInt16(u'trigger_offset'),
construct.ULInt16(u'error_retry_count'),
construct.ULInt16(u'error_retry_interval'),
construct.ULInt16(u'idle_deadline'), construct.ULInt16(u'idle_wait'),
construct.ULInt32(u'priority'), construct.ULInt32(u'max_run_time'),
construct.ULInt32(u'exit_code'), construct.ULInt32(u'status'),
construct.ULInt32(u'flags'),
construct.Struct(u'last_run_time', construct.ULInt16(u'year'),
construct.ULInt16(u'month'),
construct.ULInt16(u'weekday'),
construct.ULInt16(u'day'),
construct.ULInt16(u'hours'),
construct.ULInt16(u'minutes'),
construct.ULInt16(u'seconds'),
construct.ULInt16(u'milliseconds')))
# Using Construct's utf-16 encoding here will create strings with their
# null terminators exposed. Instead, we'll read these variables raw and
# convert them using Plaso's ReadUTF16() for proper formatting.
_JOB_VARIABLE_STRUCT = construct.Struct(
u'job_variable_length_section',
construct.ULInt16(u'running_instance_count'),
construct.ULInt16(u'application_length'),
construct.String(u'application',
lambda ctx: ctx.application_length * 2),
construct.ULInt16(u'parameter_length'),
construct.String(u'parameter', lambda ctx: ctx.parameter_length * 2),
construct.ULInt16(u'working_directory_length'),
construct.String(u'working_directory',
lambda ctx: ctx.working_directory_length * 2),
construct.ULInt16(u'username_length'),
construct.String(u'username', lambda ctx: ctx.username_length * 2),
construct.ULInt16(u'comment_length'),
construct.String(u'comment', lambda ctx: ctx.comment_length * 2),
construct.ULInt16(u'userdata_length'),
construct.String(u'userdata', lambda ctx: ctx.userdata_length),
construct.ULInt16(u'reserved_length'),
construct.String(u'reserved', lambda ctx: ctx.reserved_length),
construct.ULInt16(u'number_of_triggers'))
_TRIGGER_STRUCT = construct.Struct(u'trigger', construct.ULInt16(u'size'),
construct.ULInt16(u'reserved1'),
construct.ULInt16(u'start_year'),
construct.ULInt16(u'start_month'),
construct.ULInt16(u'start_day'),
construct.ULInt16(u'end_year'),
construct.ULInt16(u'end_month'),
construct.ULInt16(u'end_day'),
construct.ULInt16(u'start_hour'),
construct.ULInt16(u'start_minute'),
construct.ULInt32(u'duration'),
construct.ULInt32(u'interval'),
construct.ULInt32(u'trigger_flags'),
construct.ULInt32(u'trigger_type'),
construct.ULInt16(u'trigger_arg0'),
construct.ULInt16(u'trigger_arg1'),
construct.ULInt16(u'trigger_arg2'),
construct.ULInt16(u'trigger_padding'),
construct.ULInt16(u'trigger_reserved2'),
construct.ULInt16(u'trigger_reserved3'))
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses a Windows job file-like object.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): a file-like object.
Raises:
UnableToParseFile: when the file cannot be parsed.
"""
try:
header_struct = self._JOB_FIXED_LENGTH_SECTION_STRUCT.parse_stream(
file_object)
except (IOError, construct.FieldError) as exception:
raise errors.UnableToParseFile(
u'Unable to parse fixed-length section with error: {0:s}'.
format(exception))
if not header_struct.product_version in self._PRODUCT_VERSIONS:
raise errors.UnableToParseFile(
u'Unsupported product version in: 0x{0:04x}'.format(
header_struct.product_version))
if not header_struct.format_version == 1:
raise errors.UnableToParseFile(
u'Unsupported format version in: {0:d}'.format(
header_struct.format_version))
try:
job_variable_struct = self._JOB_VARIABLE_STRUCT.parse_stream(
file_object)
except (IOError, construct.FieldError) as exception:
raise errors.UnableToParseFile(
u'Unable to parse variable-length section with error: {0:s}'.
format(exception))
event_data = WinJobEventData()
event_data.application = binary.ReadUTF16(
job_variable_struct.application)
event_data.comment = binary.ReadUTF16(job_variable_struct.comment)
event_data.parameters = binary.ReadUTF16(job_variable_struct.parameter)
event_data.username = binary.ReadUTF16(job_variable_struct.username)
event_data.working_directory = binary.ReadUTF16(
job_variable_struct.working_directory)
systemtime_struct = header_struct.last_run_time
system_time_tuple = (systemtime_struct.year, systemtime_struct.month,
systemtime_struct.weekday, systemtime_struct.day,
systemtime_struct.hours,
systemtime_struct.minutes,
systemtime_struct.seconds,
systemtime_struct.milliseconds)
date_time = None
if system_time_tuple != self._EMPTY_SYSTEM_TIME_TUPLE:
try:
date_time = dfdatetime_systemtime.Systemtime(
system_time_tuple=system_time_tuple)
except ValueError:
parser_mediator.ProduceExtractionError(
u'invalid last run time: {0!s}'.format(system_time_tuple))
if date_time:
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_LAST_RUN)
parser_mediator.ProduceEventWithEventData(event, event_data)
for index in range(job_variable_struct.number_of_triggers):
try:
trigger_struct = self._TRIGGER_STRUCT.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
parser_mediator.ProduceExtractionError(
u'unable to parse trigger: {0:d} with error: {1:s}'.format(
index, exception))
return
event_data.trigger_type = trigger_struct.trigger_type
time_elements_tuple = (trigger_struct.start_year,
trigger_struct.start_month,
trigger_struct.start_day,
trigger_struct.start_hour,
trigger_struct.start_minute, 0)
if time_elements_tuple != (0, 0, 0, 0, 0, 0):
try:
date_time = dfdatetime_time_elements.TimeElements(
time_elements_tuple=time_elements_tuple)
date_time.is_local_time = True
date_time.precision = dfdatetime_definitions.PRECISION_1_MINUTE
except ValueError:
date_time = None
parser_mediator.ProduceExtractionError(
u'invalid trigger start time: {0!s}'.format(
time_elements_tuple))
if date_time:
event = time_events.DateTimeValuesEvent(
date_time,
u'Scheduled to start',
time_zone=parser_mediator.timezone)
parser_mediator.ProduceEventWithEventData(
event, event_data)
time_elements_tuple = (trigger_struct.end_year,
trigger_struct.end_month,
trigger_struct.end_day, 0, 0, 0)
if time_elements_tuple != (0, 0, 0, 0, 0, 0):
try:
date_time = dfdatetime_time_elements.TimeElements(
time_elements_tuple=time_elements_tuple)
date_time.is_local_time = True
date_time.precision = dfdatetime_definitions.PRECISION_1_DAY
except ValueError:
date_time = None
parser_mediator.ProduceExtractionError(
u'invalid trigger end time: {0!s}'.format(
time_elements_tuple))
if date_time:
event = time_events.DateTimeValuesEvent(
date_time,
u'Scheduled to end',
time_zone=parser_mediator.timezone)
parser_mediator.ProduceEventWithEventData(
event, event_data)
class CupsIppParser(interface.FileObjectParser):
"""Parser for CUPS IPP files. """
NAME = u'cups_ipp'
DESCRIPTION = u'Parser for CUPS IPP files.'
# INFO:
# For each file, we have only one document with three different timestamps:
# Created, process and finished.
# Format:
# [HEADER: MAGIC + KNOWN_TYPE][GROUP A]...[GROUP Z][GROUP_END: 0x03]
# GROUP: [GROUP ID][PAIR A]...[PAIR Z] where [PAIR: NAME + VALUE]
# GROUP ID: [1byte ID]
# PAIR: [TagID][\x00][Name][Value])
# TagID: 1 byte integer with the type of "Value".
# Name: [Length][Text][\00]
# Name can be empty when the name has more than one value.
# Example: family name "lopez mata" with more than one surname.
# Type_Text + [0x06, family, 0x00] + [0x05, lopez, 0x00] +
# Type_Text + [0x00, 0x00] + [0x04, mata, 0x00]
# Value: can be integer, boolean, or text provided by TagID.
# If boolean, Value: [\x01][0x00(False)] or [\x01(True)]
# If integer, Value: [\x04][Integer]
# If text, Value: [Length text][Text][\00]
# Magic number that identify the CUPS IPP supported version.
IPP_MAJOR_VERSION = 2
IPP_MINOR_VERSION = 0
# Supported Operation ID.
IPP_OP_ID = 5
# CUPS IPP File header.
CUPS_IPP_HEADER = construct.Struct(u'cups_ipp_header_struct',
construct.UBInt8(u'major_version'),
construct.UBInt8(u'minor_version'),
construct.UBInt16(u'operation_id'),
construct.UBInt32(u'request_id'))
# Group ID that indicates the end of the IPP Control file.
GROUP_END = 3
# Identification Groups.
GROUP_LIST = [1, 2, 4, 5, 6, 7]
# Type ID, per cups source file ipp-support.c.
TYPE_GENERAL_INTEGER = 0x20
TYPE_INTEGER = 0x21
TYPE_BOOL = 0x22
TYPE_ENUMERATION = 0x23
TYPE_DATETIME = 0x31
# Type of values that can be extracted.
INTEGER_8 = construct.UBInt8(u'integer')
INTEGER_32 = construct.UBInt32(u'integer')
TEXT = construct.PascalString(u'text',
encoding='utf-8',
length_field=construct.UBInt8(u'length'))
BOOLEAN = construct.Struct(u'boolean_value', construct.Padding(1),
INTEGER_8)
INTEGER = construct.Struct(u'integer_value', construct.Padding(1),
INTEGER_32)
# This is an RFC 2579 datetime.
DATETIME = construct.Struct(
u'datetime',
construct.Padding(1),
construct.UBInt16(u'year'),
construct.UBInt8(u'month'),
construct.UBInt8(u'day'),
construct.UBInt8(u'hour'),
construct.UBInt8(u'minutes'),
construct.UBInt8(u'seconds'),
construct.UBInt8(u'deciseconds'),
construct.String(u'direction_from_utc', length=1, encoding='ascii'),
construct.UBInt8(u'hours_from_utc'),
construct.UBInt8(u'minutes_from_utc'),
)
# Name of the pair.
PAIR_NAME = construct.Struct(u'pair_name', TEXT, construct.Padding(1))
# Specific CUPS IPP to generic name.
NAME_PAIR_TRANSLATION = {
u'printer-uri': u'uri',
u'job-uuid': u'job_id',
u'DestinationPrinterID': u'printer_id',
u'job-originating-user-name': u'user',
u'job-name': u'job_name',
u'document-format': u'doc_type',
u'job-originating-host-name': u'computer_name',
u'com.apple.print.JobInfo.PMApplicationName': u'application',
u'com.apple.print.JobInfo.PMJobOwner': u'owner'
}
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses a CUPS IPP file-like object.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
file_object: A file-like object.
Raises:
UnableToParseFile: when the file cannot be parsed.
"""
try:
header = self.CUPS_IPP_HEADER.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
raise errors.UnableToParseFile(
u'Unable to parse CUPS IPP Header with error: {0:s}'.format(
exception))
if (header.major_version != self.IPP_MAJOR_VERSION
or header.minor_version != self.IPP_MINOR_VERSION):
raise errors.UnableToParseFile(
u'[{0:s}] Unsupported version number.'.format(self.NAME))
if header.operation_id != self.IPP_OP_ID:
# Warn if the operation ID differs from the standard one. We should be
# able to parse the file nonetheless.
logging.debug(
u'[{0:s}] Unsupported operation identifier in file: {1:s}.'.
format(self.NAME, parser_mediator.GetDisplayName()))
# Read the pairs extracting the name and the value.
data_dict = {}
name, value = self.ReadPair(parser_mediator, file_object)
while name or value:
# Translate the known "name" CUPS IPP to a generic name value.
pretty_name = self.NAME_PAIR_TRANSLATION.get(name, name)
data_dict.setdefault(pretty_name, []).append(value)
name, value = self.ReadPair(parser_mediator, file_object)
# TODO: Refactor to use a lookup table to do event production.
time_dict = {}
for key, value in data_dict.items():
if key.startswith(u'date-time-') or key.startswith(u'time-'):
time_dict[key] = value
del data_dict[key]
if u'date-time-at-creation' in time_dict:
event_object = CupsIppEvent(time_dict[u'date-time-at-creation'][0],
eventdata.EventTimestamp.CREATION_TIME,
data_dict)
parser_mediator.ProduceEvent(event_object)
if u'date-time-at-processing' in time_dict:
event_object = CupsIppEvent(
time_dict[u'date-time-at-processing'][0],
eventdata.EventTimestamp.START_TIME, data_dict)
parser_mediator.ProduceEvent(event_object)
if u'date-time-at-completed' in time_dict:
event_object = CupsIppEvent(
time_dict[u'date-time-at-completed'][0],
eventdata.EventTimestamp.END_TIME, data_dict)
parser_mediator.ProduceEvent(event_object)
if u'time-at-creation' in time_dict:
time_value = time_dict[u'time-at-creation'][0]
timestamp = timelib.Timestamp.FromPosixTime(time_value)
event_object = CupsIppEvent(timestamp,
eventdata.EventTimestamp.CREATION_TIME,
data_dict)
parser_mediator.ProduceEvent(event_object)
if u'time-at-processing' in time_dict:
time_value = time_dict[u'time-at-processing'][0]
timestamp = timelib.Timestamp.FromPosixTime(time_value)
event_object = CupsIppEvent(timestamp,
eventdata.EventTimestamp.START_TIME,
data_dict)
parser_mediator.ProduceEvent(event_object)
if u'time-at-completed' in time_dict:
time_value = time_dict[u'time-at-completed'][0]
timestamp = timelib.Timestamp.FromPosixTime(time_value)
event_object = CupsIppEvent(timestamp,
eventdata.EventTimestamp.END_TIME,
data_dict)
parser_mediator.ProduceEvent(event_object)
def ReadPair(self, parser_mediator, file_object):
"""Reads an attribute name and value pair from a CUPS IPP event.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
file_object: a file-like object that points to a file.
Returns:
A list of name and value. If name and value cannot be read both are
set to None.
"""
# Pair = Type ID + Name + Value.
try:
# Can be:
# Group ID + IDtag = Group ID (1byte) + Tag ID (1byte) + '0x00'.
# IDtag = Tag ID (1byte) + '0x00'.
type_id = self.INTEGER_8.parse_stream(file_object)
if type_id == self.GROUP_END:
return None, None
elif type_id in self.GROUP_LIST:
# If it is a group ID we must read the next byte that contains
# the first TagID.
type_id = self.INTEGER_8.parse_stream(file_object)
# 0x00 separator character.
_ = self.INTEGER_8.parse_stream(file_object)
except (IOError, construct.FieldError):
logging.warning(
u'[{0:s}] Unsupported identifier in file: {1:s}.'.format(
self.NAME, parser_mediator.GetDisplayName()))
return None, None
# Name = Length name + name + 0x00
try:
name = self.PAIR_NAME.parse_stream(file_object).text
except (IOError, construct.FieldError):
logging.warning(u'[{0:s}] Unsupported name in file: {1:s}.'.format(
self.NAME, parser_mediator.GetDisplayName()))
return None, None
# Value: can be integer, boolean or text select by Type ID.
try:
if type_id in [
self.TYPE_GENERAL_INTEGER, self.TYPE_INTEGER,
self.TYPE_ENUMERATION
]:
value = self.INTEGER.parse_stream(file_object).integer
elif type_id == self.TYPE_BOOL:
value = bool(self.BOOLEAN.parse_stream(file_object).integer)
elif type_id == self.TYPE_DATETIME:
datetime = self.DATETIME.parse_stream(file_object)
value = timelib.Timestamp.FromRFC2579Datetime(
datetime.year, datetime.month, datetime.day, datetime.hour,
datetime.minutes, datetime.seconds, datetime.deciseconds,
datetime.direction_from_utc, datetime.hours_from_utc,
datetime.minutes_from_utc)
else:
value = self.TEXT.parse_stream(file_object)
except (IOError, UnicodeDecodeError, construct.FieldError):
logging.warning(
u'[{0:s}] Unsupported value in file: {1:s}.'.format(
self.NAME, parser_mediator.GetDisplayName()))
return None, None
return name, value
class MsgTweet(SBP):
"""SBP class for message MSG_TWEET (0x0012).
You can have MSG_TWEET inherit its fields directly
from an inherited SBP object, or construct it inline using a dict
of its fields.
All the news fit to tweet.
Parameters
----------
sbp : SBP
SBP parent object to inherit from.
tweet : string
Human-readable string
sender : int
Optional sender ID, defaults to SENDER_ID (see sbp/msg.py).
"""
_parser = construct.Struct('tweet' /
construct.String(140, paddir='left'), )
__slots__ = [
'tweet',
]
def __init__(self, sbp=None, **kwargs):
if sbp:
super(MsgTweet, self).__init__(sbp.msg_type, sbp.sender,
sbp.length, sbp.payload, sbp.crc)
self.from_binary(sbp.payload)
else:
super(MsgTweet, self).__init__()
self.msg_type = SBP_MSG_TWEET
self.sender = kwargs.pop('sender', SENDER_ID)
self.tweet = kwargs.pop('tweet')
def __repr__(self):
return fmt_repr(self)
@staticmethod
def from_json(s):
"""Given a JSON-encoded string s, build a message object.
"""
d = json.loads(s)
return MsgTweet.from_json_dict(d)
@staticmethod
def from_json_dict(d):
sbp = SBP.from_json_dict(d)
return MsgTweet(sbp, **d)
def from_binary(self, d):
"""Given a binary payload d, update the appropriate payload fields of
the message.
"""
p = MsgTweet._parser.parse(d)
for n in self.__class__.__slots__:
setattr(self, n, getattr(p, n))
def to_binary(self):
"""Produce a framed/packed SBP message.
"""
c = containerize(exclude_fields(self))
self.payload = MsgTweet._parser.build(c)
return self.pack()
def to_json_dict(self):
self.to_binary()
d = super(MsgTweet, self).to_json_dict()
j = walk_json_dict(exclude_fields(self))
d.update(j)
return d
class CustomDestinationsParser(interface.FileObjectParser):
"""Parses .customDestinations-ms files."""
_INITIAL_FILE_OFFSET = None
NAME = u'custom_destinations'
DESCRIPTION = u'Parser for *.customDestinations-ms files.'
# We cannot use the parser registry here since winlnk could be disabled.
# TODO: see if there is a more elegant solution for this.
_WINLNK_PARSER = winlnk.WinLnkParser()
_LNK_GUID = (
b'\x01\x14\x02\x00\x00\x00\x00\x00\xc0\x00\x00\x00\x00\x00\x00\x46')
_FOOTER_SIGNATURE = 0xbabffbab
_FILE_HEADER = construct.Struct(u'file_header',
construct.ULInt32(u'unknown1'),
construct.ULInt32(u'unknown2'),
construct.ULInt32(u'unknown3'),
construct.ULInt32(u'header_values_type'))
_HEADER_VALUE_TYPE_0 = construct.Struct(
u'header_value_type_0', construct.ULInt32(u'number_of_characters'),
construct.String(u'string', lambda ctx: ctx.number_of_characters * 2),
construct.ULInt32(u'unknown1'))
_HEADER_VALUE_TYPE_1_OR_2 = construct.Struct(
u'header_value_type_1_or_2', construct.ULInt32(u'unknown1'))
_ENTRY_HEADER = construct.Struct(u'entry_header',
construct.String(u'guid', 16))
_FILE_FOOTER = construct.Struct(u'file_footer',
construct.ULInt32(u'signature'))
def _ParseLNKFile(self, parser_mediator, file_entry, file_offset,
remaining_file_size):
"""Parses a LNK file stored within the .customDestinations-ms file.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
file_entry: A file entry object (instance of dfvfs.FileEntry).
file_offset: The offset of the LNK file data.
remaining_file_size: The size of the data remaining in the
.customDestinations-ms file.
Returns:
The size of the LNK file data or 0 if the LNK file could not be read.
"""
path_spec = path_spec_factory.Factory.NewPathSpec(
definitions.TYPE_INDICATOR_DATA_RANGE,
range_offset=file_offset,
range_size=remaining_file_size,
parent=file_entry.path_spec)
display_name = u'{0:s} # 0x{1:08x}'.format(file_entry.name,
file_offset)
try:
lnk_file_object = resolver.Resolver.OpenFileObject(path_spec)
except (dfvfs_errors.BackEndError, RuntimeError) as exception:
message = (
u'Unable to open LNK file: {0:s} with error {1:s}').format(
display_name, exception)
parser_mediator.ProduceParseError(message)
return 0
self._WINLNK_PARSER.Parse(parser_mediator,
lnk_file_object,
display_name=display_name)
# We cannot trust the file size in the LNK data so we get the last offset
# that was read instead.
lnk_file_size = lnk_file_object.get_offset()
lnk_file_object.close()
return lnk_file_size
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses a .customDestinations-ms file-like object.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
file_object: A file-like object.
Raises:
UnableToParseFile: when the file cannot be parsed.
"""
file_entry = parser_mediator.GetFileEntry()
display_name = parser_mediator.GetDisplayName()
file_object.seek(0, os.SEEK_SET)
try:
file_header = self._FILE_HEADER.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
raise errors.UnableToParseFile(
(u'Invalid Custom Destination: {0:s} - unable to parse '
u'file header with error: {1:s}').format(
display_name, exception))
if file_header.unknown1 != 2:
raise errors.UnableToParseFile((
u'Unsupported Custom Destination file: {0:s} - invalid unknown1: '
u'{1:d}.').format(display_name, file_header.unknown1))
if file_header.header_values_type > 2:
raise errors.UnableToParseFile((
u'Unsupported Custom Destination file: {0:s} - invalid header value '
u'type: {1:d}.').format(display_name,
file_header.header_values_type))
if file_header.header_values_type == 0:
data_structure = self._HEADER_VALUE_TYPE_0
else:
data_structure = self._HEADER_VALUE_TYPE_1_OR_2
try:
_ = data_structure.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
raise errors.UnableToParseFile(
(u'Invalid Custom Destination file: {0:s} - unable to parse '
u'header value with error: {1:s}').format(
display_name, exception))
file_size = file_object.get_size()
file_offset = file_object.get_offset()
remaining_file_size = file_size - file_offset
# The Custom Destination file does not have a unique signature in
# the file header that is why we use the first LNK class identifier (GUID)
# as a signature.
first_guid_checked = False
while remaining_file_size > 4:
try:
entry_header = self._ENTRY_HEADER.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
error_message = (
u'Invalid Custom Destination file: {0:s} - unable to parse '
u'entry header with error: {1:s}').format(
display_name, exception)
if not first_guid_checked:
raise errors.UnableToParseFile(error_message)
logging.warning(error_message)
break
if entry_header.guid != self._LNK_GUID:
error_message = (
u'Unsupported Custom Destination file: {0:s} - invalid entry '
u'header.').format(display_name)
if not first_guid_checked:
raise errors.UnableToParseFile(error_message)
file_object.seek(-16, os.SEEK_CUR)
try:
file_footer = self._FILE_FOOTER.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
raise IOError(
(u'Unable to parse file footer at offset: 0x{0:08x} '
u'with error: {1:s}').format(file_offset, exception))
if file_footer.signature != self._FOOTER_SIGNATURE:
logging.warning(error_message)
file_object.seek(-4, os.SEEK_CUR)
# TODO: add support for Jump List LNK file recovery.
break
first_guid_checked = True
file_offset += 16
remaining_file_size -= 16
lnk_file_size = self._ParseLNKFile(parser_mediator, file_entry,
file_offset,
remaining_file_size)
file_offset += lnk_file_size
remaining_file_size -= lnk_file_size
file_object.seek(file_offset, os.SEEK_SET)
try:
file_footer = self._FILE_FOOTER.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
logging.warning(
(u'Invalid Custom Destination file: {0:s} - unable to parse '
u'footer with error: {1:s}').format(display_name, exception))
if file_footer.signature != self._FOOTER_SIGNATURE:
logging.warning((
u'Unsupported Custom Destination file: {0:s} - invalid footer '
u'signature.').format(display_name))
class WinPrefetchParser(parser.BaseParser):
"""A parser for Windows Prefetch files."""
NAME = 'prefetch'
FILE_SIGNATURE = 'SCCA'
FILE_HEADER_STRUCT = construct.Struct('file_header',
construct.ULInt32('version'),
construct.String('signature', 4),
construct.Padding(4),
construct.ULInt32('file_size'),
construct.String('executable', 60),
construct.ULInt32('prefetch_hash'),
construct.ULInt32('flags'))
FILE_INFORMATION_V17 = construct.Struct(
'file_information_v17', construct.Padding(16),
construct.ULInt32('filenames_offset'),
construct.ULInt32('filenames_size'),
construct.ULInt32('volumes_information_offset'),
construct.ULInt32('number_of_volumes'),
construct.ULInt32('volumes_information_size'),
construct.ULInt64('last_run_time'), construct.Padding(16),
construct.ULInt32('run_count'), construct.Padding(4))
FILE_INFORMATION_V23 = construct.Struct(
'file_information_v23', construct.Padding(16),
construct.ULInt32('filenames_offset'),
construct.ULInt32('filenames_size'),
construct.ULInt32('volumes_information_offset'),
construct.ULInt32('number_of_volumes'),
construct.ULInt32('volumes_information_size'), construct.Padding(8),
construct.ULInt64('last_run_time'), construct.Padding(16),
construct.ULInt32('run_count'), construct.Padding(84))
FILE_INFORMATION_V26 = construct.Struct(
'file_information_v26', construct.Padding(16),
construct.ULInt32('filenames_offset'),
construct.ULInt32('filenames_size'),
construct.ULInt32('volumes_information_offset'),
construct.ULInt32('number_of_volumes'),
construct.ULInt32('volumes_information_size'), construct.Padding(8),
construct.ULInt64('last_run_time'),
construct.ULInt64('last_run_time1'),
construct.ULInt64('last_run_time2'),
construct.ULInt64('last_run_time3'),
construct.ULInt64('last_run_time4'),
construct.ULInt64('last_run_time5'),
construct.ULInt64('last_run_time6'),
construct.ULInt64('last_run_time7'), construct.Padding(16),
construct.ULInt32('run_count'), construct.Padding(96))
VOLUME_INFORMATION_V17 = construct.Struct(
'volume_information_v17', construct.ULInt32('device_path_offset'),
construct.ULInt32('device_path_number_of_characters'),
construct.ULInt64('creation_time'), construct.ULInt32('serial_number'),
construct.Padding(8), construct.ULInt32('directory_strings_offset'),
construct.ULInt32('number_of_directory_strings'), construct.Padding(4))
# Note that at the moment for the purpose of this parser
# the v23 and v26 volume information structures are the same.
VOLUME_INFORMATION_V23 = construct.Struct(
'volume_information_v23', construct.ULInt32('device_path_offset'),
construct.ULInt32('device_path_number_of_characters'),
construct.ULInt64('creation_time'), construct.ULInt32('serial_number'),
construct.Padding(8), construct.ULInt32('directory_strings_offset'),
construct.ULInt32('number_of_directory_strings'),
construct.Padding(68))
def _ParseFileHeader(self, file_object):
"""Parses the file header.
Args:
file_object: A file-like object to read data from.
Returns:
The file header construct object.
"""
try:
file_header = self.FILE_HEADER_STRUCT.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
raise errors.UnableToParseFile(
u'Unable to parse file header with error: {0:s}'.format(
exception))
if not file_header:
raise errors.UnableToParseFile('Unable to read file header')
if file_header.get('signature', None) != self.FILE_SIGNATURE:
raise errors.UnableToParseFile('Unsupported file signature')
return file_header
def _ParseFileInformation(self, file_object, format_version):
"""Parses the file information.
Args:
file_object: A file-like object to read data from.
format_version: The format version.
Returns:
The file information construct object.
"""
try:
if format_version == 17:
file_information = self.FILE_INFORMATION_V17.parse_stream(
file_object)
elif format_version == 23:
file_information = self.FILE_INFORMATION_V23.parse_stream(
file_object)
elif format_version == 26:
file_information = self.FILE_INFORMATION_V26.parse_stream(
file_object)
else:
file_information = None
except (IOError, construct.FieldError) as exception:
raise errors.UnableToParseFile(
u'Unable to parse file information v{0:d} with error: {1:s}'.
format(format_version, exception))
if not file_information:
raise errors.UnableToParseFile('Unable to read file information')
return file_information
def _ParseFilenames(self, file_object, file_information):
"""Parses the filenames.
Args:
file_object: A file-like object to read data from.
file_information: The file information construct object.
Returns:
A list of filenames.
"""
filenames_offset = file_information.get('filenames_offset', 0)
filenames_size = file_information.get('filenames_size', 0)
if filenames_offset > 0 and filenames_size > 0:
file_object.seek(filenames_offset, os.SEEK_SET)
filenames_data = file_object.read(filenames_size)
filenames = binary.ArrayOfUt16StreamCopyToString(filenames_data)
else:
filenames = []
return filenames
def _ParseVolumesInformationSection(self, file_object, format_version,
file_information):
"""Parses the volumes information section.
Args:
file_object: A file-like object to read data from.
format_version: The format version.
file_information: The file information construct object.
Yields:
A volume information construct object.
"""
volumes_information_offset = file_information.get(
'volumes_information_offset', 0)
if volumes_information_offset > 0:
number_of_volumes = file_information.get('number_of_volumes', 0)
file_object.seek(volumes_information_offset, os.SEEK_SET)
while number_of_volumes > 0:
try:
if format_version == 17:
yield self.VOLUME_INFORMATION_V17.parse_stream(
file_object)
else:
yield self.VOLUME_INFORMATION_V23.parse_stream(
file_object)
except (IOError, construct.FieldError) as exception:
raise errors.UnableToParseFile((
u'Unable to parse volume information v{0:d} with error: '
u'{1:s}').format(format_version, exception))
number_of_volumes -= 1
def _ParseVolumeDevicePath(self, file_object, file_information,
volume_information):
"""Parses the volume device path.
This function expects the current offset of the file-like object to point
as the end of the volume information structure.
Args:
file_object: A file-like object to read data from.
file_information: The file information construct object.
volume_information: The volume information construct object.
Returns:
A Unicode string containing the device path or None if not available.
"""
volumes_information_offset = file_information.get(
'volumes_information_offset', 0)
device_path = None
if volumes_information_offset > 0:
device_path_offset = volume_information.get(
'device_path_offset', 0)
device_path_size = 2 * volume_information.get(
'device_path_number_of_characters', 0)
if device_path_offset >= 36 and device_path_size > 0:
device_path_offset += volumes_information_offset
current_offset = file_object.tell()
file_object.seek(device_path_offset, os.SEEK_SET)
device_path = binary.ReadUtf16Stream(
file_object, byte_size=device_path_size)
file_object.seek(current_offset, os.SEEK_SET)
return device_path
def Parse(self, file_entry):
"""Extracts events from a Windows Prefetch file.
Args:
file_entry: A file entry object (instance of dfvfs.FileEntry).
Yields:
Event objects (instances of EventObject) of the extracted events.
"""
file_object = file_entry.GetFileObject()
file_header = self._ParseFileHeader(file_object)
format_version = file_header.get('version', None)
if format_version not in [17, 23, 26]:
raise errors.UnableToParseFile(
u'Unsupported format version: {0:d}'.format(format_version))
file_information = self._ParseFileInformation(file_object,
format_version)
mapped_files = self._ParseFilenames(file_object, file_information)
executable = binary.Ut16StreamCopyToString(
file_header.get('executable', u''))
volume_serial_numbers = []
volume_device_paths = []
prefetch_events = []
path = u''
for volume_information in self._ParseVolumesInformationSection(
file_object, format_version, file_information):
volume_serial_number = volume_information.get('serial_number', 0)
volume_device_path = self._ParseVolumeDevicePath(
file_object, file_information, volume_information)
volume_serial_numbers.append(volume_serial_number)
volume_device_paths.append(volume_device_path)
timestamp = volume_information.get('creation_time', 0)
if timestamp:
prefetch_events.append(
(timestamp, eventdata.EventTimestamp.CREATION_TIME))
for mapped_file in mapped_files:
if (mapped_file.startswith(volume_device_path)
and mapped_file.endswith(executable)):
_, _, path = mapped_file.partition(volume_device_path)
for prefetch_timestamp, prefetch_description in prefetch_events:
yield WinPrefetchExecutionEvent(prefetch_timestamp,
prefetch_description, file_header,
file_information, mapped_files,
path, volume_serial_numbers,
volume_device_paths)
timestamp = file_information.get('last_run_time', 0)
if timestamp:
yield WinPrefetchExecutionEvent(
timestamp, eventdata.EventTimestamp.LAST_RUNTIME, file_header,
file_information, mapped_files, path, volume_serial_numbers,
volume_device_paths)
# Check for the 7 older last run time values available in v26.
if format_version == 26:
for last_run_time_index in range(1, 8):
last_run_time_identifier = 'last_run_time{0:d}'.format(
last_run_time_index)
timestamp = file_information.get(last_run_time_identifier, 0)
if timestamp:
yield WinPrefetchExecutionEvent(
timestamp, u'Previous {0:s}'.format(
eventdata.EventTimestamp.LAST_RUNTIME),
file_header, file_information, mapped_files, path,
volume_serial_numbers, volume_device_paths)
file_object.close()
class UtmpParser(interface.SingleFileBaseParser):
"""Parser for Linux/Unix UTMP files."""
_INITIAL_FILE_OFFSET = None
NAME = 'utmp'
DESCRIPTION = u'Parser for Linux/Unix UTMP files.'
LINUX_UTMP_ENTRY = construct.Struct('utmp_linux',
construct.ULInt32('type'),
construct.ULInt32('pid'),
construct.String('terminal', 32),
construct.ULInt32('terminal_id'),
construct.String('username', 32),
construct.String('hostname', 256),
construct.ULInt16('termination'),
construct.ULInt16('exit'),
construct.ULInt32('session'),
construct.ULInt32('timestamp'),
construct.ULInt32('microsecond'),
construct.ULInt32('address_a'),
construct.ULInt32('address_b'),
construct.ULInt32('address_c'),
construct.ULInt32('address_d'),
construct.Padding(20))
LINUX_UTMP_ENTRY_SIZE = LINUX_UTMP_ENTRY.sizeof()
STATUS_TYPE = {
0: 'EMPTY',
1: 'RUN_LVL',
2: 'BOOT_TIME',
3: 'NEW_TIME',
4: 'OLD_TIME',
5: 'INIT_PROCESS',
6: 'LOGIN_PROCESS',
7: 'USER_PROCESS',
8: 'DEAD_PROCESS',
9: 'ACCOUNTING'
}
# Set a default test value for few fields, this is supposed to be a text
# that is highly unlikely to be seen in a terminal field, or a username field.
# It is important that this value does show up in such fields, but otherwise
# it can be a free flowing text field.
_DEFAULT_TEST_VALUE = u'Ekki Fraedilegur Moguleiki, thetta er bull ! = + _<>'
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses an UTMP file-like object.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
file_object: The file-like object to extract data from.
Raises:
UnableToParseFile: when the file cannot be parsed.
"""
file_object.seek(0, os.SEEK_SET)
try:
structure = self.LINUX_UTMP_ENTRY.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
raise errors.UnableToParseFile(
u'Unable to parse UTMP Header with error: {0:s}'.format(
exception))
if structure.type not in self.STATUS_TYPE:
raise errors.UnableToParseFile(
(u'Not an UTMP file, unknown type '
u'[{0:d}].').format(structure.type))
if not self._VerifyTextField(structure.terminal):
raise errors.UnableToParseFile(
u'Not an UTMP file, unknown terminal.')
if not self._VerifyTextField(structure.username):
raise errors.UnableToParseFile(
u'Not an UTMP file, unknown username.')
if not self._VerifyTextField(structure.hostname):
raise errors.UnableToParseFile(
u'Not an UTMP file, unknown hostname.')
# Check few values.
terminal = self._GetTextFromNullTerminatedString(
structure.terminal, self._DEFAULT_TEST_VALUE)
if terminal == self._DEFAULT_TEST_VALUE:
raise errors.UnableToParseFile(
u'Not an UTMP file, no terminal set.')
username = self._GetTextFromNullTerminatedString(
structure.username, self._DEFAULT_TEST_VALUE)
if username == self._DEFAULT_TEST_VALUE:
raise errors.UnableToParseFile(
u'Not an UTMP file, no username set.')
if not structure.timestamp:
raise errors.UnableToParseFile(
u'Not an UTMP file, no timestamp set in the first record.')
file_object.seek(0, os.SEEK_SET)
event_object = self._ReadUtmpEvent(file_object)
while event_object:
event_object.offset = file_object.tell()
parser_mediator.ProduceEvent(event_object)
event_object = self._ReadUtmpEvent(file_object)
def _VerifyTextField(self, text):
"""Check if a byte stream is a null terminated string.
Args:
event_object: text field from the structure.
Return:
True if it is a null terminated string, False otherwise.
"""
_, _, null_chars = text.partition(b'\x00')
if not null_chars:
return False
return len(null_chars) == null_chars.count(b'\x00')
def _ReadUtmpEvent(self, file_object):
"""Returns an UtmpEvent from a single UTMP entry.
Args:
file_object: a file-like object that points to an UTMP file.
Returns:
An event object constructed from a single UTMP record or None if we
have reached the end of the file (or EOF).
"""
offset = file_object.tell()
data = file_object.read(self.LINUX_UTMP_ENTRY_SIZE)
if not data or len(data) != self.LINUX_UTMP_ENTRY_SIZE:
return
try:
entry = self.LINUX_UTMP_ENTRY.parse(data)
except (IOError, construct.FieldError):
logging.warning(
(u'UTMP entry at 0x{:x} couldn\'t be parsed.').format(offset))
return self._ReadUtmpEvent(file_object)
user = self._GetTextFromNullTerminatedString(entry.username)
terminal = self._GetTextFromNullTerminatedString(entry.terminal)
if terminal == '~':
terminal = u'system boot'
computer_name = self._GetTextFromNullTerminatedString(entry.hostname)
if computer_name == u'N/A' or computer_name == u':0':
computer_name = u'localhost'
status = self.STATUS_TYPE.get(entry.type, u'N/A')
if not entry.address_b:
try:
ip_address = socket.inet_ntoa(
construct.ULInt32('int').build(entry.address_a))
if ip_address == '0.0.0.0':
ip_address = u'localhost'
except (IOError, construct.FieldError, socket.error):
ip_address = u'N/A'
else:
ip_address = u'{0:d}.{1:d}.{2:d}.{3:d}'.format(
entry.address_a, entry.address_b, entry.address_c,
entry.address_d)
return UtmpEvent(entry.timestamp, entry.microsecond, user,
computer_name, terminal, status, ip_address, entry)
def _GetTextFromNullTerminatedString(self,
null_terminated_string,
default_string=u'N/A'):
"""Get a UTF-8 text from a raw null terminated string.
Args:
null_terminated_string: Raw string terminated with null character.
default_string: The default string returned if the parser fails.
Returns:
A decoded UTF-8 string or if unable to decode, the supplied default
string.
"""
text, _, _ = null_terminated_string.partition('\x00')
try:
text = text.decode('utf-8')
except UnicodeDecodeError:
logging.warning(
u'[UTMP] Decode UTF8 failed, the message string may be cut short.'
)
text = text.decode('utf-8', 'ignore')
if not text:
return default_string
return text
class VolumeBootRecord(BootRecord):
_NTFS_VBR_STRUCT = construct.Struct(
'NTFS-VBR',
construct.Field('JumpOverBPB', 3),
construct.String("OemId", 8),
construct.Struct(
'BiosParameterBlock',
construct.ULInt16('SectorSize'),
construct.ULInt8('SectorsPerCluster'),
construct.Field('Reserved1', 2),
construct.Field('MustBeZero1', 3),
construct.Field('MustBeZero2', 2),
construct.ULInt8('MediaDescriptor'),
construct.Field('MustBeZero3', 2),
construct.ULInt16('SectorsPerTrack'),
construct.ULInt16('NumberOfHeads'),
construct.ULInt32('HiddenSectors'),
construct.Field('NotUsed1', 4),
construct.Const(construct.Field('DriveNumber', 1),
'80'.decode('hex')),
construct.Field('Reserved2', 3),
construct.ULInt64('TotalSectors'),
construct.ULInt64('MFTCluster'),
construct.ULInt64('MFTMirrCluster'),
construct.SLInt8('ClustersPerMFTRecord'),
construct.Field('NotUsed2', 3),
construct.SLInt8('ClustersPerIdxBuffer'),
construct.Field('NotUsed3', 3),
construct.ULInt64('VolumneSN'),
construct.Field('NotUsed4', 4),
),
construct.HexDumpAdapter(construct.Bytes("Code", 426)),
construct.Const(construct.Bytes("signature", 2), '55aa'.decode('hex')),
)
_BITLOCKER_VBR_STRUCT = construct.Struct(
'FVE-VBR',
construct.Field('JumpOverBPB', 3),
construct.Const(construct.String("OemId", 8),
'-FVE-FS-'.encode('utf8')),
construct.Struct(
'BiosParameterBlock',
construct.ULInt16('SectorSize'),
construct.ULInt8('SectorsPerCluster'),
construct.Field('Reserved1', 2),
construct.Field('MustBeZero1', 3),
construct.Field('MustBeZero2', 2),
construct.ULInt8('MediaDescriptor'),
construct.Field('MustBeZero3', 2),
construct.ULInt16('SectorsPerTrack'),
construct.ULInt16('NumberOfHeads'),
construct.ULInt32('HiddenSectors'),
construct.ULInt32('TotalSectors'),
construct.ULInt32('SectorsPerFAT'),
construct.ULInt16('FATFlags'),
construct.ULInt16('Version'),
construct.ULInt32('RootDirCluster'),
construct.ULInt16('FSInfoSector'),
construct.ULInt16('BackupSector'),
construct.Field('Reserved2', 12),
construct.Const(construct.Field('DriveNumber', 1),
'80'.decode('hex')),
construct.Field('Reserved3', 1),
construct.Field('ExtendedBootSignature', 1),
construct.ULInt32('VolumneSN'),
construct.Const(construct.String("VolumeLabel", 11),
'NO NAME '.encode('utf8')),
construct.Const(construct.String("SystemId", 8),
'FAT32 '.encode('utf8')),
),
construct.HexDumpAdapter(construct.Bytes("Code1", 70)),
construct.Field('BitlockerGUID', 16),
construct.ULInt64('FVEMetadataBlockOffset1'),
construct.ULInt64('FVEMetadataBlockOffset2'),
construct.ULInt64('FVEMetadataBlockOffset3'),
construct.HexDumpAdapter(construct.Bytes("Code2", 307)),
construct.ULInt8('FirstStrOffset'),
construct.ULInt8('SecondStrOffset'),
construct.ULInt8('ThirdStrOffset'),
construct.Const(construct.Bytes("signature", 2), '55aa'.decode('hex')),
)
def __init__(self, filePath, size, offset=None, whitelist=()):
self._type = 'VBR'
super(VolumeBootRecord, self).__init__(filePath, size, offset,
whitelist)
def _parse(self):
"""
Main method in charge of parsing the VBR.
It will try to parse the boot record according to known structures (NTFS and Bitlocker supported).
It will then try to narrow down invariant code, hash it and match the hash against a whitelist.
If no match was found, it will try some simple heuristics to detect malicious behaviours.
Finally it will compare the HiddenSectors value in BPB to that of the record's dump offset.
Returns: nothing
"""
try:
# This will parse both NTFS and Vista bitlocker volumes since they only differ by their OEM ID
vbr = self._NTFS_VBR_STRUCT.parse(self._raw)
expectedLoader, invariantCode = self._getInvariantCode('NTFS', vbr)
except construct.core.ConstructError as e1:
# Retry with Bitlocker (Win7+) volume header structure
try:
vbr = self._BITLOCKER_VBR_STRUCT.parse(self._raw)
expectedLoader, invariantCode = self._getInvariantCode(
'bitlocker', vbr)
except construct.core.ConstructError as e2:
raise InvalidVBRError(
'Invalid VBR structure: e1={0}, e2={1}\n{2}'.format(
e1, e2, hexdump(self._raw)))
self._oemId = vbr.OemId
self._bpb = vbr.BiosParameterBlock
codeHash = hashlib.sha256(invariantCode)
self._matchHash(codeHash, expectedLoader)
# If no whitelisted signature matched, try some simple heuristics to flag this VBR as malicious
# Note that the self._checkCode method is only given the "invariant" code section to help with the
# disassembling. This will obviously leads to broken offsets, but it doesn't matter since the heuristics don't
# use them.
if len(self._signature) == 0:
self._checkCode(invariantCode)
# At last, compare the offset at which this VBR was found with the value of the BPB HiddenSectors
if self._offset is not None \
and (vbr.BiosParameterBlock.HiddenSectors * vbr.BiosParameterBlock.SectorSize) != self._offset:
self._suspiciousBehaviour.append(
'Suspicious HiddenSectors value: {0} ({1} bytes)'.format(
vbr.BiosParameterBlock.HiddenSectors,
vbr.BiosParameterBlock.HiddenSectors *
vbr.BiosParameterBlock.SectorSize))
def _getInvariantCode(self, vbrType, vbrStruct):
"""
Helper method that finds all the sections of the boot code that can be hashed and compared to a whitelist.
This means that localized strings and other variable parameters (BPB, etc...) are excluded.
Currently, this method only supports NTFS and Bitlocker VBR.
Args:
vbrType: unicode string corresponding to the VBR type ('NTFS' or 'bitlocker')
vbrStruct: construct.container of the VBR
Returns: 2-tuple (unicode string of expected loader, concatenated strings of invariant sections of code)
"""
codeStart = 0
codeEnd = None
invariantCode = str()
expectedLoader = None
if vbrType == 'NTFS':
# The first three bytes are a jump over the NTFS BPB to where the code really starts (0x54) and a NOP
invariantCode += vbrStruct.JumpOverBPB
codeStart = 0x54
# NTFS VBR contains localized strings which must be excluded from the hash computation.
# Before Windows 8, these strings are located at 4 different offsets which can be calculated by adding
# 0x100 to the values respectively stored in bytes 0x1f8, 0x1f9, 0x1fa and 0x1fb.
# Starting from Windows 8, these strings are located at 3 different offsets which are directly stored in
# little endian words respectively at 0x1f6, 0x1f8 and 0x1fa
# Since there is no easy way to tell which version of Windows we are dealing with beforehand, we first
# assume it is a Windows < 8 by testing 0x1f8 against all the known first offset. If all tests fail, assume
# it is Windows >= 8 and check 0x1f6 against the only known first offset (to date)
firstStrOffset = construct.UBInt8('FirstStringOffset').parse(
self._raw[0x1f8])
# Windows NT5
if firstStrOffset == 0x83:
expectedLoader = 'NT5.1/NT5.2 VBR'
codeEnd = 0x100 + firstStrOffset
# Windows NT6.0
elif firstStrOffset == 0x80:
expectedLoader = 'NT6.0 VBR'
codeEnd = 0x100 + firstStrOffset
# Windows NT6.1
elif firstStrOffset == 0x8c:
expectedLoader = 'NT6.1 VBR'
codeEnd = 0x100 + firstStrOffset
# Windows NT6.2+
else:
firstStrOffset = construct.ULInt16('FirstStringOffset').parse(
self._raw[0x1f6:0x1f8])
if firstStrOffset == 0x18a:
expectedLoader = 'NT6.2+ VBR'
codeEnd = firstStrOffset
if codeEnd is None:
self._suspiciousBehaviour.append(
'Invalid string offset: {0:#x}'.format(firstStrOffset))
self._logger.debug(
'First localized string offset is wrong for a NTFS VBR: {0:#x}. '
'It should be 0x83, 0x80, 0x8c or 0x18a.'.format(
firstStrOffset))
codeEnd = 0
elif vbrType == 'bitlocker':
expectedLoader = 'NT6.1+ Bitlocker VBR'
# The first three bytes are a jump over the NTFS BPB to where the code really starts (0x5A) and a NOP
invariantCode += vbrStruct.JumpOverBPB
# First section of code (_BITLOCKER_VBR_STRUCT.Code1)
invariantCode += vbrStruct.Code1
# In the second section of code, there are localized strings which must be excluded from hash computation.
# Their offsets are stored in the last 3 bytes before the VBR signature (0x55aa).
# For Windows 8, 8.1 and 10, the first string offset seems to always be 0x100 (ie. FirstStrOffset = 0x00)
if vbrStruct.FirstStrOffset != 0:
self._suspiciousBehaviour.append(
'Invalid string offset: {0:#x}'.format(
vbrStruct.FirstStrOffset))
self._logger.debug(
'First localized string offset is wrong for a Bitlocker VBR. '
'It should be 0x00) : {0:#x}'.format(
vbrStruct.FirstStrOffset))
codeStart = 0xc8 # Offset of Code2
codeEnd = 0x100 + vbrStruct.FirstStrOffset
else:
raise NotImplementedError(
'VBR type "{0}" is not implemented yet'.format(vbrType))
self._logger.debug(
'Expecting {0}. Code starts at {1:#x} and ends at {2:#x}'.format(
expectedLoader, codeStart, codeEnd))
invariantCode += self._raw[codeStart:codeEnd]
return expectedLoader, invariantCode
def _checkCode(self, code):
md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_16)
md.detail = True
for i in md.disasm(code, 0):
# Check for unknown interrupt
if i.mnemonic == 'int' and i.bytes[1] not in (0x10, 0x13, 0x18,
0x1a):
self._suspiciousBehaviour.append(
'Unknown Interrupt : {0:#x}'.format(i.bytes[1]))
