class BinaryCookieParser(interface.FileObjectParser):
"""Parser for Safari Binary Cookie files."""
NAME = u'binary_cookies'
DESCRIPTION = u'Parser for Safari Binary Cookie files.'
COOKIE_HEADER = construct.Struct(
u'binary_cookie_header', construct.UBInt32(u'pages'),
construct.Array(lambda ctx: ctx.pages,
construct.UBInt32(u'page_sizes')))
COOKIE_DATA = construct.Struct(u'binary_cookie_data',
construct.ULInt32(u'size'),
construct.Bytes(u'unknown_1', 4),
construct.ULInt32(u'flags'),
construct.Bytes(u'unknown_2', 4),
construct.ULInt32(u'url_offset'),
construct.ULInt32(u'name_offset'),
construct.ULInt32(u'path_offset'),
construct.ULInt32(u'value_offset'),
construct.Bytes(u'end_of_cookie', 8),
construct.LFloat64(u'expiration_date'),
construct.LFloat64(u'creation_date'))
PAGE_DATA = construct.Struct(
u'page_data', construct.Bytes(u'header', 4),
construct.ULInt32(u'number_of_cookies'),
construct.Array(lambda ctx: ctx.number_of_cookies,
construct.ULInt32(u'offsets')))
# Cookie flags.
COOKIE_FLAG_NONE = 0
COOKIE_FLAG_SECURE = 1
COOKIE_FLAG_UNKNOWN = 2
COOKIE_FLAG_HTTP_ONLY = 4
def __init__(self):
"""Initializes a parser object."""
super(BinaryCookieParser, self).__init__()
self._cookie_plugins = (
cookie_plugins_manager.CookiePluginsManager.GetPlugins())
def _ParsePage(self, page_data, parser_mediator):
"""Extract events from a page and produce events.
Args:
page_data: Raw bytes of the page.
file_entry: The file entry (instance of dfvfs.FileEntry).
parser_mediator: A parser mediator object (instance of ParserMediator).
"""
try:
page = self.PAGE_DATA.parse(page_data)
except construct.FieldError:
parser_mediator.ProduceParseError(u'Unable to parse page')
return
for page_offset in page.offsets:
try:
cookie = self.COOKIE_DATA.parse(page_data[page_offset:])
except construct.FieldError:
message = u'Unable to parse cookie data from offset: {0:d}'.format(
page_offset)
parser_mediator.ProduceParseError(message)
continue
# The offset is determine by the range between the start of the current
# offset until the start of the next offset. Thus we need to determine
# the proper ordering of the offsets, since they are not always in the
# same ordering.
offset_dict = {
cookie.url_offset: u'url',
cookie.name_offset: u'name',
cookie.value_offset: u'value',
cookie.path_offset: u'path'
}
offsets = sorted(offset_dict.keys())
offsets.append(cookie.size + page_offset)
# TODO: Find a better approach to parsing the data than this.
data_dict = {}
for current_offset in range(0, len(offsets) - 1):
# Get the current offset and the offset for the next entry.
start, end = offsets[current_offset:current_offset + 2]
value = offset_dict.get(offsets[current_offset])
# Read the data.
data_all = page_data[start + page_offset:end + page_offset]
data, _, _ = data_all.partition(b'\x00')
data_dict[value] = data
url = data_dict.get(u'url')
cookie_name = data_dict.get(u'name')
cookie_value = data_dict.get(u'value')
path = data_dict.get(u'path')
flags = []
flag_int = cookie.flags
if flag_int & self.COOKIE_FLAG_HTTP_ONLY:
flags.append(u'HttpOnly')
if flag_int & self.COOKIE_FLAG_UNKNOWN:
flags.append(u'Unknown')
if flag_int & self.COOKIE_FLAG_SECURE:
flags.append(u'Secure')
cookie_flags = u'|'.join(flags)
if cookie.creation_date:
event_object = BinaryCookieEvent(
cookie.creation_date,
eventdata.EventTimestamp.CREATION_TIME, cookie_flags, url,
cookie_value, cookie_name, path)
parser_mediator.ProduceEvent(event_object)
if cookie.expiration_date:
event_object = BinaryCookieEvent(
cookie.expiration_date,
eventdata.EventTimestamp.EXPIRATION_TIME, cookie_flags,
url, cookie_value, cookie_name, path)
parser_mediator.ProduceEvent(event_object)
for cookie_plugin in self._cookie_plugins:
try:
cookie_plugin.UpdateChainAndProcess(
parser_mediator,
cookie_name=data_dict.get(u'name'),
cookie_data=data_dict.get(u'value'),
url=data_dict.get(u'url'))
except errors.WrongPlugin:
pass
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses a Safari binary cookie file-like object.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
file_object: A file-like object.
Raises:
UnableToParseFile: when the file cannot be parsed.
"""
# Start by verifying magic value.
# We do this here instead of in the header parsing routine due to the
# fact that we read an integer there and create an array, which is part
# of the header. For false hits this could end up with reading large chunks
# of data, which we don't want for false hits.
magic = file_object.read(4)
if magic != b'cook':
raise errors.UnableToParseFile(
u'The file is not a Binary Cookie file. Unsupported file signature.'
)
try:
header = self.COOKIE_HEADER.parse_stream(file_object)
except (IOError, construct.ArrayError, construct.FieldError):
raise errors.UnableToParseFile(
u'The file is not a Binary Cookie file (bad header).')
for page_size in header.page_sizes:
page = file_object.read(page_size)
if len(page) != page_size:
parser_mediator.ProduceParseError(
u'Unable to continue parsing Binary Cookie file')
break
self._ParsePage(page, parser_mediator)
class PlsRecallParser(interface.SingleFileBaseParser):
"""Parse PL/SQL Recall files.
Parser is based on a::
TRecallRecord = packed record
Sequence: Integer;
TimeStamp: TDateTime;
Username: array[0..30] of Char;
Database: array[0..80] of Char;
Text: array[0..4000] of Char;
end;
Delphi TDateTime is a little endian 64-bit
floating point without any time zone information
"""
_INITIAL_FILE_OFFSET = None
_PLS_KEYWORD = frozenset([
u'begin', u'commit', u'create', u'declare', u'drop', u'end', u'exception',
u'execute', u'insert', u'replace', u'rollback', u'select', u'set',
u'update'])
# 6 * 365 * 24 * 60 * 60 * 1000000.
_SIX_YEARS_IN_MICRO_SECONDS = 189216000000000
NAME = u'pls_recall'
DESCRIPTION = u'Parser for PL/SQL Recall files.'
PLS_STRUCT = construct.Struct(
u'PL/SQL_Recall',
construct.ULInt32(u'Sequence'),
construct.LFloat64(u'TimeStamp'),
construct.String(u'Username', 31, None, b'\x00'),
construct.String(u'Database', 81, None, b'\x00'),
construct.String(u'Query', 4001, None, b'\x00'))
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses a PLSRecall.dat file-like object.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
file_object: A file-like object.
Raises:
UnableToParseFile: when the file cannot be parsed.
"""
try:
is_pls = self.VerifyFile(file_object)
except (IOError, construct.FieldError) as exception:
raise errors.UnableToParseFile((
u'Not a PLSrecall File, unable to parse.'
u'with error: {0:s}').format(exception))
if not is_pls:
raise errors.UnableToParseFile(
u'Not a PLSRecall File, unable to parse.')
file_object.seek(0, os.SEEK_SET)
pls_record = self.PLS_STRUCT.parse_stream(file_object)
while pls_record:
event_object = PlsRecallEvent(
timelib.Timestamp.FromDelphiTime(pls_record.TimeStamp),
pls_record.Sequence, pls_record.Username,
pls_record.Database, pls_record.Query)
parser_mediator.ProduceEvent(event_object)
try:
pls_record = self.PLS_STRUCT.parse_stream(file_object)
except construct.FieldError:
# The code has reached the end of file (EOF).
break
def VerifyFile(self, file_object):
"""Check if the file is a PLSRecall.dat file.
Args:
file_object: file that we want to check.
Returns:
True if this is a valid PLSRecall.dat file, otherwise False.
"""
file_object.seek(0, os.SEEK_SET)
# The file consists of PL/SQL structures that are equal
# size (4125 bytes) TRecallRecord records. It should be
# noted that the query value is free form.
try:
structure = self.PLS_STRUCT.parse_stream(file_object)
except (IOError, construct.FieldError):
return False
# Verify few entries inside the structure.
try:
timestamp = timelib.Timestamp.FromDelphiTime(structure.TimeStamp)
except ValueError:
return False
if timestamp <= 0:
return False
# Verify that the timestamp is no more than six years into the future.
# Six years is an arbitrary time length just to evaluate the timestamp
# against some value. There is no guarantee that this will catch everything.
# TODO: Add a check for similarly valid value back in time. Maybe if it the
# timestamp is before 1980 we are pretty sure it is invalid?
# TODO: This is a very flaky assumption. Find a better one.
current_timestamp = timelib.Timestamp.GetNow()
if timestamp > current_timestamp + self._SIX_YEARS_IN_MICRO_SECONDS:
return False
# TODO: Add other verification checks here. For instance make sure
# that the query actually looks like a SQL query. This structure produces a
# lot of false positives and thus we need to add additional verification to
# make sure we are not parsing non-PLSRecall files.
# Another check might be to make sure the username looks legitimate, or the
# sequence number, or the database name.
# For now we just check if all three fields pass our "is this a text" test.
if not utils.IsText(structure.Username):
return False
if not utils.IsText(structure.Query):
return False
if not utils.IsText(structure.Database):
return False
# Take the first word from the query field and attempt to match that against
# allowed queries.
first_word, _, _ = structure.Query.partition(b' ')
if first_word.lower() not in self._PLS_KEYWORD:
return False
return True
class BinaryCookieParser(interface.FileObjectParser):
"""Parser for Safari Binary Cookie files."""
NAME = 'binary_cookies'
DESCRIPTION = 'Parser for Safari Binary Cookie files.'
_FILE_HEADER = construct.Struct(
'file_header', construct.Bytes('signature', 4),
construct.UBInt32('number_of_pages'),
construct.Array(lambda ctx: ctx.number_of_pages,
construct.UBInt32('page_sizes')))
_COOKIE_RECORD = construct.Struct('cookie_record',
construct.ULInt32('size'),
construct.Bytes('unknown_1', 4),
construct.ULInt32('flags'),
construct.Bytes('unknown_2', 4),
construct.ULInt32('url_offset'),
construct.ULInt32('name_offset'),
construct.ULInt32('path_offset'),
construct.ULInt32('value_offset'),
construct.Bytes('end_of_cookie', 8),
construct.LFloat64('expiration_date'),
construct.LFloat64('creation_date'))
_PAGE_HEADER = construct.Struct(
'page_header', construct.Bytes('header', 4),
construct.ULInt32('number_of_records'),
construct.Array(lambda ctx: ctx.number_of_records,
construct.ULInt32('offsets')))
def __init__(self):
"""Initializes a parser object."""
super(BinaryCookieParser, self).__init__()
self._cookie_plugins = (
cookie_plugins_manager.CookiePluginsManager.GetPlugins())
def _ParseCookieRecord(self, parser_mediator, page_data, page_offset):
"""Parses a cookie record
Args:
parser_mediator (ParserMediator): parser mediator.
page_data (bytes): page data.
page_offset (int): offset of the cookie record relative to the start
of the page.
"""
try:
cookie = self._COOKIE_RECORD.parse(page_data[page_offset:])
except construct.FieldError:
message = 'Unable to read cookie record at offset: {0:d}'.format(
page_offset)
parser_mediator.ProduceExtractionError(message)
return
# The offset is determined by the range between the start of the current
# offset until the start of the next offset. Thus we need to determine
# the proper ordering of the offsets, since they are not always in the
# same ordering.
offset_dict = {
cookie.url_offset: 'url',
cookie.name_offset: 'name',
cookie.value_offset: 'value',
cookie.path_offset: 'path'
}
offsets = sorted(offset_dict.keys())
offsets.append(cookie.size + page_offset)
# TODO: Find a better approach to parsing the data than this.
data_dict = {}
for current_offset in range(0, len(offsets) - 1):
# Get the current offset and the offset for the next entry.
start, end = offsets[current_offset:current_offset + 2]
value = offset_dict.get(offsets[current_offset])
# Read the data.
data_all = page_data[start + page_offset:end + page_offset]
data, _, _ = data_all.partition(b'\x00')
data_dict[value] = data
event_data = SafariBinaryCookieEventData()
event_data.cookie_name = data_dict.get('name')
event_data.cookie_value = data_dict.get('value')
event_data.flags = cookie.flags
event_data.path = data_dict.get('path')
event_data.url = data_dict.get('url')
if cookie.creation_date:
date_time = dfdatetime_cocoa_time.CocoaTime(
timestamp=cookie.creation_date)
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_CREATION)
parser_mediator.ProduceEventWithEventData(event, event_data)
if cookie.expiration_date:
date_time = dfdatetime_cocoa_time.CocoaTime(
timestamp=cookie.expiration_date)
else:
date_time = dfdatetime_semantic_time.SemanticTime('Not set')
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_EXPIRATION)
parser_mediator.ProduceEventWithEventData(event, event_data)
for plugin in self._cookie_plugins:
if parser_mediator.abort:
break
if event_data.cookie_name != plugin.COOKIE_NAME:
continue
try:
plugin.UpdateChainAndProcess(
parser_mediator,
cookie_name=event_data.cookie_name,
cookie_data=event_data.cookie_value,
url=event_data.url)
except Exception as exception: # pylint: disable=broad-except
parser_mediator.ProduceExtractionError(
'plugin: {0:s} unable to parse cookie with error: {1!s}'.
format(plugin.NAME, exception))
def _ParsePage(self, parser_mediator, page_number, page_data):
"""Parses a page.
Args:
parser_mediator (ParserMediator): parser mediator.
page_number (int): page number.
page_data (bytes): page data.
"""
try:
page_header = self._PAGE_HEADER.parse(page_data)
except construct.FieldError:
# TODO: add offset
parser_mediator.ProduceExtractionError(
'unable to read header of page: {0:d} at offset: 0x{1:08x}'.
format(page_number, 0))
return
for page_offset in page_header.offsets:
if parser_mediator.abort:
break
self._ParseCookieRecord(parser_mediator, page_data, page_offset)
# TODO: check footer.
@classmethod
def GetFormatSpecification(cls):
"""Retrieves the format specification for parser selection.
Returns:
FormatSpecification: format specification.
"""
format_specification = specification.FormatSpecification(cls.NAME)
format_specification.AddNewSignature(b'cook\x00', offset=0)
return format_specification
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses a Safari binary cookie file-like object.
Args:
parser_mediator (ParserMediator): parser mediator.
file_object (dfvfs.FileIO): file-like object to be parsed.
Raises:
UnableToParseFile: when the file cannot be parsed, this will signal
the event extractor to apply other parsers.
"""
try:
file_header = self._FILE_HEADER.parse_stream(file_object)
except (IOError, construct.ArrayError,
construct.FieldError) as exception:
parser_mediator.ProduceExtractionError(
'unable to read file header with error: {0!s}.'.format(
exception))
raise errors.UnableToParseFile()
if file_header.signature != b'cook':
parser_mediator.ProduceExtractionError(
'unsupported file signature.')
raise errors.UnableToParseFile()
for index, page_size in enumerate(file_header.page_sizes):
if parser_mediator.abort:
break
page_data = file_object.read(page_size)
if len(page_data) != page_size:
parser_mediator.ProduceExtractionError(
'unable to read page: {0:d}'.format(index))
break
self._ParsePage(parser_mediator, index, page_data)
class SystemResourceUsageMonitorESEDBPlugin(interface.ESEDBPlugin):
"""Parses a System Resource Usage Monitor (SRUM) ESE database file."""
NAME = 'srum'
DESCRIPTION = (
'Parser for System Resource Usage Monitor (SRUM) ESE database files.')
# TODO: add support for tables:
# {5C8CF1C7-7257-4F13-B223-970EF5939312}
# {97C2CE28-A37B-4920-B1E9-8B76CD341EC5}
# {B6D82AF1-F780-4E17-8077-6CB9AD8A6FC4}
# {D10CA2FE-6FCF-4F6D-848E-B2E99266FA86}
# {DA73FB89-2BEA-4DDC-86B8-6E048C6DA477}
# {FEE4E14F-02A9-4550-B5CE-5FA2DA202E37}
# TODO: convert interface_luid into string representation
# TODO: convert l2_profile_flags into string representation in formatter
OPTIONAL_TABLES = {
'{973F5D5C-1D90-4944-BE8E-24B94231A174}': 'ParseNetworkDataUsage',
'{D10CA2FE-6FCF-4F6D-848E-B2E99266FA89}':
'ParseApplicationResourceUsage',
'{DD6636C4-8929-4683-974E-22C046A43763}':
'ParseNetworkConnectivityUsage'
}
REQUIRED_TABLES = {'SruDbIdMapTable': ''}
_GUID_TABLE_VALUE_MAPPINGS = {
'TimeStamp': '_ConvertValueBinaryDataToFloatingPointValue'
}
_FLOAT32_LITTLE_ENDIAN = construct.LFloat32('float32')
_FLOAT64_LITTLE_ENDIAN = construct.LFloat64('float64')
_APPLICATION_RESOURCE_USAGE_VALUES_MAP = {
'application': 'AppId',
'background_bytes_read': 'BackgroundBytesRead',
'background_bytes_written': 'BackgroundBytesWritten',
'background_context_switches': 'BackgroundContextSwitches',
'background_cycle_time': 'BackgroundCycleTime',
'background_number_for_flushes': 'BackgroundNumberOfFlushes',
'background_number_for_read_operations': 'BackgroundNumReadOperations',
'background_number_for_write_operations':
'BackgroundNumWriteOperations',
'face_time': 'FaceTime',
'foreground_bytes_read': 'ForegroundBytesRead',
'foreground_bytes_written': 'ForegroundBytesWritten',
'foreground_context_switches': 'ForegroundContextSwitches',
'foreground_cycle_time': 'ForegroundCycleTime',
'foreground_number_for_flushes': 'ForegroundNumberOfFlushes',
'foreground_number_for_read_operations': 'ForegroundNumReadOperations',
'foreground_number_for_write_operations':
'ForegroundNumWriteOperations',
'identifier': 'AutoIncId',
'user_identifier': 'UserId'
}
_NETWORK_CONNECTIVITY_USAGE_VALUES_MAP = {
'application': 'AppId',
'connected_time': 'ConnectedTime',
'identifier': 'AutoIncId',
'interface_luid': 'InterfaceLuid',
'l2_profile_flags': 'L2ProfileFlags',
'l2_profile_identifier': 'L2ProfileId',
'user_identifier': 'UserId'
}
_NETWORK_DATA_USAGE_VALUES_MAP = {
'application': 'AppId',
'bytes_recieved': 'BytesRecvd',
'bytes_sent': 'BytesSent',
'identifier': 'AutoIncId',
'interface_luid': 'InterfaceLuid',
'l2_profile_flags': 'L2ProfileFlags',
'l2_profile_identifier': 'L2ProfileId',
'user_identifier': 'UserId'
}
_SUPPORTED_IDENTIFIER_TYPES = (0, 1, 2, 3)
def _ConvertValueBinaryDataToFloatingPointValue(self, value):
"""Converts a binary data value into a floating-point value.
Args:
value (bytes): binary data value containing an ASCII string or None.
Returns:
float: floating-point representation of binary data value or None.
"""
if value:
value_length = len(value)
if value_length == 4:
return self._FLOAT32_LITTLE_ENDIAN.parse(value)
elif value_length == 8:
return self._FLOAT64_LITTLE_ENDIAN.parse(value)
return None
def _GetIdentifierMappings(self, parser_mediator, cache, database):
"""Retrieves the identifier mappings from SruDbIdMapTable table.
In the SRUM database individual tables contain numeric identifiers for
the application ("AppId") and user identifier ("UserId"). A more descriptive
string of these values can be found in the SruDbIdMapTable. For example the
numeric value of 42 mapping to DiagTrack. This method will cache the
mappings of a specific SRUM database.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
cache (ESEDBCache): cache, which contains information about
the identifiers stored in the SruDbIdMapTable table.
database (pyesedb.file): ESE database.
Returns:
dict[int, str]: mapping of numeric identifiers to their string
representation.
"""
identifier_mappings = cache.GetResults('SruDbIdMapTable',
default_value={})
if not identifier_mappings:
esedb_table = database.get_table_by_name('SruDbIdMapTable')
if not esedb_table:
parser_mediator.ProduceExtractionError(
'unable to retrieve table: SruDbIdMapTable')
else:
identifier_mappings = self._ParseIdentifierMappingsTable(
parser_mediator, esedb_table)
cache.StoreDictInCache('SruDbIdMapTable', identifier_mappings)
return identifier_mappings
def _ParseGUIDTable(self, parser_mediator, cache, database, esedb_table,
values_map, event_data_class):
"""Parses a table with a GUID as name.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
cache (ESEDBCache): cache, which contains information about
the identifiers stored in the SruDbIdMapTable table.
database (pyesedb.file): ESE database.
esedb_table (pyesedb.table): table.
values_map (dict[str, str]): mapping of table columns to event data
attribute names.
event_data_class (type): event data class.
Raises:
ValueError: if the cache, database or table value is missing.
"""
if cache is None:
raise ValueError('Missing cache value.')
if database is None:
raise ValueError('Missing database value.')
if esedb_table is None:
raise ValueError('Missing table value.')
identifier_mappings = self._GetIdentifierMappings(
parser_mediator, cache, database)
for esedb_record in esedb_table.records:
if parser_mediator.abort:
break
record_values = self._GetRecordValues(
parser_mediator,
esedb_table.name,
esedb_record,
value_mappings=self._GUID_TABLE_VALUE_MAPPINGS)
event_data = event_data_class()
for attribute_name, column_name in values_map.items():
record_value = record_values.get(column_name, None)
if attribute_name in ('application', 'user_identifier'):
# Human readable versions of AppId and UserId values are stored
# in the SruDbIdMapTable table; also referred to as identifier
# mapping. Here we look up the numeric identifier stored in the GUID
# table in SruDbIdMapTable.
record_value = identifier_mappings.get(
record_value, record_value)
setattr(event_data, attribute_name, record_value)
timestamp = record_values.get('TimeStamp')
if timestamp:
date_time = dfdatetime_ole_automation_date.OLEAutomationDate(
timestamp=timestamp)
timestamp_description = definitions.TIME_DESCRIPTION_SAMPLE
else:
date_time = dfdatetime_semantic_time.SemanticTime('Not set')
timestamp_description = definitions.TIME_DESCRIPTION_NOT_A_TIME
event = time_events.DateTimeValuesEvent(date_time,
timestamp_description)
parser_mediator.ProduceEventWithEventData(event, event_data)
timestamp = record_values.get('ConnectStartTime')
if timestamp:
date_time = dfdatetime_filetime.Filetime(timestamp=timestamp)
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_FIRST_CONNECTED)
parser_mediator.ProduceEventWithEventData(event, event_data)
def _ParseIdentifierMappingRecord(self, parser_mediator, table_name,
esedb_record):
"""Extracts an identifier mapping from a SruDbIdMapTable record.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
table_name (str): name of the table the record is stored in.
esedb_record (pyesedb.record): record.
Returns:
tuple[int, str]: numeric identifier and its string representation or
None, None if no identifier mapping can be retrieved from the record.
"""
record_values = self._GetRecordValues(parser_mediator, table_name,
esedb_record)
identifier = record_values.get('IdIndex', None)
if identifier is None:
parser_mediator.ProduceExtractionError(
'IdIndex value missing from table: SruDbIdMapTable')
return None, None
identifier_type = record_values.get('IdType', None)
if identifier_type not in self._SUPPORTED_IDENTIFIER_TYPES:
parser_mediator.ProduceExtractionError(
'unsupported IdType value: {0!s} in table: SruDbIdMapTable'.
format(identifier_type))
return None, None
mapped_value = record_values.get('IdBlob', None)
if mapped_value is None:
parser_mediator.ProduceExtractionError(
'IdBlob value missing from table: SruDbIdMapTable')
return None, None
if identifier_type == 3:
try:
fwnt_identifier = pyfwnt.security_identifier()
fwnt_identifier.copy_from_byte_stream(mapped_value)
mapped_value = fwnt_identifier.get_string()
except IOError:
parser_mediator.ProduceExtractionError(
'unable to decode IdBlob value as Windows NT security identifier'
)
return None, None
else:
try:
mapped_value = mapped_value.decode('utf-16le').rstrip('\0')
except UnicodeDecodeError:
parser_mediator.ProduceExtractionError(
'unable to decode IdBlob value as UTF-16 little-endian string'
)
return None, None
return identifier, mapped_value
def _ParseIdentifierMappingsTable(self, parser_mediator, esedb_table):
"""Extracts identifier mappings from the SruDbIdMapTable table.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
esedb_table (pyesedb.table): table.
Returns:
dict[int, str]: mapping of numeric identifiers to their string
representation.
"""
identifier_mappings = {}
for esedb_record in esedb_table.records:
if parser_mediator.abort:
break
identifier, mapped_value = self._ParseIdentifierMappingRecord(
parser_mediator, esedb_table.name, esedb_record)
if identifier is None or mapped_value is None:
continue
if identifier in identifier_mappings:
parser_mediator.ProduceExtractionError(
'identifier: {0:d} already exists in mappings.'.format(
identifier))
continue
identifier_mappings[identifier] = mapped_value
return identifier_mappings
def ParseApplicationResourceUsage(self,
parser_mediator,
cache=None,
database=None,
table=None,
**unused_kwargs):
"""Parses the application resource usage table.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
cache (Optional[ESEDBCache]): cache, which contains information about
the identifiers stored in the SruDbIdMapTable table.
database (Optional[pyesedb.file]): ESE database.
table (Optional[pyesedb.table]): table.
"""
self._ParseGUIDTable(parser_mediator, cache, database, table,
self._APPLICATION_RESOURCE_USAGE_VALUES_MAP,
SRUMApplicationResourceUsageEventData)
def ParseNetworkDataUsage(self,
parser_mediator,
cache=None,
database=None,
table=None,
**unused_kwargs):
"""Parses the network data usage monitor table.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
cache (Optional[ESEDBCache]): cache, which contains information about
the identifiers stored in the SruDbIdMapTable table.
database (Optional[pyesedb.file]): ESE database.
table (Optional[pyesedb.table]): table.
"""
self._ParseGUIDTable(parser_mediator, cache, database, table,
self._NETWORK_DATA_USAGE_VALUES_MAP,
SRUMNetworkDataUsageEventData)
def ParseNetworkConnectivityUsage(self,
parser_mediator,
cache=None,
database=None,
table=None,
**unused_kwargs):
"""Parses the network connectivity usage monitor table.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
cache (Optional[ESEDBCache]): cache, which contains information about
the identifiers stored in the SruDbIdMapTable table.
database (Optional[pyesedb.file]): ESE database.
table (Optional[pyesedb.table]): table.
"""
# TODO: consider making ConnectStartTime + ConnectedTime an event.
self._ParseGUIDTable(parser_mediator, cache, database, table,
self._NETWORK_CONNECTIVITY_USAGE_VALUES_MAP,
SRUMNetworkConnectivityUsageEventData)
class PlsRecallParser(interface.BaseParser):
"""Parse PL-SQL Recall files.
Parser is based on a:
TRecallRecord = packed record
Sequence: Integer;
TimeStamp: TDateTime;
Username: array[0..30] of Char;
Database: array[0..80] of Char;
Text: array[0..4000] of Char;
end;
Delphi TDateTime is a little endian 64-bit
floating point without any time zone information
"""
NAME = 'pls_recall'
DESCRIPTION = u'Parser for PL-SQL Recall files.'
PLS_STRUCT = construct.Struct(
'PL-SQL_Recall', construct.ULInt32('Sequence'),
construct.LFloat64('TimeStamp'),
construct.String('Username', 31, None, '\x00'),
construct.String('Database', 81, None, '\x00'),
construct.String('Query', 4001, None, '\x00'))
def Parse(self, parser_context, file_entry):
"""Extract entries from a PLSRecall.dat file.
Args:
parser_context: A parser context object (instance of ParserContext).
file_entry: A file entry object (instance of dfvfs.FileEntry).
"""
file_object = file_entry.GetFileObject()
try:
is_pls = self.VerifyFile(file_object)
except (IOError, construct.FieldError) as exception:
file_object.close()
raise errors.UnableToParseFile(
(u'Not a PLSrecall File, unable to parse.'
u'with error: {0:s}').format(exception))
if not is_pls:
file_object.close()
raise errors.UnableToParseFile(
u'Not a PLSRecall File, unable to parse.')
file_object.seek(0, os.SEEK_SET)
pls_record = self.PLS_STRUCT.parse_stream(file_object)
while pls_record:
event_object = PlsRecallEvent(
timelib.Timestamp.FromDelphiTime(pls_record.TimeStamp),
pls_record.Sequence, pls_record.Username, pls_record.Database,
pls_record.Query)
parser_context.ProduceEvent(event_object,
parser_name=self.NAME,
file_entry=file_entry)
try:
pls_record = self.PLS_STRUCT.parse_stream(file_object)
except construct.FieldError as exception:
# The code has reached the end of file (EOF).
break
file_object.close()
def VerifyFile(self, file_object):
"""Check if the file is a PLSRecall.dat file.
Args:
file_object: file that we want to check.
Returns:
True if this is a valid PLSRecall.dat file, otherwise False.
"""
file_object.seek(0, os.SEEK_SET)
# The file consists of PL-SQL structures that are equal
# size (4125 bytes) TRecallRecord records. It should be
# noted that the query value is free form.
try:
structure = self.PLS_STRUCT.parse_stream(file_object)
except (IOError, construct.FieldError):
return False
# Verify few entries inside the structure.
try:
timestamp = timelib.Timestamp.FromDelphiTime(structure.TimeStamp)
except ValueError:
return False
if timestamp <= 0:
return False
# TODO: Add other verification checks here. For instance make sure
# that the query actually looks like a SQL query. This structure produces a
# lot of false positives and thus we need to add additional verification to
# make sure we are not parsing non-PLSRecall files.
# Another check might be to make sure the username looks legitimate, or the
# sequence number, or the database name.
# For now we just check if all three fields pass our "is this a text" test.
if not utils.IsText(structure.Username):
return False
if not utils.IsText(structure.Query):
return False
if not utils.IsText(structure.Database):
return False
return True
class BinaryCookiesFile(object):
"""Class that contains a Cookies.binarycookies file."""
_FILE_HEADER = construct.Struct(u'file_header',
construct.Bytes(u'signature', 4),
construct.UBInt32(u'number_of_pages'))
_PAGE_HEADER = construct.Struct(
u'page_header', construct.ULInt32(u'signature'),
construct.ULInt32(u'number_of_records'),
construct.Array(lambda ctx: ctx.number_of_records,
construct.ULInt32(u'offsets')))
_RECORD_HEADER = construct.Struct(u'record_header',
construct.ULInt32(u'size'),
construct.ULInt32(u'unknown1'),
construct.ULInt32(u'flags'),
construct.ULInt32(u'unknown2'),
construct.ULInt32(u'url_offset'),
construct.ULInt32(u'name_offset'),
construct.ULInt32(u'path_offset'),
construct.ULInt32(u'value_offset'),
construct.ULInt64(u'unknown3'),
construct.LFloat64(u'expiration_time'),
construct.LFloat64(u'creation_time'))
_FILE_FOOTER = construct.Struct(u'file_footer',
construct.Bytes(u'unknown1', 8))
def __init__(self, debug=False):
"""Initializes a file.
Args:
debug (Optional[bool]): True if debug information should be printed.
"""
super(BinaryCookiesFile, self).__init__()
self._debug = debug
self._file_object = None
self._file_object_opened_in_object = False
self._file_size = 0
self._page_sizes = []
def _ReadFileFooter(self):
"""Reads the file footer.
Raises:
IOError: if the file footer cannot be read.
"""
file_footer_data = self._file_object.read(self._FILE_FOOTER.sizeof())
if self._debug:
print(u'File footer data:')
print(hexdump.Hexdump(file_footer_data))
def _ReadFileHeader(self):
"""Reads the file header.
Raises:
IOError: if the file header cannot be read.
"""
if self._debug:
print(u'Seeking file header offset: 0x{0:08x}'.format(0))
self._file_object.seek(0, os.SEEK_SET)
file_header_data = self._file_object.read(self._FILE_HEADER.sizeof())
if self._debug:
print(u'File header data:')
print(hexdump.Hexdump(file_header_data))
try:
file_header_struct = self._FILE_HEADER.parse(file_header_data)
except construct.FieldError as exception:
raise IOError(
u'Unable to parse file header with error: {0:s}'.format(
exception))
if self._debug:
print(u'Signature\t\t\t\t\t\t\t: {0!s}'.format(
file_header_struct.signature))
print(u'Number of pages\t\t\t\t\t\t\t: {0:d}'.format(
file_header_struct.number_of_pages))
print(u'')
page_sizes_data_size = file_header_struct.number_of_pages * 4
page_sizes_data = self._file_object.read(page_sizes_data_size)
if self._debug:
print(u'Page sizes data:')
print(hexdump.Hexdump(page_sizes_data))
try:
page_sizes_array = construct.Array(
file_header_struct.number_of_pages,
construct.UBInt32(u'page_sizes')).parse(page_sizes_data)
except construct.FieldError as exception:
raise IOError(
u'Unable to parse page sizes array with error: {0:s}'.format(
exception))
self._page_sizes = []
for page_index in range(file_header_struct.number_of_pages):
self._page_sizes.append(page_sizes_array[page_index])
if self._debug:
print(u'Page: {0:d} size\t\t\t\t\t\t\t: {1:d}'.format(
page_index, page_sizes_array[page_index]))
if self._debug:
print(u'')
def _ReadPages(self):
"""Reads the pages."""
for page_size in iter(self._page_sizes):
self._ReadPage(page_size)
def _ReadPage(self, page_size):
"""Reads the page.
Args:
page_size (int): page size.
"""
page_data = self._file_object.read(page_size)
try:
page_header_struct = self._PAGE_HEADER.parse(page_data)
except construct.FieldError as exception:
raise IOError(
u'Unable to parse file header with error: {0:s}'.format(
exception))
page_header_data_size = 8 + (4 * page_header_struct.number_of_records)
if self._debug:
print(u'Page header data:')
print(hexdump.Hexdump(page_data[:page_header_data_size]))
if self._debug:
print(u'Signature\t\t\t\t\t\t\t: 0x{0:08x}'.format(
page_header_struct.signature))
print(u'Number of records\t\t\t\t\t\t: {0:d}'.format(
page_header_struct.number_of_records))
record_offsets = []
for record_index in range(page_header_struct.number_of_records):
record_offsets.append(page_header_struct.offsets[record_index])
if self._debug:
print(u'Record: {0:d} offset\t\t\t\t\t\t: {1:d}'.format(
record_index, page_header_struct.offsets[record_index]))
if self._debug:
print(u'')
for record_offset in iter(record_offsets):
self._ParseRecord(page_data, record_offset)
def _ParseRecord(self, page_data, record_offset):
"""Reads a record from the page data.
Args:
page_data (bytes): page data.
record_offset (int): record offset.
"""
try:
record_header_struct = self._RECORD_HEADER.parse(
page_data[record_offset:])
except construct.FieldError as exception:
raise IOError(
u'Unable to parse record header with error: {0:s}'.format(
exception))
record_data_size = record_offset + record_header_struct.size
if self._debug:
print(u'Record data:')
print(hexdump.Hexdump(page_data[record_offset:record_data_size]))
if self._debug:
print(u'Size\t\t\t\t\t\t\t\t: {0:d}'.format(
record_header_struct.size))
print(u'Unknown1\t\t\t\t\t\t\t: 0x{0:08x}'.format(
record_header_struct.unknown1))
print(u'Flags\t\t\t\t\t\t\t\t: 0x{0:08x}'.format(
record_header_struct.flags))
print(u'Unknown2\t\t\t\t\t\t\t: 0x{0:08x}'.format(
record_header_struct.unknown2))
print(u'URL offset\t\t\t\t\t\t\t: {0:d}'.format(
record_header_struct.url_offset))
print(u'name offset\t\t\t\t\t\t\t: {0:d}'.format(
record_header_struct.name_offset))
print(u'path offset\t\t\t\t\t\t\t: {0:d}'.format(
record_header_struct.path_offset))
print(u'value offset\t\t\t\t\t\t\t: {0:d}'.format(
record_header_struct.value_offset))
print(u'Unknown3\t\t\t\t\t\t\t: 0x{0:08x}'.format(
record_header_struct.unknown3))
date_time = (datetime.datetime(2001, 1, 1) + datetime.timedelta(
seconds=int(record_header_struct.expiration_time)))
print(u'expiration time\t\t\t\t\t\t\t: {0!s} ({1:f})'.format(
date_time, record_header_struct.expiration_time))
date_time = (datetime.datetime(2001, 1, 1) + datetime.timedelta(
seconds=int(record_header_struct.creation_time)))
print(u'creation time\t\t\t\t\t\t\t: {0!s} ({1:f})'.format(
date_time, record_header_struct.creation_time))
print(u'')
if record_header_struct.url_offset:
data_offset = record_offset + record_header_struct.url_offset
string = construct.CString(u'string').parse(
page_data[data_offset:record_data_size])
else:
sting = u''
print(u'URL\t\t\t\t\t\t\t\t: {0:s}'.format(string))
if record_header_struct.name_offset:
data_offset = record_offset + record_header_struct.name_offset
string = construct.CString(u'string').parse(
page_data[data_offset:record_data_size])
else:
sting = u''
print(u'Name\t\t\t\t\t\t\t\t: {0:s}'.format(string))
if record_header_struct.path_offset:
data_offset = record_offset + record_header_struct.path_offset
string = construct.CString(u'string').parse(
page_data[data_offset:record_data_size])
else:
sting = u''
print(u'Path\t\t\t\t\t\t\t\t: {0:s}'.format(string))
if record_header_struct.value_offset:
data_offset = record_offset + record_header_struct.value_offset
string = construct.CString(u'string').parse(
page_data[data_offset:record_data_size])
else:
sting = u''
print(u'Value\t\t\t\t\t\t\t\t: {0:s}'.format(string))
print(u'')
def Close(self):
"""Closes a file."""
if self._file_object_opened_in_object:
self._file_object.close()
self._file_object = None
def Open(self, filename):
"""Opens a file.
Args:
filename (str): filename.
"""
stat_object = os.stat(filename)
self._file_size = stat_object.st_size
self._file_object = open(filename, 'rb')
self._file_object_opened_in_object = True
self._ReadFileHeader()
self._ReadPages()
self._ReadFileFooter()
class PlsRecallParser(interface.FileObjectParser):
"""Parse PL/SQL Recall files.
Parser is based on:
TRecallRecord = packed record
Sequence: Integer;
TimeStamp: TDateTime;
Username: array[0..30] of Char;
Database: array[0..80] of Char;
Text: array[0..4000] of Char;
end;
Delphi TDateTime is a little-endian 64-bit floating-point value without
time zone information.
"""
_INITIAL_FILE_OFFSET = None
_PLS_KEYWORD = frozenset([
'begin', 'commit', 'create', 'declare', 'drop', 'end', 'exception',
'execute', 'insert', 'replace', 'rollback', 'select', 'set', 'update'
])
# 6 * 365 * 24 * 60 * 60 * 1000000.
_SIX_YEARS_IN_MICRO_SECONDS = 189216000000000
NAME = 'pls_recall'
DESCRIPTION = 'Parser for PL/SQL Recall files.'
_PLS_RECALL_RECORD = construct.Struct(
'PL/SQL_Recall', construct.ULInt32('Sequence'),
construct.LFloat64('TimeStamp'),
construct.String('Username', 31, None, b'\x00'),
construct.String('Database', 81, None, b'\x00'),
construct.String('Query', 4001, None, b'\x00'))
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses a PLSRecall.dat file-like object.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): a file-like object.
Raises:
UnableToParseFile: when the file cannot be parsed.
"""
try:
is_pls = self.VerifyFile(file_object)
except (IOError, construct.FieldError) as exception:
raise errors.UnableToParseFile(
('Not a PLSrecall File, unable to parse.'
'with error: {0!s}').format(exception))
if not is_pls:
raise errors.UnableToParseFile(
'Not a PLSRecall File, unable to parse.')
file_object.seek(0, os.SEEK_SET)
pls_record = self._PLS_RECALL_RECORD.parse_stream(file_object)
while pls_record:
event_data = PlsRecallEventData()
event_data.database_name = pls_record.Database
event_data.sequence_number = pls_record.Sequence
event_data.query = pls_record.Query
event_data.username = pls_record.Username
date_time = dfdatetime_delphi_date_time.DelphiDateTime(
timestamp=pls_record.TimeStamp)
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_WRITTEN)
parser_mediator.ProduceEventWithEventData(event, event_data)
try:
pls_record = self._PLS_RECALL_RECORD.parse_stream(file_object)
except construct.FieldError:
# The code has reached the end of file (EOF).
break
def VerifyFile(self, file_object):
"""Check if the file is a PLSRecall.dat file.
Args:
file_object (dfvfs.FileIO): a file-like object.
Returns:
bool: True if this is a valid PLSRecall.dat file, False otherwise.
"""
file_object.seek(0, os.SEEK_SET)
# The file consists of PL/SQL structures that are equal
# size (4125 bytes) TRecallRecord records. It should be
# noted that the query value is free form.
try:
structure = self._PLS_RECALL_RECORD.parse_stream(file_object)
except (IOError, construct.FieldError):
return False
# Verify that the timestamp is no more than six years into the future.
# Six years is an arbitrary time length just to evaluate the timestamp
# against some value. There is no guarantee that this will catch everything.
# TODO: Add a check for similarly valid value back in time. Maybe if it the
# timestamp is before 1980 we are pretty sure it is invalid?
# TODO: This is a very flaky assumption. Find a better one.
future_timestamp = (timelib.Timestamp.GetNow() +
self._SIX_YEARS_IN_MICRO_SECONDS)
if structure.TimeStamp > future_timestamp:
return False
# TODO: Add other verification checks here. For instance make sure
# that the query actually looks like a SQL query. This structure produces a
# lot of false positives and thus we need to add additional verification to
# make sure we are not parsing non-PLSRecall files.
# Another check might be to make sure the username looks legitimate, or the
# sequence number, or the database name.
# For now we just check if all three fields pass our "is this a text" test.
if not utils.IsText(structure.Username):
return False
if not utils.IsText(structure.Query):
return False
if not utils.IsText(structure.Database):
return False
# Take the first word from the query field and attempt to match that against
# allowed queries.
first_word, _, _ = structure.Query.partition(b' ')
if first_word.lower() not in self._PLS_KEYWORD:
return False
return True