def ParseRow(self, parser_mediator, row_offset, row):
"""Parses a line of the log file and produces events.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
row_offset (int): line number of the row.
row (dict[str, str]): fields of a single row, as specified in COLUMNS.
"""
try:
date_time = self._CreateDateTime(row['date'], row['time'])
except errors.TimestampError as exception:
parser_mediator.ProduceExtractionWarning(
'Unable to create date time with error: {0!s}'.format(exception))
date_time = dfdatetime_semantic_time.InvalidTime()
event_data = McafeeAVEventData()
event_data.action = row['action']
event_data.filename = row['filename']
event_data.offset = row_offset
event_data.rule = row['rule']
event_data.status = row['status']
event_data.trigger_location = row['trigger_location']
event_data.username = row['username']
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_WRITTEN)
parser_mediator.ProduceEventWithEventData(event, event_data)
def _ParseLayerConfigJSON(self, parser_mediator, file_object):
"""Extracts events from a Docker filesystem layer configuration file.
The path of each filesystem layer config file is:
DOCKER_DIR/graph/<layer_id>/json
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): a file-like object.
Raises:
UnableToParseFile: when the file is not a valid layer config file.
"""
file_content = file_object.read()
file_content = codecs.decode(file_content, self._ENCODING)
json_dict = json.loads(file_content)
if 'docker_version' not in json_dict:
raise errors.UnableToParseFile(
'not a valid Docker layer configuration file, missing '
'\'docker_version\' key.')
time_string = json_dict.get('created', None)
if time_string is not None:
layer_creation_command_array = [
x.strip() for x in json_dict['container_config']['Cmd']
]
layer_creation_command = ' '.join(
layer_creation_command_array).replace('\t', '')
event_data = DockerJSONLayerEventData()
event_data.command = layer_creation_command
event_data.layer_id = self._GetIdentifierFromPath(parser_mediator)
try:
date_time = dfdatetime_time_elements.TimeElementsInMicroseconds(
)
date_time.CopyFromStringISO8601(time_string)
except ValueError as exception:
parser_mediator.ProduceExtractionWarning(
('Unable to parse created time string: {0:s} with error: '
'{1!s}').format(time_string, exception))
date_time = dfdatetime_semantic_time.InvalidTime()
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_ADDED)
parser_mediator.ProduceEventWithEventData(event, event_data)
def _ParseContainerLogJSON(self, parser_mediator, file_object):
"""Extract events from a Docker container log files.
The format is one JSON formatted log message per line.
The path of each container log file (which logs the container stdout and
stderr) is:
DOCKER_DIR/containers/<container_id>/<container_id>-json.log
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): a file-like object.
"""
container_id = self._GetIdentifierFromPath(parser_mediator)
text_file_object = text_file.TextFile(file_object)
for log_line in text_file_object:
json_log_line = json.loads(log_line)
time_string = json_log_line.get('time', None)
if time_string is None:
continue
event_data = DockerJSONContainerLogEventData()
event_data.container_id = container_id
event_data.log_line = json_log_line.get('log', None)
event_data.log_source = json_log_line.get('stream', None)
# TODO: pass line number to offset or remove.
event_data.offset = 0
try:
date_time = dfdatetime_time_elements.TimeElementsInMicroseconds(
)
date_time.CopyFromStringISO8601(time_string)
except ValueError as exception:
parser_mediator.ProduceExtractionWarning(
('Unable to parse written time string: {0:s} with error: '
'{1!s}').format(time_string, exception))
date_time = dfdatetime_semantic_time.InvalidTime()
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_WRITTEN)
parser_mediator.ProduceEventWithEventData(event, event_data)
def testFormatDateTime(self):
"""Tests the _FormatDateTime function with dynamic time."""
output_mediator = self._CreateOutputMediator(dynamic_time=True)
test_helper = formatting_helper.FieldFormattingHelper(output_mediator)
event, event_data, event_data_stream = (
containers_test_lib.CreateEventFromValues(self._TEST_EVENTS[0]))
date_time_string = test_helper._FormatDateTime(event, event_data,
event_data_stream)
self.assertEqual(date_time_string, '2012-06-27T18:17:01.000000+00:00')
output_mediator.SetTimeZone('Europe/Amsterdam')
date_time_string = test_helper._FormatDateTime(event, event_data,
event_data_stream)
self.assertEqual(date_time_string, '2012-06-27T20:17:01.000000+02:00')
output_mediator.SetTimeZone('UTC')
event, event_data, event_data_stream = (
containers_test_lib.CreateEventFromValues(self._TEST_EVENTS[1]))
event.date_time._time_zone_offset = 120
date_time_string = test_helper._FormatDateTime(event, event_data,
event_data_stream)
self.assertEqual(date_time_string, '2012-06-27T18:17:01.000000+00:00')
event.date_time = dfdatetime_semantic_time.InvalidTime()
date_time_string = test_helper._FormatDateTime(event, event_data,
event_data_stream)
self.assertEqual(date_time_string, 'Invalid')
# Test with event.is_local_time
event, event_data, event_data_stream = (
containers_test_lib.CreateEventFromValues(self._TEST_EVENTS[0]))
event.timestamp -= 120 * 60 * 1000000
event.date_time.is_local_time = True
date_time_string = test_helper._FormatDateTime(event, event_data,
event_data_stream)
self.assertEqual(date_time_string, '2012-06-27T16:17:01.000000+00:00')
def ParseFileObject(self, parser_mediator, file_object):
"""Parses a Java WebStart Cache IDX file-like object.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dvfvs.FileIO): a file-like object to parse.
Raises:
UnableToParseFile: when the file cannot be parsed.
"""
file_header_map = self._GetDataTypeMap('java_idx_file_header')
try:
file_header, file_offset = self._ReadStructureFromFileObject(
file_object, 0, file_header_map)
except (ValueError, errors.ParseError) as exception:
raise errors.UnableToParseFile(
'Unable to parse file header with error: {0!s}'.format(
exception))
if not file_header.format_version in self._SUPPORTED_FORMAT_VERSIONS:
raise errors.UnableToParseFile('Unsupported format version.')
if file_header.format_version == 602:
section1_map = self._GetDataTypeMap('java_idx_602_section1')
elif file_header.format_version in (603, 604):
section1_map = self._GetDataTypeMap('java_idx_603_section1')
elif file_header.format_version == 605:
section1_map = self._GetDataTypeMap('java_idx_605_section1')
try:
section1, data_size = self._ReadStructureFromFileObject(
file_object, file_offset, section1_map)
except (ValueError, errors.ParseError) as exception:
raise errors.UnableToParseFile((
'Unable to parse section 1 (format version: {0:d}) with error: '
'{1!s}').format(file_header.format_version, exception))
file_offset += data_size
if file_header.format_version == 602:
section2_map = self._GetDataTypeMap('java_idx_602_section2')
elif file_header.format_version in (603, 604, 605):
file_offset = 128
section2_map = self._GetDataTypeMap('java_idx_603_section2')
try:
section2, data_size = self._ReadStructureFromFileObject(
file_object, file_offset, section2_map)
except (ValueError, errors.ParseError) as exception:
raise errors.UnableToParseFile((
'Unable to parse section 2 (format version: {0:d}) with error: '
'{1!s}').format(file_header.format_version, exception))
file_offset += data_size
if not section2.url:
raise errors.UnableToParseFile('URL not found in file.')
date_http_header = None
for _ in range(section2.number_of_http_headers):
http_header_map = self._GetDataTypeMap('java_idx_http_header')
try:
http_header, data_size = self._ReadStructureFromFileObject(
file_object, file_offset, http_header_map)
except (ValueError, errors.ParseError) as exception:
parser_mediator.ProduceExtractionWarning(
'Unable to parse HTTP header value at offset: 0x{0:08x}'.
format(file_offset))
break
file_offset += data_size
if http_header.name == 'date':
date_http_header = http_header
break
event_data = JavaIDXEventData()
event_data.idx_version = file_header.format_version
event_data.ip_address = getattr(section2, 'ip_address', None)
event_data.url = section2.url
date_time = dfdatetime_java_time.JavaTime(
timestamp=section1.modification_time)
# TODO: Move the timestamp description into definitions.
event = time_events.DateTimeValuesEvent(date_time, 'File Hosted Date')
parser_mediator.ProduceEventWithEventData(event, event_data)
if section1.expiration_time:
date_time = dfdatetime_java_time.JavaTime(
timestamp=section1.expiration_time)
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_EXPIRATION)
parser_mediator.ProduceEventWithEventData(event, event_data)
if date_http_header:
# A HTTP header date and time should be formatted according to RFC 1123.
# The date "should" be in UTC or have associated time zone information
# in the string itself. If that is not the case then there is no reliable
# method for Plaso to determine the proper time zone, so the assumption
# is that the date and time is in UTC.
try:
date_time = dfdatetime_time_elements.TimeElements()
date_time.CopyFromStringRFC1123(date_http_header.value)
except ValueError as exception:
parser_mediator.ProduceExtractionWarning((
'Unable to parse date HTTP header string: {0:s} with error: '
'{1!s}').format(date_http_header.value, exception))
date_time = dfdatetime_semantic_time.InvalidTime()
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_FILE_DOWNLOADED)
parser_mediator.ProduceEventWithEventData(event, event_data)
def testInitialize(self): """Tests the __init__ function.""" invalid_time_object = semantic_time.InvalidTime() self.assertEqual(invalid_time_object.string, 'Invalid')
def _ParseContainerConfigJSON(self, parser_mediator, file_object):
"""Extracts events from a Docker container configuration file.
The path of each container config file is:
DOCKER_DIR/containers/<container_id>/config.json
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): a file-like object.
Raises:
UnableToParseFile: when the file is not a valid container config file.
"""
file_content = file_object.read()
file_content = codecs.decode(file_content, self._ENCODING)
json_dict = json.loads(file_content)
if 'Driver' not in json_dict:
raise errors.UnableToParseFile(
'not a valid Docker container configuration file, '
'missing '
'\'Driver\' key.')
container_id_from_path = self._GetIdentifierFromPath(parser_mediator)
container_id_from_json = json_dict.get('ID', None)
if not container_id_from_json:
raise errors.UnableToParseFile(
'not a valid Docker layer configuration file, the \'ID\' key is '
'missing from the JSON dict (should be {0:s})'.format(
container_id_from_path))
if container_id_from_json != container_id_from_path:
raise errors.UnableToParseFile(
'not a valid Docker container configuration file. The \'ID\' key of '
'the JSON dict ({0:s}) is different from the layer ID taken from the'
' path to the file ({1:s}) JSON file.)'.format(
container_id_from_json, container_id_from_path))
if 'Config' in json_dict and 'Hostname' in json_dict['Config']:
container_name = json_dict['Config']['Hostname']
else:
container_name = 'Unknown container name'
event_data = DockerJSONContainerEventData()
event_data.container_id = container_id_from_path
event_data.container_name = container_name
json_state = json_dict.get('State', None)
if json_state is not None:
time_string = json_state.get('StartedAt', None)
if time_string is not None:
event_data.action = 'Container Started'
try:
date_time = dfdatetime_time_elements.TimeElementsInMicroseconds(
)
date_time.CopyFromStringISO8601(time_string)
except ValueError as exception:
parser_mediator.ProduceExtractionWarning((
'Unable to parse container start time string: {0:s} with error: '
'{1!s}').format(time_string, exception))
date_time = dfdatetime_semantic_time.InvalidTime()
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_START)
parser_mediator.ProduceEventWithEventData(event, event_data)
time_string = json_state.get('FinishedAt', None)
if time_string is not None:
# If the timestamp is 0001-01-01T00:00:00Z, the container
# is still running, so we don't generate a Finished event
if time_string != '0001-01-01T00:00:00Z':
event_data.action = 'Container Finished'
try:
date_time = dfdatetime_time_elements.TimeElementsInMicroseconds(
)
date_time.CopyFromStringISO8601(time_string)
except ValueError as exception:
parser_mediator.ProduceExtractionWarning((
'Unable to parse container finish time string: {0:s} with '
'error: {1!s}').format(time_string, exception))
date_time = dfdatetime_semantic_time.InvalidTime()
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_END)
parser_mediator.ProduceEventWithEventData(
event, event_data)
time_string = json_dict.get('Created', None)
if time_string is not None:
event_data.action = 'Container Created'
try:
date_time = dfdatetime_time_elements.TimeElementsInMicroseconds(
)
date_time.CopyFromStringISO8601(time_string)
except ValueError as exception:
parser_mediator.ProduceExtractionWarning((
'Unable to parse container created time string: {0:s} with error: '
'{1!s}').format(time_string, exception))
date_time = dfdatetime_semantic_time.InvalidTime()
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_ADDED)
parser_mediator.ProduceEventWithEventData(event, event_data)
def ParseReceiverData(self, parser_mediator, query, row, **unused_kwargs):
"""Parses a single row from the receiver and cache response table.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
query (str): query that created the row.
row (sqlite3.Row): row.
"""
query_hash = hash(query)
data = {}
key_url = self._GetRowValue(query_hash, row, 'request_key')
data_dict = {}
description = 'MacKeeper Entry'
# Check the URL, since that contains vital information about the type of
# event we are dealing with.
if key_url.endswith('plist'):
description = 'Configuration Definition'
data['text'] = 'Plist content added to cache.'
elif key_url.startswith('http://event.zeobit.com'):
description = 'MacKeeper Event'
try:
_, _, part = key_url.partition('?')
data['text'] = part.replace('&', ' ')
except UnicodeDecodeError:
data['text'] = 'N/A'
elif key_url.startswith('http://account.zeobit.com'):
description = 'Account Activity'
_, _, activity = key_url.partition('#')
if activity:
data['text'] = 'Action started: {0:s}'.format(activity)
else:
data['text'] = 'Unknown activity.'
elif key_url.startswith('http://support.') and 'chat' in key_url:
description = 'Chat '
try:
jquery = self._GetRowValue(query_hash, row, 'data')
jquery = codecs.decode(jquery, 'utf-8')
except UnicodeDecodeError:
jquery = ''
data_dict = self._ExtractJQuery(jquery)
data = self._ParseChatData(data_dict)
data['entry_type'] = data_dict.get('type', '')
if data['entry_type'] == 'comment':
description += 'Comment'
elif data['entry_type'] == 'outgoing':
description += 'Outgoing Message'
elif data['entry_type'] == 'incoming':
description += 'Incoming Message'
else:
# Empty or not known entry type, generic status message.
description += 'Entry'
data['text'] = ';'.join(self._DictToListOfStrings(data_dict))
if not data['text']:
data['text'] = 'No additional data.'
event_data = MacKeeperCacheEventData()
event_data.description = description
event_data.event_type = data.get('event_type', None)
event_data.offset = self._GetRowValue(query_hash, row, 'id')
event_data.query = query
event_data.record_id = data.get('id', None)
event_data.room = data.get('room', None)
event_data.text = data.get('text', None)
event_data.url = key_url
event_data.user_name = data.get('user', None)
event_data.user_sid = data.get('sid', None)
time_value = self._GetRowValue(query_hash, row, 'time_string')
if isinstance(time_value, int):
date_time = dfdatetime_java_time.JavaTime(timestamp=time_value)
else:
try:
date_time = dfdatetime_time_elements.TimeElements()
date_time.CopyFromDateTimeString(time_value)
except ValueError as exception:
parser_mediator.ProduceExtractionWarning(
'Unable to parse time string: {0:s} with error: {1!s}'.
format(time_value, exception))
date_time = dfdatetime_semantic_time.InvalidTime()
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_ADDED)
parser_mediator.ProduceEventWithEventData(event, event_data)
def _ParseRecord(self, parser_mediator, text_file_object):
"""Parses an Opera global history record.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
text_file_object (dfvfs.TextFile): text file.
Returns:
bool: True if the record was successfully parsed.
"""
try:
title = text_file_object.readline()
except UnicodeDecodeError:
parser_mediator.ProduceExtractionWarning(
'unable to read and decode title')
return False
if not title:
return False
try:
url = text_file_object.readline()
except UnicodeDecodeError:
parser_mediator.ProduceExtractionWarning(
'unable to read and decode url')
return False
try:
timestamp = text_file_object.readline()
except UnicodeDecodeError:
parser_mediator.ProduceExtractionWarning(
'unable to read and decode timestamp')
return False
try:
popularity_index = text_file_object.readline()
except UnicodeDecodeError:
parser_mediator.ProduceExtractionWarning(
'unable to read and decode popularity index')
return False
event_data = OperaGlobalHistoryEventData()
event_data.url = url.strip()
title = title.strip()
if title != event_data.url:
event_data.title = title
popularity_index = popularity_index.strip()
try:
event_data.popularity_index = int(popularity_index, 10)
except ValueError:
parser_mediator.ProduceExtractionWarning(
'unable to convert popularity index: {0:s}'.format(popularity_index))
if event_data.popularity_index < 0:
event_data.description = 'First and Only Visit'
else:
event_data.description = 'Last Visit'
timestamp = timestamp.strip()
try:
timestamp = int(timestamp, 10)
except ValueError:
parser_mediator.ProduceExtractionWarning(
'unable to convert timestamp: {0:s}'.format(timestamp))
timestamp = None
if timestamp is None:
date_time = dfdatetime_semantic_time.InvalidTime()
else:
date_time = dfdatetime_posix_time.PosixTime(timestamp=timestamp)
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_LAST_VISITED)
parser_mediator.ProduceEventWithEventData(event, event_data)
return True
