def _ParseTriggerEndTime(self, parser_mediator, trigger):
"""Parses the end time from a trigger.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
trigger (job_trigger): a trigger.
Returns:
dfdatetime.DateTimeValues: last run date and time or None if not
available.
"""
time_elements_tuple = (trigger.end_date.year, trigger.end_date.month,
trigger.end_date.day_of_month, 0, 0, 0)
date_time = None
if time_elements_tuple != (0, 0, 0, 0, 0, 0):
try:
date_time = dfdatetime_time_elements.TimeElements(
time_elements_tuple=time_elements_tuple)
date_time.is_local_time = True
# TODO: add functionality to dfdatetime to control precision.
date_time._precision = dfdatetime_definitions.PRECISION_1_DAY # pylint: disable=protected-access
except ValueError:
parser_mediator.ProduceExtractionWarning(
'invalid trigger end time: {0!s}'.format(
time_elements_tuple))
return date_time
def VerifyStructure(self, parser_mediator, line):
"""Verify that this file is a securityd log file.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
line (str): line from a text file.
Returns:
bool: True if the line is in the expected format, False if not.
"""
self._last_month = 0
self._year_use = parser_mediator.GetEstimatedYear()
try:
structure = self.SECURITYD_LINE.parseString(line)
except pyparsing.ParseException:
logger.debug('Not a MacOS securityd log file')
return False
time_elements_tuple = self._GetTimeElementsTuple(structure)
try:
dfdatetime_time_elements.TimeElements(
time_elements_tuple=time_elements_tuple)
except ValueError:
logger.debug(
'Not a MacOS securityd log file, invalid date and time: {0!s}'.format(
structure.date_time))
return False
self._last_month = time_elements_tuple[1]
return True
def _ParseLogLine(self, parser_mediator, structure):
"""Parses a log line.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
structure (pyparsing.ParseResults): structure of tokens derived from
a line of a text file.
"""
try:
date_time = dfdatetime_time_elements.TimeElements(
time_elements_tuple=structure.date_time)
# TODO: check if date and time values are local time or in UTC.
date_time.is_local_time = True
except ValueError:
parser_mediator.ProduceExtractionError(
'invalid date time value: {0!s}'.format(structure.date_time))
return
event_data = SophosAVLogEventData()
event_data.text = structure.text
event = time_events.DateTimeValuesEvent(
date_time,
definitions.TIME_DESCRIPTION_ADDED,
time_zone=parser_mediator.timezone)
parser_mediator.ProduceEventWithEventData(event, event_data)
def _ParseLogLine(self, parser_mediator, structure):
"""Parse a single log line and produce an event object.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
structure (pyparsing.ParseResults): structure parsed from the log file.
"""
time_elements_structure = structure.get('date_time', None)
if time_elements_structure:
# Ensure time_elements_tuple is not a pyparsing.ParseResults otherwise
# copy.deepcopy() of the dfDateTime object will fail on Python 3.8 with:
# "TypeError: 'str' object is not callable" due to pyparsing.ParseResults
# overriding __getattr__ with a function that returns an empty string when
# named token does not exists.
year, month, day_of_month, hours, minutes, seconds = (
time_elements_structure)
time_elements_tuple = (year, month, day_of_month, hours, minutes,
seconds)
else:
time_tuple = self._GetValueFromStructure(structure, 'time')
if not time_tuple:
parser_mediator.ProduceExtractionWarning('missing time values')
return
date_tuple = self._GetValueFromStructure(structure, 'date')
if not date_tuple:
time_elements_tuple = (self._year, self._month,
self._day_of_month, time_tuple[0],
time_tuple[1], time_tuple[2])
else:
time_elements_tuple = (date_tuple[0], date_tuple[1],
date_tuple[2], time_tuple[0],
time_tuple[1], time_tuple[2])
try:
date_time = dfdatetime_time_elements.TimeElements(
time_elements_tuple=time_elements_tuple)
except ValueError:
parser_mediator.ProduceExtractionWarning(
'invalid date time value: {0!s}'.format(time_elements_tuple))
return
event_data = IISEventData()
for key, value in structure.items():
if key in ('date', 'date_time', 'time') or value == '-':
continue
if isinstance(value, pyparsing.ParseResults):
value = ''.join(value)
setattr(event_data, key, value)
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_WRITTEN)
parser_mediator.ProduceEventWithEventData(event, event_data)
def _ParseLogLine(self, parser_mediator, structure, key):
"""Parse a single log line and produce an event object.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
structure (pyparsing.ParseResults): structure of tokens derived from
a line of a text file.
key (str): name of the parsed structure.
"""
time_elements_tuple = self._GetTimeElementsTuple(structure)
try:
date_time = dfdatetime_time_elements.TimeElements(
time_elements_tuple=time_elements_tuple)
except ValueError:
parser_mediator.ProduceExtractionWarning(
'invalid date time value: {0!s}'.format(time_elements_tuple))
return
self._last_month = time_elements_tuple[1]
if key == 'logline':
self._previous_structure = structure
message = self._GetValueFromStructure(structure, 'message')
else:
repeat_count = self._GetValueFromStructure(structure, 'times')
previous_message = self._GetValueFromStructure(
self._previous_structure, 'message')
message = 'Repeated {0:d} times: {1:s}'.format(
repeat_count, previous_message)
structure = self._previous_structure
# It uses CarsNotIn structure which leaves whitespaces
# at the beginning of the sender and the caller.
caller = self._GetValueFromStructure(structure, 'caller')
if caller:
caller = caller.strip()
# TODO: move this to formatter.
if not caller:
caller = 'unknown'
sender = self._GetValueFromStructure(structure, 'sender')
if sender:
sender = sender.strip()
event_data = MacOSSecuritydLogEventData()
event_data.caller = caller
event_data.facility = self._GetValueFromStructure(structure, 'facility')
event_data.level = self._GetValueFromStructure(structure, 'level')
event_data.message = message
event_data.security_api = self._GetValueFromStructure(
structure, 'security_api', default_value='unknown')
event_data.sender_pid = self._GetValueFromStructure(structure, 'sender_pid')
event_data.sender = sender
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_ADDED)
parser_mediator.ProduceEventWithEventData(event, event_data)
def _ParseUpdateKeyValue(self, parser_mediator, registry_value):
"""Parses the UpdateKey value.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
registry_value (dfwinreg.WinRegistryValue): Windows Registry value.
Returns:
dfdatetime_time_elements.TimeElements: date and time value or None
if not available.
"""
if not registry_value.DataIsString():
parser_mediator.ProduceExtractionWarning(
'unsupported UpdateKey value data type: {0:s}'.format(
registry_value.data_type_string))
return None
date_time_string = registry_value.GetDataAsObject()
if not date_time_string:
parser_mediator.ProduceExtractionWarning('missing UpdateKey value data')
return None
re_match = self._UPDATE_DATE_TIME_RE.match(date_time_string)
if not re_match:
parser_mediator.ProduceExtractionWarning(
'unsupported UpdateKey value data: {0!s}'.format(date_time_string))
return None
month, day_of_month, year, hours, minutes, seconds, part_of_day = (
re_match.groups())
try:
year = int(year, 10)
month = int(month, 10)
day_of_month = int(day_of_month, 10)
hours = int(hours, 10)
minutes = int(minutes, 10)
seconds = int(seconds, 10)
except (TypeError, ValueError):
parser_mediator.ProduceExtractionWarning(
'invalid UpdateKey date time value: {0!s}'.format(date_time_string))
return None
if part_of_day == 'PM':
hours += 12
time_elements_tuple = (year, month, day_of_month, hours, minutes, seconds)
try:
date_time = dfdatetime_time_elements.TimeElements(
time_elements_tuple=time_elements_tuple)
date_time.is_local_time = True
except ValueError:
parser_mediator.ProduceExtractionWarning(
'invalid UpdateKey date time value: {0!s}'.format(
time_elements_tuple))
return None
return date_time
def _ParseLogLine(self, parser_mediator, structure):
"""Parses a log line.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
structure (pyparsing.ParseResults): structure of tokens derived from
a line of a text file.
"""
time_elements_tuple = self._GetTimeElementsTuple(structure)
try:
date_time = dfdatetime_time_elements.TimeElements(
time_elements_tuple=time_elements_tuple)
date_time.is_local_time = True
except ValueError:
parser_mediator.ProduceExtractionWarning(
'invalid date time value: {0!s}'.format(time_elements_tuple))
return
event_data = VsftpdEventData()
event_data.text = self._GetValueFromStructure(structure, 'text')
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_ADDED,
time_zone=parser_mediator.timezone)
parser_mediator.ProduceEventWithEventData(event, event_data)
def VerifyStructure(self, parser_mediator, line):
"""Verify that this file is a vsftpd log file.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfVFS.
line (str): line from a text file.
Returns:
bool: True if the line is in the expected format, False if not.
"""
try:
structure = self._LOG_LINE.parseString(line)
except pyparsing.ParseException:
return False
if (' [pid ' not in line) or (': Client ' not in line):
return False
time_elements_tuple = self._GetTimeElementsTuple(structure)
try:
dfdatetime_time_elements.TimeElements(
time_elements_tuple=time_elements_tuple)
except ValueError:
logger.debug((
'Not a vsftpd log file, invalid date and time: '
'{0!s}').format(time_elements_tuple))
return False
return True
def VerifyStructure(self, parser_mediator, line):
"""Verify that this file is a XChat log file.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
line (bytes): line from a text file.
Returns:
bool: True if the line is in the expected format, False if not.
"""
try:
structure = self._HEADER.parseString(line)
except pyparsing.ParseException:
logging.debug(u'Not a XChat log file')
return False
_, month, day, hours, minutes, seconds, year = structure.date_time
month = timelib.MONTH_DICT.get(month.lower(), 0)
time_elements_tuple = (year, month, day, hours, minutes, seconds)
try:
dfdatetime_time_elements.TimeElements(
time_elements_tuple=time_elements_tuple)
except ValueError:
logging.debug(
u'Not a XChat log file, invalid date and time: {0!s}'.format(
structure.date_time))
return False
return True
def testCopyToDateTimeStringISO8601(self):
"""Tests the CopyToDateTimeStringISO8601 function."""
time_elements_object = time_elements.TimeElements(
time_elements_tuple=(2010, 8, 12, 20, 6, 31))
date_time_string = time_elements_object.CopyToDateTimeStringISO8601()
self.assertEqual(date_time_string, '2010-08-12T20:06:31Z')
def VerifyStructure(self, parser_mediator, line):
"""Verify that this file is a Sophos Anti-Virus log file.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfVFS.
line (str): line from a text file.
Returns:
bool: True if the line is in the expected format, False if not.
"""
try:
structure = self._LOG_LINE.parseString(line)
except pyparsing.ParseException:
logger.debug('Not a Sophos Anti-Virus log file')
return False
# Expect spaces at position 9 and 16.
if ' ' not in (line[8], line[15]):
logger.debug('Not a Sophos Anti-Virus log file')
return False
try:
dfdatetime_time_elements.TimeElements(
time_elements_tuple=structure.date_time)
except ValueError:
logger.debug(
('Not a Sophos Anti-Virus log file, invalid date and time: '
'{0!s}').format(structure.date_time))
return False
return True
def ParseRecord(self, parser_mediator, key, structure):
"""Parses a matching entry.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
key (str): name of the parsed structure.
structure (pyparsing.ParseResults): elements parsed from the file.
Raises:
ParseError: when the structure type is unknown.
"""
if key not in self._SUPPORTED_KEYS:
raise errors.ParseError(
'Unable to parse record, unknown structure: {0:s}'.format(key))
date_time = dfdatetime_time_elements.TimeElements()
date_time_string = self._GetValueFromStructure(structure, 'date_time')
try:
iso_date_time = self._GetISO8601String(date_time_string)
date_time.CopyFromStringISO8601(iso_date_time)
except ValueError:
parser_mediator.ProduceExtractionWarning(
'invalid date time value: {0!s}'.format(date_time_string))
return
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_RECORDED)
event_data = ApacheAccessEventData()
event_data.ip_address = self._GetValueFromStructure(
structure, 'ip_address')
event_data.remote_name = self._GetValueFromStructure(
structure, 'remote_name')
event_data.user_name = self._GetValueFromStructure(
structure, 'user_name')
event_data.http_request = self._GetValueFromStructure(
structure, 'http_request')
event_data.http_response_code = self._GetValueFromStructure(
structure, 'response_code')
event_data.http_response_bytes = self._GetValueFromStructure(
structure, 'response_bytes')
if key in ('combined_log_format', 'vhost_combined_log_format'):
event_data.http_request_referer = self._GetValueFromStructure(
structure, 'referer')
event_data.http_request_user_agent = self._GetValueFromStructure(
structure, 'user_agent')
if key == 'vhost_combined_log_format':
event_data.server_name = self._GetValueFromStructure(
structure, 'server_name')
event_data.port_number = self._GetValueFromStructure(
structure, 'port_number')
parser_mediator.ProduceEventWithEventData(event, event_data)
def ParseFileObject(self, parser_mediator, file_object):
"""Parses an Opera typed history file-like object.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): file-like object.
Raises:
UnableToParseFile: when the file cannot be parsed.
"""
data = file_object.read(self._HEADER_READ_SIZE)
if not data.startswith(b'<?xml'):
raise errors.UnableToParseFile(
'Not an Opera typed history file [not a XML]')
_, _, data = data.partition(b'\n')
if not data.startswith(b'<typed_history'):
raise errors.UnableToParseFile(
'Not an Opera typed history file [wrong XML root key]')
# For ElementTree to work we need to work on a file object seeked
# to the beginning.
file_object.seek(0, os.SEEK_SET)
xml = ElementTree.parse(file_object)
for history_item in xml.iterfind('typed_history_item'):
event_data = OperaTypedHistoryEventData()
event_data.entry_type = history_item.get('type', None)
event_data.url = history_item.get('content', None)
if event_data.entry_type == 'selected':
event_data.entry_selection = 'Filled from autocomplete.'
elif event_data.entry_type == 'text':
event_data.entry_selection = 'Manually typed.'
last_typed_time = history_item.get('last_typed', None)
if last_typed_time is None:
parser_mediator.ProduceExtractionWarning(
'missing last typed time.')
continue
date_time = dfdatetime_time_elements.TimeElements()
try:
date_time.CopyFromStringISO8601(last_typed_time)
except ValueError as exception:
parser_mediator.ProduceExtractionWarning(
'unsupported last typed time: {0:s} with error: {1!s}.'.
format(last_typed_time, exception))
continue
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_LAST_VISITED)
parser_mediator.ProduceEventWithEventData(event, event_data)
def testCopyFromDateTimeString(self):
"""Tests the CopyFromDateTimeString function."""
time_elements_object = time_elements.TimeElements()
expected_time_elements_tuple = (2010, 8, 12, 0, 0, 0)
expected_number_of_seconds = 1281571200
time_elements_object.CopyFromDateTimeString('2010-08-12')
self.assertEqual(time_elements_object._time_elements_tuple,
expected_time_elements_tuple)
self.assertEqual(time_elements_object._number_of_seconds,
expected_number_of_seconds)
expected_time_elements_tuple = (2010, 8, 12, 21, 6, 31)
expected_number_of_seconds = 1281647191
time_elements_object.CopyFromDateTimeString('2010-08-12 21:06:31')
self.assertEqual(time_elements_object._time_elements_tuple,
expected_time_elements_tuple)
self.assertEqual(time_elements_object._number_of_seconds,
expected_number_of_seconds)
expected_time_elements_tuple = (2010, 8, 12, 21, 6, 31)
expected_number_of_seconds = 1281647191
time_elements_object.CopyFromDateTimeString(
'2010-08-12 21:06:31.546875')
self.assertEqual(time_elements_object._time_elements_tuple,
expected_time_elements_tuple)
self.assertEqual(time_elements_object._number_of_seconds,
expected_number_of_seconds)
expected_time_elements_tuple = (2010, 8, 12, 21, 6, 31)
expected_number_of_seconds = 1281650791
time_elements_object.CopyFromDateTimeString(
'2010-08-12 21:06:31.546875-01:00')
self.assertEqual(time_elements_object._time_elements_tuple,
expected_time_elements_tuple)
self.assertEqual(time_elements_object._number_of_seconds,
expected_number_of_seconds)
self.assertEqual(time_elements_object.time_zone_offset, -60)
expected_time_elements_tuple = (2010, 8, 12, 21, 6, 31)
expected_number_of_seconds = 1281643591
time_elements_object.CopyFromDateTimeString(
'2010-08-12 21:06:31.546875+01:00')
self.assertEqual(time_elements_object._time_elements_tuple,
expected_time_elements_tuple)
self.assertEqual(time_elements_object._number_of_seconds,
expected_number_of_seconds)
self.assertEqual(time_elements_object.time_zone_offset, 60)
expected_time_elements_tuple = (1601, 1, 2, 0, 0, 0)
expected_number_of_seconds = -11644387200
time_elements_object.CopyFromDateTimeString('1601-01-02 00:00:00')
self.assertEqual(time_elements_object._time_elements_tuple,
expected_time_elements_tuple)
self.assertEqual(time_elements_object._number_of_seconds,
expected_number_of_seconds)
def testInitialize(self):
"""Tests the initialization function."""
time_elements_object = time_elements.TimeElements()
self.assertIsNotNone(time_elements_object)
expected_time_elements_tuple = (2010, 8, 12, 20, 6, 31)
time_elements_object = time_elements.TimeElements(
time_elements_tuple=(2010, 8, 12, 20, 6, 31))
self.assertIsNotNone(time_elements_object)
self.assertEqual(
time_elements_object._time_elements_tuple, expected_time_elements_tuple)
with self.assertRaises(ValueError):
time_elements.TimeElements(
time_elements_tuple=(2010, 8, 12, 20, 6))
with self.assertRaises(ValueError):
time_elements.TimeElements(
time_elements_tuple=(2010, 13, 12, 20, 6, 31))
def testCopyFromString(self):
"""Tests the CopyFromString function."""
time_elements_object = time_elements.TimeElements()
expected_time_elements_tuple = (2010, 8, 12, 0, 0, 0)
expected_number_of_seconds = 1281571200
time_elements_object.CopyFromString('2010-08-12')
self.assertEqual(time_elements_object._time_elements_tuple,
expected_time_elements_tuple)
self.assertEqual(time_elements_object._number_of_seconds,
expected_number_of_seconds)
def _ConvertToTimestamp(self, date, time):
"""Converts date and time strings into a timestamp.
Recent versions of Office Scan write a log field with a Unix timestamp.
Older versions may not write this field; their logs only provide a date and
a time expressed in the local time zone. This functions handles the latter
case.
Args:
date (str): date as an 8-character string in the YYYYMMDD format.
time (str): time as a 3 or 4-character string in the [H]HMM format or a
6-character string in the HHMMSS format.
Returns:
dfdatetime_time_elements.TimestampElements: the parsed timestamp.
Raises:
ValueError: if the date and time values cannot be parsed.
"""
# Check that the strings have the correct length.
if len(date) != 8:
raise ValueError('Unsupported length of date string: {0!s}'.format(
repr(date)))
if len(time) < 3 or len(time) > 4:
raise ValueError('Unsupported length of time string: {0!s}'.format(
repr(time)))
# Extract the date.
try:
year = int(date[:4], 10)
month = int(date[4:6], 10)
day = int(date[6:8], 10)
except (TypeError, ValueError):
raise ValueError('Unable to parse date string: {0!s}'.format(
repr(date)))
# Extract the time. Note that a single-digit hour value has no leading zero.
try:
hour = int(time[:-2], 10)
minutes = int(time[-2:], 10)
except (TypeError, ValueError):
raise ValueError('Unable to parse time string: {0!s}'.format(
repr(date)))
time_elements_tuple = (year, month, day, hour, minutes, 0)
date_time = dfdatetime_time_elements.TimeElements(
time_elements_tuple=time_elements_tuple)
date_time.is_local_time = True
# TODO: add functionality to dfdatetime to control precision.
date_time._precision = dfdatetime_definitions.PRECISION_1_MINUTE # pylint: disable=protected-access
return date_time
def _ParseLogLine(self, parser_mediator, structure):
"""Parse a single log line and and produce an event object.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
structure (pyparsing.ParseResults): structure of tokens derived from
a line of a text file.
"""
time_elements_tuple = self._GetValueFromStructure(
structure, 'date_time')
try:
date_time = dfdatetime_time_elements.TimeElements(
time_elements_tuple=time_elements_tuple)
date_time.is_local_time = True
except ValueError:
parser_mediator.ProduceExtractionWarning(
'invalid date time value: {0!s}'.format(time_elements_tuple))
return
event_data = WinFirewallEventData()
event_data.action = self._GetValueFromStructure(structure, 'action')
event_data.dest_ip = self._GetValueFromStructure(structure, 'dest_ip')
event_data.dest_port = self._GetValueFromStructure(
structure, 'dest_port')
event_data.flags = self._GetValueFromStructure(structure, 'flags')
event_data.icmp_code = self._GetValueFromStructure(
structure, 'icmp_code')
event_data.icmp_type = self._GetValueFromStructure(
structure, 'icmp_type')
event_data.info = self._GetValueFromStructure(structure, 'info')
event_data.path = self._GetValueFromStructure(structure, 'path')
event_data.protocol = self._GetValueFromStructure(
structure, 'protocol')
event_data.size = self._GetValueFromStructure(structure, 'size')
event_data.source_ip = self._GetValueFromStructure(
structure, 'source_ip')
event_data.source_port = self._GetValueFromStructure(
structure, 'source_port')
event_data.tcp_ack = self._GetValueFromStructure(structure, 'tcp_ack')
event_data.tcp_seq = self._GetValueFromStructure(structure, 'tcp_seq')
event_data.tcp_win = self._GetValueFromStructure(structure, 'tcp_win')
if self._use_local_timezone:
time_zone = parser_mediator.timezone
else:
time_zone = pytz.UTC
event = time_events.DateTimeValuesEvent(
date_time,
definitions.TIME_DESCRIPTION_WRITTEN,
time_zone=time_zone)
parser_mediator.ProduceEventWithEventData(event, event_data)
def _ParseHeader(self, parser_mediator, structure):
"""Parses a log header.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
structure (pyparsing.ParseResults): structure of tokens derived from
a line of a text file.
"""
time_elements_tuple = self._GetValueFromStructure(
structure, 'date_time')
# TODO: what if time_elements_tuple is None.
_, month, day, hours, minutes, seconds, year = time_elements_tuple
month = timelib.MONTH_DICT.get(month.lower(), 0)
time_elements_tuple = (year, month, day, hours, minutes, seconds)
try:
date_time = dfdatetime_time_elements.TimeElements(
time_elements_tuple=time_elements_tuple)
date_time.is_local_time = True
except ValueError:
parser_mediator.ProduceExtractionWarning(
'invalid date time value: {0!s}'.format(time_elements_tuple))
return
self._last_month = month
event_data = XChatLogEventData()
log_action = self._GetValueFromStructure(structure,
'log_action',
default_value=[])
if log_action[0] == 'BEGIN':
self._xchat_year = year
event_data.text = 'XChat start logging'
elif log_action[0] == 'END':
self._xchat_year = None
event_data.text = 'XChat end logging'
else:
logger.debug('Unknown log action: {0:s}.'.format(
' '.join(log_action)))
return
event = time_events.DateTimeValuesEvent(
date_time,
definitions.TIME_DESCRIPTION_ADDED,
time_zone=parser_mediator.timezone)
parser_mediator.ProduceEventWithEventData(event, event_data)
def VerifyStructure(self, parser_mediator, line):
"""Verify that this file is a Mac AppFirewall log file.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
line (str): line from a text file.
Returns:
bool: True if the line is in the expected format, False if not.
"""
self._last_month = 0
self._year_use = parser_mediator.GetEstimatedYear()
try:
structure = self.FIREWALL_LINE.parseString(line)
except pyparsing.ParseException as exception:
logger.debug((
'Unable to parse file as a Mac AppFirewall log file with error: '
'{0!s}').format(exception))
return False
action = self._GetValueFromStructure(structure, 'action')
if action != 'creating /var/log/appfirewall.log':
logger.debug(
'Not a Mac AppFirewall log file, invalid action: {0!s}'.format(
action))
return False
status = self._GetValueFromStructure(structure, 'status')
if status != 'Error':
logger.debug(
'Not a Mac AppFirewall log file, invalid status: {0!s}'.format(
status))
return False
time_elements_tuple = self._GetTimeElementsTuple(structure)
try:
dfdatetime_time_elements.TimeElements(
time_elements_tuple=time_elements_tuple)
except ValueError:
logger.debug(
('Not a Mac AppFirewall log file, invalid date and time: '
'{0!s}').format(time_elements_tuple))
return False
self._last_month = time_elements_tuple[1]
return True
def testGetMessages(self):
"""Tests the GetMessages method."""
date_time = dfdatetime_time_elements.TimeElements()
date_time.CopyFromString(u'2016-11-14 20:36:37.222')
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_MODIFICATION)
event.data_type = u'mac:wifilog:line'
event.action = u'Interface en0 turn up.'
event.agent = u'airportd[88]'
event.function = u'airportdProcessDLILEvent'
event.text = u'en0 attached (up)'
expected_messages = (
u'Action: Interface en0 turn up. '
u'Agent: airportd[88] '
u'(airportdProcessDLILEvent) '
u'Log: en0 attached (up)',
u'Action: Interface en0 turn up.')
messages = self._formatter.GetMessages(None, event)
self.assertEqual(messages, expected_messages)
date_time = dfdatetime_time_elements.TimeElements()
date_time.CopyFromString(u'2017-01-02 00:10:15')
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_MODIFICATION)
event.data_type = u'mac:wifilog:line'
event.text = u'test-macbookpro newsyslog[50498]: logfile turned over'
expected_messages = (
u'Log: test-macbookpro newsyslog[50498]: logfile turned over',
u'Log: test-macbookpro newsyslog[50498]: logfile turned over')
messages = self._formatter.GetMessages(None, event)
self.assertEqual(messages, expected_messages)
def _ParseLogLine(self, parser_mediator, structure, key):
"""Parse a single log line and produce an event object.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
key (str): identifier of the structure of tokens.
structure (pyparsing.ParseResults): structure of tokens derived from
a line of a text file.
"""
time_elements_tuple = self._GetTimeElementsTuple(structure)
try:
date_time = dfdatetime_time_elements.TimeElements(
time_elements_tuple=time_elements_tuple)
except ValueError:
parser_mediator.ProduceExtractionError(
'invalid date time value: {0!s}'.format(structure.date_time))
return
self._last_month = time_elements_tuple[1]
# If the actual entry is a repeated entry, we take the basic information
# from the previous entry, but using the timestmap from the actual entry.
if key == 'logline':
self._previous_structure = structure
else:
structure = self._previous_structure
# Pyparsing reads in RAW, but the text is in UTF8.
try:
action = structure.action.decode('utf-8')
except UnicodeDecodeError:
logging.warning(
'Decode UTF8 failed, the message string may be cut short.')
action = structure.action.decode('utf-8', 'ignore')
event_data = MacAppFirewallLogEventData()
event_data.action = action
event_data.agent = structure.agent
event_data.computer_name = structure.computer_name
# Due to the use of CharsNotIn pyparsing structure contains whitespaces
# that need to be removed.
event_data.process_name = structure.process_name.strip()
event_data.status = structure.status
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_ADDED)
parser_mediator.ProduceEventWithEventData(event, event_data)
def _ParseLogLine(self, parser_mediator, structure):
"""Parse a single log line and produce an event object.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
structure (pyparsing.ParseResults): structure parsed from the log file.
"""
if structure.date_time:
time_elements_tuple = structure.date_time
elif structure.date and structure.time:
year, month, day_of_month = structure.date
hours, minutes, seconds = structure.time
time_elements_tuple = (year, month, day_of_month, hours, minutes,
seconds)
elif structure.time:
hours, minutes, seconds = structure.time
time_elements_tuple = (self._year, self._month, self._day_of_month,
hours, minutes, seconds)
else:
parser_mediator.ProduceExtractionError(
'missing date and time values')
return
try:
date_time = dfdatetime_time_elements.TimeElements(
time_elements_tuple=time_elements_tuple)
except ValueError:
parser_mediator.ProduceExtractionError(
'invalid date time value: {0!s}'.format(time_elements_tuple))
return
event_data = IISEventData()
for key, value in iter(structure.items()):
if key in ('date', 'date_time', 'time') or value == '-':
continue
if isinstance(value, pyparsing.ParseResults):
value = ''.join(value)
setattr(event_data, key, value)
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_WRITTEN)
parser_mediator.ProduceEventWithEventData(event, event_data)
def ParseRecord(self, parser_mediator, key, structure):
"""Parses a structure of tokens derived from a line of a text file.
Args:
parser_mediator (ParserMediator): parser mediator.
key (str): identifier of the structure of tokens.
structure (pyparsing.ParseResults): structure of tokens derived from
a line of a text file.
Raises:
ParseError: when the structure type is unknown.
"""
if key != 'line':
raise errors.ParseError(
'Unable to parse record, unknown structure: {0:s}'.format(key))
# Ensure time_elements_tuple is not a pyparsing.ParseResults otherwise
# copy.deepcopy() of the dfDateTime object will fail on Python 3.8 with:
# "TypeError: 'str' object is not callable" due to pyparsing.ParseResults
# overriding __getattr__ with a function that returns an empty string when
# named token does not exists.
time_elements_structure = structure.get('date_time', None)
try:
year, month, day_of_month, hours, minutes, seconds = (
time_elements_structure)
date_time = dfdatetime_time_elements.TimeElements(
time_elements_tuple=(year, month, day_of_month, hours, minutes,
seconds))
except (TypeError, ValueError):
parser_mediator.ProduceExtractionWarning(
'invalid date time value: {0!s}'.format(
time_elements_structure))
return
body_text = self._GetValueFromStructure(structure, 'body')
if not body_text:
parser_mediator.ProduceExtractionWarning('missing body text')
return
event_data = DpkgEventData()
event_data.body = body_text
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_ADDED)
parser_mediator.ProduceEventWithEventData(event, event_data)
def _GetDateTime(self, structure):
"""Retrieves the date and time from a date and time values structure.
The date and time values in Apache access log files are formatted as:
"[18/Sep/2011:19:18:28 -0400]".
Args:
structure (pyparsing.ParseResults): structure of tokens derived from a
line of a text file.
Returns:
dfdatetime.DateTimeValues: date and time.
Raises:
ValueError: if the structure cannot be converted into a date time string.
"""
year = self._GetValueFromStructure(structure, 'year')
month = self._GetValueFromStructure(structure, 'month')
try:
month = self._MONTH_DICT.get(month.lower(), 0)
except AttributeError as exception:
raise ValueError(
'unable to parse month with error: {0!s}.'.format(exception))
day_of_month = self._GetValueFromStructure(structure, 'day')
hours = self._GetValueFromStructure(structure, 'hours')
minutes = self._GetValueFromStructure(structure, 'minutes')
seconds = self._GetValueFromStructure(structure, 'seconds')
time_offset = self._GetValueFromStructure(structure, 'time_offset')
try:
time_zone_offset = int(time_offset[1:3], 10) * 60
time_zone_offset += int(time_offset[3:5], 10)
if time_offset[0] == '-':
time_zone_offset *= -1
except (TypeError, ValueError) as exception:
raise ValueError(
'unable to parse time zone offset with error: {0!s}.'.format(
exception))
time_elements_tuple = (year, month, day_of_month, hours, minutes,
seconds)
return dfdatetime_time_elements.TimeElements(
time_elements_tuple=time_elements_tuple,
time_zone_offset=time_zone_offset)
def _ParseHeader(self, parser_mediator, structure):
"""Parses a log header.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
structure (pyparsing.ParseResults): structure of tokens derived from
a line of a text file.
"""
_, month, day, hours, minutes, seconds, year = structure.date_time
month = timelib.MONTH_DICT.get(month.lower(), 0)
time_elements_tuple = (year, month, day, hours, minutes, seconds)
try:
date_time = dfdatetime_time_elements.TimeElements(
time_elements_tuple=time_elements_tuple)
date_time.is_local_time = True
except ValueError:
parser_mediator.ProduceExtractionError(
u'invalid date time value: {0!s}'.format(structure.date_time))
return
self._last_month = month
event_data = XChatLogEventData()
if structure.log_action[0] == u'BEGIN':
self._xchat_year = year
event_data.text = u'XChat start logging'
elif structure.log_action[0] == u'END':
self._xchat_year = None
event_data.text = u'XChat end logging'
else:
logging.debug(u'Unknown log action: {0:s}.'.format(u' '.join(
structure.log_action)))
return
event = time_events.DateTimeValuesEvent(
date_time,
eventdata.EventTimestamp.ADDED_TIME,
time_zone=parser_mediator.timezone)
parser_mediator.ProduceEventWithEventData(event, event_data)
def _ParseLogLine(self, parser_mediator, structure, key):
"""Parse a single log line and produce an event object.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
file_object (dfvfs.FileIO): a file-like object.
structure (pyparsing.ParseResults): structure of tokens derived from
a line of a text file.
"""
time_elements_tuple = self._GetTimeElementsTuple(structure)
try:
date_time = dfdatetime_time_elements.TimeElements(
time_elements_tuple=time_elements_tuple)
except ValueError:
parser_mediator.ProduceExtractionError(
u'invalid date time value: {0!s}'.format(structure.date_time))
return
self._last_month = time_elements_tuple[1]
if key == u'logline':
self._previous_structure = structure
message = structure.message
else:
message = u'Repeated {0:d} times: {1:s}'.format(
structure.times, self._previous_structure.message)
structure = self._previous_structure
# It uses CarsNotIn structure which leaves whitespaces
# at the beginning of the sender and the caller.
event_data = MacSecuritydLogEventData()
event_data.caller = structure.caller.strip() or u'unknown'
event_data.facility = structure.facility
event_data.level = structure.level
event_data.message = message
event_data.security_api = structure.security_api or u'unknown'
event_data.sender_pid = structure.sender_pid
event_data.sender = structure.sender.strip()
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_ADDED)
parser_mediator.ProduceEventWithEventData(event, event_data)
def _ParseLogLine(self, parser_mediator, structure, key):
"""Parse a single log line and produce an event object.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
key (str): identifier of the structure of tokens.
structure (pyparsing.ParseResults): structure of tokens derived from
a line of a text file.
"""
time_elements_tuple = self._GetTimeElementsTuple(structure)
try:
date_time = dfdatetime_time_elements.TimeElements(
time_elements_tuple=time_elements_tuple)
except ValueError:
parser_mediator.ProduceExtractionWarning(
'invalid date time value: {0!s}'.format(time_elements_tuple))
return
self._last_month = time_elements_tuple[1]
# If the actual entry is a repeated entry, we take the basic information
# from the previous entry, but use the timestamp from the actual entry.
if key == 'logline':
self._previous_structure = structure
else:
structure = self._previous_structure
event_data = MacAppFirewallLogEventData()
event_data.action = self._GetValueFromStructure(structure, 'action')
event_data.agent = self._GetValueFromStructure(structure, 'agent')
event_data.computer_name = self._GetValueFromStructure(
structure, 'computer_name')
event_data.status = self._GetValueFromStructure(structure, 'status')
# Due to the use of CharsNotIn pyparsing structure contains whitespaces
# that need to be removed.
process_name = self._GetValueFromStructure(structure, 'process_name')
if process_name:
event_data.process_name = process_name.strip()
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_ADDED)
parser_mediator.ProduceEventWithEventData(event, event_data)
def _ParseLogLine(self, parser_mediator, structure):
"""Parse a single log line and and produce an event object.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
structure (pyparsing.ParseResults): structure of tokens derived from
a line of a text file.
"""
try:
date_time = dfdatetime_time_elements.TimeElements(
time_elements_tuple=structure.date_time)
date_time.is_local_time = True
except ValueError:
parser_mediator.ProduceExtractionError(
u'invalid date time value: {0!s}'.format(structure.date_time))
return
event_data = WinFirewallEventData()
event_data.action = structure.action
event_data.dest_ip = structure.dest_ip
event_data.dest_port = structure.dest_port
event_data.flags = structure.flags
event_data.icmp_code = structure.icmp_code
event_data.icmp_type = structure.icmp_type
event_data.info = structure.info
event_data.path = structure.path
event_data.protocol = structure.protocol
event_data.size = structure.size
event_data.source_ip = structure.source_ip
event_data.source_port = structure.source_port
event_data.tcp_ack = structure.tcp_ack
event_data.tcp_seq = structure.tcp_seq
event_data.tcp_win = structure.tcp_win
if self._use_local_timezone:
time_zone = parser_mediator.timezone
else:
time_zone = pytz.UTC
event = time_events.DateTimeValuesEvent(
date_time,
eventdata.EventTimestamp.WRITTEN_TIME,
time_zone=time_zone)
parser_mediator.ProduceEventWithEventData(event, event_data)
def _BuildDateTime(time_elements_structure):
"""Builds time elements from an APT History time stamp.
Args:
time_elements_structure (pyparsing.ParseResults): structure of tokens
derived from an APT History time stamp.
Returns:
dfdatetime.TimeElements: date and time extracted from the structure or
None f the structure does not represent a valid string.
"""
try:
date_time = dfdatetime_time_elements.TimeElements(
time_elements_tuple=time_elements_structure)
# APT History logs store date and time values in local time.
date_time.is_local_time = True
return date_time
except ValueError:
return None
