def _ParseTriggerEndTime(self, parser_mediator, trigger): """Parses the end time from a trigger. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. trigger (job_trigger): a trigger. Returns: dfdatetime.DateTimeValues: last run date and time or None if not available. """ time_elements_tuple = (trigger.end_date.year, trigger.end_date.month, trigger.end_date.day_of_month, 0, 0, 0) date_time = None if time_elements_tuple != (0, 0, 0, 0, 0, 0): try: date_time = dfdatetime_time_elements.TimeElements( time_elements_tuple=time_elements_tuple) date_time.is_local_time = True # TODO: add functionality to dfdatetime to control precision. date_time._precision = dfdatetime_definitions.PRECISION_1_DAY # pylint: disable=protected-access except ValueError: parser_mediator.ProduceExtractionWarning( 'invalid trigger end time: {0!s}'.format( time_elements_tuple)) return date_time
def VerifyStructure(self, parser_mediator, line): """Verify that this file is a securityd log file. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. line (str): line from a text file. Returns: bool: True if the line is in the expected format, False if not. """ self._last_month = 0 self._year_use = parser_mediator.GetEstimatedYear() try: structure = self.SECURITYD_LINE.parseString(line) except pyparsing.ParseException: logger.debug('Not a MacOS securityd log file') return False time_elements_tuple = self._GetTimeElementsTuple(structure) try: dfdatetime_time_elements.TimeElements( time_elements_tuple=time_elements_tuple) except ValueError: logger.debug( 'Not a MacOS securityd log file, invalid date and time: {0!s}'.format( structure.date_time)) return False self._last_month = time_elements_tuple[1] return True
def _ParseLogLine(self, parser_mediator, structure): """Parses a log line. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. structure (pyparsing.ParseResults): structure of tokens derived from a line of a text file. """ try: date_time = dfdatetime_time_elements.TimeElements( time_elements_tuple=structure.date_time) # TODO: check if date and time values are local time or in UTC. date_time.is_local_time = True except ValueError: parser_mediator.ProduceExtractionError( 'invalid date time value: {0!s}'.format(structure.date_time)) return event_data = SophosAVLogEventData() event_data.text = structure.text event = time_events.DateTimeValuesEvent( date_time, definitions.TIME_DESCRIPTION_ADDED, time_zone=parser_mediator.timezone) parser_mediator.ProduceEventWithEventData(event, event_data)
def _ParseLogLine(self, parser_mediator, structure): """Parse a single log line and produce an event object. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. structure (pyparsing.ParseResults): structure parsed from the log file. """ time_elements_structure = structure.get('date_time', None) if time_elements_structure: # Ensure time_elements_tuple is not a pyparsing.ParseResults otherwise # copy.deepcopy() of the dfDateTime object will fail on Python 3.8 with: # "TypeError: 'str' object is not callable" due to pyparsing.ParseResults # overriding __getattr__ with a function that returns an empty string when # named token does not exists. year, month, day_of_month, hours, minutes, seconds = ( time_elements_structure) time_elements_tuple = (year, month, day_of_month, hours, minutes, seconds) else: time_tuple = self._GetValueFromStructure(structure, 'time') if not time_tuple: parser_mediator.ProduceExtractionWarning('missing time values') return date_tuple = self._GetValueFromStructure(structure, 'date') if not date_tuple: time_elements_tuple = (self._year, self._month, self._day_of_month, time_tuple[0], time_tuple[1], time_tuple[2]) else: time_elements_tuple = (date_tuple[0], date_tuple[1], date_tuple[2], time_tuple[0], time_tuple[1], time_tuple[2]) try: date_time = dfdatetime_time_elements.TimeElements( time_elements_tuple=time_elements_tuple) except ValueError: parser_mediator.ProduceExtractionWarning( 'invalid date time value: {0!s}'.format(time_elements_tuple)) return event_data = IISEventData() for key, value in structure.items(): if key in ('date', 'date_time', 'time') or value == '-': continue if isinstance(value, pyparsing.ParseResults): value = ''.join(value) setattr(event_data, key, value) event = time_events.DateTimeValuesEvent( date_time, definitions.TIME_DESCRIPTION_WRITTEN) parser_mediator.ProduceEventWithEventData(event, event_data)
def _ParseLogLine(self, parser_mediator, structure, key): """Parse a single log line and produce an event object. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. structure (pyparsing.ParseResults): structure of tokens derived from a line of a text file. key (str): name of the parsed structure. """ time_elements_tuple = self._GetTimeElementsTuple(structure) try: date_time = dfdatetime_time_elements.TimeElements( time_elements_tuple=time_elements_tuple) except ValueError: parser_mediator.ProduceExtractionWarning( 'invalid date time value: {0!s}'.format(time_elements_tuple)) return self._last_month = time_elements_tuple[1] if key == 'logline': self._previous_structure = structure message = self._GetValueFromStructure(structure, 'message') else: repeat_count = self._GetValueFromStructure(structure, 'times') previous_message = self._GetValueFromStructure( self._previous_structure, 'message') message = 'Repeated {0:d} times: {1:s}'.format( repeat_count, previous_message) structure = self._previous_structure # It uses CarsNotIn structure which leaves whitespaces # at the beginning of the sender and the caller. caller = self._GetValueFromStructure(structure, 'caller') if caller: caller = caller.strip() # TODO: move this to formatter. if not caller: caller = 'unknown' sender = self._GetValueFromStructure(structure, 'sender') if sender: sender = sender.strip() event_data = MacOSSecuritydLogEventData() event_data.caller = caller event_data.facility = self._GetValueFromStructure(structure, 'facility') event_data.level = self._GetValueFromStructure(structure, 'level') event_data.message = message event_data.security_api = self._GetValueFromStructure( structure, 'security_api', default_value='unknown') event_data.sender_pid = self._GetValueFromStructure(structure, 'sender_pid') event_data.sender = sender event = time_events.DateTimeValuesEvent( date_time, definitions.TIME_DESCRIPTION_ADDED) parser_mediator.ProduceEventWithEventData(event, event_data)
def _ParseUpdateKeyValue(self, parser_mediator, registry_value): """Parses the UpdateKey value. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. registry_value (dfwinreg.WinRegistryValue): Windows Registry value. Returns: dfdatetime_time_elements.TimeElements: date and time value or None if not available. """ if not registry_value.DataIsString(): parser_mediator.ProduceExtractionWarning( 'unsupported UpdateKey value data type: {0:s}'.format( registry_value.data_type_string)) return None date_time_string = registry_value.GetDataAsObject() if not date_time_string: parser_mediator.ProduceExtractionWarning('missing UpdateKey value data') return None re_match = self._UPDATE_DATE_TIME_RE.match(date_time_string) if not re_match: parser_mediator.ProduceExtractionWarning( 'unsupported UpdateKey value data: {0!s}'.format(date_time_string)) return None month, day_of_month, year, hours, minutes, seconds, part_of_day = ( re_match.groups()) try: year = int(year, 10) month = int(month, 10) day_of_month = int(day_of_month, 10) hours = int(hours, 10) minutes = int(minutes, 10) seconds = int(seconds, 10) except (TypeError, ValueError): parser_mediator.ProduceExtractionWarning( 'invalid UpdateKey date time value: {0!s}'.format(date_time_string)) return None if part_of_day == 'PM': hours += 12 time_elements_tuple = (year, month, day_of_month, hours, minutes, seconds) try: date_time = dfdatetime_time_elements.TimeElements( time_elements_tuple=time_elements_tuple) date_time.is_local_time = True except ValueError: parser_mediator.ProduceExtractionWarning( 'invalid UpdateKey date time value: {0!s}'.format( time_elements_tuple)) return None return date_time
def _ParseLogLine(self, parser_mediator, structure): """Parses a log line. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. structure (pyparsing.ParseResults): structure of tokens derived from a line of a text file. """ time_elements_tuple = self._GetTimeElementsTuple(structure) try: date_time = dfdatetime_time_elements.TimeElements( time_elements_tuple=time_elements_tuple) date_time.is_local_time = True except ValueError: parser_mediator.ProduceExtractionWarning( 'invalid date time value: {0!s}'.format(time_elements_tuple)) return event_data = VsftpdEventData() event_data.text = self._GetValueFromStructure(structure, 'text') event = time_events.DateTimeValuesEvent( date_time, definitions.TIME_DESCRIPTION_ADDED, time_zone=parser_mediator.timezone) parser_mediator.ProduceEventWithEventData(event, event_data)
def VerifyStructure(self, parser_mediator, line): """Verify that this file is a vsftpd log file. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfVFS. line (str): line from a text file. Returns: bool: True if the line is in the expected format, False if not. """ try: structure = self._LOG_LINE.parseString(line) except pyparsing.ParseException: return False if (' [pid ' not in line) or (': Client ' not in line): return False time_elements_tuple = self._GetTimeElementsTuple(structure) try: dfdatetime_time_elements.TimeElements( time_elements_tuple=time_elements_tuple) except ValueError: logger.debug(( 'Not a vsftpd log file, invalid date and time: ' '{0!s}').format(time_elements_tuple)) return False return True
def VerifyStructure(self, parser_mediator, line): """Verify that this file is a XChat log file. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. line (bytes): line from a text file. Returns: bool: True if the line is in the expected format, False if not. """ try: structure = self._HEADER.parseString(line) except pyparsing.ParseException: logging.debug(u'Not a XChat log file') return False _, month, day, hours, minutes, seconds, year = structure.date_time month = timelib.MONTH_DICT.get(month.lower(), 0) time_elements_tuple = (year, month, day, hours, minutes, seconds) try: dfdatetime_time_elements.TimeElements( time_elements_tuple=time_elements_tuple) except ValueError: logging.debug( u'Not a XChat log file, invalid date and time: {0!s}'.format( structure.date_time)) return False return True
def testCopyToDateTimeStringISO8601(self): """Tests the CopyToDateTimeStringISO8601 function.""" time_elements_object = time_elements.TimeElements( time_elements_tuple=(2010, 8, 12, 20, 6, 31)) date_time_string = time_elements_object.CopyToDateTimeStringISO8601() self.assertEqual(date_time_string, '2010-08-12T20:06:31Z')
def VerifyStructure(self, parser_mediator, line): """Verify that this file is a Sophos Anti-Virus log file. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfVFS. line (str): line from a text file. Returns: bool: True if the line is in the expected format, False if not. """ try: structure = self._LOG_LINE.parseString(line) except pyparsing.ParseException: logger.debug('Not a Sophos Anti-Virus log file') return False # Expect spaces at position 9 and 16. if ' ' not in (line[8], line[15]): logger.debug('Not a Sophos Anti-Virus log file') return False try: dfdatetime_time_elements.TimeElements( time_elements_tuple=structure.date_time) except ValueError: logger.debug( ('Not a Sophos Anti-Virus log file, invalid date and time: ' '{0!s}').format(structure.date_time)) return False return True
def ParseRecord(self, parser_mediator, key, structure): """Parses a matching entry. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. key (str): name of the parsed structure. structure (pyparsing.ParseResults): elements parsed from the file. Raises: ParseError: when the structure type is unknown. """ if key not in self._SUPPORTED_KEYS: raise errors.ParseError( 'Unable to parse record, unknown structure: {0:s}'.format(key)) date_time = dfdatetime_time_elements.TimeElements() date_time_string = self._GetValueFromStructure(structure, 'date_time') try: iso_date_time = self._GetISO8601String(date_time_string) date_time.CopyFromStringISO8601(iso_date_time) except ValueError: parser_mediator.ProduceExtractionWarning( 'invalid date time value: {0!s}'.format(date_time_string)) return event = time_events.DateTimeValuesEvent( date_time, definitions.TIME_DESCRIPTION_RECORDED) event_data = ApacheAccessEventData() event_data.ip_address = self._GetValueFromStructure( structure, 'ip_address') event_data.remote_name = self._GetValueFromStructure( structure, 'remote_name') event_data.user_name = self._GetValueFromStructure( structure, 'user_name') event_data.http_request = self._GetValueFromStructure( structure, 'http_request') event_data.http_response_code = self._GetValueFromStructure( structure, 'response_code') event_data.http_response_bytes = self._GetValueFromStructure( structure, 'response_bytes') if key in ('combined_log_format', 'vhost_combined_log_format'): event_data.http_request_referer = self._GetValueFromStructure( structure, 'referer') event_data.http_request_user_agent = self._GetValueFromStructure( structure, 'user_agent') if key == 'vhost_combined_log_format': event_data.server_name = self._GetValueFromStructure( structure, 'server_name') event_data.port_number = self._GetValueFromStructure( structure, 'port_number') parser_mediator.ProduceEventWithEventData(event, event_data)
def ParseFileObject(self, parser_mediator, file_object): """Parses an Opera typed history file-like object. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. file_object (dfvfs.FileIO): file-like object. Raises: UnableToParseFile: when the file cannot be parsed. """ data = file_object.read(self._HEADER_READ_SIZE) if not data.startswith(b'<?xml'): raise errors.UnableToParseFile( 'Not an Opera typed history file [not a XML]') _, _, data = data.partition(b'\n') if not data.startswith(b'<typed_history'): raise errors.UnableToParseFile( 'Not an Opera typed history file [wrong XML root key]') # For ElementTree to work we need to work on a file object seeked # to the beginning. file_object.seek(0, os.SEEK_SET) xml = ElementTree.parse(file_object) for history_item in xml.iterfind('typed_history_item'): event_data = OperaTypedHistoryEventData() event_data.entry_type = history_item.get('type', None) event_data.url = history_item.get('content', None) if event_data.entry_type == 'selected': event_data.entry_selection = 'Filled from autocomplete.' elif event_data.entry_type == 'text': event_data.entry_selection = 'Manually typed.' last_typed_time = history_item.get('last_typed', None) if last_typed_time is None: parser_mediator.ProduceExtractionWarning( 'missing last typed time.') continue date_time = dfdatetime_time_elements.TimeElements() try: date_time.CopyFromStringISO8601(last_typed_time) except ValueError as exception: parser_mediator.ProduceExtractionWarning( 'unsupported last typed time: {0:s} with error: {1!s}.'. format(last_typed_time, exception)) continue event = time_events.DateTimeValuesEvent( date_time, definitions.TIME_DESCRIPTION_LAST_VISITED) parser_mediator.ProduceEventWithEventData(event, event_data)
def testCopyFromDateTimeString(self): """Tests the CopyFromDateTimeString function.""" time_elements_object = time_elements.TimeElements() expected_time_elements_tuple = (2010, 8, 12, 0, 0, 0) expected_number_of_seconds = 1281571200 time_elements_object.CopyFromDateTimeString('2010-08-12') self.assertEqual(time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual(time_elements_object._number_of_seconds, expected_number_of_seconds) expected_time_elements_tuple = (2010, 8, 12, 21, 6, 31) expected_number_of_seconds = 1281647191 time_elements_object.CopyFromDateTimeString('2010-08-12 21:06:31') self.assertEqual(time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual(time_elements_object._number_of_seconds, expected_number_of_seconds) expected_time_elements_tuple = (2010, 8, 12, 21, 6, 31) expected_number_of_seconds = 1281647191 time_elements_object.CopyFromDateTimeString( '2010-08-12 21:06:31.546875') self.assertEqual(time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual(time_elements_object._number_of_seconds, expected_number_of_seconds) expected_time_elements_tuple = (2010, 8, 12, 21, 6, 31) expected_number_of_seconds = 1281650791 time_elements_object.CopyFromDateTimeString( '2010-08-12 21:06:31.546875-01:00') self.assertEqual(time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual(time_elements_object._number_of_seconds, expected_number_of_seconds) self.assertEqual(time_elements_object.time_zone_offset, -60) expected_time_elements_tuple = (2010, 8, 12, 21, 6, 31) expected_number_of_seconds = 1281643591 time_elements_object.CopyFromDateTimeString( '2010-08-12 21:06:31.546875+01:00') self.assertEqual(time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual(time_elements_object._number_of_seconds, expected_number_of_seconds) self.assertEqual(time_elements_object.time_zone_offset, 60) expected_time_elements_tuple = (1601, 1, 2, 0, 0, 0) expected_number_of_seconds = -11644387200 time_elements_object.CopyFromDateTimeString('1601-01-02 00:00:00') self.assertEqual(time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual(time_elements_object._number_of_seconds, expected_number_of_seconds)
def testInitialize(self): """Tests the initialization function.""" time_elements_object = time_elements.TimeElements() self.assertIsNotNone(time_elements_object) expected_time_elements_tuple = (2010, 8, 12, 20, 6, 31) time_elements_object = time_elements.TimeElements( time_elements_tuple=(2010, 8, 12, 20, 6, 31)) self.assertIsNotNone(time_elements_object) self.assertEqual( time_elements_object._time_elements_tuple, expected_time_elements_tuple) with self.assertRaises(ValueError): time_elements.TimeElements( time_elements_tuple=(2010, 8, 12, 20, 6)) with self.assertRaises(ValueError): time_elements.TimeElements( time_elements_tuple=(2010, 13, 12, 20, 6, 31))
def testCopyFromString(self): """Tests the CopyFromString function.""" time_elements_object = time_elements.TimeElements() expected_time_elements_tuple = (2010, 8, 12, 0, 0, 0) expected_number_of_seconds = 1281571200 time_elements_object.CopyFromString('2010-08-12') self.assertEqual(time_elements_object._time_elements_tuple, expected_time_elements_tuple) self.assertEqual(time_elements_object._number_of_seconds, expected_number_of_seconds)
def _ConvertToTimestamp(self, date, time): """Converts date and time strings into a timestamp. Recent versions of Office Scan write a log field with a Unix timestamp. Older versions may not write this field; their logs only provide a date and a time expressed in the local time zone. This functions handles the latter case. Args: date (str): date as an 8-character string in the YYYYMMDD format. time (str): time as a 3 or 4-character string in the [H]HMM format or a 6-character string in the HHMMSS format. Returns: dfdatetime_time_elements.TimestampElements: the parsed timestamp. Raises: ValueError: if the date and time values cannot be parsed. """ # Check that the strings have the correct length. if len(date) != 8: raise ValueError('Unsupported length of date string: {0!s}'.format( repr(date))) if len(time) < 3 or len(time) > 4: raise ValueError('Unsupported length of time string: {0!s}'.format( repr(time))) # Extract the date. try: year = int(date[:4], 10) month = int(date[4:6], 10) day = int(date[6:8], 10) except (TypeError, ValueError): raise ValueError('Unable to parse date string: {0!s}'.format( repr(date))) # Extract the time. Note that a single-digit hour value has no leading zero. try: hour = int(time[:-2], 10) minutes = int(time[-2:], 10) except (TypeError, ValueError): raise ValueError('Unable to parse time string: {0!s}'.format( repr(date))) time_elements_tuple = (year, month, day, hour, minutes, 0) date_time = dfdatetime_time_elements.TimeElements( time_elements_tuple=time_elements_tuple) date_time.is_local_time = True # TODO: add functionality to dfdatetime to control precision. date_time._precision = dfdatetime_definitions.PRECISION_1_MINUTE # pylint: disable=protected-access return date_time
def _ParseLogLine(self, parser_mediator, structure): """Parse a single log line and and produce an event object. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. structure (pyparsing.ParseResults): structure of tokens derived from a line of a text file. """ time_elements_tuple = self._GetValueFromStructure( structure, 'date_time') try: date_time = dfdatetime_time_elements.TimeElements( time_elements_tuple=time_elements_tuple) date_time.is_local_time = True except ValueError: parser_mediator.ProduceExtractionWarning( 'invalid date time value: {0!s}'.format(time_elements_tuple)) return event_data = WinFirewallEventData() event_data.action = self._GetValueFromStructure(structure, 'action') event_data.dest_ip = self._GetValueFromStructure(structure, 'dest_ip') event_data.dest_port = self._GetValueFromStructure( structure, 'dest_port') event_data.flags = self._GetValueFromStructure(structure, 'flags') event_data.icmp_code = self._GetValueFromStructure( structure, 'icmp_code') event_data.icmp_type = self._GetValueFromStructure( structure, 'icmp_type') event_data.info = self._GetValueFromStructure(structure, 'info') event_data.path = self._GetValueFromStructure(structure, 'path') event_data.protocol = self._GetValueFromStructure( structure, 'protocol') event_data.size = self._GetValueFromStructure(structure, 'size') event_data.source_ip = self._GetValueFromStructure( structure, 'source_ip') event_data.source_port = self._GetValueFromStructure( structure, 'source_port') event_data.tcp_ack = self._GetValueFromStructure(structure, 'tcp_ack') event_data.tcp_seq = self._GetValueFromStructure(structure, 'tcp_seq') event_data.tcp_win = self._GetValueFromStructure(structure, 'tcp_win') if self._use_local_timezone: time_zone = parser_mediator.timezone else: time_zone = pytz.UTC event = time_events.DateTimeValuesEvent( date_time, definitions.TIME_DESCRIPTION_WRITTEN, time_zone=time_zone) parser_mediator.ProduceEventWithEventData(event, event_data)
def _ParseHeader(self, parser_mediator, structure): """Parses a log header. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. structure (pyparsing.ParseResults): structure of tokens derived from a line of a text file. """ time_elements_tuple = self._GetValueFromStructure( structure, 'date_time') # TODO: what if time_elements_tuple is None. _, month, day, hours, minutes, seconds, year = time_elements_tuple month = timelib.MONTH_DICT.get(month.lower(), 0) time_elements_tuple = (year, month, day, hours, minutes, seconds) try: date_time = dfdatetime_time_elements.TimeElements( time_elements_tuple=time_elements_tuple) date_time.is_local_time = True except ValueError: parser_mediator.ProduceExtractionWarning( 'invalid date time value: {0!s}'.format(time_elements_tuple)) return self._last_month = month event_data = XChatLogEventData() log_action = self._GetValueFromStructure(structure, 'log_action', default_value=[]) if log_action[0] == 'BEGIN': self._xchat_year = year event_data.text = 'XChat start logging' elif log_action[0] == 'END': self._xchat_year = None event_data.text = 'XChat end logging' else: logger.debug('Unknown log action: {0:s}.'.format( ' '.join(log_action))) return event = time_events.DateTimeValuesEvent( date_time, definitions.TIME_DESCRIPTION_ADDED, time_zone=parser_mediator.timezone) parser_mediator.ProduceEventWithEventData(event, event_data)
def VerifyStructure(self, parser_mediator, line): """Verify that this file is a Mac AppFirewall log file. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. line (str): line from a text file. Returns: bool: True if the line is in the expected format, False if not. """ self._last_month = 0 self._year_use = parser_mediator.GetEstimatedYear() try: structure = self.FIREWALL_LINE.parseString(line) except pyparsing.ParseException as exception: logger.debug(( 'Unable to parse file as a Mac AppFirewall log file with error: ' '{0!s}').format(exception)) return False action = self._GetValueFromStructure(structure, 'action') if action != 'creating /var/log/appfirewall.log': logger.debug( 'Not a Mac AppFirewall log file, invalid action: {0!s}'.format( action)) return False status = self._GetValueFromStructure(structure, 'status') if status != 'Error': logger.debug( 'Not a Mac AppFirewall log file, invalid status: {0!s}'.format( status)) return False time_elements_tuple = self._GetTimeElementsTuple(structure) try: dfdatetime_time_elements.TimeElements( time_elements_tuple=time_elements_tuple) except ValueError: logger.debug( ('Not a Mac AppFirewall log file, invalid date and time: ' '{0!s}').format(time_elements_tuple)) return False self._last_month = time_elements_tuple[1] return True
def testGetMessages(self): """Tests the GetMessages method.""" date_time = dfdatetime_time_elements.TimeElements() date_time.CopyFromString(u'2016-11-14 20:36:37.222') event = time_events.DateTimeValuesEvent( date_time, definitions.TIME_DESCRIPTION_MODIFICATION) event.data_type = u'mac:wifilog:line' event.action = u'Interface en0 turn up.' event.agent = u'airportd[88]' event.function = u'airportdProcessDLILEvent' event.text = u'en0 attached (up)' expected_messages = ( u'Action: Interface en0 turn up. ' u'Agent: airportd[88] ' u'(airportdProcessDLILEvent) ' u'Log: en0 attached (up)', u'Action: Interface en0 turn up.') messages = self._formatter.GetMessages(None, event) self.assertEqual(messages, expected_messages) date_time = dfdatetime_time_elements.TimeElements() date_time.CopyFromString(u'2017-01-02 00:10:15') event = time_events.DateTimeValuesEvent( date_time, definitions.TIME_DESCRIPTION_MODIFICATION) event.data_type = u'mac:wifilog:line' event.text = u'test-macbookpro newsyslog[50498]: logfile turned over' expected_messages = ( u'Log: test-macbookpro newsyslog[50498]: logfile turned over', u'Log: test-macbookpro newsyslog[50498]: logfile turned over') messages = self._formatter.GetMessages(None, event) self.assertEqual(messages, expected_messages)
def _ParseLogLine(self, parser_mediator, structure, key): """Parse a single log line and produce an event object. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. key (str): identifier of the structure of tokens. structure (pyparsing.ParseResults): structure of tokens derived from a line of a text file. """ time_elements_tuple = self._GetTimeElementsTuple(structure) try: date_time = dfdatetime_time_elements.TimeElements( time_elements_tuple=time_elements_tuple) except ValueError: parser_mediator.ProduceExtractionError( 'invalid date time value: {0!s}'.format(structure.date_time)) return self._last_month = time_elements_tuple[1] # If the actual entry is a repeated entry, we take the basic information # from the previous entry, but using the timestmap from the actual entry. if key == 'logline': self._previous_structure = structure else: structure = self._previous_structure # Pyparsing reads in RAW, but the text is in UTF8. try: action = structure.action.decode('utf-8') except UnicodeDecodeError: logging.warning( 'Decode UTF8 failed, the message string may be cut short.') action = structure.action.decode('utf-8', 'ignore') event_data = MacAppFirewallLogEventData() event_data.action = action event_data.agent = structure.agent event_data.computer_name = structure.computer_name # Due to the use of CharsNotIn pyparsing structure contains whitespaces # that need to be removed. event_data.process_name = structure.process_name.strip() event_data.status = structure.status event = time_events.DateTimeValuesEvent( date_time, definitions.TIME_DESCRIPTION_ADDED) parser_mediator.ProduceEventWithEventData(event, event_data)
def _ParseLogLine(self, parser_mediator, structure): """Parse a single log line and produce an event object. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. structure (pyparsing.ParseResults): structure parsed from the log file. """ if structure.date_time: time_elements_tuple = structure.date_time elif structure.date and structure.time: year, month, day_of_month = structure.date hours, minutes, seconds = structure.time time_elements_tuple = (year, month, day_of_month, hours, minutes, seconds) elif structure.time: hours, minutes, seconds = structure.time time_elements_tuple = (self._year, self._month, self._day_of_month, hours, minutes, seconds) else: parser_mediator.ProduceExtractionError( 'missing date and time values') return try: date_time = dfdatetime_time_elements.TimeElements( time_elements_tuple=time_elements_tuple) except ValueError: parser_mediator.ProduceExtractionError( 'invalid date time value: {0!s}'.format(time_elements_tuple)) return event_data = IISEventData() for key, value in iter(structure.items()): if key in ('date', 'date_time', 'time') or value == '-': continue if isinstance(value, pyparsing.ParseResults): value = ''.join(value) setattr(event_data, key, value) event = time_events.DateTimeValuesEvent( date_time, definitions.TIME_DESCRIPTION_WRITTEN) parser_mediator.ProduceEventWithEventData(event, event_data)
def ParseRecord(self, parser_mediator, key, structure): """Parses a structure of tokens derived from a line of a text file. Args: parser_mediator (ParserMediator): parser mediator. key (str): identifier of the structure of tokens. structure (pyparsing.ParseResults): structure of tokens derived from a line of a text file. Raises: ParseError: when the structure type is unknown. """ if key != 'line': raise errors.ParseError( 'Unable to parse record, unknown structure: {0:s}'.format(key)) # Ensure time_elements_tuple is not a pyparsing.ParseResults otherwise # copy.deepcopy() of the dfDateTime object will fail on Python 3.8 with: # "TypeError: 'str' object is not callable" due to pyparsing.ParseResults # overriding __getattr__ with a function that returns an empty string when # named token does not exists. time_elements_structure = structure.get('date_time', None) try: year, month, day_of_month, hours, minutes, seconds = ( time_elements_structure) date_time = dfdatetime_time_elements.TimeElements( time_elements_tuple=(year, month, day_of_month, hours, minutes, seconds)) except (TypeError, ValueError): parser_mediator.ProduceExtractionWarning( 'invalid date time value: {0!s}'.format( time_elements_structure)) return body_text = self._GetValueFromStructure(structure, 'body') if not body_text: parser_mediator.ProduceExtractionWarning('missing body text') return event_data = DpkgEventData() event_data.body = body_text event = time_events.DateTimeValuesEvent( date_time, definitions.TIME_DESCRIPTION_ADDED) parser_mediator.ProduceEventWithEventData(event, event_data)
def _GetDateTime(self, structure): """Retrieves the date and time from a date and time values structure. The date and time values in Apache access log files are formatted as: "[18/Sep/2011:19:18:28 -0400]". Args: structure (pyparsing.ParseResults): structure of tokens derived from a line of a text file. Returns: dfdatetime.DateTimeValues: date and time. Raises: ValueError: if the structure cannot be converted into a date time string. """ year = self._GetValueFromStructure(structure, 'year') month = self._GetValueFromStructure(structure, 'month') try: month = self._MONTH_DICT.get(month.lower(), 0) except AttributeError as exception: raise ValueError( 'unable to parse month with error: {0!s}.'.format(exception)) day_of_month = self._GetValueFromStructure(structure, 'day') hours = self._GetValueFromStructure(structure, 'hours') minutes = self._GetValueFromStructure(structure, 'minutes') seconds = self._GetValueFromStructure(structure, 'seconds') time_offset = self._GetValueFromStructure(structure, 'time_offset') try: time_zone_offset = int(time_offset[1:3], 10) * 60 time_zone_offset += int(time_offset[3:5], 10) if time_offset[0] == '-': time_zone_offset *= -1 except (TypeError, ValueError) as exception: raise ValueError( 'unable to parse time zone offset with error: {0!s}.'.format( exception)) time_elements_tuple = (year, month, day_of_month, hours, minutes, seconds) return dfdatetime_time_elements.TimeElements( time_elements_tuple=time_elements_tuple, time_zone_offset=time_zone_offset)
def _ParseHeader(self, parser_mediator, structure): """Parses a log header. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. structure (pyparsing.ParseResults): structure of tokens derived from a line of a text file. """ _, month, day, hours, minutes, seconds, year = structure.date_time month = timelib.MONTH_DICT.get(month.lower(), 0) time_elements_tuple = (year, month, day, hours, minutes, seconds) try: date_time = dfdatetime_time_elements.TimeElements( time_elements_tuple=time_elements_tuple) date_time.is_local_time = True except ValueError: parser_mediator.ProduceExtractionError( u'invalid date time value: {0!s}'.format(structure.date_time)) return self._last_month = month event_data = XChatLogEventData() if structure.log_action[0] == u'BEGIN': self._xchat_year = year event_data.text = u'XChat start logging' elif structure.log_action[0] == u'END': self._xchat_year = None event_data.text = u'XChat end logging' else: logging.debug(u'Unknown log action: {0:s}.'.format(u' '.join( structure.log_action))) return event = time_events.DateTimeValuesEvent( date_time, eventdata.EventTimestamp.ADDED_TIME, time_zone=parser_mediator.timezone) parser_mediator.ProduceEventWithEventData(event, event_data)
def _ParseLogLine(self, parser_mediator, structure, key): """Parse a single log line and produce an event object. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. file_object (dfvfs.FileIO): a file-like object. structure (pyparsing.ParseResults): structure of tokens derived from a line of a text file. """ time_elements_tuple = self._GetTimeElementsTuple(structure) try: date_time = dfdatetime_time_elements.TimeElements( time_elements_tuple=time_elements_tuple) except ValueError: parser_mediator.ProduceExtractionError( u'invalid date time value: {0!s}'.format(structure.date_time)) return self._last_month = time_elements_tuple[1] if key == u'logline': self._previous_structure = structure message = structure.message else: message = u'Repeated {0:d} times: {1:s}'.format( structure.times, self._previous_structure.message) structure = self._previous_structure # It uses CarsNotIn structure which leaves whitespaces # at the beginning of the sender and the caller. event_data = MacSecuritydLogEventData() event_data.caller = structure.caller.strip() or u'unknown' event_data.facility = structure.facility event_data.level = structure.level event_data.message = message event_data.security_api = structure.security_api or u'unknown' event_data.sender_pid = structure.sender_pid event_data.sender = structure.sender.strip() event = time_events.DateTimeValuesEvent( date_time, definitions.TIME_DESCRIPTION_ADDED) parser_mediator.ProduceEventWithEventData(event, event_data)
def _ParseLogLine(self, parser_mediator, structure, key): """Parse a single log line and produce an event object. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. key (str): identifier of the structure of tokens. structure (pyparsing.ParseResults): structure of tokens derived from a line of a text file. """ time_elements_tuple = self._GetTimeElementsTuple(structure) try: date_time = dfdatetime_time_elements.TimeElements( time_elements_tuple=time_elements_tuple) except ValueError: parser_mediator.ProduceExtractionWarning( 'invalid date time value: {0!s}'.format(time_elements_tuple)) return self._last_month = time_elements_tuple[1] # If the actual entry is a repeated entry, we take the basic information # from the previous entry, but use the timestamp from the actual entry. if key == 'logline': self._previous_structure = structure else: structure = self._previous_structure event_data = MacAppFirewallLogEventData() event_data.action = self._GetValueFromStructure(structure, 'action') event_data.agent = self._GetValueFromStructure(structure, 'agent') event_data.computer_name = self._GetValueFromStructure( structure, 'computer_name') event_data.status = self._GetValueFromStructure(structure, 'status') # Due to the use of CharsNotIn pyparsing structure contains whitespaces # that need to be removed. process_name = self._GetValueFromStructure(structure, 'process_name') if process_name: event_data.process_name = process_name.strip() event = time_events.DateTimeValuesEvent( date_time, definitions.TIME_DESCRIPTION_ADDED) parser_mediator.ProduceEventWithEventData(event, event_data)
def _ParseLogLine(self, parser_mediator, structure): """Parse a single log line and and produce an event object. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. structure (pyparsing.ParseResults): structure of tokens derived from a line of a text file. """ try: date_time = dfdatetime_time_elements.TimeElements( time_elements_tuple=structure.date_time) date_time.is_local_time = True except ValueError: parser_mediator.ProduceExtractionError( u'invalid date time value: {0!s}'.format(structure.date_time)) return event_data = WinFirewallEventData() event_data.action = structure.action event_data.dest_ip = structure.dest_ip event_data.dest_port = structure.dest_port event_data.flags = structure.flags event_data.icmp_code = structure.icmp_code event_data.icmp_type = structure.icmp_type event_data.info = structure.info event_data.path = structure.path event_data.protocol = structure.protocol event_data.size = structure.size event_data.source_ip = structure.source_ip event_data.source_port = structure.source_port event_data.tcp_ack = structure.tcp_ack event_data.tcp_seq = structure.tcp_seq event_data.tcp_win = structure.tcp_win if self._use_local_timezone: time_zone = parser_mediator.timezone else: time_zone = pytz.UTC event = time_events.DateTimeValuesEvent( date_time, eventdata.EventTimestamp.WRITTEN_TIME, time_zone=time_zone) parser_mediator.ProduceEventWithEventData(event, event_data)
def _BuildDateTime(time_elements_structure): """Builds time elements from an APT History time stamp. Args: time_elements_structure (pyparsing.ParseResults): structure of tokens derived from an APT History time stamp. Returns: dfdatetime.TimeElements: date and time extracted from the structure or None f the structure does not represent a valid string. """ try: date_time = dfdatetime_time_elements.TimeElements( time_elements_tuple=time_elements_structure) # APT History logs store date and time values in local time. date_time.is_local_time = True return date_time except ValueError: return None