def ParseRow(self, parser_mediator, row_offset, row): """Parses a line of the log file and produces events. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. row_offset (int): line number of the row. row (dict[str, str]): fields of a single row, as specified in COLUMNS. """ try: date_time = self._CreateDateTime(row['date'], row['time']) except errors.TimestampError as exception: parser_mediator.ProduceExtractionWarning( 'Unable to create date time with error: {0!s}'.format(exception)) date_time = dfdatetime_semantic_time.InvalidTime() event_data = McafeeAVEventData() event_data.action = row['action'] event_data.filename = row['filename'] event_data.offset = row_offset event_data.rule = row['rule'] event_data.status = row['status'] event_data.trigger_location = row['trigger_location'] event_data.username = row['username'] event = time_events.DateTimeValuesEvent( date_time, definitions.TIME_DESCRIPTION_WRITTEN) parser_mediator.ProduceEventWithEventData(event, event_data)
def _ParseLayerConfigJSON(self, parser_mediator, file_object): """Extracts events from a Docker filesystem layer configuration file. The path of each filesystem layer config file is: DOCKER_DIR/graph/<layer_id>/json Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. file_object (dfvfs.FileIO): a file-like object. Raises: UnableToParseFile: when the file is not a valid layer config file. """ file_content = file_object.read() file_content = codecs.decode(file_content, self._ENCODING) json_dict = json.loads(file_content) if 'docker_version' not in json_dict: raise errors.UnableToParseFile( 'not a valid Docker layer configuration file, missing ' '\'docker_version\' key.') time_string = json_dict.get('created', None) if time_string is not None: layer_creation_command_array = [ x.strip() for x in json_dict['container_config']['Cmd'] ] layer_creation_command = ' '.join( layer_creation_command_array).replace('\t', '') event_data = DockerJSONLayerEventData() event_data.command = layer_creation_command event_data.layer_id = self._GetIdentifierFromPath(parser_mediator) try: date_time = dfdatetime_time_elements.TimeElementsInMicroseconds( ) date_time.CopyFromStringISO8601(time_string) except ValueError as exception: parser_mediator.ProduceExtractionWarning( ('Unable to parse created time string: {0:s} with error: ' '{1!s}').format(time_string, exception)) date_time = dfdatetime_semantic_time.InvalidTime() event = time_events.DateTimeValuesEvent( date_time, definitions.TIME_DESCRIPTION_ADDED) parser_mediator.ProduceEventWithEventData(event, event_data)
def _ParseContainerLogJSON(self, parser_mediator, file_object): """Extract events from a Docker container log files. The format is one JSON formatted log message per line. The path of each container log file (which logs the container stdout and stderr) is: DOCKER_DIR/containers/<container_id>/<container_id>-json.log Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. file_object (dfvfs.FileIO): a file-like object. """ container_id = self._GetIdentifierFromPath(parser_mediator) text_file_object = text_file.TextFile(file_object) for log_line in text_file_object: json_log_line = json.loads(log_line) time_string = json_log_line.get('time', None) if time_string is None: continue event_data = DockerJSONContainerLogEventData() event_data.container_id = container_id event_data.log_line = json_log_line.get('log', None) event_data.log_source = json_log_line.get('stream', None) # TODO: pass line number to offset or remove. event_data.offset = 0 try: date_time = dfdatetime_time_elements.TimeElementsInMicroseconds( ) date_time.CopyFromStringISO8601(time_string) except ValueError as exception: parser_mediator.ProduceExtractionWarning( ('Unable to parse written time string: {0:s} with error: ' '{1!s}').format(time_string, exception)) date_time = dfdatetime_semantic_time.InvalidTime() event = time_events.DateTimeValuesEvent( date_time, definitions.TIME_DESCRIPTION_WRITTEN) parser_mediator.ProduceEventWithEventData(event, event_data)
def testFormatDateTime(self): """Tests the _FormatDateTime function with dynamic time.""" output_mediator = self._CreateOutputMediator(dynamic_time=True) test_helper = formatting_helper.FieldFormattingHelper(output_mediator) event, event_data, event_data_stream = ( containers_test_lib.CreateEventFromValues(self._TEST_EVENTS[0])) date_time_string = test_helper._FormatDateTime(event, event_data, event_data_stream) self.assertEqual(date_time_string, '2012-06-27T18:17:01.000000+00:00') output_mediator.SetTimeZone('Europe/Amsterdam') date_time_string = test_helper._FormatDateTime(event, event_data, event_data_stream) self.assertEqual(date_time_string, '2012-06-27T20:17:01.000000+02:00') output_mediator.SetTimeZone('UTC') event, event_data, event_data_stream = ( containers_test_lib.CreateEventFromValues(self._TEST_EVENTS[1])) event.date_time._time_zone_offset = 120 date_time_string = test_helper._FormatDateTime(event, event_data, event_data_stream) self.assertEqual(date_time_string, '2012-06-27T18:17:01.000000+00:00') event.date_time = dfdatetime_semantic_time.InvalidTime() date_time_string = test_helper._FormatDateTime(event, event_data, event_data_stream) self.assertEqual(date_time_string, 'Invalid') # Test with event.is_local_time event, event_data, event_data_stream = ( containers_test_lib.CreateEventFromValues(self._TEST_EVENTS[0])) event.timestamp -= 120 * 60 * 1000000 event.date_time.is_local_time = True date_time_string = test_helper._FormatDateTime(event, event_data, event_data_stream) self.assertEqual(date_time_string, '2012-06-27T16:17:01.000000+00:00')
def ParseFileObject(self, parser_mediator, file_object): """Parses a Java WebStart Cache IDX file-like object. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. file_object (dvfvs.FileIO): a file-like object to parse. Raises: UnableToParseFile: when the file cannot be parsed. """ file_header_map = self._GetDataTypeMap('java_idx_file_header') try: file_header, file_offset = self._ReadStructureFromFileObject( file_object, 0, file_header_map) except (ValueError, errors.ParseError) as exception: raise errors.UnableToParseFile( 'Unable to parse file header with error: {0!s}'.format( exception)) if not file_header.format_version in self._SUPPORTED_FORMAT_VERSIONS: raise errors.UnableToParseFile('Unsupported format version.') if file_header.format_version == 602: section1_map = self._GetDataTypeMap('java_idx_602_section1') elif file_header.format_version in (603, 604): section1_map = self._GetDataTypeMap('java_idx_603_section1') elif file_header.format_version == 605: section1_map = self._GetDataTypeMap('java_idx_605_section1') try: section1, data_size = self._ReadStructureFromFileObject( file_object, file_offset, section1_map) except (ValueError, errors.ParseError) as exception: raise errors.UnableToParseFile(( 'Unable to parse section 1 (format version: {0:d}) with error: ' '{1!s}').format(file_header.format_version, exception)) file_offset += data_size if file_header.format_version == 602: section2_map = self._GetDataTypeMap('java_idx_602_section2') elif file_header.format_version in (603, 604, 605): file_offset = 128 section2_map = self._GetDataTypeMap('java_idx_603_section2') try: section2, data_size = self._ReadStructureFromFileObject( file_object, file_offset, section2_map) except (ValueError, errors.ParseError) as exception: raise errors.UnableToParseFile(( 'Unable to parse section 2 (format version: {0:d}) with error: ' '{1!s}').format(file_header.format_version, exception)) file_offset += data_size if not section2.url: raise errors.UnableToParseFile('URL not found in file.') date_http_header = None for _ in range(section2.number_of_http_headers): http_header_map = self._GetDataTypeMap('java_idx_http_header') try: http_header, data_size = self._ReadStructureFromFileObject( file_object, file_offset, http_header_map) except (ValueError, errors.ParseError) as exception: parser_mediator.ProduceExtractionWarning( 'Unable to parse HTTP header value at offset: 0x{0:08x}'. format(file_offset)) break file_offset += data_size if http_header.name == 'date': date_http_header = http_header break event_data = JavaIDXEventData() event_data.idx_version = file_header.format_version event_data.ip_address = getattr(section2, 'ip_address', None) event_data.url = section2.url date_time = dfdatetime_java_time.JavaTime( timestamp=section1.modification_time) # TODO: Move the timestamp description into definitions. event = time_events.DateTimeValuesEvent(date_time, 'File Hosted Date') parser_mediator.ProduceEventWithEventData(event, event_data) if section1.expiration_time: date_time = dfdatetime_java_time.JavaTime( timestamp=section1.expiration_time) event = time_events.DateTimeValuesEvent( date_time, definitions.TIME_DESCRIPTION_EXPIRATION) parser_mediator.ProduceEventWithEventData(event, event_data) if date_http_header: # A HTTP header date and time should be formatted according to RFC 1123. # The date "should" be in UTC or have associated time zone information # in the string itself. If that is not the case then there is no reliable # method for Plaso to determine the proper time zone, so the assumption # is that the date and time is in UTC. try: date_time = dfdatetime_time_elements.TimeElements() date_time.CopyFromStringRFC1123(date_http_header.value) except ValueError as exception: parser_mediator.ProduceExtractionWarning(( 'Unable to parse date HTTP header string: {0:s} with error: ' '{1!s}').format(date_http_header.value, exception)) date_time = dfdatetime_semantic_time.InvalidTime() event = time_events.DateTimeValuesEvent( date_time, definitions.TIME_DESCRIPTION_FILE_DOWNLOADED) parser_mediator.ProduceEventWithEventData(event, event_data)
def testInitialize(self): """Tests the __init__ function.""" invalid_time_object = semantic_time.InvalidTime() self.assertEqual(invalid_time_object.string, 'Invalid')
def _ParseContainerConfigJSON(self, parser_mediator, file_object): """Extracts events from a Docker container configuration file. The path of each container config file is: DOCKER_DIR/containers/<container_id>/config.json Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. file_object (dfvfs.FileIO): a file-like object. Raises: UnableToParseFile: when the file is not a valid container config file. """ file_content = file_object.read() file_content = codecs.decode(file_content, self._ENCODING) json_dict = json.loads(file_content) if 'Driver' not in json_dict: raise errors.UnableToParseFile( 'not a valid Docker container configuration file, ' 'missing ' '\'Driver\' key.') container_id_from_path = self._GetIdentifierFromPath(parser_mediator) container_id_from_json = json_dict.get('ID', None) if not container_id_from_json: raise errors.UnableToParseFile( 'not a valid Docker layer configuration file, the \'ID\' key is ' 'missing from the JSON dict (should be {0:s})'.format( container_id_from_path)) if container_id_from_json != container_id_from_path: raise errors.UnableToParseFile( 'not a valid Docker container configuration file. The \'ID\' key of ' 'the JSON dict ({0:s}) is different from the layer ID taken from the' ' path to the file ({1:s}) JSON file.)'.format( container_id_from_json, container_id_from_path)) if 'Config' in json_dict and 'Hostname' in json_dict['Config']: container_name = json_dict['Config']['Hostname'] else: container_name = 'Unknown container name' event_data = DockerJSONContainerEventData() event_data.container_id = container_id_from_path event_data.container_name = container_name json_state = json_dict.get('State', None) if json_state is not None: time_string = json_state.get('StartedAt', None) if time_string is not None: event_data.action = 'Container Started' try: date_time = dfdatetime_time_elements.TimeElementsInMicroseconds( ) date_time.CopyFromStringISO8601(time_string) except ValueError as exception: parser_mediator.ProduceExtractionWarning(( 'Unable to parse container start time string: {0:s} with error: ' '{1!s}').format(time_string, exception)) date_time = dfdatetime_semantic_time.InvalidTime() event = time_events.DateTimeValuesEvent( date_time, definitions.TIME_DESCRIPTION_START) parser_mediator.ProduceEventWithEventData(event, event_data) time_string = json_state.get('FinishedAt', None) if time_string is not None: # If the timestamp is 0001-01-01T00:00:00Z, the container # is still running, so we don't generate a Finished event if time_string != '0001-01-01T00:00:00Z': event_data.action = 'Container Finished' try: date_time = dfdatetime_time_elements.TimeElementsInMicroseconds( ) date_time.CopyFromStringISO8601(time_string) except ValueError as exception: parser_mediator.ProduceExtractionWarning(( 'Unable to parse container finish time string: {0:s} with ' 'error: {1!s}').format(time_string, exception)) date_time = dfdatetime_semantic_time.InvalidTime() event = time_events.DateTimeValuesEvent( date_time, definitions.TIME_DESCRIPTION_END) parser_mediator.ProduceEventWithEventData( event, event_data) time_string = json_dict.get('Created', None) if time_string is not None: event_data.action = 'Container Created' try: date_time = dfdatetime_time_elements.TimeElementsInMicroseconds( ) date_time.CopyFromStringISO8601(time_string) except ValueError as exception: parser_mediator.ProduceExtractionWarning(( 'Unable to parse container created time string: {0:s} with error: ' '{1!s}').format(time_string, exception)) date_time = dfdatetime_semantic_time.InvalidTime() event = time_events.DateTimeValuesEvent( date_time, definitions.TIME_DESCRIPTION_ADDED) parser_mediator.ProduceEventWithEventData(event, event_data)
def ParseReceiverData(self, parser_mediator, query, row, **unused_kwargs): """Parses a single row from the receiver and cache response table. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. query (str): query that created the row. row (sqlite3.Row): row. """ query_hash = hash(query) data = {} key_url = self._GetRowValue(query_hash, row, 'request_key') data_dict = {} description = 'MacKeeper Entry' # Check the URL, since that contains vital information about the type of # event we are dealing with. if key_url.endswith('plist'): description = 'Configuration Definition' data['text'] = 'Plist content added to cache.' elif key_url.startswith('http://event.zeobit.com'): description = 'MacKeeper Event' try: _, _, part = key_url.partition('?') data['text'] = part.replace('&', ' ') except UnicodeDecodeError: data['text'] = 'N/A' elif key_url.startswith('http://account.zeobit.com'): description = 'Account Activity' _, _, activity = key_url.partition('#') if activity: data['text'] = 'Action started: {0:s}'.format(activity) else: data['text'] = 'Unknown activity.' elif key_url.startswith('http://support.') and 'chat' in key_url: description = 'Chat ' try: jquery = self._GetRowValue(query_hash, row, 'data') jquery = codecs.decode(jquery, 'utf-8') except UnicodeDecodeError: jquery = '' data_dict = self._ExtractJQuery(jquery) data = self._ParseChatData(data_dict) data['entry_type'] = data_dict.get('type', '') if data['entry_type'] == 'comment': description += 'Comment' elif data['entry_type'] == 'outgoing': description += 'Outgoing Message' elif data['entry_type'] == 'incoming': description += 'Incoming Message' else: # Empty or not known entry type, generic status message. description += 'Entry' data['text'] = ';'.join(self._DictToListOfStrings(data_dict)) if not data['text']: data['text'] = 'No additional data.' event_data = MacKeeperCacheEventData() event_data.description = description event_data.event_type = data.get('event_type', None) event_data.offset = self._GetRowValue(query_hash, row, 'id') event_data.query = query event_data.record_id = data.get('id', None) event_data.room = data.get('room', None) event_data.text = data.get('text', None) event_data.url = key_url event_data.user_name = data.get('user', None) event_data.user_sid = data.get('sid', None) time_value = self._GetRowValue(query_hash, row, 'time_string') if isinstance(time_value, int): date_time = dfdatetime_java_time.JavaTime(timestamp=time_value) else: try: date_time = dfdatetime_time_elements.TimeElements() date_time.CopyFromDateTimeString(time_value) except ValueError as exception: parser_mediator.ProduceExtractionWarning( 'Unable to parse time string: {0:s} with error: {1!s}'. format(time_value, exception)) date_time = dfdatetime_semantic_time.InvalidTime() event = time_events.DateTimeValuesEvent( date_time, definitions.TIME_DESCRIPTION_ADDED) parser_mediator.ProduceEventWithEventData(event, event_data)
def _ParseRecord(self, parser_mediator, text_file_object): """Parses an Opera global history record. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. text_file_object (dfvfs.TextFile): text file. Returns: bool: True if the record was successfully parsed. """ try: title = text_file_object.readline() except UnicodeDecodeError: parser_mediator.ProduceExtractionWarning( 'unable to read and decode title') return False if not title: return False try: url = text_file_object.readline() except UnicodeDecodeError: parser_mediator.ProduceExtractionWarning( 'unable to read and decode url') return False try: timestamp = text_file_object.readline() except UnicodeDecodeError: parser_mediator.ProduceExtractionWarning( 'unable to read and decode timestamp') return False try: popularity_index = text_file_object.readline() except UnicodeDecodeError: parser_mediator.ProduceExtractionWarning( 'unable to read and decode popularity index') return False event_data = OperaGlobalHistoryEventData() event_data.url = url.strip() title = title.strip() if title != event_data.url: event_data.title = title popularity_index = popularity_index.strip() try: event_data.popularity_index = int(popularity_index, 10) except ValueError: parser_mediator.ProduceExtractionWarning( 'unable to convert popularity index: {0:s}'.format(popularity_index)) if event_data.popularity_index < 0: event_data.description = 'First and Only Visit' else: event_data.description = 'Last Visit' timestamp = timestamp.strip() try: timestamp = int(timestamp, 10) except ValueError: parser_mediator.ProduceExtractionWarning( 'unable to convert timestamp: {0:s}'.format(timestamp)) timestamp = None if timestamp is None: date_time = dfdatetime_semantic_time.InvalidTime() else: date_time = dfdatetime_posix_time.PosixTime(timestamp=timestamp) event = time_events.DateTimeValuesEvent( date_time, definitions.TIME_DESCRIPTION_LAST_VISITED) parser_mediator.ProduceEventWithEventData(event, event_data) return True