def _GetSanitizedEventValues(self, event_object): """Builds a dictionary from an event_object. The event object need to be sanitized to prevent certain values from causing problems when indexing with Elasticsearch. For example the value of the pathspec attribute is a nested dictionary which will cause problems for Elasticsearch automatic indexing. Args: event_object: the event object (instance of EventObject). Returns: Dictionary with sanitized event object values. """ event_values = {} for attribute_name, attribute_value in event_object.GetAttributes(): # Ignore the regvalue attribute as it cause issues when indexing if attribute_name == u'regvalue': continue if attribute_name == u'pathspec': try: attribute_value = JsonPathSpecSerializer.WriteSerialized( attribute_value) except TypeError: continue event_values[attribute_name] = attribute_value # Add string representation of the timestamp attribute_value = timelib.Timestamp.RoundToSeconds( event_object.timestamp) attribute_value = timelib.Timestamp.CopyToIsoFormat( attribute_value, timezone=self._output_mediator.timezone) event_values[u'datetime'] = attribute_value message, _ = self._output_mediator.GetFormattedMessages(event_object) if message is None: raise errors.NoFormatterFound( u'Unable to find event formatter for: {0:s}.'.format( getattr(event_object, u'data_type', u'UNKNOWN'))) event_values[u'message'] = message # Tags needs to be a list for Elasticsearch to index correctly. try: labels = list(event_values[u'tag'].labels) except (KeyError, AttributeError): labels = [] event_values[u'tag'] = labels source_short, source = self._output_mediator.GetFormattedSources( event_object) if source is None or source_short is None: raise errors.NoFormatterFound( u'Unable to find event formatter for: {0:s}.'.format( getattr(event_object, u'data_type', u'UNKNOWN'))) event_values[u'source_short'] = source_short event_values[u'source_long'] = source return event_values
def _EventToDict(self, event_object): """Returns a dict built from an event object. Args: event_object: the event object (instance of EventObject). """ ret_dict = event_object.GetValues() # Get rid of few attributes that cause issues (and need correcting). if 'pathspec' in ret_dict: del ret_dict['pathspec'] #if 'tag' in ret_dict: # del ret_dict['tag'] # tag = getattr(event_object, 'tag', None) # if tag: # tags = tag.tags # ret_dict['tag'] = tags # if getattr(tag, 'comment', ''): # ret_dict['comment'] = tag.comment ret_dict['tag'] = [] # To not overload the index, remove the regvalue index. if 'regvalue' in ret_dict: del ret_dict['regvalue'] # Adding attributes in that are calculated/derived. # We want to remove millisecond precision (causes some issues in # conversion). ret_dict['datetime'] = timelib.Timestamp.CopyToIsoFormat( timelib.Timestamp.RoundToSeconds(event_object.timestamp), timezone=self._output_mediator.timezone) message, _ = self._output_mediator.GetFormattedMessages(event_object) if message is None: raise errors.NoFormatterFound( u'Unable to find event formatter for: {0:s}.'.format( getattr(event_object, u'data_type', u'UNKNOWN'))) ret_dict['message'] = message source_short, source = self._output_mediator.GetFormattedSources( event_object) if source is None or source_short is None: raise errors.NoFormatterFound( u'Unable to find event formatter for: {0:s}.'.format( getattr(event_object, u'data_type', u'UNKNOWN'))) ret_dict['source_short'] = source_short ret_dict['source_long'] = source hostname = self._output_mediator.GetHostname(event_object) ret_dict['hostname'] = hostname username = self._output_mediator.GetUsername(event_object) ret_dict['username'] = username return ret_dict
def _GetSanitizedEventValues(self, event_object): """Builds a dictionary from an event_object. The values from the event object need to be sanitized to prevent values causing problems when indexing them with ElasticSearch. Args: event_object: the event object (instance of EventObject). Returns: Dictionary with sanitized event object values. """ event_values = event_object.GetValues() # Get rid of few attributes that cause issues (and need correcting). if u'pathspec' in event_values: del event_values[u'pathspec'] # To not overload the index, remove the regvalue index. # TODO: Investigate if this is really necessary? if u'regvalue' in event_values: del event_values[u'regvalue'] # Adding attributes in that are calculated/derived. # We want to remove millisecond precision (causes some issues in # conversion). event_values[u'datetime'] = timelib.Timestamp.CopyToIsoFormat( timelib.Timestamp.RoundToSeconds(event_object.timestamp), timezone=self._output_mediator.timezone) message, _ = self._output_mediator.GetFormattedMessages(event_object) if message is None: raise errors.NoFormatterFound( u'Unable to find event formatter for: {0:s}.'.format( getattr(event_object, u'data_type', u'UNKNOWN'))) event_values[u'message'] = message try: tags = list(event_values[u'tag'].tags) except (KeyError, AttributeError): tags = [] event_values[u'tag'] = tags source_short, source = self._output_mediator.GetFormattedSources( event_object) if source is None or source_short is None: raise errors.NoFormatterFound( u'Unable to find event formatter for: {0:s}.'.format( getattr(event_object, u'data_type', u'UNKNOWN'))) event_values[u'source_short'] = source_short event_values[u'source_long'] = source return event_values
def _GetSanitizedEventValues(self, event_object): """Builds a dictionary from an event_object. The values from the event object need to be sanitized to prevent values causing problems when indexing them with ElasticSearch. Args: event_object: the event object (instance of EventObject). Returns: Dictionary with sanitized event object values. """ event_values = {} for attribute_name, attribute_value in event_object.GetAttributes(): # Ignore attributes that cause issues (and need correcting). if attribute_name in (u'pathspec', u'regvalue'): continue event_values[attribute_name] = attribute_value # Adding attributes in that are calculated/derived. # We want to remove millisecond precision (causes some issues in # conversion). attribute_value = timelib.Timestamp.RoundToSeconds( event_object.timestamp) attribute_value = timelib.Timestamp.CopyToIsoFormat( attribute_value, timezone=self._output_mediator.timezone) event_values[u'datetime'] = attribute_value message, _ = self._output_mediator.GetFormattedMessages(event_object) if message is None: raise errors.NoFormatterFound( u'Unable to find event formatter for: {0:s}.'.format( getattr(event_object, u'data_type', u'UNKNOWN'))) event_values[u'message'] = message try: labels = list(event_values[u'tag'].labels) except (KeyError, AttributeError): labels = [] event_values[u'tag'] = labels source_short, source = self._output_mediator.GetFormattedSources( event_object) if source is None or source_short is None: raise errors.NoFormatterFound( u'Unable to find event formatter for: {0:s}.'.format( getattr(event_object, u'data_type', u'UNKNOWN'))) event_values[u'source_short'] = source_short event_values[u'source_long'] = source return event_values
def _EventToDict(self, event_object): """Returns a dict built from an event object. Args: event_object: the event object (instance of EventObject). """ event_values = {} for attribute_name, attribute_value in event_object.GetAttributes(): # Ignore attributes that cause issues (and need correcting). if attribute_name in (u'pathspec', u'regvalue', u'tag'): continue event_values[attribute_name] = attribute_value # Adding attributes in that are derived. # We want to remove millisecond precision (causes some issues in # conversion). attribute_value = timelib.Timestamp.RoundToSeconds( event_object.timestamp) attribute_value = timelib.Timestamp.CopyToIsoFormat( attribute_value, timezone=self._output_mediator.timezone) event_values[u'datetime'] = attribute_value message, _ = self._output_mediator.GetFormattedMessages(event_object) if message is None: raise errors.NoFormatterFound( u'Unable to find event formatter for: {0:s}.'.format( getattr(event_object, u'data_type', u'UNKNOWN'))) event_values[u'message'] = message source_short, source = self._output_mediator.GetFormattedSources( event_object) if source is None or source_short is None: raise errors.NoFormatterFound( u'Unable to find event formatter for: {0:s}.'.format( getattr(event_object, u'data_type', u'UNKNOWN'))) event_values[u'source_short'] = source_short event_values[u'source_long'] = source hostname = self._output_mediator.GetHostname(event_object) event_values[u'hostname'] = hostname username = self._output_mediator.GetUsername(event_object) event_values[u'username'] = username return event_values
def _FormatDescription(self, event, event_data): """Formats the description. Args: event (EventObject): event. event_data (EventData): event data. Returns: str: formatted description field. Raises: NoFormatterFound: If no event formatter can be found to match the data type in the event data. """ date_time_string = timelib.Timestamp.CopyToIsoFormat( event.timestamp, timezone=self._output_mediator.timezone) timestamp_description = event.timestamp_desc or 'UNKNOWN' message, _ = self._output_mediator.GetFormattedMessages(event_data) if message is None: data_type = getattr(event_data, 'data_type', 'UNKNOWN') raise errors.NoFormatterFound( 'Unable to find event formatter for: {0:s}.'.format(data_type)) description = '{0:s}; {1:s}; {2:s}'.format( date_time_string, timestamp_description, message.replace(self._DESCRIPTION_FIELD_DELIMITER, ' ')) return self._SanitizeField(description)
def _FormatMessageShort(self, event, event_data, event_data_stream): """Formats a short message field. Args: event (EventObject): event. event_data (EventData): event data. event_data_stream (EventDataStream): event data stream. Returns: str: short message field. Raises: NoFormatterFound: if no message formatter can be found to match the data type in the event data. WrongFormatter: if the event data cannot be formatted by the message formatter. """ message_formatter = self._output_mediator.GetMessageFormatter( event_data.data_type) if not message_formatter: raise errors.NoFormatterFound( ('Unable to find message formatter event with data type: ' '{0:s}.').format(event_data.data_type)) event_values = event_data.CopyToDict() message_formatter.FormatEventValues(event_values) if event_data.data_type in ('windows:evt:record', 'windows:evtx:record'): event_values[ 'message_string'] = self._FormatWindowsEventLogMessage( event, event_data, event_data_stream) return message_formatter.GetMessageShort(event_values)
def WriteEventBody(self, event_object): """Writes the body of an event object to the output. Each event object contains both attributes that are considered "reserved" and others that aren't. The 'raw' representation of the object makes a distinction between these two types as well as extracting the format strings from the object. Args: event_object: the event object (instance of EventObject). Raises: errors.NoFormatterFound: If no formatter for that event is found. """ if not hasattr(event_object, 'timestamp'): return # TODO: move this to an output module interface. event_formatter = formatters_manager.FormattersManager.GetFormatterObject( event_object.data_type) if not event_formatter: raise errors.NoFormatterFound( u'Unable to find event formatter for: {0:s}.'.format( event_object.data_type)) msg, _ = event_formatter.GetMessages(self._formatter_mediator, event_object) source_short, _ = event_formatter.GetSources(event_object) date_use = timelib.Timestamp.CopyToPosix(event_object.timestamp) hostname = getattr(event_object, 'hostname', u'') username = getattr(event_object, 'username', u'') if self.store: if not hostname: hostname = self._hostnames.get(event_object.store_number, '') pre_obj = self._preprocesses.get(event_object.store_number) if pre_obj: check_user = pre_obj.GetUsernameById(username) if check_user != '-': username = check_user notes = getattr(event_object, 'notes', u'') if not notes: notes = u'File: {0:s} inode: {1!s}'.format( getattr(event_object, 'display_name', u''), getattr(event_object, 'inode', u'')) out_write = u'{0!s}|{1:s}|{2:s}|{3:s}|{4:s}|{5:s}|{6!s}\n'.format( date_use, source_short.replace(self.DELIMITER, u' '), hostname.replace(self.DELIMITER, u' '), username.replace(self.DELIMITER, u' '), msg.replace(self.DELIMITER, u' '), self.zone, notes.replace(self.DELIMITER, u' ')) self.filehandle.WriteLine(out_write)
def ParseSourceShort(self, event_object): """Return the source string.""" event_formatter = eventdata.EventFormatterManager.GetFormatter(event_object) if not event_formatter: raise errors.NoFormatterFound( u'Unable to find no event formatter for: {0:s}.'.format( event_object.DATA_TYPE)) source, _ = event_formatter.GetSources(event_object) return source
def _FormatExtraAttributes(self, event, event_data, event_data_stream): """Formats an extra attributes field. Args: event (EventObject): event. event_data (EventData): event data. event_data_stream (EventDataStream): event data stream. Returns: str: extra attributes field. Raises: NoFormatterFound: if no event formatter can be found to match the data type in the event data. """ message_formatter = self._output_mediator.GetMessageFormatter( event_data.data_type) if not message_formatter: raise errors.NoFormatterFound( ('Unable to find message formatter event with data type: ' '{0:s}.').format(event_data.data_type)) formatted_attribute_names = ( message_formatter.GetFormatStringAttributeNames()) formatted_attribute_names.update(definitions.RESERVED_VARIABLE_NAMES) extra_attributes = [] for attribute_name, attribute_value in event_data.GetAttributes(): if attribute_name not in formatted_attribute_names: # Some parsers have written bytes values to storage. if isinstance(attribute_value, bytes): attribute_value = attribute_value.decode( 'utf-8', 'replace') logger.warning( 'Found bytes value for attribute "{0:s}" for data type: ' '{1!s}. Value was converted to UTF-8: "{2:s}"'.format( attribute_name, event_data.data_type, attribute_value)) # With ! in {1!s} we force a string conversion since some of # the extra attributes values can be integer, float point or # boolean values. extra_attributes.append('{0:s}: {1!s}'.format( attribute_name, attribute_value)) if event_data_stream: for attribute_name, attribute_value in event_data_stream.GetAttributes( ): if attribute_name != 'path_spec': extra_attributes.append('{0:s}: {1!s}'.format( attribute_name, attribute_value)) extra_attributes = '; '.join(sorted(extra_attributes)) return extra_attributes.replace('\n', '-').replace('\r', '')
def _FormatExtraAttributes(self, event, event_data, event_data_stream): """Formats an extra attributes field. Args: event (EventObject): event. event_data (EventData): event data. event_data_stream (EventDataStream): event data stream. Returns: str: extra attributes field. Raises: NoFormatterFound: if no event formatter can be found to match the data type in the event data. """ event_attributes = list(event_data.GetAttributes()) if event_data_stream: event_attributes.extend(event_data_stream.GetAttributes()) # TODO: reverse logic and get formatted attributes instead. unformatted_attributes = (formatters_manager.FormattersManager. GetUnformattedAttributes(event_data)) if unformatted_attributes is None: raise errors.NoFormatterFound( 'Unable to find event formatter for: {0:s}.'.format( event_data.data_type)) event_attributes = list(event_data.GetAttributes()) if event_data_stream: event_attributes.extend(event_data_stream.GetAttributes()) extra_attributes = [] for attribute_name, attribute_value in sorted(event_attributes): if attribute_name in unformatted_attributes: # Some parsers have written bytes values to storage. if isinstance(attribute_value, bytes): attribute_value = attribute_value.decode( 'utf-8', 'replace') logger.warning( 'Found bytes value for attribute "{0:s}" for data type: ' '{1!s}. Value was converted to UTF-8: "{2:s}"'.format( attribute_name, event_data.data_type, attribute_value)) # With ! in {1!s} we force a string conversion since some of # the extra attributes values can be integer, float point or # boolean values. extra_attributes.append('{0:s}: {1!s}'.format( attribute_name, attribute_value)) extra_attributes = '; '.join(extra_attributes) return extra_attributes.replace('\n', '-').replace('\r', '')
def ParseSourceShort(self, event_object): """Return the source string.""" # TODO: move this to an output module interface. event_formatter = formatters_manager.FormattersManager.GetFormatterObject( event_object.data_type) if not event_formatter: raise errors.NoFormatterFound( u'Unable to find no event formatter for: {0:s}.'.format( event_object.data_type)) source, _ = event_formatter.GetSources(event_object) return source
def ParseSource(self, event_object): """Return the source string.""" # TODO: move this to an output module interface. event_formatter = formatters_manager.EventFormatterManager.GetFormatter( event_object) if not event_formatter: raise errors.NoFormatterFound( u'Unable to find no event formatter for: {0:s}.'.format( event_object.DATA_TYPE)) _, source = event_formatter.GetSources(event_object) return source
def EventBody(self, event_object): """Formats data as TLN and writes to the filehandle from OutputFormater. Args: event_object: The event object (EventObject). Raises: errors.NoFormatterFound: If no formatter for that event is found. """ if not hasattr(event_object, 'timestamp'): return # TODO: move this to an output module interface. event_formatter = formatters_manager.EventFormatterManager.GetFormatter( event_object) if not event_formatter: raise errors.NoFormatterFound( u'Unable to find event formatter for: {0:s}.'.format( event_object.DATA_TYPE)) msg, _ = event_formatter.GetMessages(event_object) source_short, _ = event_formatter.GetSources(event_object) date_use = timelib.Timestamp.CopyToPosix(event_object.timestamp) hostname = getattr(event_object, 'hostname', u'') username = getattr(event_object, 'username', u'') if self.store: if not hostname: hostname = self._hostnames.get(event_object.store_number, '') pre_obj = self._preprocesses.get(event_object.store_number) if pre_obj: check_user = pre_obj.GetUsernameById(username) if check_user != '-': username = check_user notes = getattr(event_object, 'notes', u'') if not notes: notes = u'File: {0:s} inode: {1!s}'.format( getattr(event_object, 'display_name', u''), getattr(event_object, 'inode', u'')) out_write = u'{0!s}|{1:s}|{2:s}|{3:s}|{4:s}|{5:s}|{6!s}\n'.format( date_use, source_short.replace(self.DELIMITER, u' '), hostname.replace(self.DELIMITER, u' '), username.replace(self.DELIMITER, u' '), msg.replace(self.DELIMITER, u' '), self.zone, notes.replace(self.DELIMITER, u' ')) self.filehandle.WriteLine(out_write)
def _FormatSource(self, event_object): """Formats the source. Args: event_object: the event object (instance of EventObject). Returns: A string containing the value for the source field. """ source_short, _ = self._output_mediator.GetFormattedSources(event_object) if source_short is None: raise errors.NoFormatterFound( u'Unable to find event formatter for: {0:s}.'.format( getattr(event_object, u'data_type', u'UNKNOWN'))) return self._SanitizeField(source_short)
def _FormatSource(self, event): """Formats the source. Args: event (EventObject): event. Returns: str: formatted source field. """ source_short, _ = self._output_mediator.GetFormattedSources(event) if source_short is None: data_type = getattr(event, 'data_type', 'UNKNOWN') raise errors.NoFormatterFound( 'Unable to find event formatter for: {0:s}.'.format(data_type)) return self._SanitizeField(source_short)
def ParseMessageShort(self, event_object): """Return the message string from the EventObject. Args: event_object: The event object (EventObject). Raises: errors.NoFormatterFound: If no formatter for that event is found. """ event_formatter = eventdata.EventFormatterManager.GetFormatter(event_object) if not event_formatter: raise errors.NoFormatterFound( u'Unable to find no event formatter for: {0:s}.'.format( event_object.DATA_TYPE)) _, msg_short = event_formatter.GetMessages(event_object) return msg_short
def ParseMessage(self, event_object): """Return the message string from the EventObject. Args: event_object: The event object (EventObject). Raises: errors.NoFormatterFound: If no formatter for that event is found. """ # TODO: move this to an output module interface. event_formatter = formatters_manager.FormattersManager.GetFormatterObject( event_object.data_type) if not event_formatter: raise errors.NoFormatterFound( u'Unable to find no event formatter for: {0:s}.'.format( event_object.data_type)) msg, _ = event_formatter.GetMessages(self._formatter_mediator, event_object) return msg
def EventBody(self, event_object): """Formats data as TLN and writes to the filehandle from OutputFormater. Args: event_object: The event object (EventObject). Raises: errors.NoFormatterFound: If no formatter for that event is found. """ if not hasattr(event_object, 'timestamp'): return event_formatter = eventdata.EventFormatterManager.GetFormatter( event_object) if not event_formatter: raise errors.NoFormatterFound( u'Unable to find event formatter for: {0:s}.'.format( event_object.DATA_TYPE)) msg, _ = event_formatter.GetMessages(event_object) source_short, _ = event_formatter.GetSources(event_object) date_use = timelib.Timestamp.CopyToPosix(event_object.timestamp) hostname = getattr(event_object, 'hostname', u'') username = getattr(event_object, 'username', u'') if self.store: if not hostname: hostname = self._hostnames.get(event_object.store_number, u'') pre_obj = self._preprocesses.get(event_object.store_number) if pre_obj: check_user = pre_obj.GetUsernameById(username) if check_user != '-': username = check_user out_write = u'{}|{}|{}|{}|{}\n'.format( date_use, source_short.replace(self.DELIMITER, u' '), hostname.replace(self.DELIMITER, u' '), username.replace(self.DELIMITER, u' '), msg.replace(self.DELIMITER, u' ')) self.filehandle.WriteLine(out_write)
def _FormatSourceShort(self, event): """Formats the short source. Args: event (EventObject): event. Returns: str: short source field. Raises: NoFormatterFound: If no event formatter can be found to match the data type in the event. """ source_short, _ = self._output_mediator.GetFormattedSources(event) if source_short is None: data_type = getattr(event, 'data_type', 'UNKNOWN') raise errors.NoFormatterFound( 'Unable to find event formatter for: {0:s}.'.format(data_type)) return source_short
def _FormatMessage(self, event): """Formats the message. Args: event (EventObject): event. Returns: str: message field. Raises: NoFormatterFound: if no event formatter can be found to match the data type in the event. """ message, _ = self._output_mediator.GetFormattedMessages(event) if message is None: data_type = getattr(event, 'data_type', 'UNKNOWN') raise errors.NoFormatterFound( 'Unable to find event formatter for: {0:s}.'.format(data_type)) return message
def _FormatSource(self, event): """Formats the source. Args: event (EventObject): event. Returns: str: source field. Raises: NoFormatterFound: If no event formatter can be found to match the data type in the event. """ _, source = self._output_mediator.GetFormattedSources(event) if source is None: raise errors.NoFormatterFound( u'Unable to find event formatter for: {0:s}.'.format( getattr(event, u'data_type', u'UNKNOWN'))) return source
def _FormatMessageShort(self, event): """Formats the short message. Args: event (EventObject): event. Returns: str: short message field. Raises: NoFormatterFound: If no event formatter can be found to match the data type in the event. """ _, message_short = self._output_mediator.GetFormattedMessages(event) if message_short is None: raise errors.NoFormatterFound( u'Unable to find event formatter for: {0:s}.'.format( getattr(event, u'data_type', u'UNKNOWN'))) return message_short
def _FormatSource(self, event_object): """Formats the source. Args: event_object: an event object (instance of EventObject). Returns: A string containing the value for the source field. Raises: NoFormatterFound: If no event formatter can be found to match the data type in the event object. """ _, source = self._output_mediator.GetFormattedSources(event_object) if source is None: raise errors.NoFormatterFound( u'Unable to find event formatter for: {0:s}.'.format( getattr(event_object, u'data_type', u'UNKNOWN'))) return source
def _FormatMessage(self, event_object): """Formats the message. Args: event_object: the event object (instance of EventObject). Returns: A string containing the value for the message field. Raises: NoFormatterFound: If no event formatter can be found to match the data type in the event object. """ message, _ = self._output_mediator.GetFormattedMessages(event_object) if message is None: raise errors.NoFormatterFound( u'Unable to find event formatter for: {0:s}.'.format( getattr(event_object, u'data_type', u'UNKNOWN'))) return self._SanitizeField(message)
def _FormatSource(self, event, event_data): """Formats the source. Args: event (EventObject): event. event_data (EventData): event data. Returns: str: source field. Raises: NoFormatterFound: if no event formatter can be found to match the data type in the event data. """ _, source = self._output_mediator.GetFormattedSources(event, event_data) if source is None: data_type = getattr(event_data, 'data_type', 'UNKNOWN') raise errors.NoFormatterFound( 'Unable to create source for event with data type: {0:s}.'.format( data_type)) return source
def _FormatMessage(self, event, event_data, event_data_stream): """Formats a message field. Args: event (EventObject): event. event_data (EventData): event data. event_data_stream (EventDataStream): event data stream. Returns: str: message field. Raises: NoFormatterFound: if no event formatter can be found to match the data type in the event data. """ # TODO: refactor GetFormattedMessages by GetFormattedMessage. message, _ = self._output_mediator.GetFormattedMessages(event_data) if message is None: raise errors.NoFormatterFound( 'Unable to create message for event with data type: {0:s}.'. format(event_data.data_type)) return message
def _FormatDescription(self, event_object): """Formats the description. Args: event_object: the event object (instance of EventObject). Returns: A string containing the value for the description field. """ date_time_string = timelib.Timestamp.CopyToIsoFormat( event_object.timestamp, timezone=self._output_mediator.timezone) timestamp_description = getattr(event_object, u'timestamp_desc', u'UNKNOWN') message, _ = self._output_mediator.GetFormattedMessages(event_object) if message is None: raise errors.NoFormatterFound( u'Unable to find event formatter for: {0:s}.'.format( getattr(event_object, u'data_type', u'UNKNOWN'))) description = u'{0:s}; {1:s}; {2:s}'.format( date_time_string, timestamp_description, message.replace(self._DESCRIPTION_FIELD_DELIMITER, u' ')) return self._SanitizeField(description)
def _GetSanitizedEventValues(self, event): """Sanitizes the event object for use in 4n6time. Args: event (EventObject): event. Returns: A dictionary object containing the sanitized values. Raises: NoFormatterFound: If no event formatter can be found to match the data type in the event object. """ data_type = getattr(event, u'data_type', u'UNKNOWN') event_formatter = self._output_mediator.GetEventFormatter(event) if not event_formatter: raise errors.NoFormatterFound( u'Unable to find event formatter for: {0:s}.'.format( data_type)) message, _ = self._output_mediator.GetFormattedMessages(event) if message is None: raise errors.NoFormatterFound( u'Unable to find event formatter for: {0:s}.'.format( data_type)) source_short, source = self._output_mediator.GetFormattedSources(event) if source is None or source_short is None: raise errors.NoFormatterFound( u'Unable to find event formatter for: {0:s}.'.format( data_type)) datetime_object = None if event.timestamp is not None: datetime_object = timelib.Timestamp.CopyToDatetime( event.timestamp, self._output_mediator.timezone) if not datetime_object: self._ReportEventError( event, (u'unable to copy timestamp: {0:d} to datetime object.')) return format_variables = self._output_mediator.GetFormatStringAttributeNames( event) if format_variables is None: raise errors.NoFormatterFound( u'Unable to find event formatter for: {0:s}.'.format( data_type)) extra_attributes = [] for attribute_name, attribute_value in sorted(event.GetAttributes()): if (attribute_name in definitions.RESERVED_VARIABLE_NAMES or attribute_name in format_variables): continue extra_attributes.append(u'{0:s}: {1!s} '.format( attribute_name, attribute_value)) extra_attributes = u' '.join(extra_attributes) inode = event.inode if inode is None and hasattr(event, u'pathspec'): inode = getattr(event.pathspec, u'inode', u'-') if inode is None: inode = u'-' datetime_string = u'N/A' if datetime_object: datetime_string = ( u'{0:04d}-{1:02d}-{2:02d} {3:02d}:{4:02d}:{5:02d}'.format( datetime_object.year, datetime_object.month, datetime_object.day, datetime_object.hour, datetime_object.minute, datetime_object.second)) tags = None if getattr(event, u'tag', None): tags = getattr(event.tag, u'tags', None) taglist = u'' if isinstance(tags, (list, tuple)): taglist = u','.join(tags) offset = event.offset if offset is None: offset = 0 row = { u'timezone': u'{0!s}'.format(self._output_mediator.timezone), u'MACB': self._output_mediator.GetMACBRepresentation(event), u'source': source_short, u'sourcetype': source, u'type': getattr(event, u'timestamp_desc', u'-'), u'user': getattr(event, u'username', u'-'), u'host': getattr(event, u'hostname', u'-'), u'description': message, u'filename': getattr(event, u'filename', u'-'), u'inode': inode, u'notes': getattr(event, u'notes', u'-'), u'format': getattr(event, u'parser', u'-'), u'extra': extra_attributes, u'datetime': datetime_string, u'reportnotes': u'', u'inreport': u'', u'tag': taglist, u'offset': offset, u'vss_store_number': self._GetVSSNumber(event), u'URL': getattr(event, u'url', u'-'), u'record_number': getattr(event, u'record_number', 0), u'event_identifier': getattr(event, u'event_identifier', u'-'), u'event_type': getattr(event, u'event_type', u'-'), u'source_name': getattr(event, u'source_name', u'-'), u'user_sid': getattr(event, u'user_sid', u'-'), u'computer_name': getattr(event, u'computer_name', u'-'), u'evidence': self._evidence } return row
def EventBody(self, event_object): """Formats data as 4n6time database table format and writes to the db. Args: event_object: The event object (EventObject). Raises: raise errors.NoFormatterFound: If no formatter for this event is found. """ if not hasattr(event_object, 'timestamp'): return formatter = eventdata.EventFormatterManager.GetFormatter(event_object) if not formatter: raise errors.NoFormatterFound( u'Unable to output event, no formatter found.') if (isinstance( formatter, formatters.winreg.WinRegistryGenericFormatter) and formatter.FORMAT_STRING.find('<|>') == -1): formatter.FORMAT_STRING = u'[{keyname}]<|>{text}<|>' elif isinstance(formatter, eventdata.ConditionalEventFormatter): formatter.FORMAT_STRING_SEPARATOR = u'<|>' elif isinstance(formatter, eventdata.EventFormatter): formatter.format_string = formatter.format_string.replace('}', '}<|>') msg, msg_short = formatter.GetMessages(event_object) source_short, source_long = formatter.GetSources(event_object) date_use = timelib.Timestamp.CopyToDatetime( event_object.timestamp, self.zone) if not date_use: logging.error(u'Unable to process date for entry: {0:s}'.format(msg)) return extra = [] format_variables = self.FORMAT_ATTRIBUTE_RE.findall( formatter.format_string) for key in event_object.GetAttributes(): if key in utils.RESERVED_VARIABLES or key in format_variables: continue extra.append(u'{0:s}: {1!s} '.format( key, getattr(event_object, key, None))) extra = u' '.join(extra) inode = getattr(event_object, 'inode', '-') if inode == '-': if (hasattr(event_object, 'pathspec') and hasattr(event_object.pathspec, 'image_inode')): inode = event_object.pathspec.image_inode date_use_string = u'{0}-{1}-{2} {3}:{4}:{5}'.format( date_use.year, date_use.month, date_use.day, date_use.hour, date_use.minute, date_use.second) tags = [] if hasattr(event_object, 'tag'): if hasattr(event_object.tag, 'tags'): tags = event_object.tag.tags else: tags = u'' else: tags = u'' taglist = u','.join(tags) row = (str(self.zone), helper.GetLegacy(event_object), source_short, source_long, getattr(event_object, 'timestamp_desc', '-'), getattr(event_object, 'username', '-'), getattr(event_object, 'hostname', '-'), msg, getattr(event_object, 'filename', '-'), inode, getattr(event_object, 'notes', '-'), getattr(event_object, 'parser', '-'), extra, date_use_string, '', '', taglist, '', getattr(event_object, 'offset', 0), event_object.store_number, event_object.store_index, self.GetVSSNumber(event_object), getattr(event_object, 'url', '-'), getattr(event_object, 'record_number', 0), getattr(event_object, 'event_identifier', '-'), getattr(event_object, 'event_type', '-'), getattr(event_object, 'source_name', '-'), getattr(event_object, 'user_sid', '-'), getattr(event_object, 'computer_name', '-'), self.evidence) try: self.curs.execute( 'INSERT INTO log2timeline(timezone, MACB, source, ' 'sourcetype, type, user, host, description, filename, ' 'inode, notes, format, extra, datetime, reportnotes, ' 'inreport, tag, color, offset, store_number, ' 'store_index, vss_store_number, URL, record_number, ' 'event_identifier, event_type, source_name, user_sid, ' 'computer_name, evidence) VALUES (' '%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, ' '%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, ' '%s, %s, %s, %s)', row) except MySQLdb.Error as exception: logging.warning( u'Unable to insert into database with error: {0:s}.'.format( exception)) self.count += 1 # TODO: Experiment if committing the current transaction # every 10000 inserts is the optimal approach. if self.count % 10000 == 0: self.conn.commit() if self.set_status: self.set_status(u'Inserting event: {0:d}'.format(self.count))