def _GetSanitizedEventValues(self, event_object):
"""Builds a dictionary from an event_object.
The event object need to be sanitized to prevent certain values from causing
problems when indexing with Elasticsearch. For example the value of
the pathspec attribute is a nested dictionary which will cause problems for
Elasticsearch automatic indexing.
Args:
event_object: the event object (instance of EventObject).
Returns:
Dictionary with sanitized event object values.
"""
event_values = {}
for attribute_name, attribute_value in event_object.GetAttributes():
# Ignore the regvalue attribute as it cause issues when indexing
if attribute_name == u'regvalue':
continue
if attribute_name == u'pathspec':
try:
attribute_value = JsonPathSpecSerializer.WriteSerialized(
attribute_value)
except TypeError:
continue
event_values[attribute_name] = attribute_value
# Add string representation of the timestamp
attribute_value = timelib.Timestamp.RoundToSeconds(
event_object.timestamp)
attribute_value = timelib.Timestamp.CopyToIsoFormat(
attribute_value, timezone=self._output_mediator.timezone)
event_values[u'datetime'] = attribute_value
message, _ = self._output_mediator.GetFormattedMessages(event_object)
if message is None:
raise errors.NoFormatterFound(
u'Unable to find event formatter for: {0:s}.'.format(
getattr(event_object, u'data_type', u'UNKNOWN')))
event_values[u'message'] = message
# Tags needs to be a list for Elasticsearch to index correctly.
try:
labels = list(event_values[u'tag'].labels)
except (KeyError, AttributeError):
labels = []
event_values[u'tag'] = labels
source_short, source = self._output_mediator.GetFormattedSources(
event_object)
if source is None or source_short is None:
raise errors.NoFormatterFound(
u'Unable to find event formatter for: {0:s}.'.format(
getattr(event_object, u'data_type', u'UNKNOWN')))
event_values[u'source_short'] = source_short
event_values[u'source_long'] = source
return event_values
def _EventToDict(self, event_object):
"""Returns a dict built from an event object.
Args:
event_object: the event object (instance of EventObject).
"""
ret_dict = event_object.GetValues()
# Get rid of few attributes that cause issues (and need correcting).
if 'pathspec' in ret_dict:
del ret_dict['pathspec']
#if 'tag' in ret_dict:
# del ret_dict['tag']
# tag = getattr(event_object, 'tag', None)
# if tag:
# tags = tag.tags
# ret_dict['tag'] = tags
# if getattr(tag, 'comment', ''):
# ret_dict['comment'] = tag.comment
ret_dict['tag'] = []
# To not overload the index, remove the regvalue index.
if 'regvalue' in ret_dict:
del ret_dict['regvalue']
# Adding attributes in that are calculated/derived.
# We want to remove millisecond precision (causes some issues in
# conversion).
ret_dict['datetime'] = timelib.Timestamp.CopyToIsoFormat(
timelib.Timestamp.RoundToSeconds(event_object.timestamp),
timezone=self._output_mediator.timezone)
message, _ = self._output_mediator.GetFormattedMessages(event_object)
if message is None:
raise errors.NoFormatterFound(
u'Unable to find event formatter for: {0:s}.'.format(
getattr(event_object, u'data_type', u'UNKNOWN')))
ret_dict['message'] = message
source_short, source = self._output_mediator.GetFormattedSources(
event_object)
if source is None or source_short is None:
raise errors.NoFormatterFound(
u'Unable to find event formatter for: {0:s}.'.format(
getattr(event_object, u'data_type', u'UNKNOWN')))
ret_dict['source_short'] = source_short
ret_dict['source_long'] = source
hostname = self._output_mediator.GetHostname(event_object)
ret_dict['hostname'] = hostname
username = self._output_mediator.GetUsername(event_object)
ret_dict['username'] = username
return ret_dict
def _GetSanitizedEventValues(self, event_object):
"""Builds a dictionary from an event_object.
The values from the event object need to be sanitized to prevent
values causing problems when indexing them with ElasticSearch.
Args:
event_object: the event object (instance of EventObject).
Returns:
Dictionary with sanitized event object values.
"""
event_values = event_object.GetValues()
# Get rid of few attributes that cause issues (and need correcting).
if u'pathspec' in event_values:
del event_values[u'pathspec']
# To not overload the index, remove the regvalue index.
# TODO: Investigate if this is really necessary?
if u'regvalue' in event_values:
del event_values[u'regvalue']
# Adding attributes in that are calculated/derived.
# We want to remove millisecond precision (causes some issues in
# conversion).
event_values[u'datetime'] = timelib.Timestamp.CopyToIsoFormat(
timelib.Timestamp.RoundToSeconds(event_object.timestamp),
timezone=self._output_mediator.timezone)
message, _ = self._output_mediator.GetFormattedMessages(event_object)
if message is None:
raise errors.NoFormatterFound(
u'Unable to find event formatter for: {0:s}.'.format(
getattr(event_object, u'data_type', u'UNKNOWN')))
event_values[u'message'] = message
try:
tags = list(event_values[u'tag'].tags)
except (KeyError, AttributeError):
tags = []
event_values[u'tag'] = tags
source_short, source = self._output_mediator.GetFormattedSources(
event_object)
if source is None or source_short is None:
raise errors.NoFormatterFound(
u'Unable to find event formatter for: {0:s}.'.format(
getattr(event_object, u'data_type', u'UNKNOWN')))
event_values[u'source_short'] = source_short
event_values[u'source_long'] = source
return event_values
def _GetSanitizedEventValues(self, event_object):
"""Builds a dictionary from an event_object.
The values from the event object need to be sanitized to prevent
values causing problems when indexing them with ElasticSearch.
Args:
event_object: the event object (instance of EventObject).
Returns:
Dictionary with sanitized event object values.
"""
event_values = {}
for attribute_name, attribute_value in event_object.GetAttributes():
# Ignore attributes that cause issues (and need correcting).
if attribute_name in (u'pathspec', u'regvalue'):
continue
event_values[attribute_name] = attribute_value
# Adding attributes in that are calculated/derived.
# We want to remove millisecond precision (causes some issues in
# conversion).
attribute_value = timelib.Timestamp.RoundToSeconds(
event_object.timestamp)
attribute_value = timelib.Timestamp.CopyToIsoFormat(
attribute_value, timezone=self._output_mediator.timezone)
event_values[u'datetime'] = attribute_value
message, _ = self._output_mediator.GetFormattedMessages(event_object)
if message is None:
raise errors.NoFormatterFound(
u'Unable to find event formatter for: {0:s}.'.format(
getattr(event_object, u'data_type', u'UNKNOWN')))
event_values[u'message'] = message
try:
labels = list(event_values[u'tag'].labels)
except (KeyError, AttributeError):
labels = []
event_values[u'tag'] = labels
source_short, source = self._output_mediator.GetFormattedSources(
event_object)
if source is None or source_short is None:
raise errors.NoFormatterFound(
u'Unable to find event formatter for: {0:s}.'.format(
getattr(event_object, u'data_type', u'UNKNOWN')))
event_values[u'source_short'] = source_short
event_values[u'source_long'] = source
return event_values
def _EventToDict(self, event_object):
"""Returns a dict built from an event object.
Args:
event_object: the event object (instance of EventObject).
"""
event_values = {}
for attribute_name, attribute_value in event_object.GetAttributes():
# Ignore attributes that cause issues (and need correcting).
if attribute_name in (u'pathspec', u'regvalue', u'tag'):
continue
event_values[attribute_name] = attribute_value
# Adding attributes in that are derived.
# We want to remove millisecond precision (causes some issues in
# conversion).
attribute_value = timelib.Timestamp.RoundToSeconds(
event_object.timestamp)
attribute_value = timelib.Timestamp.CopyToIsoFormat(
attribute_value, timezone=self._output_mediator.timezone)
event_values[u'datetime'] = attribute_value
message, _ = self._output_mediator.GetFormattedMessages(event_object)
if message is None:
raise errors.NoFormatterFound(
u'Unable to find event formatter for: {0:s}.'.format(
getattr(event_object, u'data_type', u'UNKNOWN')))
event_values[u'message'] = message
source_short, source = self._output_mediator.GetFormattedSources(
event_object)
if source is None or source_short is None:
raise errors.NoFormatterFound(
u'Unable to find event formatter for: {0:s}.'.format(
getattr(event_object, u'data_type', u'UNKNOWN')))
event_values[u'source_short'] = source_short
event_values[u'source_long'] = source
hostname = self._output_mediator.GetHostname(event_object)
event_values[u'hostname'] = hostname
username = self._output_mediator.GetUsername(event_object)
event_values[u'username'] = username
return event_values
def _FormatDescription(self, event, event_data):
"""Formats the description.
Args:
event (EventObject): event.
event_data (EventData): event data.
Returns:
str: formatted description field.
Raises:
NoFormatterFound: If no event formatter can be found to match the data
type in the event data.
"""
date_time_string = timelib.Timestamp.CopyToIsoFormat(
event.timestamp, timezone=self._output_mediator.timezone)
timestamp_description = event.timestamp_desc or 'UNKNOWN'
message, _ = self._output_mediator.GetFormattedMessages(event_data)
if message is None:
data_type = getattr(event_data, 'data_type', 'UNKNOWN')
raise errors.NoFormatterFound(
'Unable to find event formatter for: {0:s}.'.format(data_type))
description = '{0:s}; {1:s}; {2:s}'.format(
date_time_string, timestamp_description,
message.replace(self._DESCRIPTION_FIELD_DELIMITER, ' '))
return self._SanitizeField(description)
def _FormatMessageShort(self, event, event_data, event_data_stream):
"""Formats a short message field.
Args:
event (EventObject): event.
event_data (EventData): event data.
event_data_stream (EventDataStream): event data stream.
Returns:
str: short message field.
Raises:
NoFormatterFound: if no message formatter can be found to match the data
type in the event data.
WrongFormatter: if the event data cannot be formatted by the message
formatter.
"""
message_formatter = self._output_mediator.GetMessageFormatter(
event_data.data_type)
if not message_formatter:
raise errors.NoFormatterFound(
('Unable to find message formatter event with data type: '
'{0:s}.').format(event_data.data_type))
event_values = event_data.CopyToDict()
message_formatter.FormatEventValues(event_values)
if event_data.data_type in ('windows:evt:record',
'windows:evtx:record'):
event_values[
'message_string'] = self._FormatWindowsEventLogMessage(
event, event_data, event_data_stream)
return message_formatter.GetMessageShort(event_values)
def WriteEventBody(self, event_object):
"""Writes the body of an event object to the output.
Each event object contains both attributes that are considered "reserved"
and others that aren't. The 'raw' representation of the object makes a
distinction between these two types as well as extracting the format
strings from the object.
Args:
event_object: the event object (instance of EventObject).
Raises:
errors.NoFormatterFound: If no formatter for that event is found.
"""
if not hasattr(event_object, 'timestamp'):
return
# TODO: move this to an output module interface.
event_formatter = formatters_manager.FormattersManager.GetFormatterObject(
event_object.data_type)
if not event_formatter:
raise errors.NoFormatterFound(
u'Unable to find event formatter for: {0:s}.'.format(
event_object.data_type))
msg, _ = event_formatter.GetMessages(self._formatter_mediator,
event_object)
source_short, _ = event_formatter.GetSources(event_object)
date_use = timelib.Timestamp.CopyToPosix(event_object.timestamp)
hostname = getattr(event_object, 'hostname', u'')
username = getattr(event_object, 'username', u'')
if self.store:
if not hostname:
hostname = self._hostnames.get(event_object.store_number, '')
pre_obj = self._preprocesses.get(event_object.store_number)
if pre_obj:
check_user = pre_obj.GetUsernameById(username)
if check_user != '-':
username = check_user
notes = getattr(event_object, 'notes', u'')
if not notes:
notes = u'File: {0:s} inode: {1!s}'.format(
getattr(event_object, 'display_name', u''),
getattr(event_object, 'inode', u''))
out_write = u'{0!s}|{1:s}|{2:s}|{3:s}|{4:s}|{5:s}|{6!s}\n'.format(
date_use, source_short.replace(self.DELIMITER, u' '),
hostname.replace(self.DELIMITER, u' '),
username.replace(self.DELIMITER, u' '),
msg.replace(self.DELIMITER, u' '), self.zone,
notes.replace(self.DELIMITER, u' '))
self.filehandle.WriteLine(out_write)
def ParseSourceShort(self, event_object):
"""Return the source string."""
event_formatter = eventdata.EventFormatterManager.GetFormatter(event_object)
if not event_formatter:
raise errors.NoFormatterFound(
u'Unable to find no event formatter for: {0:s}.'.format(
event_object.DATA_TYPE))
source, _ = event_formatter.GetSources(event_object)
return source
def _FormatExtraAttributes(self, event, event_data, event_data_stream):
"""Formats an extra attributes field.
Args:
event (EventObject): event.
event_data (EventData): event data.
event_data_stream (EventDataStream): event data stream.
Returns:
str: extra attributes field.
Raises:
NoFormatterFound: if no event formatter can be found to match the data
type in the event data.
"""
message_formatter = self._output_mediator.GetMessageFormatter(
event_data.data_type)
if not message_formatter:
raise errors.NoFormatterFound(
('Unable to find message formatter event with data type: '
'{0:s}.').format(event_data.data_type))
formatted_attribute_names = (
message_formatter.GetFormatStringAttributeNames())
formatted_attribute_names.update(definitions.RESERVED_VARIABLE_NAMES)
extra_attributes = []
for attribute_name, attribute_value in event_data.GetAttributes():
if attribute_name not in formatted_attribute_names:
# Some parsers have written bytes values to storage.
if isinstance(attribute_value, bytes):
attribute_value = attribute_value.decode(
'utf-8', 'replace')
logger.warning(
'Found bytes value for attribute "{0:s}" for data type: '
'{1!s}. Value was converted to UTF-8: "{2:s}"'.format(
attribute_name, event_data.data_type,
attribute_value))
# With ! in {1!s} we force a string conversion since some of
# the extra attributes values can be integer, float point or
# boolean values.
extra_attributes.append('{0:s}: {1!s}'.format(
attribute_name, attribute_value))
if event_data_stream:
for attribute_name, attribute_value in event_data_stream.GetAttributes(
):
if attribute_name != 'path_spec':
extra_attributes.append('{0:s}: {1!s}'.format(
attribute_name, attribute_value))
extra_attributes = '; '.join(sorted(extra_attributes))
return extra_attributes.replace('\n', '-').replace('\r', '')
def _FormatExtraAttributes(self, event, event_data, event_data_stream):
"""Formats an extra attributes field.
Args:
event (EventObject): event.
event_data (EventData): event data.
event_data_stream (EventDataStream): event data stream.
Returns:
str: extra attributes field.
Raises:
NoFormatterFound: if no event formatter can be found to match the data
type in the event data.
"""
event_attributes = list(event_data.GetAttributes())
if event_data_stream:
event_attributes.extend(event_data_stream.GetAttributes())
# TODO: reverse logic and get formatted attributes instead.
unformatted_attributes = (formatters_manager.FormattersManager.
GetUnformattedAttributes(event_data))
if unformatted_attributes is None:
raise errors.NoFormatterFound(
'Unable to find event formatter for: {0:s}.'.format(
event_data.data_type))
event_attributes = list(event_data.GetAttributes())
if event_data_stream:
event_attributes.extend(event_data_stream.GetAttributes())
extra_attributes = []
for attribute_name, attribute_value in sorted(event_attributes):
if attribute_name in unformatted_attributes:
# Some parsers have written bytes values to storage.
if isinstance(attribute_value, bytes):
attribute_value = attribute_value.decode(
'utf-8', 'replace')
logger.warning(
'Found bytes value for attribute "{0:s}" for data type: '
'{1!s}. Value was converted to UTF-8: "{2:s}"'.format(
attribute_name, event_data.data_type,
attribute_value))
# With ! in {1!s} we force a string conversion since some of
# the extra attributes values can be integer, float point or
# boolean values.
extra_attributes.append('{0:s}: {1!s}'.format(
attribute_name, attribute_value))
extra_attributes = '; '.join(extra_attributes)
return extra_attributes.replace('\n', '-').replace('\r', '')
def ParseSourceShort(self, event_object):
"""Return the source string."""
# TODO: move this to an output module interface.
event_formatter = formatters_manager.FormattersManager.GetFormatterObject(
event_object.data_type)
if not event_formatter:
raise errors.NoFormatterFound(
u'Unable to find no event formatter for: {0:s}.'.format(
event_object.data_type))
source, _ = event_formatter.GetSources(event_object)
return source
def ParseSource(self, event_object):
"""Return the source string."""
# TODO: move this to an output module interface.
event_formatter = formatters_manager.EventFormatterManager.GetFormatter(
event_object)
if not event_formatter:
raise errors.NoFormatterFound(
u'Unable to find no event formatter for: {0:s}.'.format(
event_object.DATA_TYPE))
_, source = event_formatter.GetSources(event_object)
return source
def EventBody(self, event_object):
"""Formats data as TLN and writes to the filehandle from OutputFormater.
Args:
event_object: The event object (EventObject).
Raises:
errors.NoFormatterFound: If no formatter for that event is found.
"""
if not hasattr(event_object, 'timestamp'):
return
# TODO: move this to an output module interface.
event_formatter = formatters_manager.EventFormatterManager.GetFormatter(
event_object)
if not event_formatter:
raise errors.NoFormatterFound(
u'Unable to find event formatter for: {0:s}.'.format(
event_object.DATA_TYPE))
msg, _ = event_formatter.GetMessages(event_object)
source_short, _ = event_formatter.GetSources(event_object)
date_use = timelib.Timestamp.CopyToPosix(event_object.timestamp)
hostname = getattr(event_object, 'hostname', u'')
username = getattr(event_object, 'username', u'')
if self.store:
if not hostname:
hostname = self._hostnames.get(event_object.store_number, '')
pre_obj = self._preprocesses.get(event_object.store_number)
if pre_obj:
check_user = pre_obj.GetUsernameById(username)
if check_user != '-':
username = check_user
notes = getattr(event_object, 'notes', u'')
if not notes:
notes = u'File: {0:s} inode: {1!s}'.format(
getattr(event_object, 'display_name', u''),
getattr(event_object, 'inode', u''))
out_write = u'{0!s}|{1:s}|{2:s}|{3:s}|{4:s}|{5:s}|{6!s}\n'.format(
date_use, source_short.replace(self.DELIMITER, u' '),
hostname.replace(self.DELIMITER, u' '),
username.replace(self.DELIMITER, u' '),
msg.replace(self.DELIMITER, u' '), self.zone,
notes.replace(self.DELIMITER, u' '))
self.filehandle.WriteLine(out_write)
def _FormatSource(self, event_object):
"""Formats the source.
Args:
event_object: the event object (instance of EventObject).
Returns:
A string containing the value for the source field.
"""
source_short, _ = self._output_mediator.GetFormattedSources(event_object)
if source_short is None:
raise errors.NoFormatterFound(
u'Unable to find event formatter for: {0:s}.'.format(
getattr(event_object, u'data_type', u'UNKNOWN')))
return self._SanitizeField(source_short)
def _FormatSource(self, event):
"""Formats the source.
Args:
event (EventObject): event.
Returns:
str: formatted source field.
"""
source_short, _ = self._output_mediator.GetFormattedSources(event)
if source_short is None:
data_type = getattr(event, 'data_type', 'UNKNOWN')
raise errors.NoFormatterFound(
'Unable to find event formatter for: {0:s}.'.format(data_type))
return self._SanitizeField(source_short)
def ParseMessageShort(self, event_object):
"""Return the message string from the EventObject.
Args:
event_object: The event object (EventObject).
Raises:
errors.NoFormatterFound: If no formatter for that event is found.
"""
event_formatter = eventdata.EventFormatterManager.GetFormatter(event_object)
if not event_formatter:
raise errors.NoFormatterFound(
u'Unable to find no event formatter for: {0:s}.'.format(
event_object.DATA_TYPE))
_, msg_short = event_formatter.GetMessages(event_object)
return msg_short
def ParseMessage(self, event_object):
"""Return the message string from the EventObject.
Args:
event_object: The event object (EventObject).
Raises:
errors.NoFormatterFound: If no formatter for that event is found.
"""
# TODO: move this to an output module interface.
event_formatter = formatters_manager.FormattersManager.GetFormatterObject(
event_object.data_type)
if not event_formatter:
raise errors.NoFormatterFound(
u'Unable to find no event formatter for: {0:s}.'.format(
event_object.data_type))
msg, _ = event_formatter.GetMessages(self._formatter_mediator, event_object)
return msg
def EventBody(self, event_object):
"""Formats data as TLN and writes to the filehandle from OutputFormater.
Args:
event_object: The event object (EventObject).
Raises:
errors.NoFormatterFound: If no formatter for that event is found.
"""
if not hasattr(event_object, 'timestamp'):
return
event_formatter = eventdata.EventFormatterManager.GetFormatter(
event_object)
if not event_formatter:
raise errors.NoFormatterFound(
u'Unable to find event formatter for: {0:s}.'.format(
event_object.DATA_TYPE))
msg, _ = event_formatter.GetMessages(event_object)
source_short, _ = event_formatter.GetSources(event_object)
date_use = timelib.Timestamp.CopyToPosix(event_object.timestamp)
hostname = getattr(event_object, 'hostname', u'')
username = getattr(event_object, 'username', u'')
if self.store:
if not hostname:
hostname = self._hostnames.get(event_object.store_number, u'')
pre_obj = self._preprocesses.get(event_object.store_number)
if pre_obj:
check_user = pre_obj.GetUsernameById(username)
if check_user != '-':
username = check_user
out_write = u'{}|{}|{}|{}|{}\n'.format(
date_use, source_short.replace(self.DELIMITER, u' '),
hostname.replace(self.DELIMITER, u' '),
username.replace(self.DELIMITER, u' '),
msg.replace(self.DELIMITER, u' '))
self.filehandle.WriteLine(out_write)
def _FormatSourceShort(self, event):
"""Formats the short source.
Args:
event (EventObject): event.
Returns:
str: short source field.
Raises:
NoFormatterFound: If no event formatter can be found to match the data
type in the event.
"""
source_short, _ = self._output_mediator.GetFormattedSources(event)
if source_short is None:
data_type = getattr(event, 'data_type', 'UNKNOWN')
raise errors.NoFormatterFound(
'Unable to find event formatter for: {0:s}.'.format(data_type))
return source_short
def _FormatMessage(self, event):
"""Formats the message.
Args:
event (EventObject): event.
Returns:
str: message field.
Raises:
NoFormatterFound: if no event formatter can be found to match the data
type in the event.
"""
message, _ = self._output_mediator.GetFormattedMessages(event)
if message is None:
data_type = getattr(event, 'data_type', 'UNKNOWN')
raise errors.NoFormatterFound(
'Unable to find event formatter for: {0:s}.'.format(data_type))
return message
def _FormatSource(self, event):
"""Formats the source.
Args:
event (EventObject): event.
Returns:
str: source field.
Raises:
NoFormatterFound: If no event formatter can be found to match the data
type in the event.
"""
_, source = self._output_mediator.GetFormattedSources(event)
if source is None:
raise errors.NoFormatterFound(
u'Unable to find event formatter for: {0:s}.'.format(
getattr(event, u'data_type', u'UNKNOWN')))
return source
def _FormatMessageShort(self, event):
"""Formats the short message.
Args:
event (EventObject): event.
Returns:
str: short message field.
Raises:
NoFormatterFound: If no event formatter can be found to match the data
type in the event.
"""
_, message_short = self._output_mediator.GetFormattedMessages(event)
if message_short is None:
raise errors.NoFormatterFound(
u'Unable to find event formatter for: {0:s}.'.format(
getattr(event, u'data_type', u'UNKNOWN')))
return message_short
def _FormatSource(self, event_object):
"""Formats the source.
Args:
event_object: an event object (instance of EventObject).
Returns:
A string containing the value for the source field.
Raises:
NoFormatterFound: If no event formatter can be found to match the data
type in the event object.
"""
_, source = self._output_mediator.GetFormattedSources(event_object)
if source is None:
raise errors.NoFormatterFound(
u'Unable to find event formatter for: {0:s}.'.format(
getattr(event_object, u'data_type', u'UNKNOWN')))
return source
def _FormatMessage(self, event_object):
"""Formats the message.
Args:
event_object: the event object (instance of EventObject).
Returns:
A string containing the value for the message field.
Raises:
NoFormatterFound: If no event formatter can be found to match the data
type in the event object.
"""
message, _ = self._output_mediator.GetFormattedMessages(event_object)
if message is None:
raise errors.NoFormatterFound(
u'Unable to find event formatter for: {0:s}.'.format(
getattr(event_object, u'data_type', u'UNKNOWN')))
return self._SanitizeField(message)
def _FormatSource(self, event, event_data):
"""Formats the source.
Args:
event (EventObject): event.
event_data (EventData): event data.
Returns:
str: source field.
Raises:
NoFormatterFound: if no event formatter can be found to match the data
type in the event data.
"""
_, source = self._output_mediator.GetFormattedSources(event, event_data)
if source is None:
data_type = getattr(event_data, 'data_type', 'UNKNOWN')
raise errors.NoFormatterFound(
'Unable to create source for event with data type: {0:s}.'.format(
data_type))
return source
def _FormatMessage(self, event, event_data, event_data_stream):
"""Formats a message field.
Args:
event (EventObject): event.
event_data (EventData): event data.
event_data_stream (EventDataStream): event data stream.
Returns:
str: message field.
Raises:
NoFormatterFound: if no event formatter can be found to match the data
type in the event data.
"""
# TODO: refactor GetFormattedMessages by GetFormattedMessage.
message, _ = self._output_mediator.GetFormattedMessages(event_data)
if message is None:
raise errors.NoFormatterFound(
'Unable to create message for event with data type: {0:s}.'.
format(event_data.data_type))
return message
def _FormatDescription(self, event_object):
"""Formats the description.
Args:
event_object: the event object (instance of EventObject).
Returns:
A string containing the value for the description field.
"""
date_time_string = timelib.Timestamp.CopyToIsoFormat(
event_object.timestamp, timezone=self._output_mediator.timezone)
timestamp_description = getattr(event_object, u'timestamp_desc', u'UNKNOWN')
message, _ = self._output_mediator.GetFormattedMessages(event_object)
if message is None:
raise errors.NoFormatterFound(
u'Unable to find event formatter for: {0:s}.'.format(
getattr(event_object, u'data_type', u'UNKNOWN')))
description = u'{0:s}; {1:s}; {2:s}'.format(
date_time_string, timestamp_description,
message.replace(self._DESCRIPTION_FIELD_DELIMITER, u' '))
return self._SanitizeField(description)
def _GetSanitizedEventValues(self, event):
"""Sanitizes the event object for use in 4n6time.
Args:
event (EventObject): event.
Returns:
A dictionary object containing the sanitized values.
Raises:
NoFormatterFound: If no event formatter can be found to match the data
type in the event object.
"""
data_type = getattr(event, u'data_type', u'UNKNOWN')
event_formatter = self._output_mediator.GetEventFormatter(event)
if not event_formatter:
raise errors.NoFormatterFound(
u'Unable to find event formatter for: {0:s}.'.format(
data_type))
message, _ = self._output_mediator.GetFormattedMessages(event)
if message is None:
raise errors.NoFormatterFound(
u'Unable to find event formatter for: {0:s}.'.format(
data_type))
source_short, source = self._output_mediator.GetFormattedSources(event)
if source is None or source_short is None:
raise errors.NoFormatterFound(
u'Unable to find event formatter for: {0:s}.'.format(
data_type))
datetime_object = None
if event.timestamp is not None:
datetime_object = timelib.Timestamp.CopyToDatetime(
event.timestamp, self._output_mediator.timezone)
if not datetime_object:
self._ReportEventError(
event,
(u'unable to copy timestamp: {0:d} to datetime object.'))
return
format_variables = self._output_mediator.GetFormatStringAttributeNames(
event)
if format_variables is None:
raise errors.NoFormatterFound(
u'Unable to find event formatter for: {0:s}.'.format(
data_type))
extra_attributes = []
for attribute_name, attribute_value in sorted(event.GetAttributes()):
if (attribute_name in definitions.RESERVED_VARIABLE_NAMES
or attribute_name in format_variables):
continue
extra_attributes.append(u'{0:s}: {1!s} '.format(
attribute_name, attribute_value))
extra_attributes = u' '.join(extra_attributes)
inode = event.inode
if inode is None and hasattr(event, u'pathspec'):
inode = getattr(event.pathspec, u'inode', u'-')
if inode is None:
inode = u'-'
datetime_string = u'N/A'
if datetime_object:
datetime_string = (
u'{0:04d}-{1:02d}-{2:02d} {3:02d}:{4:02d}:{5:02d}'.format(
datetime_object.year, datetime_object.month,
datetime_object.day, datetime_object.hour,
datetime_object.minute, datetime_object.second))
tags = None
if getattr(event, u'tag', None):
tags = getattr(event.tag, u'tags', None)
taglist = u''
if isinstance(tags, (list, tuple)):
taglist = u','.join(tags)
offset = event.offset
if offset is None:
offset = 0
row = {
u'timezone': u'{0!s}'.format(self._output_mediator.timezone),
u'MACB': self._output_mediator.GetMACBRepresentation(event),
u'source': source_short,
u'sourcetype': source,
u'type': getattr(event, u'timestamp_desc', u'-'),
u'user': getattr(event, u'username', u'-'),
u'host': getattr(event, u'hostname', u'-'),
u'description': message,
u'filename': getattr(event, u'filename', u'-'),
u'inode': inode,
u'notes': getattr(event, u'notes', u'-'),
u'format': getattr(event, u'parser', u'-'),
u'extra': extra_attributes,
u'datetime': datetime_string,
u'reportnotes': u'',
u'inreport': u'',
u'tag': taglist,
u'offset': offset,
u'vss_store_number': self._GetVSSNumber(event),
u'URL': getattr(event, u'url', u'-'),
u'record_number': getattr(event, u'record_number', 0),
u'event_identifier': getattr(event, u'event_identifier', u'-'),
u'event_type': getattr(event, u'event_type', u'-'),
u'source_name': getattr(event, u'source_name', u'-'),
u'user_sid': getattr(event, u'user_sid', u'-'),
u'computer_name': getattr(event, u'computer_name', u'-'),
u'evidence': self._evidence
}
return row
def EventBody(self, event_object):
"""Formats data as 4n6time database table format and writes to the db.
Args:
event_object: The event object (EventObject).
Raises:
raise errors.NoFormatterFound: If no formatter for this event is found.
"""
if not hasattr(event_object, 'timestamp'):
return
formatter = eventdata.EventFormatterManager.GetFormatter(event_object)
if not formatter:
raise errors.NoFormatterFound(
u'Unable to output event, no formatter found.')
if (isinstance(
formatter, formatters.winreg.WinRegistryGenericFormatter) and
formatter.FORMAT_STRING.find('<|>') == -1):
formatter.FORMAT_STRING = u'[{keyname}]<|>{text}<|>'
elif isinstance(formatter, eventdata.ConditionalEventFormatter):
formatter.FORMAT_STRING_SEPARATOR = u'<|>'
elif isinstance(formatter, eventdata.EventFormatter):
formatter.format_string = formatter.format_string.replace('}', '}<|>')
msg, msg_short = formatter.GetMessages(event_object)
source_short, source_long = formatter.GetSources(event_object)
date_use = timelib.Timestamp.CopyToDatetime(
event_object.timestamp, self.zone)
if not date_use:
logging.error(u'Unable to process date for entry: {0:s}'.format(msg))
return
extra = []
format_variables = self.FORMAT_ATTRIBUTE_RE.findall(
formatter.format_string)
for key in event_object.GetAttributes():
if key in utils.RESERVED_VARIABLES or key in format_variables:
continue
extra.append(u'{0:s}: {1!s} '.format(
key, getattr(event_object, key, None)))
extra = u' '.join(extra)
inode = getattr(event_object, 'inode', '-')
if inode == '-':
if (hasattr(event_object, 'pathspec') and
hasattr(event_object.pathspec, 'image_inode')):
inode = event_object.pathspec.image_inode
date_use_string = u'{0}-{1}-{2} {3}:{4}:{5}'.format(
date_use.year, date_use.month, date_use.day, date_use.hour,
date_use.minute, date_use.second)
tags = []
if hasattr(event_object, 'tag'):
if hasattr(event_object.tag, 'tags'):
tags = event_object.tag.tags
else:
tags = u''
else:
tags = u''
taglist = u','.join(tags)
row = (str(self.zone),
helper.GetLegacy(event_object),
source_short,
source_long,
getattr(event_object, 'timestamp_desc', '-'),
getattr(event_object, 'username', '-'),
getattr(event_object, 'hostname', '-'),
msg,
getattr(event_object, 'filename', '-'),
inode,
getattr(event_object, 'notes', '-'),
getattr(event_object, 'parser', '-'),
extra,
date_use_string,
'',
'',
taglist,
'',
getattr(event_object, 'offset', 0),
event_object.store_number,
event_object.store_index,
self.GetVSSNumber(event_object),
getattr(event_object, 'url', '-'),
getattr(event_object, 'record_number', 0),
getattr(event_object, 'event_identifier', '-'),
getattr(event_object, 'event_type', '-'),
getattr(event_object, 'source_name', '-'),
getattr(event_object, 'user_sid', '-'),
getattr(event_object, 'computer_name', '-'),
self.evidence)
try:
self.curs.execute(
'INSERT INTO log2timeline(timezone, MACB, source, '
'sourcetype, type, user, host, description, filename, '
'inode, notes, format, extra, datetime, reportnotes, '
'inreport, tag, color, offset, store_number, '
'store_index, vss_store_number, URL, record_number, '
'event_identifier, event_type, source_name, user_sid, '
'computer_name, evidence) VALUES ('
'%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, '
'%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, '
'%s, %s, %s, %s)', row)
except MySQLdb.Error as exception:
logging.warning(
u'Unable to insert into database with error: {0:s}.'.format(
exception))
self.count += 1
# TODO: Experiment if committing the current transaction
# every 10000 inserts is the optimal approach.
if self.count % 10000 == 0:
self.conn.commit()
if self.set_status:
self.set_status(u'Inserting event: {0:d}'.format(self.count))
