def WriteEventBody(self, event):
"""Writes the body of an event object to the output.
Args:
event (EventObject): event.
"""
if not hasattr(event, 'timestamp'):
return
row = self._GetSanitizedEventValues(event)
try:
self._cursor.execute(self._INSERT_QUERY, row)
except MySQLdb.Error as exception:
logger.warning(
'Unable to insert into database with error: {0!s}.'.format(
exception))
self._count += 1
# TODO: Experiment if committing the current transaction
# every 10000 inserts is the optimal approach.
if self._count % 10000 == 0:
self._connection.commit()
if self._set_status:
self._set_status('Inserting event: {0:d}'.format(self._count))
def _FlushEvents(self):
"""Inserts the buffered event documents into Elasticsearch."""
try:
# pylint: disable=unexpected-keyword-arg
bulk_arguments = {
'body': self._event_documents,
'index': self._index_name,
'request_timeout': self._DEFAULT_REQUEST_TIMEOUT
}
# TODO: Remove once Elasticsearch v6.x is deprecated.
if self._GetClientMajorVersion() < 7:
bulk_arguments['doc_type'] = self._document_type
self._client.bulk(**bulk_arguments)
except (ValueError,
elasticsearch.exceptions.ElasticsearchException) as exception:
# Ignore problematic events
logger.warning(
'Unable to bulk insert with error: {0!s}'.format(exception))
logger.debug('Inserted {0:d} events into Elasticsearch'.format(
self._number_of_buffered_events))
self._event_documents = []
self._number_of_buffered_events = 0
def WriteEventBody(self, event, event_data, event_tag):
"""Writes event values to the output.
Args:
event (EventObject): event.
event_data (EventData): event data.
event_tag (EventTag): event tag.
"""
if not hasattr(event, 'timestamp'):
return
row = self._GetSanitizedEventValues(event, event_data, event_tag)
try:
self._cursor.execute(self._INSERT_QUERY, row)
except MySQLdb.Error as exception:
logger.warning(
'Unable to insert into database with error: {0!s}.'.format(
exception))
self._count += 1
# TODO: Experiment if committing the current transaction
# every 10000 inserts is the optimal approach.
if self._count % 10000 == 0:
self._connection.commit()
if self._set_status:
self._set_status('Inserting event: {0:d}'.format(self._count))
def WriteHeader(self):
"""Sets up the Elasticsearch index and the Timesketch database object.
Creates the Elasticsearch index with Timesketch specific settings and the
Timesketch SearchIndex database object.
"""
# This cannot be static because we use the value of self._document_type
# from arguments.
mappings = {
'properties': {
'timesketch_label': {
'type': 'nested'
},
'datetime': {
'type': 'date'
}
}
}
# Get Elasticsearch host and port from Timesketch configuration.
with self._timesketch.app_context():
self._host = current_app.config['ELASTIC_HOST']
self._port = current_app.config['ELASTIC_PORT']
self._Connect()
self._CreateIndexIfNotExists(self._index_name, mappings)
user = None
if self._timeline_owner:
user = timesketch_user.User.query.filter_by(
username=self._timeline_owner).first()
if not user:
raise RuntimeError(
'Unknown Timesketch user: {0:s}'.format(self._timeline_owner))
else:
logger.warning('Timeline will be visible to all Timesketch users')
with self._timesketch.app_context():
search_index = timesketch_sketch.SearchIndex.get_or_create(
name=self._timeline_name, description=self._timeline_name, user=user,
index_name=self._index_name)
# Grant the user read permission on the mapping object and set status.
# If user is None the timeline becomes visible to all users.
search_index.grant_permission(user=user, permission='read')
# In case we have a user grant additional permissions.
if user:
search_index.grant_permission(user=user, permission='write')
search_index.grant_permission(user=user, permission='delete')
# Let the Timesketch UI know that the timeline is processing.
search_index.set_status('processing')
# Save the mapping object to the Timesketch database.
timesketch_db_session.add(search_index)
timesketch_db_session.commit()
logger.debug('Adding events to Timesketch.')
def WriteHeader(self):
"""Sets up the Elasticsearch index and the Timesketch database object.
Creates the Elasticsearch index with Timesketch specific settings and the
Timesketch SearchIndex database object.
"""
# This cannot be static because we use the value of self._document_type
# from arguments.
mappings = {
self._document_type: {
'properties': {
'timesketch_label': {
'type': 'nested'
}
}
}
}
# Get Elasticsearch host and port from Timesketch configuration.
with self._timesketch.app_context():
self._host = current_app.config['ELASTIC_HOST']
self._port = current_app.config['ELASTIC_PORT']
self._Connect()
self._CreateIndexIfNotExists(self._index_name, mappings)
user = None
if self._timeline_owner:
user = timesketch_user.User.query.filter_by(
username=self._timeline_owner).first()
if not user:
raise RuntimeError(
'Unknown Timesketch user: {0:s}'.format(self._timeline_owner))
else:
logger.warning('Timeline will be visible to all Timesketch users')
with self._timesketch.app_context():
search_index = timesketch_sketch.SearchIndex.get_or_create(
name=self._timeline_name, description=self._timeline_name, user=user,
index_name=self._index_name)
# Grant the user read permission on the mapping object and set status.
# If user is None the timeline becomes visible to all users.
search_index.grant_permission(user=user, permission='read')
# In case we have a user grant additional permissions.
if user:
search_index.grant_permission(user=user, permission='write')
search_index.grant_permission(user=user, permission='delete')
# Let the Timesketch UI know that the timeline is processing.
search_index.set_status('processing')
# Save the mapping object to the Timesketch database.
timesketch_db_session.add(search_index)
timesketch_db_session.commit()
logger.debug('Adding events to Timesketch.')
def _FormatExtraAttributes(self, event, event_data, event_data_stream):
"""Formats an extra attributes field.
Args:
event (EventObject): event.
event_data (EventData): event data.
event_data_stream (EventDataStream): event data stream.
Returns:
str: extra attributes field.
Raises:
NoFormatterFound: if no event formatter can be found to match the data
type in the event data.
"""
message_formatter = self._output_mediator.GetMessageFormatter(
event_data.data_type)
if not message_formatter:
raise errors.NoFormatterFound(
('Unable to find message formatter event with data type: '
'{0:s}.').format(event_data.data_type))
formatted_attribute_names = (
message_formatter.GetFormatStringAttributeNames())
formatted_attribute_names.update(definitions.RESERVED_VARIABLE_NAMES)
extra_attributes = []
for attribute_name, attribute_value in event_data.GetAttributes():
if attribute_name not in formatted_attribute_names:
# Some parsers have written bytes values to storage.
if isinstance(attribute_value, bytes):
attribute_value = attribute_value.decode(
'utf-8', 'replace')
logger.warning(
'Found bytes value for attribute "{0:s}" for data type: '
'{1!s}. Value was converted to UTF-8: "{2:s}"'.format(
attribute_name, event_data.data_type,
attribute_value))
# With ! in {1!s} we force a string conversion since some of
# the extra attributes values can be integer, float point or
# boolean values.
extra_attributes.append('{0:s}: {1!s}'.format(
attribute_name, attribute_value))
if event_data_stream:
for attribute_name, attribute_value in event_data_stream.GetAttributes(
):
if attribute_name != 'path_spec':
extra_attributes.append('{0:s}: {1!s}'.format(
attribute_name, attribute_value))
extra_attributes = '; '.join(sorted(extra_attributes))
return extra_attributes.replace('\n', '-').replace('\r', '')
def _FormatExtraAttributes(self, event, event_data, event_data_stream):
"""Formats an extra attributes field.
Args:
event (EventObject): event.
event_data (EventData): event data.
event_data_stream (EventDataStream): event data stream.
Returns:
str: extra attributes field.
Raises:
NoFormatterFound: if no event formatter can be found to match the data
type in the event data.
"""
event_attributes = list(event_data.GetAttributes())
if event_data_stream:
event_attributes.extend(event_data_stream.GetAttributes())
# TODO: reverse logic and get formatted attributes instead.
unformatted_attributes = (formatters_manager.FormattersManager.
GetUnformattedAttributes(event_data))
if unformatted_attributes is None:
raise errors.NoFormatterFound(
'Unable to find event formatter for: {0:s}.'.format(
event_data.data_type))
event_attributes = list(event_data.GetAttributes())
if event_data_stream:
event_attributes.extend(event_data_stream.GetAttributes())
extra_attributes = []
for attribute_name, attribute_value in sorted(event_attributes):
if attribute_name in unformatted_attributes:
# Some parsers have written bytes values to storage.
if isinstance(attribute_value, bytes):
attribute_value = attribute_value.decode(
'utf-8', 'replace')
logger.warning(
'Found bytes value for attribute "{0:s}" for data type: '
'{1!s}. Value was converted to UTF-8: "{2:s}"'.format(
attribute_name, event_data.data_type,
attribute_value))
# With ! in {1!s} we force a string conversion since some of
# the extra attributes values can be integer, float point or
# boolean values.
extra_attributes.append('{0:s}: {1!s}'.format(
attribute_name, attribute_value))
extra_attributes = '; '.join(extra_attributes)
return extra_attributes.replace('\n', '-').replace('\r', '')
def _FlushEventsToElasticSearch(self):
"""Insert events in bulk to Elasticsearch."""
try:
self.client.bulk(index=self._index,
doc_type=self._doc_type,
body=self._events)
except ValueError as e:
# Ignore problematic events
logger.warning('{0:s}'.format(e))
# Clear the events list
self._events = []
logger.info('{0:d} events added'.format(self._counter['events']))
def WriteEventBody_old(self, event):
"""
WriteEventBody write event data at database
:param event:
:return:
:raises pymysql.Error: Unable action
"""
if not hasattr(event, 'timestamp'):
return
row = self._GetSanitizedEventValues(event)
if row['datetime'] == 'N/A':
row['datetime'] = '1900-01-01 00:00:00.000'
row['case_id'] = self._case_id
row['evd_id'] = self._evd_id
row['par_id'] = self._par_id
_INSERT_QUERY = (
'INSERT INTO log2timeline(timezone, MACB, source, '
'sourcetype, type, user, host, description, filename, '
'inode, notes, format, extra, datetime, reportnotes, inreport, '
'tag, offset, vss_store_number, URL, record_number, '
'event_identifier, event_type, source_name, user_sid, computer_name, '
'evidence, case_id, evd_id, par_id) '
'VALUES (%(timezone)s, %(MACB)s, %(source)s, %(sourcetype)s, %(type)s, %(user)s, %(host)s, '
'%(description)s, %(filename)s, %(inode)s, %(notes)s, %(format)s, %(extra)s, %(datetime)s, '
'%(reportnotes)s, %(inreport)s, %(tag)s, %(offset)s, %(vss_store_number)s, %(URL)s, '
'%(record_number)s, %(event_identifier)s, %(event_type)s, %(source_name)s, '
'%(user_sid)s, %(computer_name)s, %(evidence)s, %(case_id)s, '
'%(evd_id)s, %(par_id)s)')
try:
self._cursor.execute(_INSERT_QUERY, row)
except pymysql.Error as exception:
logger.warning(
'Unable to insert into database with error: {0!s}.'.format(
exception))
print(row['description'])
self._count += 1
# TODO: Experiment if committing the current transaction
# every 10000 inserts is the optimal approach.
if self._count % 10000 == 0:
self._connection.commit()
if self._set_status:
self._set_status('Inserting event: {0:d}'.format(self._count))
def GetMessageFormatter(self, data_type):
"""Retrieves the message formatter for a specific data type.
Args:
data_type (str): data type.
Returns:
EventFormatter: corresponding message formatter or the default message
formatter if not available.
"""
data_type = data_type.lower()
message_formatter = self._message_formatters.get(data_type, None)
if not message_formatter:
logger.warning(
'Using default message formatter for data type: {0:s}'.format(
data_type))
message_formatter = self._DEFAULT_MESSAGE_FORMATTER
return message_formatter
def _SanitizeField(self, data_type, attribute_name, field):
"""Sanitizes a field for output.
Args:
data_type (str): event data type.
attribute_name (str): name of the event attribute.
field (object): value of the field to sanitize.
Returns:
object: sanitized value of the field.
"""
# Some parsers have written bytes values to storage.
if isinstance(field, bytes):
field = field.decode('utf-8', 'replace')
logger.warning(
'Found bytes value for attribute: {0:s} of data type: '
'{1!s}. Value was converted to UTF-8: "{2:s}"'.format(
attribute_name, data_type, field))
return field
def _FlushEvents(self):
"""Inserts the buffered event documents into Elasticsearch."""
try:
# pylint: disable=unexpected-keyword-arg
# pylint does not recognizes request_timeout as a valid kwarg. According
# to http://elasticsearch-py.readthedocs.io/en/master/api.html#timeout
# it should be supported.
self._client.bulk(
body=self._event_documents, doc_type=self._document_type,
index=self._index_name, request_timeout=self._DEFAULT_REQUEST_TIMEOUT)
except ValueError as exception:
# Ignore problematic events
logger.warning('Unable to bulk insert with error: {0!s}'.format(
exception))
logger.debug('Inserted {0:d} events into Elasticsearch'.format(
self._number_of_buffered_events))
self._event_documents = []
self._number_of_buffered_events = 0
def _GetSanitizedEventValues(self, event, event_data, event_tag):
"""Sanitizes the event for use in Elasticsearch.
The event values need to be sanitized to prevent certain values from
causing problems when indexing with Elasticsearch. For example the path
specification is a nested dictionary which will cause problems for
Elasticsearch automatic indexing.
Args:
event (EventObject): event.
event_data (EventData): event data.
event_tag (EventTag): event tag.
Returns:
dict[str, object]: sanitized event values.
Raises:
NoFormatterFound: if no event formatter can be found to match the data
type in the event data.
"""
event_values = {}
for attribute_name, attribute_value in event_data.GetAttributes():
# TODO: remove regvalue, which is kept for backwards compatibility.
# Ignore the regvalue attribute as it cause issues when indexing.
if attribute_name == 'regvalue':
continue
if attribute_name == 'pathspec':
try:
attribute_value = JsonPathSpecSerializer.WriteSerialized(
attribute_value)
except TypeError:
continue
event_values[attribute_name] = attribute_value
if isinstance(attribute_value, py2to3.BYTES_TYPE):
# Some parsers have written bytes values to storage.
attribute_value = attribute_value.decode('utf-8', 'replace')
logger.warning(
'Found bytes value for attribute "{0:s}" for data type: '
'{1!s}. Value was converted to UTF-8: "{2:s}"'.format(
attribute_name, event_data.data_type, attribute_value))
event_values[attribute_name] = attribute_value
# Add a string representation of the timestamp.
try:
attribute_value = timelib.Timestamp.RoundToSeconds(event.timestamp)
except TypeError as exception:
logger.warning(('Unable to round timestamp {0!s}. error: {1!s}. '
'Defaulting to 0').format(event.timestamp,
exception))
attribute_value = 0
attribute_value = timelib.Timestamp.CopyToIsoFormat(
attribute_value, timezone=self._output_mediator.timezone)
event_values['datetime'] = attribute_value
event_values['timestamp'] = event.timestamp
event_values['timestamp_desc'] = event.timestamp_desc
message, _ = self._output_mediator.GetFormattedMessages(event_data)
if message is None:
data_type = getattr(event_data, 'data_type', 'UNKNOWN')
raise errors.NoFormatterFound(
'Unable to find event formatter for: {0:s}.'.format(data_type))
event_values['message'] = message
# Tags needs to be a list for Elasticsearch to index correctly.
labels = []
if event_tag:
try:
labels = list(event_tag.labels)
except (AttributeError, KeyError):
pass
event_values['tag'] = labels
source_short, source = self._output_mediator.GetFormattedSources(
event, event_data)
if source is None or source_short is None:
data_type = getattr(event_data, 'data_type', 'UNKNOWN')
raise errors.NoFormatterFound(
'Unable to find event formatter for: {0:s}.'.format(data_type))
event_values['source_short'] = source_short
event_values['source_long'] = source
return event_values
def GetFormattedEvent(self, event, event_data, event_data_stream, event_tag):
"""Retrieves a string representation of the event.
Args:
event (EventObject): event.
event_data (EventData): event data.
event_data_stream (EventDataStream): event data stream.
event_tag (EventTag): event tag.
Returns:
str: string representation of the event.
"""
date_time = dfdatetime_posix_time.PosixTimeInMicroseconds(
timestamp=event.timestamp)
date_time_string = date_time.CopyToDateTimeStringISO8601()
lines_of_text = [
'+-' * 40,
'[Timestamp]:',
' {0:s}'.format(date_time_string)]
path_specification = getattr(event_data_stream, 'path_spec', None)
if not path_specification:
# Note that support for event_data.pathspec is kept for backwards
# compatibility.
path_specification = getattr(event_data, 'pathspec', None)
if path_specification:
lines_of_text.extend([
'',
'[Pathspec]:'])
lines_of_text.extend([
' {0:s}'.format(line)
for line in path_specification.comparable.split('\n')])
# Remove additional empty line.
lines_of_text.pop()
reserved_attributes = [
'',
'[Reserved attributes]:']
additional_attributes = [
'',
'[Additional attributes]:']
event_attributes = list(event_data.GetAttributes())
if event_data_stream:
event_attributes.extend(event_data_stream.GetAttributes())
event_attribute_names = [name for name, _ in event_attributes]
if 'display_name' not in event_attribute_names:
attribute_value = self._field_formatting_helper.GetFormattedField(
'display_name', event, event_data, event_data_stream, event_tag)
event_attributes.append(('display_name', attribute_value))
if 'filename' not in event_attribute_names:
attribute_value = self._field_formatting_helper.GetFormattedField(
'filename', event, event_data, event_data_stream, event_tag)
event_attributes.append(('filename', attribute_value))
if 'inode' not in event_attribute_names:
attribute_value = self._field_formatting_helper.GetFormattedField(
'inode', event, event_data, event_data_stream, event_tag)
event_attributes.append(('inode', attribute_value))
for attribute_name, attribute_value in sorted(event_attributes):
# Some parsers have written bytes values to storage.
if isinstance(attribute_value, bytes):
attribute_value = attribute_value.decode('utf-8', 'replace')
logger.warning(
'Found bytes value for attribute "{0:s}" for data type: '
'{1!s}. Value was converted to UTF-8: "{2:s}"'.format(
attribute_name, event_data.data_type, attribute_value))
# Note that support for event_data.pathspec is kept for backwards
# compatibility. The current value is event_data_stream.path_spec.
if attribute_name in ('path_spec', 'pathspec'):
continue
attribute_string = ' {{{0!s}}} {1!s}'.format(
attribute_name, attribute_value)
if attribute_name in definitions.RESERVED_VARIABLE_NAMES:
reserved_attributes.append(attribute_string)
else:
additional_attributes.append(attribute_string)
lines_of_text.extend(reserved_attributes)
lines_of_text.extend(additional_attributes)
if event_tag:
labels = [
'\'{0:s}\''.format(label) for label in event_tag.labels]
lines_of_text.extend([
'',
'[Tag]:',
' {{labels}} [{0:s}]'.format(', '.join(labels))])
lines_of_text.append('')
return '\n'.join(lines_of_text)
def _GetSanitizedEventValues(self, event):
"""Sanitizes the event for use in Elasticsearch.
The event values need to be sanitized to prevent certain values from
causing problems when indexing with Elasticsearch. For example the path
specification is a nested dictionary which will cause problems for
Elasticsearch automatic indexing.
Args:
event (EventObject): event.
Returns:
dict[str, object]: sanitized event values.
Raises:
NoFormatterFound: if no event formatter can be found to match the data
type in the event.
"""
event_values = {}
for attribute_name, attribute_value in event.GetAttributes():
# Ignore the regvalue attribute as it cause issues when indexing.
if attribute_name == 'regvalue':
continue
if attribute_name == 'pathspec':
try:
attribute_value = JsonPathSpecSerializer.WriteSerialized(
attribute_value)
except TypeError:
continue
event_values[attribute_name] = attribute_value
# Add a string representation of the timestamp.
try:
attribute_value = timelib.Timestamp.RoundToSeconds(event.timestamp)
except TypeError as exception:
logger.warning((
'Unable to round timestamp {0!s}. error: {1!s}. '
'Defaulting to 0').format(event.timestamp, exception))
attribute_value = 0
attribute_value = timelib.Timestamp.CopyToIsoFormat(
attribute_value, timezone=self._output_mediator.timezone)
event_values['datetime'] = attribute_value
message, _ = self._output_mediator.GetFormattedMessages(event)
if message is None:
data_type = getattr(event, 'data_type', 'UNKNOWN')
raise errors.NoFormatterFound(
'Unable to find event formatter for: {0:s}.'.format(data_type))
event_values['message'] = message
# Tags needs to be a list for Elasticsearch to index correctly.
try:
labels = list(event_values['tag'].labels)
except (KeyError, AttributeError):
labels = []
event_values['tag'] = labels
source_short, source = self._output_mediator.GetFormattedSources(event)
if source is None or source_short is None:
data_type = getattr(event, 'data_type', 'UNKNOWN')
raise errors.NoFormatterFound(
'Unable to find event formatter for: {0:s}.'.format(data_type))
event_values['source_short'] = source_short
event_values['source_long'] = source
return event_values
def GetFormattedEvent(cls, event, event_data, event_tag):
"""Retrieves a string representation of the event.
Args:
event (EventObject): event.
event_data (EventData): event data.
event_tag (EventTag): event tag.
Returns:
str: string representation of the event.
"""
date_time = dfdatetime_posix_time.PosixTimeInMicroseconds(
timestamp=event.timestamp)
date_time_string = date_time.CopyToDateTimeStringISO8601()
lines_of_text = [
'+-' * 40,
'[Timestamp]:',
' {0:s}'.format(date_time_string)]
pathspec = getattr(event_data, 'pathspec', None)
if pathspec:
lines_of_text.extend([
'',
'[Pathspec]:'])
lines_of_text.extend([
' {0:s}'.format(line) for line in pathspec.comparable.split('\n')])
# Remove additional empty line.
lines_of_text.pop()
reserved_attributes = [
'',
'[Reserved attributes]:']
additional_attributes = [
'',
'[Additional attributes]:']
for attribute_name, attribute_value in sorted(event_data.GetAttributes()):
# Some parsers have written bytes values to storage.
if isinstance(attribute_value, py2to3.BYTES_TYPE):
attribute_value = attribute_value.decode('utf-8', 'replace')
logger.warning(
'Found bytes value for attribute "{0:s}" for data type: '
'{1!s}. Value was converted to UTF-8: "{2:s}"'.format(
attribute_name, event_data.data_type, attribute_value))
if attribute_name == 'pathspec':
continue
attribute_string = ' {{{0!s}}} {1!s}'.format(
attribute_name, attribute_value)
if attribute_name in definitions.RESERVED_VARIABLE_NAMES:
reserved_attributes.append(attribute_string)
else:
additional_attributes.append(attribute_string)
lines_of_text.extend(reserved_attributes)
lines_of_text.extend(additional_attributes)
if event_tag:
labels = [
'\'{0:s}\''.format(label) for label in event_tag.labels]
lines_of_text.extend([
'',
'[Tag]:',
' {{labels}} [{0:s}]'.format(', '.join(labels))])
lines_of_text.extend(['', ''])
return '\n'.join(lines_of_text)
def _GetSanitizedEventValues(self, event, event_data, event_tag):
"""Sanitizes the event for use in 4n6time.
Args:
event (EventObject): event.
event_data (EventData): event data.
event_tag (EventTag): event tag.
Returns:
dict[str, object]: dictionary containing the sanitized event values.
Raises:
NoFormatterFound: If no event formatter can be found to match the data
type in the event data.
"""
data_type = getattr(event_data, 'data_type', 'UNKNOWN')
event_formatter = self._output_mediator.GetEventFormatter(event_data)
if not event_formatter:
raise errors.NoFormatterFound(
'Unable to find event formatter for: {0:s}.'.format(data_type))
message, _ = self._output_mediator.GetFormattedMessages(event_data)
if message is None:
raise errors.NoFormatterFound(
'Unable to find event formatter for: {0:s}.'.format(data_type))
source_short, source = self._output_mediator.GetFormattedSources(
event, event_data)
if source is None or source_short is None:
raise errors.NoFormatterFound(
'Unable to find event formatter for: {0:s}.'.format(data_type))
datetime_string = self._FormatDateTime(event, event_data)
unformatted_attributes = (formatters_manager.FormattersManager.
GetUnformattedAttributes(event_data))
if unformatted_attributes is None:
raise errors.NoFormatterFound(
'Unable to find event formatter for: {0:s}.'.format(data_type))
extra_attributes = []
for attribute_name, attribute_value in sorted(
event_data.GetAttributes()):
if attribute_name in unformatted_attributes:
# Some parsers have written bytes values to storage.
if isinstance(attribute_value, py2to3.BYTES_TYPE):
attribute_value = attribute_value.decode(
'utf-8', 'replace')
logger.warning(
'Found bytes value for attribute "{0:s}" for data type: '
'{1!s}. Value was converted to UTF-8: "{2:s}"'.format(
attribute_name, event_data.data_type,
attribute_value))
extra_attributes.append('{0:s}: {1!s} '.format(
attribute_name, attribute_value))
extra_attributes = ' '.join(extra_attributes)
inode = self._FormatInode(event_data)
vss_store_number = self._FormatVSSNumber(event_data)
tag = self._FormatTag(event_tag)
offset = event_data.offset
if offset is None:
offset = 0
row = {
'timezone': '{0!s}'.format(self._output_mediator.timezone),
'MACB':
self._output_mediator.GetMACBRepresentation(event, event_data),
'source': source_short,
'sourcetype': source,
'type': event.timestamp_desc or '-',
'user': getattr(event_data, 'username', '-'),
'host': getattr(event_data, 'hostname', '-'),
'description': message,
'filename': getattr(event_data, 'filename', '-'),
'inode': inode,
'notes': getattr(event_data, 'notes', '-'),
'format': getattr(event_data, 'parser', '-'),
'extra': extra_attributes,
'datetime': datetime_string,
'reportnotes': '',
'inreport': '',
'tag': tag,
'offset': offset,
'vss_store_number': vss_store_number,
'URL': getattr(event_data, 'url', '-'),
'record_number': getattr(event_data, 'record_number', 0),
'event_identifier': getattr(event_data, 'event_identifier', '-'),
'event_type': getattr(event_data, 'event_type', '-'),
'source_name': getattr(event_data, 'source_name', '-'),
'user_sid': getattr(event_data, 'user_sid', '-'),
'computer_name': getattr(event_data, 'computer_name', '-'),
'evidence': self._evidence
}
return row
def _GetSanitizedEventValues(self, event, event_data, event_data_stream,
event_tag):
"""Sanitizes the event for use in Elasticsearch.
The event values need to be sanitized to prevent certain values from
causing problems when indexing with Elasticsearch. For example the path
specification is a nested dictionary which will cause problems for
Elasticsearch automatic indexing.
Args:
event (EventObject): event.
event_data (EventData): event data.
event_data_stream (EventDataStream): event data stream.
event_tag (EventTag): event tag.
Returns:
dict[str, object]: sanitized event values.
Raises:
NoFormatterFound: if no event formatter can be found to match the data
type in the event data.
"""
event_attributes = list(event_data.GetAttributes())
if event_data_stream:
event_attributes.extend(event_data_stream.GetAttributes())
event_values = {}
for attribute_name, attribute_value in sorted(event_attributes):
# Note that support for event_data.pathspec is kept for backwards
# compatibility. The current value is event_data_stream.path_spec.
if attribute_name in ('path_spec', 'pathspec'):
try:
attribute_value = JsonPathSpecSerializer.WriteSerialized(
attribute_value)
except TypeError:
continue
event_values[attribute_name] = attribute_value
if isinstance(attribute_value, bytes):
# Some parsers have written bytes values to storage.
attribute_value = attribute_value.decode('utf-8', 'replace')
logger.warning(
'Found bytes value for attribute "{0:s}" for data type: '
'{1!s}. Value was converted to UTF-8: "{2:s}"'.format(
attribute_name, event_data.data_type, attribute_value))
event_values[attribute_name] = attribute_value
event_values['timestamp'] = event.timestamp
event_values['timestamp_desc'] = event.timestamp_desc
# Add a date and time string representation of the timestamp.
date_time = dfdatetime_posix_time.PosixTimeInMicroseconds(
timestamp=event.timestamp)
event_values['datetime'] = date_time.CopyToDateTimeStringISO8601()
message, _ = self._output_mediator.GetFormattedMessages(event_data)
if message is None:
data_type = getattr(event_data, 'data_type', 'UNKNOWN')
raise errors.NoFormatterFound(
'Unable to find event formatter for: {0:s}.'.format(data_type))
event_values['message'] = message
# Tags needs to be a list for Elasticsearch to index correctly.
labels = []
if event_tag:
try:
labels = list(event_tag.labels)
except (AttributeError, KeyError):
pass
event_values['tag'] = labels
source_short, source = self._output_mediator.GetFormattedSources(
event, event_data)
if source is None or source_short is None:
data_type = getattr(event_data, 'data_type', 'UNKNOWN')
raise errors.NoFormatterFound(
'Unable to find event formatter for: {0:s}.'.format(data_type))
event_values['source_short'] = source_short
event_values['source_long'] = source
return event_values
def WriteHeader(self):
"""Setup the Elasticsearch index and the Timesketch database object.
Creates the Elasticsearch index with Timesketch specific settings and the
Timesketch SearchIndex database object.
"""
# This cannot be static because we use the value of self._doc_type from
# arguments.
_document_mapping = {
self._doc_type: {
'properties': {
'timesketch_label': {
'type': 'nested'
}
}
}
}
# Get Elasticsearch host and port from Timesketch configuration.
with self._timesketch.app_context():
_host = current_app.config['ELASTIC_HOST']
_port = current_app.config['ELASTIC_PORT']
self._elastic = ElasticSearchHelper(self._output_mediator, _host,
_port, self._flush_interval,
self._index_name,
_document_mapping, self._doc_type)
user = None
if self._username:
user = User.query.filter_by(username=self._username).first()
if not user:
raise RuntimeError('Unknown Timesketch user: {0:s}'.format(
self._username))
else:
logger.warning('Timeline will be visible to all Timesketch users')
with self._timesketch.app_context():
search_index = SearchIndex.get_or_create(
name=self._timeline_name,
description=self._timeline_name,
user=user,
index_name=self._index_name)
# Grant the user read permission on the mapping object and set status.
# If user is None the timeline becomes visible to all users.
search_index.grant_permission(user=user, permission='read')
# In case we have a user grant additional permissions.
if user:
search_index.grant_permission(user=user, permission='write')
search_index.grant_permission(user=user, permission='delete')
# Let the Timesketch UI know that the timeline is processing.
search_index.set_status('processing')
# Save the mapping object to the Timesketch database.
db_session.add(search_index)
db_session.commit()
logger.info('Adding events to Timesketch.')
def _GetOutputValues(self, event, event_data, event_tag):
"""Retrieves output values.
Args:
event (EventObject): event.
event_data (EventData): event data.
event_tag (EventTag): event tag.
Returns:
list[str]: output values or None if no timestamp was present in the event.
Raises:
NoFormatterFound: If no event formatter can be found to match the data
type in the event data.
"""
if not hasattr(event, 'timestamp'):
logger.error('Unable to output event without timestamp.')
return None
data_type = getattr(event_data, 'data_type', 'UNKNOWN')
# TODO: add function to pass event_values to GetFormattedMessages.
message, message_short = self._output_mediator.GetFormattedMessages(
event_data)
if message is None or message_short is None:
raise errors.NoFormatterFound(
'Unable to find event formatter for: {0:s}.'.format(data_type))
# TODO: add function to pass event_values to GetFormattedSources.
source_short, source = self._output_mediator.GetFormattedSources(
event, event_data)
if source is None or source_short is None:
raise errors.NoFormatterFound(
'Unable to find event formatter for: {0:s}.'.format(data_type))
# TODO: preserve dfdatetime as an object.
# TODO: add support for self._output_mediator.timezone
date_time = dfdatetime_posix_time.PosixTimeInMicroseconds(
timestamp=event.timestamp)
unformatted_attributes = (
formatters_manager.FormattersManager.GetUnformattedAttributes(
event_data))
if unformatted_attributes is None:
raise errors.NoFormatterFound(
'Unable to find event formatter for: {0:s}.'.format(data_type))
extra_attributes = []
for attribute_name, attribute_value in sorted(event_data.GetAttributes()):
if attribute_name in unformatted_attributes:
# Some parsers have written bytes values to storage.
if isinstance(attribute_value, bytes):
attribute_value = attribute_value.decode('utf-8', 'replace')
logger.warning(
'Found bytes value for attribute "{0:s}" for data type: '
'{1!s}. Value was converted to UTF-8: "{2:s}"'.format(
attribute_name, event_data.data_type, attribute_value))
# With ! in {1!s} we force a string conversion since some of
# the extra attributes values can be integer, float point or
# boolean values.
extra_attributes.append('{0:s}: {1!s}'.format(
attribute_name, attribute_value))
extra_attributes = '; '.join(extra_attributes)
extra_attributes = extra_attributes.replace('\n', '-').replace('\r', '')
inode = self._FormatInode(event_data)
hostname = self._FormatHostname(event_data)
username = self._FormatUsername(event_data)
if event_tag:
notes = ' '.join(event_tag.labels) or '-'
else:
notes = '-'
year, month, day_of_month = date_time.GetDate()
hours, minutes, seconds = date_time.GetTimeOfDay()
try:
date_string = '{0:02d}/{1:02d}/{2:04d}'.format(month, day_of_month, year)
time_string = '{0:02d}:{1:02d}:{2:02d}'.format(hours, minutes, seconds)
except (TypeError, ValueError):
self._ReportEventError(event, event_data, (
'unable to copy timestamp: {0!s} to a human readable date and time. '
'Defaulting to: "00/00/0000" "--:--:--"').format(event.timestamp))
date_string = '00/00/0000'
time_string = '--:--:--'
output_values = [
date_string,
time_string,
'{0!s}'.format(self._output_mediator.timezone),
'....',
source_short,
source,
'-',
username,
hostname,
message_short,
message,
'2',
getattr(event_data, 'display_name', '-'),
'{0!s}'.format(inode),
notes,
getattr(event_data, 'parser', '-'),
extra_attributes]
return output_values
