def SetTimelineName(self, timeline_name):
"""Sets the timeline name.
Args:
timeline_name (str): timeline name.
"""
self._timeline_name = timeline_name
logger.info('Timeline name: {0:s}'.format(self._timeline_name))
def SetTimelineOwner(self, username):
"""Sets the username of the user that should own the timeline.
Args:
username (str): username.
"""
self._timeline_owner = username
logger.info('Owner of the timeline: {0!s}'.format(self._timeline_owner))
def SetTimelineOwner(self, username):
"""Sets the username of the user that should own the timeline.
Args:
username (str): username.
"""
self._timeline_owner = username
logger.info('Owner of the timeline: {0!s}'.format(self._timeline_owner))
def SetUserName(self, username):
"""Sets the username of the user that should own the timeline.
Args:
username (str): username.
"""
self._username = username
logger.info('Owner of the timeline: {0:s}'.format(self._username))
def SetFlushInterval(self, flush_interval):
"""Set the flush interval.
Args:
flush_interval (int): Number of events to buffer before bulk insert.
"""
self._flush_interval = flush_interval
logger.info('Flush interval: {0:d}'.format(self._flush_interval))
def SetElasticPassword(self, elastic_password):
"""Set the Elastic password.
Args:
elastic_password (str): Elastic password to authenticate with.
"""
self._elastic_password = elastic_password
logger.info('Elastic password: {0:s}'.format('****'))
def SetDocType(self, doc_type):
"""Sets the Elasticsearch document type.
Args:
doc_type (str): document type.
"""
self._doc_type = doc_type
logger.info('Document type: {0:s}'.format(self._doc_type))
def SetElasticUser(self, elastic_user):
"""Set the Elastic username.
Args:
elastic_user (str): Elastic user to authenticate with.
"""
self._elastic_user = elastic_user
logger.info('Elastic user: {0:s}'.format(self._elastic_user))
def SetFlushInterval(self, flush_interval):
"""Sets the flush interval.
Args:
flush_interval (int): flush interval.
"""
self._flush_interval = flush_interval
logger.info('Flush interval: {0:d}'.format(self._flush_interval))
def SetDocType(self, doc_type):
"""Set the port.
Args:
doc_type (str): The document type to use when indexing.
"""
self._doc_type = doc_type
logger.info('Document type: {0:s}'.format(self._doc_type))
def SetIndexName(self, index_name):
"""Sets the index name.
Args:
index_name (str): index name.
"""
self._index_name = index_name
logger.info('Index name: {0:s}'.format(self._index_name))
def SetTimelineName(self, timeline_name):
"""Sets the timeline name.
Args:
timeline_name (str): timeline name.
"""
self._timeline_name = timeline_name
logger.info('Timeline name: {0:s}'.format(self._timeline_name))
def SetTimelineIdentifier(self, timeline_identifier):
"""Sets the timeline identifier.
Args:
timeline_identifier (int): timeline identifier.
"""
self._timeline_identifier = timeline_identifier
logger.info('Timeline identifier: {0:d}'.format(
self._timeline_identifier))
def SetServerInformation(self, server, port):
"""Set the Elasticsearch server information.
Args:
server (str): IP address or hostname of the server.
port (int): Port number of the server.
"""
self._host = server
self._port = port
logger.info('Server address: {0:s}'.format(self._host))
logger.info('Server port: {0:d}'.format(self._port))
def SetRawFields(self, raw_fields):
"""Set raw (not analyzed) fields.
This is used for sorting and aggregations in Elasticsearch.
https://www.elastic.co/guide/en/elasticsearch/guide/current/
multi-fields.html
Args:
raw_fields (bool): Add not-analyzed index for string fields.
"""
self._raw_fields = raw_fields
logger.info('Add non analyzed string fields: {0!s}'.format(
self._raw_fields))
def _FlushEventsToElasticSearch(self):
"""Insert events in bulk to Elasticsearch."""
try:
self.client.bulk(index=self._index,
doc_type=self._doc_type,
body=self._events)
except ValueError as e:
# Ignore problematic events
logger.warning('{0:s}'.format(e))
# Clear the events list
self._events = []
logger.info('{0:d} events added'.format(self._counter['events']))
def WriteHeader(self):
"""Setup the Elasticsearch index."""
if not self._mapping:
self._mapping = {}
if self._raw_fields:
if self._doc_type not in self._mapping:
self._mapping[self._doc_type] = {}
_raw_field_mapping = [{
'strings': {
'match_mapping_type': 'string',
'mapping': {
'fields': {
'raw': {
'type': 'text',
'index': 'not_analyzed',
'ignore_above':
self._ELASTIC_ANALYZER_STRING_LIMIT
}
}
}
}
}]
self._mapping[
self._doc_type]['dynamic_templates'] = _raw_field_mapping
self._elastic = ElasticSearchHelper(
self._output_mediator,
self._host,
self._port,
self._flush_interval,
self._index_name,
self._mapping,
self._doc_type,
elastic_password=self._elastic_password,
elastic_user=self._elastic_user)
logger.info('Adding events to Elasticsearch..')
def __init__(self, output_mediator):
"""Initializes a Timesketch output module.
Args:
output_mediator (OutputMediator): mediates interactions between output
modules and other components, such as storage and dfvfs.
"""
super(TimesketchOutputModule, self).__init__(output_mediator)
self._doc_type = None
self._elastic = None
self._flush_interval = None
self._host = None
self._index_name = None
self._mapping = None
self._port = None
self._timesketch = timesketch.create_app()
self._username = None
hostname = self._output_mediator.GetStoredHostname()
if hostname:
logger.info('Hostname: {0:s}'.format(hostname))
self._timeline_name = hostname
else:
self._timeline_name = None
def WriteHeader(self):
"""Setup the Elasticsearch index and the Timesketch database object.
Creates the Elasticsearch index with Timesketch specific settings and the
Timesketch SearchIndex database object.
"""
# This cannot be static because we use the value of self._doc_type from
# arguments.
_document_mapping = {
self._doc_type: {
'properties': {
'timesketch_label': {
'type': 'nested'
}
}
}
}
# Get Elasticsearch host and port from Timesketch configuration.
with self._timesketch.app_context():
_host = current_app.config['ELASTIC_HOST']
_port = current_app.config['ELASTIC_PORT']
self._elastic = ElasticSearchHelper(self._output_mediator, _host,
_port, self._flush_interval,
self._index_name,
_document_mapping, self._doc_type)
user = None
if self._username:
user = User.query.filter_by(username=self._username).first()
if not user:
raise RuntimeError('Unknown Timesketch user: {0:s}'.format(
self._username))
else:
logger.warning('Timeline will be visible to all Timesketch users')
with self._timesketch.app_context():
search_index = SearchIndex.get_or_create(
name=self._timeline_name,
description=self._timeline_name,
user=user,
index_name=self._index_name)
# Grant the user read permission on the mapping object and set status.
# If user is None the timeline becomes visible to all users.
search_index.grant_permission(user=user, permission='read')
# In case we have a user grant additional permissions.
if user:
search_index.grant_permission(user=user, permission='write')
search_index.grant_permission(user=user, permission='delete')
# Let the Timesketch UI know that the timeline is processing.
search_index.set_status('processing')
# Save the mapping object to the Timesketch database.
db_session.add(search_index)
db_session.commit()
logger.info('Adding events to Timesketch.')
