def WriteHeader(self):
"""Setup the Elasticsearch index and the Timesketch database object.
Creates the Elasticsearch index with Timesketch specific settings and the
Timesketch SearchIndex database object.
"""
# This cannot be static because we use the value of self._doc_type from
# arguments.
_document_mapping = {
self._doc_type: {
u'properties': {
u'timesketch_label': {
u'type': u'nested'
}
}
}
}
# Get Elasticsearch host and port from Timesketch configuration.
with self._timesketch.app_context():
_host = current_app.config[u'ELASTIC_HOST']
_port = current_app.config[u'ELASTIC_PORT']
self._elastic = ElasticSearchHelper(
self._output_mediator, _host, _port, self._flush_interval,
self._index_name, _document_mapping, self._doc_type)
user = None
if self._username:
user = User.query.filter_by(username=self._username).first()
if not user:
raise RuntimeError(
u'Unknown Timesketch user: {0:s}'.format(self._username))
else:
logging.warning(u'Timeline will be visible to all Timesketch users')
with self._timesketch.app_context():
search_index = SearchIndex.get_or_create(
name=self._timeline_name, description=self._timeline_name, user=user,
index_name=self._index_name)
# Grant the user read permission on the mapping object and set status.
# If user is None the timeline becomes visible to all users.
search_index.grant_permission(user=user, permission=u'read')
# In case we have a user grant additional permissions.
if user:
search_index.grant_permission(user=user, permission=u'write')
search_index.grant_permission(user=user, permission=u'delete')
# Let the Timesketch UI know that the timeline is processing.
search_index.set_status(u'processing')
# Save the mapping object to the Timesketch database.
db_session.add(search_index)
db_session.commit()
logging.info(u'Adding events to Timesketch.')
def _create_searchindex(self, name, user, acl=False):
"""Create a searchindex in the database.
Args:
name: Name of the searchindex (string)
user: A user (instance of timesketch.models.user.User)
acl: Boolean value to decide if ACL permissions should be set
Returns:
A searchindex (instance of timesketch.models.sketch.SearchIndex)
"""
searchindex = SearchIndex(
name=name, description=name, index_name=name, user=user)
if acl:
for permission in [u'read', u'write', u'delete']:
searchindex.grant_permission(user=user, permission=permission)
self._commit_to_database(searchindex)
return searchindex
def Close(self):
"""Closes the connection to TimeSketch Elasticsearch database.
Sends the remaining events for indexing and adds the timeline to Timesketch.
"""
self._FlushEventsToElasticsearch()
with self._timesketch.app_context():
# Get Timesketch user object, or None if user do not exist. This is a
# SQLAlchemy query against the Timesketch database.
user_query = User.query.filter_by(username=self._timeline_owner)
user = user_query.first()
search_index = SearchIndex(
name=self._timeline_name, description=self._timeline_name, user=user,
index_name=self._index_name)
# Grant all users read permission on the mapping object.
search_index.grant_permission(None, u'read')
# Save the mapping object to the Timesketch database.
db_session.add(search_index)
db_session.commit()
def run(self, name, index, username):
"""Create the SearchIndex."""
es = ElasticsearchDataStore(
host=current_app.config['ELASTIC_HOST'],
port=current_app.config['ELASTIC_PORT'])
user = User.query.filter_by(username=username).first()
if not user:
sys.stderr.write('User does not exist\n')
sys.exit(1)
if not es.client.indices.exists(index=index):
sys.stderr.write('Index does not exist in the datastore\n')
sys.exit(1)
if SearchIndex.query.filter_by(name=name, index_name=index).first():
sys.stderr.write(
'Index with this name already exist in Timesketch\n')
sys.exit(1)
searchindex = SearchIndex(
name=name, description=name, user=user, index_name=index)
searchindex.grant_permission('read')
db_session.add(searchindex)
db_session.commit()
sys.stdout.write('Search index {0:s} created\n'.format(name))
def post(self):
"""Handles POST request to the resource.
Returns:
A search index in JSON (instance of flask.wrappers.Response)
"""
form = forms.SearchIndexForm.build(request)
searchindex_name = form.searchindex_name.data
es_index_name = form.es_index_name.data
public = form.public.data
if not form.validate_on_submit():
abort(HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to validate form data')
searchindex = SearchIndex.query.filter_by(
index_name=es_index_name).first()
metadata = {'created': True}
if searchindex:
metadata['created'] = False
metadata['deleted'] = searchindex.get_status.status == 'deleted'
status_code = HTTP_STATUS_CODE_OK
else:
searchindex = SearchIndex.get_or_create(
name=searchindex_name,
description=searchindex_name,
user=current_user,
index_name=es_index_name)
searchindex.grant_permission(permission='read', user=current_user)
searchindex.grant_permission(permission='write', user=current_user)
searchindex.grant_permission(permission='delete',
user=current_user)
if public:
searchindex.grant_permission(permission='read', user=None)
datastore = self.datastore
if not datastore.client.indices.exists(index=es_index_name):
# Create the index in Elasticsearch
self.datastore.create_index(index_name=es_index_name,
doc_type='generic_event')
db_session.add(searchindex)
db_session.commit()
status_code = HTTP_STATUS_CODE_CREATED
return self.to_json(searchindex,
meta=metadata,
status_code=status_code)
def post(self):
"""Handles POST request to the resource.
Returns:
A view in JSON (instance of flask.wrappers.Response)
Raises:
ApiHTTPError
"""
UPLOAD_ENABLED = current_app.config[u'UPLOAD_ENABLED']
UPLOAD_FOLDER = current_app.config[u'UPLOAD_FOLDER']
form = UploadFileForm()
if form.validate_on_submit() and UPLOAD_ENABLED:
from timesketch.lib.tasks import run_plaso
file_storage = form.file.data
timeline_name = form.name.data
# We do not need a human readable filename or
# datastore index name, so we use UUIDs here.
filename = unicode(uuid.uuid4().hex)
index_name = unicode(uuid.uuid4().hex)
file_path = os.path.join(UPLOAD_FOLDER, filename)
file_storage.save(file_path)
search_index = SearchIndex.get_or_create(
name=timeline_name, description=timeline_name,
user=current_user, index_name=index_name)
search_index.grant_permission(permission=u'read', user=current_user)
search_index.grant_permission(
permission=u'write', user=current_user)
search_index.grant_permission(
permission=u'delete', user=current_user)
search_index.set_status(u'processing')
db_session.add(search_index)
db_session.commit()
run_plaso.apply_async(
(file_path, timeline_name, index_name), task_id=index_name)
return self.to_json(
search_index, status_code=HTTP_STATUS_CODE_CREATED)
else:
raise ApiHTTPError(
message=form.errors[u'file'][0],
status_code=HTTP_STATUS_CODE_BAD_REQUEST)
def timelines(sketch_id):
"""Generates the sketch explore view template.
Returns:
Template with context.
"""
TIMELINES_TO_SHOW = 20
sketch = Sketch.query.get_with_acl(sketch_id)
searchindices_in_sketch = [t.searchindex.id for t in sketch.timelines]
query = request.args.get(u'q', None)
indices = SearchIndex.all_with_acl(
current_user).order_by(
desc(SearchIndex.created_at)).filter(
not_(SearchIndex.id.in_(searchindices_in_sketch)))
filtered = False
if query:
indices = indices.filter(SearchIndex.name.contains(query)).limit(500)
filtered = True
if not filtered:
indices = indices.limit(TIMELINES_TO_SHOW)
# Setup the form
form = AddTimelineForm()
form.timelines.choices = set((i.id, i.name) for i in indices.all())
# Create new timeline form POST
if form.validate_on_submit():
if not sketch.has_permission(current_user, u'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN)
for searchindex_id in form.timelines.data:
searchindex = SearchIndex.query.get_with_acl(searchindex_id)
if searchindex not in [t.searchindex for t in sketch.timelines]:
_timeline = Timeline(
name=searchindex.name, description=searchindex.description,
sketch=sketch, user=current_user, searchindex=searchindex)
db_session.add(_timeline)
sketch.timelines.append(_timeline)
db_session.commit()
return redirect(url_for(u'sketch_views.timelines', sketch_id=sketch.id))
return render_template(
u'sketch/timelines.html', sketch=sketch, timelines=indices.all(),
form=form, filtered=filtered)
def WriteHeader(self):
"""Setup the Elasticsearch index and the Timesketch database object.
Creates the Elasticsearch index with Timesketch specific settings and the
Timesketch SearchIndex database object.
"""
# This cannot be static because we use the value of self._doc_type from
# arguments.
_document_mapping = {
self._doc_type: {
u'_timestamp': {
u'enabled': True,
u'path': u'datetime',
u'format': u'date_time_no_millis'
},
u'properties': {u'timesketch_label': {u'type': u'nested'}}
}
}
if not self._elastic_db.client.indices.exists(self._index_name):
try:
self._elastic_db.client.indices.create(
index=self._index_name, body={u'mappings': _document_mapping})
except elastic_exceptions.ConnectionError as exception:
logging.error((
u'Unable to proceed, cannot connect to Timesketch backend '
u'with error: {0:s}.\nPlease verify connection.').format(exception))
raise RuntimeError(u'Unable to connect to Timesketch backend.')
with self._timesketch.app_context():
search_index = SearchIndex.get_or_create(
name=self._timeline_name, description=self._timeline_name, user=None,
index_name=self._index_name)
# Grant all users read permission on the mapping object and set status.
search_index.grant_permission(None, u'read')
search_index.set_status(u'processing')
# Save the mapping object to the Timesketch database.
db_session.add(search_index)
db_session.commit()
logging.info(u'Adding events to Timesketch..')
def post(self, sketch_id):
"""Handles POST request to the resource.
Returns:
A sketch in JSON (instance of flask.wrappers.Response)
Raises:
ApiHTTPError
"""
sketch = Sketch.query.get_with_acl(sketch_id)
searchindices_in_sketch = [t.searchindex.id for t in sketch.timelines]
indices = SearchIndex.all_with_acl(
current_user).order_by(
desc(SearchIndex.created_at)).filter(
not_(SearchIndex.id.in_(searchindices_in_sketch)))
add_timeline_form = AddTimelineForm.build(request)
add_timeline_form.timelines.choices = set(
(i.id, i.name) for i in indices.all())
if add_timeline_form.validate_on_submit():
if not sketch.has_permission(current_user, u'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN)
for searchindex_id in add_timeline_form.timelines.data:
searchindex = SearchIndex.query.get_with_acl(searchindex_id)
if searchindex not in [t.searchindex for t in sketch.timelines]:
_timeline = Timeline(
name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=current_user,
searchindex=searchindex)
db_session.add(_timeline)
sketch.timelines.append(_timeline)
db_session.commit()
return self.to_json(sketch, status_code=HTTP_STATUS_CODE_CREATED)
else:
raise ApiHTTPError(
message=add_timeline_form.errors,
status_code=HTTP_STATUS_CODE_BAD_REQUEST)
def post(self):
"""Handles POST request to the resource.
Returns:
A search index in JSON (instance of flask.wrappers.Response)
"""
form = SearchIndexForm.build(request)
timeline_name = form.timeline_name.data
index_name = form.index_name.data
public = form.public.data
if form.validate_on_submit():
searchindex = SearchIndex.query.filter_by(
index_name=index_name).first()
if not searchindex:
searchindex = SearchIndex.get_or_create(
name=timeline_name,
description=timeline_name,
user=current_user,
index_name=index_name)
searchindex.grant_permission(permission=u'read',
user=current_user)
if public:
searchindex.grant_permission(permission=u'read', user=None)
# Create the index in Elasticsearch
self.datastore.create_index(index_name=index_name,
doc_type=u'generic_event')
db_session.add(searchindex)
db_session.commit()
return self.to_json(searchindex,
status_code=HTTP_STATUS_CODE_CREATED)
return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def timelines(sketch_id):
"""Generates the sketch explore view template.
Returns:
Template with context.
"""
sketch = Sketch.query.get_with_acl(sketch_id)
searchindices_in_sketch = [t.searchindex.id for t in sketch.timelines]
indices = SearchIndex.all_with_acl(current_user).order_by(
desc(SearchIndex.created_at)).filter(
not_(SearchIndex.id.in_(searchindices_in_sketch)))
upload_enabled = current_app.config['UPLOAD_ENABLED']
graphs_enabled = current_app.config['GRAPH_BACKEND_ENABLED']
try:
plaso_version = current_app.config['PLASO_VERSION']
except KeyError:
plaso_version = 'Unknown'
# Setup the form
form = AddTimelineForm()
form.timelines.choices = set((i.id, i.name) for i in indices.all())
# Create new timeline form POST
if form.validate_on_submit():
if not sketch.has_permission(current_user, 'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN)
for searchindex_id in form.timelines.data:
searchindex = SearchIndex.query.get_with_acl(searchindex_id)
if searchindex not in [t.searchindex for t in sketch.timelines]:
_timeline = Timeline(
name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=current_user,
searchindex=searchindex)
db_session.add(_timeline)
sketch.timelines.append(_timeline)
db_session.commit()
# If enabled, run sketch analyzers when timeline is added.
# Import here to avoid circular imports.
from timesketch.lib import tasks
sketch_analyzer_group = tasks.build_sketch_analysis_pipeline(
sketch_id)
if sketch_analyzer_group:
pipeline = (tasks.run_sketch_init.s(
[searchindex.index_name]) | sketch_analyzer_group)
pipeline.apply_async(task_id=searchindex.index_name)
return redirect(
url_for('sketch_views.timelines', sketch_id=sketch.id))
return render_template(
'sketch/timelines.html',
sketch=sketch,
timelines=indices.all(),
form=form,
upload_enabled=upload_enabled,
plaso_version=plaso_version,
graphs_enabled=graphs_enabled)
def run(self, file_path, sketch_id, username, timeline_name):
"""This is the run method."""
file_path = os.path.realpath(file_path)
file_path_no_extension, extension = os.path.splitext(file_path)
extension = extension.lstrip('.')
filename = os.path.basename(file_path_no_extension)
supported_extensions = ('plaso', 'csv', 'jsonl')
if not os.path.isfile(file_path):
sys.exit('No such file: {0:s}'.format(file_path))
if extension not in supported_extensions:
sys.exit(
'Extension {0:s} is not supported. '
'(supported extensions are: {1:s})'.format(
extension, ', '.join(supported_extensions)))
user = None
if not username:
username = pwd.getpwuid(os.stat(file_path).st_uid).pw_name
if not username == 'root':
if not isinstance(username, six.text_type):
username = codecs.decode(username, 'utf-8')
user = User.query.filter_by(username=username).first()
if not user:
sys.exit('Cannot determine user for file: {0:s}'.format(file_path))
sketch = None
# If filename starts with <number> then use that as sketch_id.
# E.g: 42_file_name.plaso means sketch_id is 42.
sketch_id_from_filename = filename.split('_')[0]
if not sketch_id and sketch_id_from_filename.isdigit():
sketch_id = sketch_id_from_filename
if sketch_id:
try:
sketch = Sketch.query.get_with_acl(sketch_id, user=user)
except Forbidden:
pass
if not timeline_name:
if not isinstance(timeline_name, six.text_type):
timeline_name = codecs.decode(timeline_name, 'utf-8')
timeline_name = timeline_name.replace('_', ' ')
# Remove sketch ID if present in the filename.
timeline_parts = timeline_name.split()
if timeline_parts[0].isdigit():
timeline_name = ' '.join(timeline_name.split()[1:])
if not sketch:
# Create a new sketch.
sketch_name = 'Sketch for: {0:s}'.format(timeline_name)
sketch = Sketch(
name=sketch_name, description=sketch_name, user=user)
# Need to commit here to be able to set permissions later.
db_session.add(sketch)
db_session.commit()
sketch.grant_permission(permission='read', user=user)
sketch.grant_permission(permission='write', user=user)
sketch.grant_permission(permission='delete', user=user)
sketch.status.append(sketch.Status(user=None, status='new'))
db_session.add(sketch)
db_session.commit()
index_name = uuid.uuid4().hex
if not isinstance(index_name, six.text_type):
index_name = codecs.decode(index_name, 'utf-8')
searchindex = SearchIndex.get_or_create(
name=timeline_name,
description=timeline_name,
user=user,
index_name=index_name)
searchindex.grant_permission(permission='read', user=user)
searchindex.grant_permission(permission='write', user=user)
searchindex.grant_permission(permission='delete', user=user)
searchindex.set_status('processing')
db_session.add(searchindex)
db_session.commit()
if sketch and sketch.has_permission(user, 'write'):
timeline = Timeline(
name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=user,
searchindex=searchindex)
timeline.set_status('processing')
sketch.timelines.append(timeline)
db_session.add(timeline)
db_session.commit()
# Start Celery pipeline for indexing and analysis.
# Import here to avoid circular imports.
from timesketch.lib import tasks
pipeline = tasks.build_index_pipeline(
file_path, timeline_name, index_name, extension, sketch.id)
pipeline.apply_async(task_id=index_name)
print('Imported {0:s} to sketch: {1:d} ({2:s})'.format(
file_path, sketch.id, sketch.name))
def setup_sketch(timeline_name, index_name, username, sketch_id=None):
"""Use existing sketch or create a new sketch.
Args:
timeline_name: (str) Name of the Timeline
index_name: (str) Name of the index
username: (str) Who should own the timeline
sketch_id: (str) Optional sketch_id to add timeline to
Returns:
(tuple) sketch ID and timeline ID as integers
"""
with app.app_context():
user = User.get_or_create(username=username)
sketch = None
if sketch_id:
try:
sketch = Sketch.query.get_with_acl(sketch_id, user=user)
logger.info('Using existing sketch: {} ({})'.format(
sketch.name, sketch.id))
except Forbidden:
pass
if not (sketch or sketch_id):
# Create a new sketch.
sketch_name = 'Turbinia: {}'.format(timeline_name)
sketch = Sketch(name=sketch_name,
description=sketch_name,
user=user)
# Need to commit here to be able to set permissions later.
db_session.add(sketch)
db_session.commit()
sketch.grant_permission(permission='read', user=user)
sketch.grant_permission(permission='write', user=user)
sketch.grant_permission(permission='delete', user=user)
sketch.status.append(sketch.Status(user=None, status='new'))
db_session.add(sketch)
db_session.commit()
logger.info('Created new sketch: {} ({})'.format(
sketch.name, sketch.id))
searchindex = SearchIndex.get_or_create(
name=timeline_name,
description='Created by Turbinia.',
user=user,
index_name=index_name)
searchindex.grant_permission(permission='read', user=user)
searchindex.grant_permission(permission='write', user=user)
searchindex.grant_permission(permission='delete', user=user)
searchindex.set_status('processing')
db_session.add(searchindex)
db_session.commit()
timeline = Timeline(name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=user,
searchindex=searchindex)
# If the user doesn't have write access to the sketch then create the
# timeline but don't attach it to the sketch.
if not sketch.has_permission(user, 'write'):
timeline.sketch = None
else:
sketch.timelines.append(timeline)
db_session.add(timeline)
db_session.commit()
timeline.set_status('processing')
return sketch.id, timeline.id
