def home():
"""Generates the home page view template.
Returns:
Template with context.
"""
form = HiddenNameDescriptionForm()
sketches = Sketch.all_with_acl().filter(
not_(Sketch.Status.status == 'deleted'),
Sketch.Status.parent).order_by(Sketch.updated_at.desc())
# Only render upload button if it is configured.
upload_enabled = current_app.config['UPLOAD_ENABLED']
# Handle form for creating a new sketch.
if form.validate_on_submit():
sketch = Sketch(
name=form.name.data,
description=form.description.data,
user=current_user)
sketch.status.append(sketch.Status(user=None, status='new'))
db_session.add(sketch)
db_session.commit()
# Give the requesting user permissions on the new sketch.
sketch.grant_permission(permission='read', user=current_user)
sketch.grant_permission(permission='write', user=current_user)
sketch.grant_permission(permission='delete', user=current_user)
return redirect(url_for('sketch_views.overview', sketch_id=sketch.id))
return render_template(
'home/home.html',
sketches=sketches,
form=form,
upload_enabled=upload_enabled)
def run_wrapper(self):
"""A wrapper method to run the analyzer.
This method is decorated to flush the bulk insert operation on the
datastore. This makes sure that all events are indexed at exit.
Returns:
Return value of the run method.
"""
result = self.run()
# Update the searchindex description with analyzer result.
# TODO: Don't overload the description field.
searchindex = SearchIndex.query.filter_by(
index_name=self.index_name).first()
# Some code paths set the description equals to the name. Remove that
# here to get a clean description with only analyzer results.
if searchindex.description == searchindex.name:
searchindex.description = ''
# Append the analyzer result.
if result:
searchindex.description = '{0:s}\n{1:s}'.format(
searchindex.description, result)
db_session.add(searchindex)
db_session.commit()
return result
def explore(sketch_id, view_id=None):
"""Generates the sketch explore view template.
Returns:
Template with context.
"""
sketch = Sketch.query.get_with_acl(sketch_id)
if view_id:
view = View.query.get(view_id)
else:
view = View.query.filter(
View.user == current_user,
View.name == u'',
View.sketch_id == sketch_id).order_by(
View.created_at.desc()).first()
if not view:
view = View(
user=current_user, name=u'', sketch=sketch, query_string=u'',
query_filter=u'{}')
db_session.add(view)
db_session.commit()
sketch_timelines = u','.join(
[t.searchindex.index_name for t in sketch.timelines])
view_form = SaveViewForm()
return render_template(
u'sketch/explore.html', sketch=sketch, view=view,
timelines=sketch_timelines, view_form=view_form)
def _set_timeline_status(index_name, status, error_msg=None):
"""Helper function to set status for searchindex and all related timelines.
Args:
index_name: Name of the datastore index.
status: Status to set.
error_msg: Error message.
"""
searchindex = SearchIndex.query.filter_by(index_name=index_name).first()
timelines = Timeline.query.filter_by(searchindex=searchindex).all()
# Set status
searchindex.set_status(status)
for timeline in timelines:
timeline.set_status(status)
db_session.add(timeline)
# Update description if there was a failure in ingestion
if error_msg and status == 'fail':
# TODO: Don't overload the description field.
searchindex.description = error_msg
# Commit changes to database
db_session.add(searchindex)
db_session.commit()
def add_view(self, view_name, analyzer_name, query_string=None,
query_dsl=None, query_filter=None):
"""Add saved view to the Sketch.
Args:
view_name: The name of the view.
analyzer_name: The name of the analyzer.
query_string: Elasticsearch query string.
query_dsl: Dictionary with Elasticsearch DSL query.
query_filter: Dictionary with Elasticsearch filters.
Raises:
ValueError: If both query_string an query_dsl are missing.
Returns: An instance of a SQLAlchemy View object.
"""
if not query_string or query_dsl:
raise ValueError('Both query_string and query_dsl are missing.')
if not query_filter:
query_filter = {'indices': '_all'}
name = '[{0:s}] {1:s}'.format(analyzer_name, view_name)
view = View.get_or_create(name=name, sketch=self.sql_sketch, user=None)
view.query_string = query_string
view.query_filter = view.validate_filter(query_filter)
view.query_dsl = query_dsl
view.searchtemplate = None
db_session.add(view)
db_session.commit()
return view
def timeline(sketch_id, timeline_id):
"""Generates the sketch timeline view template.
Returns:
Template with context.
"""
timeline_form = TimelineForm()
sketch = Sketch.query.get_with_acl(sketch_id)
sketch_timeline = Timeline.query.filter(
Timeline.id == timeline_id, Timeline.sketch == sketch).first()
if not sketch_timeline:
abort(HTTP_STATUS_CODE_NOT_FOUND)
if timeline_form.validate_on_submit():
if not sketch.has_permission(current_user, u'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN)
sketch_timeline.name = timeline_form.name.data
sketch_timeline.description = timeline_form.description.data
sketch_timeline.color = timeline_form.color.data
db_session.add(sketch_timeline)
db_session.commit()
return redirect(
url_for(u'sketch_views.timeline', sketch_id=sketch.id,
timeline_id=sketch_timeline.id))
return render_template(
u'sketch/timeline.html', sketch=sketch, timeline=sketch_timeline,
timeline_form=timeline_form)
def WriteHeader(self):
"""Sets up the Elasticsearch index and the Timesketch database object.
Creates the Elasticsearch index with Timesketch specific settings and the
Timesketch SearchIndex database object.
"""
# This cannot be static because we use the value of self._document_type
# from arguments.
mappings = {
self._document_type: {
'properties': {
'timesketch_label': {
'type': 'nested'
}
}
}
}
# Get Elasticsearch host and port from Timesketch configuration.
with self._timesketch.app_context():
self._host = current_app.config['ELASTIC_HOST']
self._port = current_app.config['ELASTIC_PORT']
self._Connect()
self._CreateIndexIfNotExists(self._index_name, mappings)
user = None
if self._timeline_owner:
user = timesketch_user.User.query.filter_by(
username=self._timeline_owner).first()
if not user:
raise RuntimeError(
'Unknown Timesketch user: {0:s}'.format(self._timeline_owner))
else:
logger.warning('Timeline will be visible to all Timesketch users')
with self._timesketch.app_context():
search_index = timesketch_sketch.SearchIndex.get_or_create(
name=self._timeline_name, description=self._timeline_name, user=user,
index_name=self._index_name)
# Grant the user read permission on the mapping object and set status.
# If user is None the timeline becomes visible to all users.
search_index.grant_permission(user=user, permission='read')
# In case we have a user grant additional permissions.
if user:
search_index.grant_permission(user=user, permission='write')
search_index.grant_permission(user=user, permission='delete')
# Let the Timesketch UI know that the timeline is processing.
search_index.set_status('processing')
# Save the mapping object to the Timesketch database.
timesketch_db_session.add(search_index)
timesketch_db_session.commit()
logger.debug('Adding events to Timesketch.')
def run(self, name):
"""Creates the group."""
if not isinstance(name, six.text_type):
name = codecs.decode(name, 'utf-8')
group = Group.get_or_create(name=name)
db_session.add(group)
db_session.commit()
sys.stdout.write('Group {0:s} created\n'.format(name))
def _commit_to_database(self, model):
"""Add object to the database session and commit.
Args:
model: Instance of timesketch.models.[model] object
"""
db_session.add(model)
db_session.commit()
def post(self, sketch_id):
"""Handles POST request to the resource.
Args:
sketch_id: Integer primary key for a sketch database model
Returns:
An annotation in JSON (instance of flask.wrappers.Response)
"""
form = EventAnnotationForm.build(request)
if form.validate_on_submit():
sketch = Sketch.query.get_with_acl(sketch_id)
indices = [t.searchindex.index_name for t in sketch.timelines]
annotation_type = form.annotation_type.data
searchindex_id = form.searchindex_id.data
searchindex = SearchIndex.query.get(searchindex_id)
event_id = form.event_id.data
if searchindex_id not in indices:
abort(HTTP_STATUS_CODE_BAD_REQUEST)
def _set_label(label, toggle=False):
"""Set label on the event in the datastore."""
self.datastore.set_label(
searchindex_id, event_id, sketch.id, current_user.id, label,
toggle=toggle)
# Get or create an event in the SQL database to have something to
# attach the annotation to.
event = Event.get_or_create(
sketch=sketch, searchindex=searchindex,
document_id=event_id)
# Add the annotation to the event object.
if u'comment' in annotation_type:
annotation = Event.Comment(
comment=form.annotation.data, user=current_user)
event.comments.append(annotation)
_set_label(u'__ts_comment')
elif u'label' in annotation_type:
annotation = Event.Label.get_or_create(
label=form.annotation.data, user=current_user)
if annotation not in event.labels:
event.labels.append(annotation)
toggle = False
if u'__ts_star' in form.annotation.data:
toggle = True
_set_label(form.annotation.data, toggle)
else:
abort(HTTP_STATUS_CODE_BAD_REQUEST)
# Save the event to the database
db_session.add(event)
db_session.commit()
return self.to_json(
annotation, status_code=HTTP_STATUS_CODE_CREATED)
return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def WriteHeader(self):
"""Setup the Elasticsearch index and the Timesketch database object.
Creates the Elasticsearch index with Timesketch specific settings and the
Timesketch SearchIndex database object.
"""
# This cannot be static because we use the value of self._doc_type from
# arguments.
_document_mapping = {
self._doc_type: {
u'properties': {
u'timesketch_label': {
u'type': u'nested'
}
}
}
}
# Get Elasticsearch host and port from Timesketch configuration.
with self._timesketch.app_context():
_host = current_app.config[u'ELASTIC_HOST']
_port = current_app.config[u'ELASTIC_PORT']
self._elastic = ElasticSearchHelper(
self._output_mediator, _host, _port, self._flush_interval,
self._index_name, _document_mapping, self._doc_type)
user = None
if self._username:
user = User.query.filter_by(username=self._username).first()
if not user:
raise RuntimeError(
u'Unknown Timesketch user: {0:s}'.format(self._username))
else:
logging.warning(u'Timeline will be visible to all Timesketch users')
with self._timesketch.app_context():
search_index = SearchIndex.get_or_create(
name=self._timeline_name, description=self._timeline_name, user=user,
index_name=self._index_name)
# Grant the user read permission on the mapping object and set status.
# If user is None the timeline becomes visible to all users.
search_index.grant_permission(user=user, permission=u'read')
# In case we have a user grant additional permissions.
if user:
search_index.grant_permission(user=user, permission=u'write')
search_index.grant_permission(user=user, permission=u'delete')
# Let the Timesketch UI know that the timeline is processing.
search_index.set_status(u'processing')
# Save the mapping object to the Timesketch database.
db_session.add(search_index)
db_session.commit()
logging.info(u'Adding events to Timesketch.')
def run(self, username, password):
"""Creates the user."""
if not password:
password = self.get_password_from_prompt()
if not isinstance(password, six.text_type):
password = codecs.decode(password, 'utf-8')
username = codecs.decode(username, 'utf-8')
user = User.get_or_create(username=username)
user.set_password(plaintext=password)
db_session.add(user)
db_session.commit()
sys.stdout.write('User {0:s} created/updated\n'.format(username))
def Close(self):
"""Closes the connection to TimeSketch Elasticsearch database.
Sends the remaining events for indexing and removes the processing status on
the Timesketch search index object.
"""
self._elastic.AddEvent(None, force_flush=True)
with self._timesketch.app_context():
search_index = SearchIndex.query.filter_by(
index_name=self._index_name).first()
search_index.status.remove(search_index.status[0])
db_session.add(search_index)
db_session.commit()
def Close(self):
"""Closes the connection to TimeSketch Elasticsearch database.
Sends the remaining events for indexing and removes the processing status on
the Timesketch search index object.
"""
super(TimesketchOutputModule, self).Close()
with self._timesketch.app_context():
search_index = timesketch_sketch.SearchIndex.query.filter_by(
index_name=self._index_name).first()
search_index.status.remove(search_index.status[0])
timesketch_db_session.add(search_index)
timesketch_db_session.commit()
def home():
"""Generates the home page view template.
Returns:
Template with context.
"""
form = HiddenNameDescriptionForm()
sketches = Sketch.all_with_acl().filter(
not_(Sketch.Status.status == u'deleted'),
Sketch.Status.parent).order_by(Sketch.updated_at.desc())
query_filter = request.args.get(u'filter', u'')
query = request.args.get(u'q', u'')
# Only render upload button if it is configured.
upload_enabled = current_app.config[u'UPLOAD_ENABLED']
last_sketch = View.query.filter_by(
user=current_user, name=u'').order_by(
View.updated_at.desc()).first()
if query_filter:
if query_filter == u'user':
sketches = sketches.filter(Sketch.user == current_user)
elif query_filter == u'shared':
sketches = sketches.filter(not_(Sketch.user == current_user))
# TODO: Figure out a better way to handle this.
if query:
if query.startswith(u'*'):
query = u''
else:
sketches = sketches.filter(Sketch.name.contains(query)).limit(100)
# Handle form for creating a new sketch.
if form.validate_on_submit():
sketch = Sketch(
name=form.name.data, description=form.description.data,
user=current_user)
sketch.status.append(sketch.Status(user=None, status=u'new'))
# Give the requesting user permissions on the new sketch.
sketch.grant_permission(current_user, u'read')
sketch.grant_permission(current_user, u'write')
sketch.grant_permission(current_user, u'delete')
db_session.add(sketch)
db_session.commit()
return redirect(url_for(u'sketch_views.overview', sketch_id=sketch.id))
return render_template(
u'home/home.html', sketches=sketches, form=form, query=query,
upload_enabled=upload_enabled, last_sketch=last_sketch)
def post(self):
"""Handles POST request to the resource.
Returns:
A view in JSON (instance of flask.wrappers.Response)
Raises:
ApiHTTPError
"""
UPLOAD_ENABLED = current_app.config[u'UPLOAD_ENABLED']
UPLOAD_FOLDER = current_app.config[u'UPLOAD_FOLDER']
form = UploadFileForm()
if form.validate_on_submit() and UPLOAD_ENABLED:
from timesketch.lib.tasks import run_plaso
file_storage = form.file.data
timeline_name = form.name.data
# We do not need a human readable filename or
# datastore index name, so we use UUIDs here.
filename = unicode(uuid.uuid4().hex)
index_name = unicode(uuid.uuid4().hex)
file_path = os.path.join(UPLOAD_FOLDER, filename)
file_storage.save(file_path)
search_index = SearchIndex.get_or_create(
name=timeline_name, description=timeline_name,
user=current_user, index_name=index_name)
search_index.grant_permission(permission=u'read', user=current_user)
search_index.grant_permission(
permission=u'write', user=current_user)
search_index.grant_permission(
permission=u'delete', user=current_user)
search_index.set_status(u'processing')
db_session.add(search_index)
db_session.commit()
run_plaso.apply_async(
(file_path, timeline_name, index_name), task_id=index_name)
return self.to_json(
search_index, status_code=HTTP_STATUS_CODE_CREATED)
else:
raise ApiHTTPError(
message=form.errors[u'file'][0],
status_code=HTTP_STATUS_CODE_BAD_REQUEST)
def timelines(sketch_id):
"""Generates the sketch explore view template.
Returns:
Template with context.
"""
TIMELINES_TO_SHOW = 20
sketch = Sketch.query.get_with_acl(sketch_id)
searchindices_in_sketch = [t.searchindex.id for t in sketch.timelines]
query = request.args.get(u'q', None)
indices = SearchIndex.all_with_acl(
current_user).order_by(
desc(SearchIndex.created_at)).filter(
not_(SearchIndex.id.in_(searchindices_in_sketch)))
filtered = False
if query:
indices = indices.filter(SearchIndex.name.contains(query)).limit(500)
filtered = True
if not filtered:
indices = indices.limit(TIMELINES_TO_SHOW)
# Setup the form
form = AddTimelineForm()
form.timelines.choices = set((i.id, i.name) for i in indices.all())
# Create new timeline form POST
if form.validate_on_submit():
if not sketch.has_permission(current_user, u'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN)
for searchindex_id in form.timelines.data:
searchindex = SearchIndex.query.get_with_acl(searchindex_id)
if searchindex not in [t.searchindex for t in sketch.timelines]:
_timeline = Timeline(
name=searchindex.name, description=searchindex.description,
sketch=sketch, user=current_user, searchindex=searchindex)
db_session.add(_timeline)
sketch.timelines.append(_timeline)
db_session.commit()
return redirect(url_for(u'sketch_views.timelines', sketch_id=sketch.id))
return render_template(
u'sketch/timelines.html', sketch=sketch, timelines=indices.all(),
form=form, filtered=filtered)
def explore(sketch_id, view_id=None):
"""Generates the sketch explore view template.
Returns:
Template with context.
"""
sketch = Sketch.query.get_with_acl(sketch_id)
sketch_timelines = [t.searchindex.index_name for t in sketch.timelines]
view_form = SaveViewForm()
# Get parameters from the GET query
url_query = request.args.get(u'q', u'')
url_time_start = request.args.get(u'time_start', None)
url_time_end = request.args.get(u'time_end', None)
if view_id:
view = View.query.get(view_id)
# Check that this view belongs to the sketch
if view.sketch_id != sketch.id:
abort(HTTP_STATUS_CODE_NOT_FOUND)
# Return 404 if view is deleted
if view.get_status.status == u'deleted':
return abort(HTTP_STATUS_CODE_NOT_FOUND)
else:
view = sketch.get_user_view(current_user)
if url_query:
view.query_string = url_query
query_filter = json.loads(view.query_filter)
query_filter[u'time_start'] = url_time_start
query_filter[u'time_end'] = url_time_end
view.query_filter = json.dumps(query_filter, ensure_ascii=False)
if not view:
query_filter = dict(indices=sketch_timelines)
view = View(
user=current_user, name=u'', sketch=sketch, query_string=u'',
query_filter=json.dumps(query_filter, ensure_ascii=False))
db_session.add(view)
db_session.commit()
return render_template(
u'sketch/explore.html', sketch=sketch, view=view,
timelines=sketch_timelines, view_form=view_form)
def post(self, sketch_id):
"""Handles POST request to the resource.
Args:
sketch_id: Integer primary key for a sketch database model
Returns:
A view in JSON (instance of flask.wrappers.Response)
"""
form = StoryForm.build(request)
if form.validate_on_submit():
sketch = Sketch.query.get_with_acl(sketch_id)
story = Story(
title=u'', content=u'', sketch=sketch, user=current_user)
db_session.add(story)
db_session.commit()
return self.to_json(story, status_code=HTTP_STATUS_CODE_CREATED)
return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def post(self):
"""Handles POST request to the resource.
Returns:
A sketch in JSON (instance of flask.wrappers.Response)
"""
form = NameDescriptionForm.build(request)
if form.validate_on_submit():
sketch = Sketch(
name=form.name.data, description=form.description.data,
user=current_user)
sketch.status.append(sketch.Status(user=None, status=u'new'))
# Give the requesting user permissions on the new sketch.
sketch.grant_permission(permission=u'read', user=current_user)
sketch.grant_permission(permission=u'write', user=current_user)
sketch.grant_permission(permission=u'delete', user=current_user)
db_session.add(sketch)
db_session.commit()
return self.to_json(sketch, status_code=HTTP_STATUS_CODE_CREATED)
return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def post(self, sketch_id):
"""Handles POST request to the resource.
Args:
sketch_id: Integer primary key for a sketch database model
Returns:
A view in JSON (instance of flask.wrappers.Response)
"""
form = SaveViewForm.build(request)
if form.validate_on_submit():
sketch = Sketch.query.get_with_acl(sketch_id)
view = View(
name=form.name.data, sketch=sketch, user=current_user,
query_string=form.query.data,
query_filter=json.dumps(form.filter.data, ensure_ascii=False))
db_session.add(view)
db_session.commit()
return self.to_json(view, status_code=HTTP_STATUS_CODE_CREATED)
return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def WriteHeader(self):
"""Setup the Elasticsearch index and the Timesketch database object.
Creates the Elasticsearch index with Timesketch specific settings and the
Timesketch SearchIndex database object.
"""
# This cannot be static because we use the value of self._doc_type from
# arguments.
_document_mapping = {
self._doc_type: {
u'_timestamp': {
u'enabled': True,
u'path': u'datetime',
u'format': u'date_time_no_millis'
},
u'properties': {u'timesketch_label': {u'type': u'nested'}}
}
}
if not self._elastic_db.client.indices.exists(self._index_name):
try:
self._elastic_db.client.indices.create(
index=self._index_name, body={u'mappings': _document_mapping})
except elastic_exceptions.ConnectionError as exception:
logging.error((
u'Unable to proceed, cannot connect to Timesketch backend '
u'with error: {0:s}.\nPlease verify connection.').format(exception))
raise RuntimeError(u'Unable to connect to Timesketch backend.')
with self._timesketch.app_context():
search_index = SearchIndex.get_or_create(
name=self._timeline_name, description=self._timeline_name, user=None,
index_name=self._index_name)
# Grant all users read permission on the mapping object and set status.
search_index.grant_permission(None, u'read')
search_index.set_status(u'processing')
# Save the mapping object to the Timesketch database.
db_session.add(search_index)
db_session.commit()
logging.info(u'Adding events to Timesketch..')
def Close(self):
"""Closes the connection to TimeSketch Elasticsearch database.
Sends the remaining events for indexing and adds the timeline to Timesketch.
"""
self._FlushEventsToElasticsearch()
with self._timesketch.app_context():
# Get Timesketch user object, or None if user do not exist. This is a
# SQLAlchemy query against the Timesketch database.
user_query = User.query.filter_by(username=self._timeline_owner)
user = user_query.first()
search_index = SearchIndex(
name=self._timeline_name, description=self._timeline_name, user=user,
index_name=self._index_name)
# Grant all users read permission on the mapping object.
search_index.grant_permission(None, u'read')
# Save the mapping object to the Timesketch database.
db_session.add(search_index)
db_session.commit()
def post(self, sketch_id):
"""Handles POST request to the resource.
Returns:
A sketch in JSON (instance of flask.wrappers.Response)
Raises:
ApiHTTPError
"""
sketch = Sketch.query.get_with_acl(sketch_id)
searchindices_in_sketch = [t.searchindex.id for t in sketch.timelines]
indices = SearchIndex.all_with_acl(
current_user).order_by(
desc(SearchIndex.created_at)).filter(
not_(SearchIndex.id.in_(searchindices_in_sketch)))
add_timeline_form = AddTimelineForm.build(request)
add_timeline_form.timelines.choices = set(
(i.id, i.name) for i in indices.all())
if add_timeline_form.validate_on_submit():
if not sketch.has_permission(current_user, u'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN)
for searchindex_id in add_timeline_form.timelines.data:
searchindex = SearchIndex.query.get_with_acl(searchindex_id)
if searchindex not in [t.searchindex for t in sketch.timelines]:
_timeline = Timeline(
name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=current_user,
searchindex=searchindex)
db_session.add(_timeline)
sketch.timelines.append(_timeline)
db_session.commit()
return self.to_json(sketch, status_code=HTTP_STATUS_CODE_CREATED)
else:
raise ApiHTTPError(
message=add_timeline_form.errors,
status_code=HTTP_STATUS_CODE_BAD_REQUEST)
def run(self, name, index, username):
"""Create the SearchIndex."""
es = ElasticsearchDataStore(
host=current_app.config['ELASTIC_HOST'],
port=current_app.config['ELASTIC_PORT'])
user = User.query.filter_by(username=username).first()
if not user:
sys.stderr.write('User does not exist\n')
sys.exit(1)
if not es.client.indices.exists(index=index):
sys.stderr.write('Index does not exist in the datastore\n')
sys.exit(1)
if SearchIndex.query.filter_by(name=name, index_name=index).first():
sys.stderr.write(
'Index with this name already exist in Timesketch\n')
sys.exit(1)
searchindex = SearchIndex(
name=name, description=name, user=user, index_name=index)
searchindex.grant_permission('read')
db_session.add(searchindex)
db_session.commit()
sys.stdout.write('Search index {0:s} created\n'.format(name))
def add_comment(self, comment):
"""Add comment to event.
Args:
comment: Comment string.
Raises:
RuntimeError: if no sketch is present.
"""
if not self.sketch:
raise RuntimeError('No sketch provided.')
searchindex = SearchIndex.query.filter_by(
index_name=self.index_name).first()
db_event = SQLEvent.get_or_create(
sketch=self.sketch.sql_sketch, searchindex=searchindex,
document_id=self.event_id)
comment = SQLEvent.Comment(comment=comment, user=None)
db_event.comments.append(comment)
db_session.add(db_event)
db_session.commit()
self.add_label(label='__ts_comment')
def explore(sketch_id, view_id=None):
"""Generates the sketch explore view template.
Returns:
Template with context.
"""
sketch = Sketch.query.get_with_acl(sketch_id)
sketch_timelines = [t.searchindex.index_name for t in sketch.timelines]
if view_id:
view = View.query.get(view_id)
else:
view = sketch.get_user_view(current_user)
if not view:
query_filter = dict(indices=sketch_timelines)
view = View(
user=current_user, name=u'', sketch=sketch, query_string=u'',
query_filter=json.dumps(query_filter, ensure_ascii=False))
db_session.add(view)
db_session.commit()
view_form = SaveViewForm()
return render_template(
u'sketch/explore.html', sketch=sketch, view=view,
timelines=sketch_timelines, view_form=view_form)
def setup_sketch(timeline_name, index_name, username, sketch_id=None):
"""Use existing sketch or create a new sketch.
Args:
timeline_name: (str) Name of the Timeline
index_name: (str) Name of the index
username: (str) Who should own the timeline
sketch_id: (str) Optional sketch_id to add timeline to
Returns:
(tuple) sketch ID and timeline ID as integers
"""
with app.app_context():
user = User.get_or_create(username=username)
sketch = None
if sketch_id:
try:
sketch = Sketch.query.get_with_acl(sketch_id, user=user)
logger.info('Using existing sketch: {} ({})'.format(
sketch.name, sketch.id))
except Forbidden:
pass
if not (sketch or sketch_id):
# Create a new sketch.
sketch_name = 'Turbinia: {}'.format(timeline_name)
sketch = Sketch(name=sketch_name,
description=sketch_name,
user=user)
# Need to commit here to be able to set permissions later.
db_session.add(sketch)
db_session.commit()
sketch.grant_permission(permission='read', user=user)
sketch.grant_permission(permission='write', user=user)
sketch.grant_permission(permission='delete', user=user)
sketch.status.append(sketch.Status(user=None, status='new'))
db_session.add(sketch)
db_session.commit()
logger.info('Created new sketch: {} ({})'.format(
sketch.name, sketch.id))
searchindex = SearchIndex.get_or_create(
name=timeline_name,
description='Created by Turbinia.',
user=user,
index_name=index_name)
searchindex.grant_permission(permission='read', user=user)
searchindex.grant_permission(permission='write', user=user)
searchindex.grant_permission(permission='delete', user=user)
searchindex.set_status('processing')
db_session.add(searchindex)
db_session.commit()
timeline = Timeline(name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=user,
searchindex=searchindex)
# If the user doesn't have write access to the sketch then create the
# timeline but don't attach it to the sketch.
if not sketch.has_permission(user, 'write'):
timeline.sketch = None
else:
sketch.timelines.append(timeline)
db_session.add(timeline)
db_session.commit()
timeline.set_status('processing')
return sketch.id, timeline.id
def post(self, sketch_id, view_id):
"""Handles POST request to the resource.
Args:
sketch_id: Integer primary key for a sketch database model
view_id: Integer primary key for a view database model
Returns:
A view in JSON (instance of flask.wrappers.Response)
"""
form = forms.SaveViewForm.build(request)
if not form.validate_on_submit():
abort(HTTP_STATUS_CODE_BAD_REQUEST,
'Unable to update view, not able to validate form data')
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.')
if not sketch.has_permission(current_user, 'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN,
'User does not have write access controls on sketch.')
view = View.query.get(view_id)
if not view:
abort(HTTP_STATUS_CODE_NOT_FOUND, 'No view found with this ID.')
if view.sketch.id != sketch.id:
abort(HTTP_STATUS_CODE_BAD_REQUEST,
'Unable to update view, view not attached to sketch.')
view.query_string = form.query.data
description = form.description.data
if description:
view.description = description
query_filter = form.filter.data
# Stripping potential pagination from views before saving it.
if 'from' in query_filter:
del query_filter['from']
view.query_filter = json.dumps(query_filter, ensure_ascii=False)
view.query_dsl = json.dumps(form.dsl.data, ensure_ascii=False)
name = form.name.data
if name:
view.name = name
view.user = current_user
view.sketch = sketch
if form.dsl.data:
view.query_string = ''
db_session.add(view)
db_session.commit()
# Update the last activity of a sketch.
utils.update_sketch_last_activity(sketch)
return self.to_json(view, status_code=HTTP_STATUS_CODE_CREATED)
def create_view_from_form(sketch, form):
"""Creates a view from form data.
Args:
sketch: Instance of timesketch.models.sketch.Sketch
form: Instance of timesketch.lib.forms.SaveViewForm
Returns:
A view (Instance of timesketch.models.sketch.View)
"""
# Default to user supplied data
view_name = form.name.data
query_string = form.query.data
query_filter_dict = form.filter.data
# Stripping potential pagination from views before saving it.
if 'from' in query_filter_dict:
del query_filter_dict['from']
query_filter = json.dumps(query_filter_dict, ensure_ascii=False)
query_dsl = json.dumps(form.dsl.data, ensure_ascii=False)
if isinstance(query_filter, tuple):
query_filter = query_filter[0]
# No search template by default (before we know if the user want to
# create a template or use an existing template when creating the view)
searchtemplate = None
# Create view from a search template
if form.from_searchtemplate_id.data:
# Get the template from the datastore
template_id = form.from_searchtemplate_id.data
searchtemplate = SearchTemplate.query.get(template_id)
# Copy values from the template
view_name = searchtemplate.name
query_string = searchtemplate.query_string
query_filter = searchtemplate.query_filter
query_dsl = searchtemplate.query_dsl
# WTF form returns a tuple for the filter. This is not
# compatible with SQLAlchemy.
if isinstance(query_filter, tuple):
query_filter = query_filter[0]
if not view_name:
abort(HTTP_STATUS_CODE_BAD_REQUEST, 'View name is missing.')
# Create a new search template based on this view (only if requested by
# the user).
if form.new_searchtemplate.data:
query_filter_dict = json.loads(query_filter)
if query_filter_dict.get('indices', None):
query_filter_dict['indices'] = '_all'
query_filter = json.dumps(query_filter_dict, ensure_ascii=False)
searchtemplate = SearchTemplate(name=view_name,
user=current_user,
query_string=query_string,
query_filter=query_filter,
query_dsl=query_dsl)
db_session.add(searchtemplate)
db_session.commit()
# Create the view in the database
view = View(name=view_name,
sketch=sketch,
user=current_user,
query_string=query_string,
query_filter=query_filter,
query_dsl=query_dsl,
searchtemplate=searchtemplate)
db_session.add(view)
db_session.commit()
return view
def _upload_and_index(self,
file_extension,
timeline_name,
index_name,
sketch,
enable_stream,
data_label='',
file_path='',
events='',
meta=None):
"""Creates a full pipeline for an uploaded file and returns the results.
Args:
file_extension: the extension of the uploaded file.
timeline_name: name the timeline will be stored under in the
datastore.
index_name: the Elastic index name for the timeline.
sketch: Instance of timesketch.models.sketch.Sketch
enable_stream: boolean indicating whether this is file is part of a
stream or not.
data_label: Optional string with a data label for the search index.
file_path: the path to the file to be uploaded (optional).
events: a string with events to upload (optional).
meta: optional dict with additional meta fields that will be
included in the return.
Returns:
A timeline if created otherwise a search index in JSON (instance
of flask.wrappers.Response)
"""
searchindex = self._get_index(name=timeline_name,
description=timeline_name,
sketch=sketch,
index_name=index_name,
data_label=data_label,
extension=file_extension)
searchindex.set_status('processing')
timelines = Timeline.query.filter_by(name=timeline_name,
sketch=sketch).all()
timeline = None
for timeline_ in timelines:
if timeline_.searchindex.index_name == searchindex.index_name:
timeline = timeline_
break
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'There is a timeline in the sketch that has the same name '
'but is stored in a different index, check the data_label '
'on the uploaded data')
if not timeline:
timeline = Timeline.get_or_create(name=timeline_name,
description=timeline_name,
sketch=sketch,
user=current_user,
searchindex=searchindex)
if not timeline:
abort(HTTP_STATUS_CODE_BAD_REQUEST,
'Unable to get or create a new Timeline object.')
timeline.set_status('processing')
sketch.timelines.append(timeline)
labels_to_prevent_deletion = current_app.config.get(
'LABELS_TO_PREVENT_DELETION', [])
for sketch_label in sketch.get_labels:
if sketch_label not in labels_to_prevent_deletion:
continue
timeline.add_label(sketch_label)
searchindex.add_label(sketch_label)
db_session.add(timeline)
db_session.commit()
sketch_id = sketch.id
# Start Celery pipeline for indexing and analysis.
# Import here to avoid circular imports.
# pylint: disable=import-outside-toplevel
from timesketch.lib import tasks
pipeline = tasks.build_index_pipeline(
file_path=file_path,
events=events,
timeline_name=timeline_name,
index_name=searchindex.index_name,
file_extension=file_extension,
sketch_id=sketch_id,
only_index=enable_stream,
timeline_id=timeline.id)
pipeline.apply_async()
return self.to_json(timeline,
status_code=HTTP_STATUS_CODE_CREATED,
meta=meta)
def explore(sketch_id, view_id=None, searchtemplate_id=None):
"""Generates the sketch explore view template.
Returns:
Template with context.
"""
save_view = False # If the view should be saved to the database.
sketch = Sketch.query.get_with_acl(sketch_id)
sketch_timelines = [t.searchindex.index_name for t in sketch.timelines]
view_form = SaveViewForm()
graphs_enabled = current_app.config['GRAPH_BACKEND_ENABLED']
similarity_enabled = current_app.config.get('ENABLE_EXPERIMENTAL_UI')
# Get parameters from the GET query
url_query = request.args.get('q', '')
url_time_start = request.args.get('time_start', None)
url_time_end = request.args.get('time_end', None)
url_index = request.args.get('index', None)
url_size = request.args.get('size', None)
if searchtemplate_id:
searchtemplate = SearchTemplate.query.get(searchtemplate_id)
view = sketch.get_user_view(current_user)
if not view:
view = View(user=current_user, name='', sketch=sketch)
view.query_string = searchtemplate.query_string
view.query_filter = searchtemplate.query_filter
view.query_dsl = searchtemplate.query_dsl
save_view = True
elif view_id:
view = View.query.get(view_id)
# Check that this view belongs to the sketch
if view.sketch_id != sketch.id:
abort(HTTP_STATUS_CODE_NOT_FOUND)
# Return 404 if view is deleted
if view.get_status.status == 'deleted':
return abort(HTTP_STATUS_CODE_NOT_FOUND)
else:
view = sketch.get_user_view(current_user)
if not view:
view = View(
user=current_user, name='', sketch=sketch, query_string='*')
view.query_filter = view.validate_filter(
dict(indices=sketch_timelines))
save_view = True
if url_query:
view.query_string = url_query
query_filter = json.loads(view.query_filter)
query_filter['from'] = 0 # if we loaded from get, start at first event
query_filter['time_start'] = url_time_start
query_filter['time_end'] = url_time_end
if url_index in sketch_timelines:
query_filter['indices'] = [url_index]
if url_size:
query_filter['size'] = url_size
view.query_filter = view.validate_filter(query_filter)
view.query_dsl = None
save_view = True
if save_view:
db_session.add(view)
db_session.commit()
return render_template(
'sketch/explore.html',
sketch=sketch,
view=view,
named_view=view_id,
timelines=sketch_timelines,
view_form=view_form,
searchtemplate_id=searchtemplate_id,
graphs_enabled=graphs_enabled,
similarity_enabled=similarity_enabled)
def post(self, sketch_id, timeline_id):
"""Handles GET request to the resource.
Args:
sketch_id: Integer primary key for a sketch database model
timeline_id: Integer primary key for a timeline database model
"""
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(
HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.')
timeline = Timeline.query.get(timeline_id)
if not timeline:
abort(
HTTP_STATUS_CODE_NOT_FOUND,
'No timeline found with this ID.')
# Check that this timeline belongs to the sketch
if timeline.sketch_id != sketch.id:
abort(
HTTP_STATUS_CODE_NOT_FOUND,
'The sketch ID ({0:d}) does not match with the timeline '
'sketch ID ({1:d})'.format(sketch.id, timeline.sketch_id))
if not sketch.has_permission(user=current_user, permission='write'):
abort(
HTTP_STATUS_CODE_FORBIDDEN,
'The user does not have write permission on the sketch.')
form = forms.TimelineForm.build(request)
if not form.validate_on_submit():
abort(
HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to validate form data.')
if form.labels.data:
label_string = form.labels.data
labels = json.loads(label_string)
if not isinstance(labels, (list, tuple)):
abort(
HTTP_STATUS_CODE_BAD_REQUEST, (
'Label needs to be a JSON string that '
'converts to a list of strings.'))
if not all([isinstance(x, str) for x in labels]):
abort(
HTTP_STATUS_CODE_BAD_REQUEST, (
'Label needs to be a JSON string that '
'converts to a list of strings (not all strings)'))
label_action = form.label_action.data
if label_action not in ('add', 'remove'):
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'Label action needs to be either add or remove.')
changed = False
if label_action == 'add':
changes = []
for label in labels:
changes.append(
self._add_label(timeline=timeline, label=label))
changed = any(changes)
elif label_action == 'remove':
if not sketch.has_permission(
user=current_user, permission='delete'):
abort(
HTTP_STATUS_CODE_FORBIDDEN,
'The user does not have delete permission on sketch.')
changes = []
for label in labels:
changes.append(
self._remove_label(timeline=timeline, label=label))
changed = any(changes)
if not changed:
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'Label [{0:s}] not {1:s}'.format(
', '.join(labels), label_action))
db_session.add(timeline)
db_session.commit()
return HTTP_STATUS_CODE_OK
timeline.name = form.name.data
timeline.description = form.description.data
timeline.color = form.color.data
db_session.add(timeline)
db_session.commit()
# Update the last activity of a sketch.
utils.update_sketch_last_activity(sketch)
return HTTP_STATUS_CODE_OK
def post(self, sketch_id):
"""Handles POST request to the resource.
Handler for /api/v1/sketches/:sketch_id/explore/
Args:
sketch_id: Integer primary key for a sketch database model
Returns:
JSON with list of matched events
"""
sketch = Sketch.query.get_with_acl(sketch_id)
form = ExploreForm.build(request)
if form.validate_on_submit():
query_dsl = form.dsl.data
query_filter = form.filter.data
sketch_indices = {
t.searchindex.index_name
for t in sketch.timelines
}
indices = query_filter.get(u'indices', sketch_indices)
# If _all in indices then execute the query on all indices
if u'_all' in indices:
indices = sketch_indices
# Make sure that the indices in the filter are part of the sketch.
# This will also remove any deleted timeline from the search result.
indices = get_validated_indices(indices, sketch_indices)
# Make sure we have a query string or star filter
if not (form.query.data, query_filter.get(u'star'),
query_filter.get(u'events'), query_dsl):
abort(HTTP_STATUS_CODE_BAD_REQUEST)
result = self.datastore.search(sketch_id,
form.query.data,
query_filter,
query_dsl,
indices,
aggregations=None,
return_results=True)
# Get labels for each event that matches the sketch.
# Remove all other labels.
for event in result[u'hits'][u'hits']:
event[u'selected'] = False
event[u'_source'][u'label'] = []
try:
for label in event[u'_source'][u'timesketch_label']:
if sketch.id != label[u'sketch_id']:
continue
event[u'_source'][u'label'].append(label[u'name'])
del event[u'_source'][u'timesketch_label']
except KeyError:
pass
# Update or create user state view. This is used in the UI to let
# the user get back to the last state in the explore view.
view = View.get_or_create(user=current_user,
sketch=sketch,
name=u'')
view.query_string = form.query.data
view.query_filter = json.dumps(query_filter, ensure_ascii=False)
view.query_dsl = json.dumps(query_dsl, ensure_ascii=False)
db_session.add(view)
db_session.commit()
# Add metadata for the query result. This is used by the UI to
# render the event correctly and to display timing and hit count
# information.
tl_colors = {}
tl_names = {}
for timeline in sketch.timelines:
tl_colors[timeline.searchindex.index_name] = timeline.color
tl_names[timeline.searchindex.index_name] = timeline.name
meta = {
u'es_time': result[u'took'],
u'es_total_count': result[u'hits'][u'total'],
u'timeline_colors': tl_colors,
u'timeline_names': tl_names,
}
schema = {u'meta': meta, u'objects': result[u'hits'][u'hits']}
return jsonify(schema)
return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def post(self, sketch_id):
"""Handles POST request to the resource.
Handler for /api/v1/sketches/:sketch_id/event/create/
Args:
sketch_id: Integer primary key for a sketch database model
Returns:
An annotation in JSON (instance of flask.wrappers.Response)
"""
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.')
if not sketch.has_permission(current_user, 'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN,
'User does not have write access controls on sketch.')
form = request.json
if not form:
form = request.data
timeline_name = 'sketch specific timeline'
index_name_seed = 'timesketch_{0:d}'.format(sketch_id)
event_type = 'user_created_event'
date_string = form.get('date_string')
if not date_string:
date = datetime.datetime.utcnow().isoformat()
else:
# derive datetime from timestamp:
try:
date = dateutil.parser.parse(date_string)
except (dateutil.parser.ParserError, OverflowError) as e:
logger.error('Unable to convert date string', exc_info=True)
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'Unable to add event, not able to convert the date '
'string. Was it properly formatted? Error: '
'{0!s}'.format(e))
timestamp = int(time.mktime(date.utctimetuple())) * 1000000
timestamp += date.microsecond
event = {
'datetime': date_string,
'timestamp': timestamp,
'timestamp_desc': form.get('timestamp_desc', 'Event Happened'),
'message': form.get('message', 'No message string'),
}
attributes = form.get('attributes', {})
if not isinstance(attributes, dict):
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'Unable to add an event where the attributes are not a '
'dict object.')
event.update(attributes)
tag = form.get('tag', [])
if not isinstance(tag, list):
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'Unable to add an event where the tags are not a '
'list of strings.')
if tag and any(not isinstance(x, str) for x in tag):
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'Unable to add an event where the tags are not a '
'list of strings.')
event['tag'] = tag
# We do not need a human readable filename or
# datastore index name, so we use UUIDs here.
index_name = hashlib.md5(index_name_seed.encode()).hexdigest()
if six.PY2:
index_name = codecs.decode(index_name, 'utf-8')
# Try to create index
timeline = None
try:
# Create the index in Elasticsearch (unless it already exists)
self.datastore.create_index(index_name=index_name,
doc_type=event_type)
# Create the search index in the Timesketch database
searchindex = SearchIndex.get_or_create(
name=timeline_name,
description='internal timeline for user-created events',
user=current_user,
index_name=index_name)
searchindex.grant_permission(permission='read', user=current_user)
searchindex.grant_permission(permission='write', user=current_user)
searchindex.grant_permission(permission='delete',
user=current_user)
searchindex.set_status('ready')
db_session.add(searchindex)
db_session.commit()
if sketch and sketch.has_permission(current_user, 'write'):
self.datastore.import_event(index_name,
event_type,
event,
flush_interval=1)
timeline = Timeline.get_or_create(
name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=current_user,
searchindex=searchindex)
if timeline not in sketch.timelines:
sketch.timelines.append(timeline)
timeline.set_status('ready')
db_session.add(timeline)
db_session.commit()
# TODO: Can this be narrowed down, both in terms of the scope it
# applies to, as well as not to catch a generic exception.
except Exception as e: # pylint: disable=broad-except
abort(HTTP_STATUS_CODE_BAD_REQUEST,
'Failed to add event ({0!s})'.format(e))
# Return Timeline if it was created.
# pylint: disable=no-else-return
if timeline:
return self.to_json(timeline, status_code=HTTP_STATUS_CODE_CREATED)
return self.to_json(searchindex, status_code=HTTP_STATUS_CODE_CREATED)
def _upload_and_index(self,
file_extension,
timeline_name,
index_name,
sketch,
enable_stream,
file_path='',
events='',
meta=None):
"""Creates a full pipeline for an uploaded file and returns the results.
Args:
file_extension: the extension of the uploaded file.
timeline_name: name the timeline will be stored under in the
datastore.
index_name: the Elastic index name for the timeline.
sketch: Instance of timesketch.models.sketch.Sketch
enable_stream: boolean indicating whether this is file is part of a
stream or not.
file_path: the path to the file to be uploaded (optional).
events: a string with events to upload (optional).
meta: optional dict with additional meta fields that will be
included in the return.
Returns:
A timeline if created otherwise a search index in JSON (instance
of flask.wrappers.Response)
"""
# Check if search index already exists.
searchindex = SearchIndex.query.filter_by(
name=timeline_name, user=current_user,
index_name=index_name).first()
timeline = None
if searchindex:
searchindex.set_status('processing')
timeline = Timeline.query.filter_by(
name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=current_user,
searchindex=searchindex).first()
else:
# Create the search index in the Timesketch database
searchindex = SearchIndex.get_or_create(name=timeline_name,
description='',
user=current_user,
index_name=index_name)
searchindex.grant_permission(permission='read', user=current_user)
searchindex.grant_permission(permission='write', user=current_user)
searchindex.grant_permission(permission='delete',
user=current_user)
searchindex.set_status('processing')
db_session.add(searchindex)
db_session.commit()
if sketch and sketch.has_permission(current_user, 'write'):
labels_to_prevent_deletion = current_app.config.get(
'LABELS_TO_PREVENT_DELETION', [])
timeline = Timeline(name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=current_user,
searchindex=searchindex)
timeline.set_status('processing')
sketch.timelines.append(timeline)
for label in sketch.get_labels:
if label not in labels_to_prevent_deletion:
continue
timeline.add_label(label)
searchindex.add_label(label)
db_session.add(timeline)
db_session.commit()
# Start Celery pipeline for indexing and analysis.
# Import here to avoid circular imports.
# pylint: disable=import-outside-toplevel
from timesketch.lib import tasks
pipeline = tasks.build_index_pipeline(file_path=file_path,
events=events,
timeline_name=timeline_name,
index_name=index_name,
file_extension=file_extension,
sketch_id=sketch.id,
only_index=enable_stream)
pipeline.apply_async()
# Return Timeline if it was created.
# pylint: disable=no-else-return
if timeline:
return self.to_json(timeline,
status_code=HTTP_STATUS_CODE_CREATED,
meta=meta)
return self.to_json(searchindex,
status_code=HTTP_STATUS_CODE_CREATED,
meta=meta)
def _get_index(self,
name,
description,
sketch,
index_name='',
data_label='',
extension=''):
"""Returns a SearchIndex object to be used for uploads.
Args:
name: the name of the searchindex.
description: the description of the searchindex.
sketch: sketch object (instance of Sketch).
index_name: optional index name, if supplied and if it exists
then the index associated with that will be returned.
data_label: optional label of the data, if supplied will be used to
determine whether an already existing index can be
used or a new one created.
extension: optional file extension if a file is being uploaded,
if supplied and no data label used, then the extension will be
used as a data label.
Returns:
A SearchIndex object.
"""
if index_name:
if not isinstance(index_name, str):
index_name = codecs.decode(index_name, 'utf-8')
searchindex = SearchIndex.query.filter_by(
name=name, index_name=index_name).first()
if searchindex and searchindex.has_permission(permission='write',
user=current_user):
return searchindex
if extension and not data_label:
data_label = extension
if not data_label:
data_label = 'generic'
# Since CSV and JSON are basically the same label, we combine it here.
if data_label in ('csv', 'json', 'jsonl'):
data_label = 'csv_jsonl'
indices = [t.searchindex for t in sketch.active_timelines]
for index in indices:
if index.has_label(data_label) and sketch.has_permission(
permission='write', user=current_user):
return index
index_name = index_name or uuid.uuid4().hex
searchindex = SearchIndex.get_or_create(name=name,
index_name=index_name,
description=description,
user=current_user)
searchindex.grant_permission(permission='read', user=current_user)
searchindex.grant_permission(permission='write', user=current_user)
searchindex.grant_permission(permission='delete', user=current_user)
searchindex.set_status('processing')
db_session.add(searchindex)
db_session.commit()
searchindex.add_label(data_label, user=current_user)
return searchindex
def _upload_and_index(self,
file_extension,
timeline_name,
index_name,
sketch,
form,
enable_stream,
original_filename='',
data_label='',
file_path='',
events='',
meta=None):
"""Creates a full pipeline for an uploaded file and returns the results.
Args:
file_extension: the extension of the uploaded file.
timeline_name: name the timeline will be stored under in the
datastore.
index_name: the Elastic index name for the timeline.
sketch: Instance of timesketch.models.sketch.Sketch
form: a dict with the configuration for the upload.
enable_stream: boolean indicating whether this is file is part of a
stream or not.
original_filename: Original filename from the upload.
data_label: Optional string with a data label for the search index.
file_path: the path to the file to be uploaded (optional).
events: a string with events to upload (optional).
meta: optional dict with additional meta fields that will be
included in the return.
Returns:
A timeline if created otherwise a search index in JSON (instance
of flask.wrappers.Response)
"""
searchindex = self._get_index(name=timeline_name,
description=timeline_name,
sketch=sketch,
index_name=index_name,
data_label=data_label,
extension=file_extension)
if not searchindex:
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'We were unable to acquire a searchindex and therefore not '
'able to upload data, please try again. If this error persist '
'please create an issue on Github: https://github.com/'
'google/timesketch/issues/new/choose')
searchindex.set_status('processing')
timelines = Timeline.query.filter_by(name=timeline_name,
sketch=sketch).all()
timeline = None
for timeline_ in timelines:
if timeline_.searchindex.index_name == searchindex.index_name:
timeline = timeline_
break
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'There is a timeline in the sketch that has the same name '
'but is stored in a different index, check the data_label '
'on the uploaded data')
if not timeline:
timeline = Timeline.get_or_create(name=timeline_name,
description=timeline_name,
sketch=sketch,
user=current_user,
searchindex=searchindex)
if not timeline:
abort(HTTP_STATUS_CODE_BAD_REQUEST,
'Unable to get or create a new Timeline object.')
timeline.set_status('processing')
sketch.timelines.append(timeline)
labels_to_prevent_deletion = current_app.config.get(
'LABELS_TO_PREVENT_DELETION', [])
for sketch_label in sketch.get_labels:
if sketch_label not in labels_to_prevent_deletion:
continue
timeline.add_label(sketch_label)
searchindex.add_label(sketch_label)
file_size = form.get('total_file_size', 0)
datasource = DataSource(timeline=timeline,
user=current_user,
provider=form.get('provider', 'N/A'),
context=form.get('context', 'N/A'),
file_on_disk=file_path,
file_size=int(file_size),
original_filename=original_filename,
data_label=data_label)
timeline.datasources.append(datasource)
db_session.add(datasource)
db_session.add(timeline)
db_session.commit()
sketch_id = sketch.id
# Start Celery pipeline for indexing and analysis.
# Import here to avoid circular imports.
# pylint: disable=import-outside-toplevel
from timesketch.lib import tasks
pipeline = tasks.build_index_pipeline(
file_path=file_path,
events=events,
timeline_name=timeline_name,
index_name=searchindex.index_name,
file_extension=file_extension,
sketch_id=sketch_id,
only_index=enable_stream,
timeline_id=timeline.id)
task_id = uuid.uuid4().hex
pipeline.apply_async(task_id=task_id)
if meta is None:
meta = {}
meta['task_id'] = task_id
return self.to_json(timeline,
status_code=HTTP_STATUS_CODE_CREATED,
meta=meta)
def post(self, sketch_id):
"""Handles POST request to the resource.
Returns:
A sketch in JSON (instance of flask.wrappers.Response)
"""
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(
HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.')
if not sketch.has_permission(current_user, 'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN,
'User does not have write access controls on sketch.')
form = forms.AddTimelineSimpleForm.build(request)
metadata = {'created': True}
searchindex_id = form.timeline.data
searchindex = SearchIndex.query.get_with_acl(searchindex_id)
if searchindex.get_status.status == 'deleted':
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'Unable to create a timeline using a deleted search index')
timeline_id = [
t.searchindex.id for t in sketch.timelines
if t.searchindex.id == searchindex_id
]
if not form.validate_on_submit():
abort(
HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to validate form data.')
if not sketch.has_permission(current_user, 'write'):
abort(
HTTP_STATUS_CODE_FORBIDDEN,
'User does not have write access to the sketch.')
if not timeline_id:
return_code = HTTP_STATUS_CODE_CREATED
timeline = Timeline(
name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=current_user,
searchindex=searchindex)
sketch.timelines.append(timeline)
labels_to_prevent_deletion = current_app.config.get(
'LABELS_TO_PREVENT_DELETION', [])
for label in sketch.get_labels:
if label not in labels_to_prevent_deletion:
continue
timeline.add_label(label)
searchindex.add_label(label)
# Set status to ready so the timeline can be queried.
timeline.set_status('ready')
db_session.add(timeline)
db_session.commit()
else:
metadata['created'] = False
return_code = HTTP_STATUS_CODE_OK
timeline = Timeline.query.get(timeline_id)
# Run sketch analyzers when timeline is added. Import here to avoid
# circular imports.
# pylint: disable=import-outside-toplevel
if current_app.config.get('AUTO_SKETCH_ANALYZERS'):
# pylint: disable=import-outside-toplevel
from timesketch.lib import tasks
sketch_analyzer_group, _ = tasks.build_sketch_analysis_pipeline(
sketch_id, searchindex_id, current_user.id)
if sketch_analyzer_group:
pipeline = (tasks.run_sketch_init.s(
[searchindex.index_name]) | sketch_analyzer_group)
pipeline.apply_async()
# Update the last activity of a sketch.
utils.update_sketch_last_activity(sketch)
return self.to_json(
timeline, meta=metadata, status_code=return_code)
def post(self, sketch_id):
"""Handles POST request to the resource.
Returns:
Graph in JSON (instance of flask.wrappers.Response)
"""
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(HTTP_STATUS_CODE_NOT_FOUND, "No sketch found with this ID.")
form = request.json
plugin_name = form.get("plugin")
graph_config = form.get("config")
refresh = form.get("refresh")
timeline_ids = form.get("timeline_ids", None)
if timeline_ids and not isinstance(timeline_ids, (list, tuple)):
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
"Timeline IDs if supplied need to be a list.",
)
if timeline_ids and not all([isinstance(x, int) for x in timeline_ids]):
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
"Timeline IDs needs to be a list of integers.",
)
sketch_indices = [
timeline.searchindex.index_name for timeline in sketch.active_timelines
]
cache = GraphCache.get_or_create(sketch=sketch, graph_plugin=plugin_name)
if graph_config:
cache_config = graph_config
else:
cache_config = cache.graph_config
if isinstance(cache_config, str):
cache_config = json.loads(cache_config)
# Refresh cache if timelines have been added/removed from the sketch.
if cache_config:
cache_graph_filter = cache_config.get("filter", {})
cache_filter_indices = cache_graph_filter.get("indices", [])
if set(sketch_indices) ^ set(cache_filter_indices):
refresh = True
if cache.graph_elements and not refresh:
return self.to_json(cache)
graph_class = manager.GraphManager.get_graph(plugin_name)
graph = graph_class(sketch=sketch, timeline_ids=timeline_ids)
cytoscape_json = graph.generate().to_cytoscape()
if cytoscape_json:
cache.graph_elements = json.dumps(cytoscape_json)
cache.graph_config = json.dumps(graph_config)
cache.update_modification_time()
db_session.add(cache)
db_session.commit()
# Update the last activity of a sketch.
utils.update_sketch_last_activity(sketch)
return self.to_json(cache)
def run(self, file_path, sketch_id, username, timeline_name):
"""This is the run method."""
file_path = os.path.realpath(file_path)
file_path_no_extension, extension = os.path.splitext(file_path)
extension = extension.lstrip('.')
filename = os.path.basename(file_path_no_extension)
supported_extensions = ('plaso', 'csv', 'jsonl')
if not os.path.isfile(file_path):
sys.exit('No such file: {0:s}'.format(file_path))
if extension not in supported_extensions:
sys.exit('Extension {0:s} is not supported. '
'(supported extensions are: {1:s})'.format(
extension, ', '.join(supported_extensions)))
user = None
if not username:
username = pwd.getpwuid(os.stat(file_path).st_uid).pw_name
if not username == 'root':
if not isinstance(username, six.text_type):
username = codecs.decode(username, 'utf-8')
user = User.query.filter_by(username=username).first()
if not user:
sys.exit('Cannot determine user for file: {0:s}'.format(file_path))
sketch = None
# If filename starts with <number> then use that as sketch_id.
# E.g: 42_file_name.plaso means sketch_id is 42.
sketch_id_from_filename = filename.split('_')[0]
if not sketch_id and sketch_id_from_filename.isdigit():
sketch_id = sketch_id_from_filename
if sketch_id:
try:
sketch = Sketch.query.get_with_acl(sketch_id, user=user)
except Forbidden:
pass
if not timeline_name:
if not isinstance(timeline_name, six.text_type):
timeline_name = codecs.decode(timeline_name, 'utf-8')
timeline_name = timeline_name.replace('_', ' ')
# Remove sketch ID if present in the filename.
timeline_parts = timeline_name.split()
if timeline_parts[0].isdigit():
timeline_name = ' '.join(timeline_name.split()[1:])
if not sketch:
# Create a new sketch.
sketch_name = 'Sketch for: {0:s}'.format(timeline_name)
sketch = Sketch(name=sketch_name,
description=sketch_name,
user=user)
# Need to commit here to be able to set permissions later.
db_session.add(sketch)
db_session.commit()
sketch.grant_permission(permission='read', user=user)
sketch.grant_permission(permission='write', user=user)
sketch.grant_permission(permission='delete', user=user)
sketch.status.append(sketch.Status(user=None, status='new'))
db_session.add(sketch)
db_session.commit()
index_name = uuid.uuid4().hex
if not isinstance(index_name, six.text_type):
index_name = codecs.decode(index_name, 'utf-8')
searchindex = SearchIndex.get_or_create(name=timeline_name,
description=timeline_name,
user=user,
index_name=index_name)
searchindex.grant_permission(permission='read', user=user)
searchindex.grant_permission(permission='write', user=user)
searchindex.grant_permission(permission='delete', user=user)
searchindex.set_status('processing')
db_session.add(searchindex)
db_session.commit()
if sketch and sketch.has_permission(user, 'write'):
timeline = Timeline(name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=user,
searchindex=searchindex)
timeline.set_status('processing')
sketch.timelines.append(timeline)
db_session.add(timeline)
db_session.commit()
# Start Celery pipeline for indexing and analysis.
# Import here to avoid circular imports.
from timesketch.lib import tasks
pipeline = tasks.build_index_pipeline(file_path, timeline_name,
index_name, extension, sketch.id)
pipeline.apply_async(task_id=index_name)
print('Imported {0:s} to sketch: {1:d} ({2:s})'.format(
file_path, sketch.id, sketch.name))
def run(self, import_location, export_location):
"""Export/Import search templates to/from file.
Args:
import_location: Path to the yaml file to import templates.
export_location: Path to the yaml file to export templates.
"""
if export_location:
search_templates = []
for search_template in SearchTemplate.query.all():
labels = []
for label in search_template.labels:
if label.label.startswith('supported_os:'):
labels.append(label.label.replace('supported_os:', ''))
search_templates.append({
'name': search_template.name,
'query_string': search_template.query_string,
'query_dsl': search_template.query_dsl,
'supported_os': labels
})
with open(export_location, 'w') as fh:
yaml.safe_dump(search_templates, stream=fh)
if import_location:
try:
with open(import_location, 'rb') as fh:
search_templates = yaml.safe_load(fh)
except IOError as e:
sys.stdout.write('Unable to open file: {0!s}\n'.format(e))
sys.exit(1)
for search_template in search_templates:
name = search_template['name']
query_string = search_template['query_string']
query_dsl = search_template['query_dsl']
# Skip search template if already exits.
if SearchTemplate.query.filter_by(name=name).first():
continue
imported_template = SearchTemplate(name=name,
user=User(None),
query_string=query_string,
query_dsl=query_dsl)
# Add supported_os labels.
for supported_os in search_template['supported_os']:
label_name = 'supported_os:{0:s}'.format(supported_os)
label = SearchTemplate.Label.get_or_create(
label=label_name, user=None)
imported_template.labels.append(label)
# Set flag to identify local vs import templates.
remote_flag = SearchTemplate.Label.get_or_create(
label='remote_template', user=None)
imported_template.labels.append(remote_flag)
db_session.add(imported_template)
db_session.commit()
def post(self, sketch_id):
"""Handles POST request to the resource.
Args:
sketch_id: Integer primary key for a sketch database model
Returns:
An annotation in JSON (instance of flask.wrappers.Response)
"""
form = EventAnnotationForm.build(request)
if form.validate_on_submit():
annotations = []
sketch = Sketch.query.get_with_acl(sketch_id)
indices = [t.searchindex.index_name for t in sketch.timelines]
annotation_type = form.annotation_type.data
events = form.events.raw_data
for _event in events:
searchindex_id = _event[u'_index']
searchindex = SearchIndex.query.filter_by(
index_name=searchindex_id).first()
event_id = _event[u'_id']
event_type = _event[u'_type']
if searchindex_id not in indices:
abort(HTTP_STATUS_CODE_BAD_REQUEST)
# Get or create an event in the SQL database to have something
# to attach the annotation to.
event = Event.get_or_create(sketch=sketch,
searchindex=searchindex,
document_id=event_id)
# Add the annotation to the event object.
if u'comment' in annotation_type:
annotation = Event.Comment(comment=form.annotation.data,
user=current_user)
event.comments.append(annotation)
self.datastore.set_label(searchindex_id,
event_id,
event_type,
sketch.id,
current_user.id,
u'__ts_comment',
toggle=False)
elif u'label' in annotation_type:
annotation = Event.Label.get_or_create(
label=form.annotation.data, user=current_user)
if annotation not in event.labels:
event.labels.append(annotation)
toggle = False
if u'__ts_star' or u'__ts_hidden' in form.annotation.data:
toggle = True
self.datastore.set_label(searchindex_id,
event_id,
event_type,
sketch.id,
current_user.id,
form.annotation.data,
toggle=toggle)
else:
abort(HTTP_STATUS_CODE_BAD_REQUEST)
annotations.append(annotation)
# Save the event to the database
db_session.add(event)
db_session.commit()
return self.to_json(annotations,
status_code=HTTP_STATUS_CODE_CREATED)
return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def create_group(group_name):
"""Create a group."""
group = Group.get_or_create(name=group_name)
db_session.add(group)
db_session.commit()
print(f"Group created: {group_name}")
def create_view_from_form(sketch, form):
"""Creates a view from form data.
Args:
sketch: Instance of timesketch.models.sketch.Sketch
form: Instance of timesketch.lib.forms.SaveViewForm
Returns:
A view (Instance of timesketch.models.sketch.View)
"""
# Default to user supplied data
view_name = form.name.data
query_string = form.query.data
query_filter = json.dumps(form.filter.data, ensure_ascii=False),
query_dsl = json.dumps(form.dsl.data, ensure_ascii=False)
# WTF forms turns the filter into a tuple for some reason.
# pylint: disable=redefined-variable-type
if isinstance(query_filter, tuple):
query_filter = query_filter[0]
# No search template by default (before we know if the user want to
# create a template or use an existing template when creating the view)
searchtemplate = None
# Create view from a search template
if form.from_searchtemplate_id.data:
# Get the template from the datastore
template_id = form.from_searchtemplate_id.data
searchtemplate = SearchTemplate.query.get(template_id)
# Copy values from the template
view_name = searchtemplate.name
query_string = searchtemplate.query_string
query_filter = searchtemplate.query_filter,
query_dsl = searchtemplate.query_dsl
# WTF form returns a tuple for the filter. This is not
# compatible with SQLAlchemy.
if isinstance(query_filter, tuple):
query_filter = query_filter[0]
# Create a new search template based on this view (only if requested by
# the user).
if form.new_searchtemplate.data:
query_filter_dict = json.loads(query_filter)
if query_filter_dict.get(u'indices', None):
query_filter_dict[u'indices'] = u'_all'
# pylint: disable=redefined-variable-type
query_filter = json.dumps(query_filter_dict, ensure_ascii=False)
searchtemplate = SearchTemplate(name=view_name,
user=current_user,
query_string=query_string,
query_filter=query_filter,
query_dsl=query_dsl)
db_session.add(searchtemplate)
db_session.commit()
# Create the view in the database
view = View(name=view_name,
sketch=sketch,
user=current_user,
query_string=query_string,
query_filter=query_filter,
query_dsl=query_dsl,
searchtemplate=searchtemplate)
db_session.add(view)
db_session.commit()
return view
def commit(self):
"""Commit changes to DB."""
self.group.orientation = self._orientation
self.group.parameters = self._parameters
db_session.add(self.group)
db_session.commit()
def build_sketch_analysis_pipeline(sketch_id,
searchindex_id,
user_id,
analyzer_names=None,
analyzer_kwargs=None):
"""Build a pipeline for sketch analysis.
If no analyzer_names is passed in then we assume auto analyzers should be
run and get this list from the configuration. Parameters to the analyzers
can be passed in to this function, otherwise they will be taken from the
configuration. Either default kwargs for auto analyzers or defaults for
manually run analyzers.
Args:
sketch_id (int): The ID of the sketch to analyze.
searchindex_id (int): The ID of the searchindex to analyze.
user_id (int): The ID of the user who started the analyzer.
analyzer_names (list): List of analyzers to run.
analyzer_kwargs (dict): Arguments to the analyzers.
Returns:
A tuple with a Celery group with analysis tasks or None if no analyzers
are enabled and an analyzer session ID.
"""
tasks = []
if not analyzer_names:
analyzer_names = current_app.config.get('AUTO_SKETCH_ANALYZERS', [])
if not analyzer_kwargs:
analyzer_kwargs = current_app.config.get(
'AUTO_SKETCH_ANALYZERS_KWARGS', {})
# Exit early if no sketch analyzers are configured to run.
if not analyzer_names:
return None, None
if not analyzer_kwargs:
analyzer_kwargs = current_app.config.get('ANALYZERS_DEFAULT_KWARGS',
{})
if user_id:
user = User.query.get(user_id)
else:
user = None
sketch = Sketch.query.get(sketch_id)
analysis_session = AnalysisSession(user, sketch)
analyzers = manager.AnalysisManager.get_analyzers(analyzer_names)
for analyzer_name, analyzer_cls in analyzers:
if not analyzer_cls.IS_SKETCH_ANALYZER:
continue
kwargs = analyzer_kwargs.get(analyzer_name, {})
searchindex = SearchIndex.query.get(searchindex_id)
timeline = Timeline.query.filter_by(sketch=sketch,
searchindex=searchindex).first()
analysis = Analysis(name=analyzer_name,
description=analyzer_name,
analyzer_name=analyzer_name,
parameters=json.dumps(kwargs),
user=user,
sketch=sketch,
timeline=timeline)
analysis.set_status('PENDING')
analysis_session.analyses.append(analysis)
db_session.add(analysis)
db_session.commit()
tasks.append(
run_sketch_analyzer.s(sketch_id, analysis.id, analyzer_name,
**kwargs))
# Commit the analysis session to the database.
db_session.add(analysis_session)
db_session.commit()
if current_app.config.get('ENABLE_EMAIL_NOTIFICATIONS'):
tasks.append(run_email_result_task.s(sketch_id))
if not tasks:
return None, None
return chain(tasks), analysis_session.id
def post(self, sketch_id, group_id):
"""Handles POST request to the resource.
Args:
sketch_id: Integer primary key for a sketch database model.
group_id: Integer primary key for an aggregation group database
model.
"""
sketch = Sketch.query.get_with_acl(sketch_id)
group = AggregationGroup.query.get(group_id)
if not group:
abort(
HTTP_STATUS_CODE_NOT_FOUND, 'No Group found with this ID.')
if not sketch:
abort(
HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.')
# Check that this group belongs to the sketch
if group.sketch_id != sketch.id:
msg = (
'The sketch ID ({0:d}) does not match with the aggregation '
'group sketch ID ({1:d})'.format(sketch.id, group.sketch_id))
abort(HTTP_STATUS_CODE_FORBIDDEN, msg)
if not sketch.has_permission(user=current_user, permission='write'):
abort(
HTTP_STATUS_CODE_FORBIDDEN,
'The user does not have write permission on the sketch.')
form = request.json
if not form:
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'No JSON data, unable to process request to create '
'a new aggregation group.')
group.name = form.get('name', group.name)
group.description = form.get('description', group.description)
group.parameters = form.get('parameters', group.parameters)
group.orientation = form.get('orientation', group.orientation)
group.user = current_user
group.sketch = sketch
agg_ids = json.loads(form.get('aggregations', group.aggregations))
aggregations = []
for agg_id in agg_ids:
aggregation = Aggregation.query.get(agg_id)
if not aggregation:
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'No aggregation found for ID: {0:d}'.format(agg_id))
aggregations.append(aggregation)
group.aggregations = aggregations
db_session.add(group)
db_session.commit()
return self.to_json(group, status_code=HTTP_STATUS_CODE_CREATED)
def timelines(sketch_id):
"""Generates the sketch explore view template.
Returns:
Template with context.
"""
sketch = Sketch.query.get_with_acl(sketch_id)
searchindices_in_sketch = [t.searchindex.id for t in sketch.timelines]
indices = SearchIndex.all_with_acl(current_user).order_by(
desc(SearchIndex.created_at)).filter(
not_(SearchIndex.id.in_(searchindices_in_sketch)))
upload_enabled = current_app.config['UPLOAD_ENABLED']
graphs_enabled = current_app.config['GRAPH_BACKEND_ENABLED']
try:
plaso_version = current_app.config['PLASO_VERSION']
except KeyError:
plaso_version = 'Unknown'
# Setup the form
form = AddTimelineForm()
form.timelines.choices = set((i.id, i.name) for i in indices.all())
# Create new timeline form POST
if form.validate_on_submit():
if not sketch.has_permission(current_user, 'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN)
for searchindex_id in form.timelines.data:
searchindex = SearchIndex.query.get_with_acl(searchindex_id)
if searchindex not in [t.searchindex for t in sketch.timelines]:
_timeline = Timeline(
name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=current_user,
searchindex=searchindex)
db_session.add(_timeline)
sketch.timelines.append(_timeline)
db_session.commit()
# If enabled, run sketch analyzers when timeline is added.
# Import here to avoid circular imports.
from timesketch.lib import tasks
sketch_analyzer_group = tasks.build_sketch_analysis_pipeline(
sketch_id)
if sketch_analyzer_group:
pipeline = (tasks.run_sketch_init.s(
[searchindex.index_name]) | sketch_analyzer_group)
pipeline.apply_async(task_id=searchindex.index_name)
return redirect(
url_for('sketch_views.timelines', sketch_id=sketch.id))
return render_template(
'sketch/timelines.html',
sketch=sketch,
timelines=indices.all(),
form=form,
upload_enabled=upload_enabled,
plaso_version=plaso_version,
graphs_enabled=graphs_enabled)
def post(self, sketch_id, aggregation_id):
"""Handles POST request to the resource.
Handler for /api/v1/sketches/:sketch_id/aggregation/:aggregation_id
Args:
sketch_id: Integer primary key for a sketch database model
aggregation_id: Integer primary key for an aggregation database
model
"""
form = request.json
if not form:
form = request.data
if not form:
abort(
HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to validate form data.')
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(
HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.')
if not sketch.has_permission(current_user, 'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN,
'User does not have write access controls on sketch.')
aggregation = Aggregation.query.get(aggregation_id)
if not aggregation:
abort(
HTTP_STATUS_CODE_NOT_FOUND,
'No aggregation found with this ID.')
if not sketch.has_permission(user=current_user, permission='write'):
abort(
HTTP_STATUS_CODE_FORBIDDEN,
'The user does not have write permission on the sketch.')
aggregation.name = form.get('name', '')
aggregation.description = form.get('description', '')
aggregation.agg_type = form.get('agg_type', aggregation.agg_type)
aggregation.chart_type = form.get('chart_type', aggregation.chart_type)
aggregation.user = current_user
aggregation.sketch = sketch
labels = form.get('labels', '')
if labels:
for label in json.loads(labels):
if aggregation.has_label(label):
continue
aggregation.add_label(label)
if form.get('parameters'):
aggregation.parameters = json.dumps(
form.get('parameters'), ensure_ascii=False)
if form.get('view_id'):
aggregation.view = form.get('view_id')
db_session.add(aggregation)
db_session.commit()
return self.to_json(aggregation, status_code=HTTP_STATUS_CODE_CREATED)
def explore(sketch_id, view_id=None, searchtemplate_id=None):
"""Generates the sketch explore view template.
Returns:
Template with context.
"""
save_view = False # If the view should be saved to the database.
sketch = Sketch.query.get_with_acl(sketch_id)
sketch_timelines = [t.searchindex.index_name for t in sketch.timelines]
view_form = SaveViewForm()
graphs_enabled = current_app.config['GRAPH_BACKEND_ENABLED']
similarity_enabled = current_app.config.get('ENABLE_EXPERIMENTAL_UI')
# Get parameters from the GET query
url_query = request.args.get('q', '')
url_time_start = request.args.get('time_start', None)
url_time_end = request.args.get('time_end', None)
url_index = request.args.get('index', None)
url_size = request.args.get('size', None)
if searchtemplate_id:
searchtemplate = SearchTemplate.query.get(searchtemplate_id)
view = sketch.get_user_view(current_user)
if not view:
view = View(user=current_user, name='', sketch=sketch)
view.query_string = searchtemplate.query_string
view.query_filter = searchtemplate.query_filter
view.query_dsl = searchtemplate.query_dsl
save_view = True
elif view_id:
view = View.query.get(view_id)
# Check that this view belongs to the sketch
if view.sketch_id != sketch.id:
abort(HTTP_STATUS_CODE_NOT_FOUND)
# Return 404 if view is deleted
if view.get_status.status == 'deleted':
return abort(HTTP_STATUS_CODE_NOT_FOUND)
else:
view = sketch.get_user_view(current_user)
if not view:
view = View(user=current_user,
name='',
sketch=sketch,
query_string='*')
view.query_filter = view.validate_filter(
dict(indices=sketch_timelines))
save_view = True
if url_query:
view.query_string = url_query
query_filter = json.loads(view.query_filter)
query_filter['from'] = 0 # if we loaded from get, start at first event
query_filter['time_start'] = url_time_start
query_filter['time_end'] = url_time_end
if url_index in sketch_timelines:
query_filter['indices'] = [url_index]
if url_size:
query_filter['size'] = url_size
view.query_filter = view.validate_filter(query_filter)
view.query_dsl = None
save_view = True
if save_view:
db_session.add(view)
db_session.commit()
return render_template('sketch/explore.html',
sketch=sketch,
view=view,
named_view=view_id,
timelines=sketch_timelines,
view_form=view_form,
searchtemplate_id=searchtemplate_id,
graphs_enabled=graphs_enabled,
similarity_enabled=similarity_enabled)
def post(self):
"""Handles POST request to the resource.
Returns:
A view in JSON (instance of flask.wrappers.Response)
"""
upload_enabled = current_app.config['UPLOAD_ENABLED']
if not upload_enabled:
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'Failed to create timeline, upload not enabled')
form = forms.CreateTimelineForm()
if not form.validate_on_submit():
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'Failed to create timeline, form data not validated')
sketch_id = form.sketch_id.data
timeline_name = form.name.data
sketch = None
if sketch_id:
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(
HTTP_STATUS_CODE_NOT_FOUND,
'No sketch found with this ID.')
# We do not need a human readable filename or
# datastore index name, so we use UUIDs here.
index_name = uuid.uuid4().hex
if not isinstance(index_name, six.text_type):
index_name = codecs.decode(index_name, 'utf-8')
# Create the search index in the Timesketch database
searchindex = SearchIndex.get_or_create(
name=timeline_name,
description=timeline_name,
user=current_user,
index_name=index_name)
searchindex.grant_permission(permission='read', user=current_user)
searchindex.grant_permission(permission='write', user=current_user)
searchindex.grant_permission(
permission='delete', user=current_user)
searchindex.set_status('processing')
db_session.add(searchindex)
db_session.commit()
timeline = None
if sketch and sketch.has_permission(current_user, 'write'):
timeline = Timeline(
name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=current_user,
searchindex=searchindex)
sketch.timelines.append(timeline)
db_session.add(timeline)
db_session.commit()
# Return Timeline if it was created.
# pylint: disable=no-else-return
if timeline:
return self.to_json(
timeline, status_code=HTTP_STATUS_CODE_CREATED)
# Update the last activity of a sketch.
utils.update_sketch_last_activity(sketch)
return self.to_json(
searchindex, status_code=HTTP_STATUS_CODE_CREATED)
def post(self, sketch_id):
"""Handles POST request to the resource.
Handler for /api/v1/sketches/:sketch_id/explore/
Args:
sketch_id: Integer primary key for a sketch database model
Returns:
JSON with list of matched events
"""
sketch = Sketch.query.get_with_acl(sketch_id)
form = ExploreForm.build(request)
if form.validate_on_submit():
query_filter = form.filter.data
sketch_indices = [
t.searchindex.index_name for t in sketch.timelines]
indices = query_filter.get(u'indices', sketch_indices)
# Make sure that the indices in the filter are part of the sketch
if set(indices) - set(sketch_indices):
abort(HTTP_STATUS_CODE_BAD_REQUEST)
# Make sure we have a query string or star filter
if not form.query.data and not query_filter.get(u'star'):
abort(HTTP_STATUS_CODE_BAD_REQUEST)
result = self.datastore.search(
sketch_id, form.query.data, query_filter, indices)
# Get labels for each event that matches the sketch.
# Remove all other labels.
for event in result[u'hits'][u'hits']:
event[u'selected'] = False
event[u'_source'][u'label'] = []
try:
for label in event[u'_source'][u'timesketch_label']:
if sketch.id != label[u'sketch_id']:
continue
event[u'_source'][u'label'].append(label[u'name'])
del event[u'_source'][u'timesketch_label']
except KeyError:
pass
# Update or create user state view. This is used in the UI to let
# the user get back to the last state in the explore view.
view = View.get_or_create(
user=current_user, sketch=sketch, name=u'')
view.query_string = form.query.data
view.query_filter = json.dumps(query_filter)
db_session.add(view)
db_session.commit()
# Add metadata for the query result. This is used by the UI to
# render the event correctly and to display timing and hit count
# information.
tl_colors = {}
tl_names = {}
for timeline in sketch.timelines:
tl_colors[timeline.searchindex.index_name] = timeline.color
tl_names[timeline.searchindex.index_name] = timeline.name
meta = {
u'es_time': result[u'took'],
u'es_total_count': result[u'hits'][u'total'],
u'timeline_colors': tl_colors,
u'timeline_names': tl_names
}
schema = {
u'meta': meta,
u'objects': result[u'hits'][u'hits']
}
return jsonify(schema)
return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def post(self, sketch_id):
"""Handles POST request to the resource.
Args:
sketch_id: Integer primary key for a sketch database model
Returns:
An annotation in JSON (instance of flask.wrappers.Response)
"""
form = forms.EventAnnotationForm.build(request)
if not form.validate_on_submit():
abort(HTTP_STATUS_CODE_BAD_REQUEST,
'Unable to validate form data.')
annotations = []
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.')
if not sketch.has_permission(current_user, 'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN,
'User does not have write access controls on sketch.')
indices = [
t.searchindex.index_name for t in sketch.timelines
if t.get_status.status.lower() == 'ready'
]
annotation_type = form.annotation_type.data
events = form.events.raw_data
for _event in events:
searchindex_id = _event['_index']
searchindex = SearchIndex.query.filter_by(
index_name=searchindex_id).first()
event_id = _event['_id']
event_type = _event['_type']
if searchindex_id not in indices:
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'Search index ID ({0!s}) does not belong to the list '
'of indices'.format(searchindex_id))
# Get or create an event in the SQL database to have something
# to attach the annotation to.
event = Event.get_or_create(sketch=sketch,
searchindex=searchindex,
document_id=event_id)
# Add the annotation to the event object.
if 'comment' in annotation_type:
annotation = Event.Comment(comment=form.annotation.data,
user=current_user)
event.comments.append(annotation)
self.datastore.set_label(searchindex_id,
event_id,
event_type,
sketch.id,
current_user.id,
'__ts_comment',
toggle=False)
elif 'label' in annotation_type:
annotation = Event.Label.get_or_create(
label=form.annotation.data, user=current_user)
if annotation not in event.labels:
event.labels.append(annotation)
toggle = False
if '__ts_star' in form.annotation.data:
toggle = True
if '__ts_hidden' in form.annotation.data:
toggle = True
if form.remove.data:
toggle = True
self.datastore.set_label(searchindex_id,
event_id,
event_type,
sketch.id,
current_user.id,
form.annotation.data,
toggle=toggle)
else:
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'Annotation type needs to be either label or comment, '
'not {0!s}'.format(annotation_type))
annotations.append(annotation)
# Save the event to the database
db_session.add(event)
db_session.commit()
return self.to_json(annotations, status_code=HTTP_STATUS_CODE_CREATED)
def WriteHeader(self):
"""Setup the Elasticsearch index and the Timesketch database object.
Creates the Elasticsearch index with Timesketch specific settings and the
Timesketch SearchIndex database object.
"""
# This cannot be static because we use the value of self._doc_type from
# arguments.
_document_mapping = {
self._doc_type: {
u'properties': {
u'timesketch_label': {
u'type': u'nested'
}
}
}
}
# Get Elasticsearch host and port from Timesketch configuration.
with self._timesketch.app_context():
_host = current_app.config[u'ELASTIC_HOST']
_port = current_app.config[u'ELASTIC_PORT']
self._elastic = ElasticSearchHelper(self._output_mediator, _host,
_port, self._flush_interval,
self._index_name,
_document_mapping, self._doc_type)
user = None
if self._username:
user = User.query.filter_by(username=self._username).first()
if not user:
raise RuntimeError(u'Unknown Timesketch user: {0:s}'.format(
self._username))
else:
logging.warning(
u'Timeline will be visible to all Timesketch users')
with self._timesketch.app_context():
search_index = SearchIndex.get_or_create(
name=self._timeline_name,
description=self._timeline_name,
user=user,
index_name=self._index_name)
# Grant the user read permission on the mapping object and set status.
# If user is None the timeline becomes visible to all users.
search_index.grant_permission(user=user, permission=u'read')
# In case we have a user grant additional permissions.
if user:
search_index.grant_permission(user=user, permission=u'write')
search_index.grant_permission(user=user, permission=u'delete')
# Let the Timesketch UI know that the timeline is processing.
search_index.set_status(u'processing')
# Save the mapping object to the Timesketch database.
db_session.add(search_index)
db_session.commit()
logging.info(u'Adding events to Timesketch.')
def post(self):
"""Handles POST request to the resource.
Returns:
A view in JSON (instance of flask.wrappers.Response)
Raises:
ApiHTTPError
"""
UPLOAD_ENABLED = current_app.config[u'UPLOAD_ENABLED']
UPLOAD_FOLDER = current_app.config[u'UPLOAD_FOLDER']
form = UploadFileForm()
if form.validate_on_submit() and UPLOAD_ENABLED:
from timesketch.lib.tasks import run_plaso
from timesketch.lib.tasks import run_csv
# Map the right task based on the file type
task_directory = {u'plaso': run_plaso, u'csv': run_csv}
sketch_id = form.sketch_id.data
file_storage = form.file.data
_filename, _extension = os.path.splitext(file_storage.filename)
file_extension = _extension.lstrip(u'.')
timeline_name = form.name.data or _filename.rstrip(u'.')
sketch = None
if sketch_id:
sketch = Sketch.query.get_with_acl(sketch_id)
# Current user
username = current_user.username
# We do not need a human readable filename or
# datastore index name, so we use UUIDs here.
filename = unicode(uuid.uuid4().hex)
index_name = unicode(uuid.uuid4().hex)
file_path = os.path.join(UPLOAD_FOLDER, filename)
file_storage.save(file_path)
# Create the search index in the Timesketch database
searchindex = SearchIndex.get_or_create(name=timeline_name,
description=timeline_name,
user=current_user,
index_name=index_name)
searchindex.grant_permission(permission=u'read', user=current_user)
searchindex.grant_permission(permission=u'write',
user=current_user)
searchindex.grant_permission(permission=u'delete',
user=current_user)
searchindex.set_status(u'processing')
db_session.add(searchindex)
db_session.commit()
if sketch and sketch.has_permission(current_user, u'write'):
timeline = Timeline(name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=current_user,
searchindex=searchindex)
db_session.add(timeline)
sketch.timelines.append(timeline)
db_session.commit()
# Run the task in the background
task = task_directory.get(file_extension)
task.apply_async((file_path, timeline_name, index_name, username),
task_id=index_name)
return self.to_json(searchindex,
status_code=HTTP_STATUS_CODE_CREATED)
else:
raise ApiHTTPError(message=form.errors[u'file'][0],
status_code=HTTP_STATUS_CODE_BAD_REQUEST)
def explore(sketch_id, view_id=None):
"""Generates the sketch explore view template.
Returns:
Template with context.
"""
save_view = False # If the view should be saved to the database.
sketch = Sketch.query.get_with_acl(sketch_id)
sketch_timelines = [t.searchindex.index_name for t in sketch.timelines]
view_form = SaveViewForm()
# Get parameters from the GET query
url_query = request.args.get(u'q', u'')
url_time_start = request.args.get(u'time_start', None)
url_time_end = request.args.get(u'time_end', None)
url_index = request.args.get(u'index', None)
url_limit = request.args.get(u'limit', None)
if view_id:
view = View.query.get(view_id)
# Check that this view belongs to the sketch
if view.sketch_id != sketch.id:
abort(HTTP_STATUS_CODE_NOT_FOUND)
# Return 404 if view is deleted
if view.get_status.status == u'deleted':
return abort(HTTP_STATUS_CODE_NOT_FOUND)
else:
view = sketch.get_user_view(current_user)
if not view:
query_filter = view.validate_filter(dict(indices=sketch_timelines))
view = View(user=current_user,
name=u'',
sketch=sketch,
query_string=u'*',
query_filter=query_filter)
save_view = True
if url_query:
view.query_string = url_query
query_filter = json.loads(view.query_filter)
query_filter[u'time_start'] = url_time_start
query_filter[u'time_end'] = url_time_end
if url_index in sketch_timelines:
query_filter[u'indices'] = [url_index]
if url_limit:
query_filter[u'limit'] = url_limit
view.query_filter = view.validate_filter(query_filter)
save_view = True
if save_view:
db_session.add(view)
db_session.commit()
return render_template(u'sketch/explore.html',
sketch=sketch,
view=view,
named_view=view_id,
timelines=sketch_timelines,
view_form=view_form)
def post(self, sketch_id):
"""Handles POST request to the resource.
Returns:
A HTTP 200 if the attribute is successfully added or modified.
"""
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(HTTP_STATUS_CODE_NOT_FOUND, "No sketch found with this ID.")
if not sketch.has_permission(current_user, "write"):
return abort(
HTTP_STATUS_CODE_FORBIDDEN,
"User does not have write permission on the sketch.",
)
form = request.json
if not form:
form = request.data
if not form:
return abort(
HTTP_STATUS_CODE_FORBIDDEN,
"Unable to add or modify an attribute from a "
"sketch without any data submitted.",
)
for check in ["name", "ontology"]:
error_message = self._validate_form_entry(form, check)
if error_message:
return abort(HTTP_STATUS_CODE_BAD_REQUEST, error_message)
name = form.get("name")
ontology = form.get("ontology", "text")
ontology_def = ontology_lib.ONTOLOGY
ontology_dict = ontology_def.get(ontology, {})
cast_as_string = ontology_dict.get("cast_as", "str")
values = form.get("values")
if not values:
return abort(HTTP_STATUS_CODE_BAD_REQUEST,
"Missing values from the request.")
if not isinstance(values, (list, tuple)):
return abort(HTTP_STATUS_CODE_BAD_REQUEST,
"Values needs to be a list.")
value_strings = [
ontology_lib.OntologyManager.encode_value(x, cast_as_string)
for x in values
]
if any([not isinstance(x, str) for x in value_strings]):
return abort(
HTTP_STATUS_CODE_BAD_REQUEST,
"All values needs to be stored as strings.",
)
attribute = None
message = ""
update_attribute = False
for attribute in sketch.attributes:
if (attribute.name == name) and (attribute.ontology == ontology):
message = "Attribute Updated"
update_attribute = True
break
if update_attribute:
_ = AttributeValue.query.filter_by(attribute=attribute).delete()
else:
attribute = Attribute(user=current_user,
sketch=sketch,
name=name,
ontology=ontology)
db_session.add(attribute)
db_session.commit()
for value in value_strings:
attribute_value = AttributeValue(user=current_user,
attribute=attribute,
value=value)
attribute.values.append(attribute_value)
db_session.add(attribute_value)
db_session.commit()
db_session.add(attribute)
db_session.commit()
return_data = {
"name": name,
"ontology": ontology,
"cast_as": cast_as_string,
}
response = None
if message:
return_data["action"] = "update"
response = jsonify(return_data)
response.status_code = HTTP_STATUS_CODE_OK
else:
return_data["action"] = "create"
response = jsonify(return_data)
response.status_code = HTTP_STATUS_CODE_CREATED
return response
def post(self, sketch_id):
"""Handles POST request to the resource.
Handler for /api/v1/sketches/:sketch_id/explore/
Args:
sketch_id: Integer primary key for a sketch database model
Returns:
JSON with list of matched events
"""
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.')
if not sketch.has_permission(current_user, 'read'):
abort(HTTP_STATUS_CODE_FORBIDDEN,
'User does not have read access controls on sketch.')
if sketch.get_status.status == 'archived':
abort(HTTP_STATUS_CODE_BAD_REQUEST,
'Unable to query on an archived sketch.')
form = forms.ExploreForm.build(request)
if not form.validate_on_submit():
abort(HTTP_STATUS_CODE_BAD_REQUEST,
'Unable to explore data, unable to validate form data')
# TODO: Remove form and use json instead.
query_dsl = form.dsl.data
enable_scroll = form.enable_scroll.data
scroll_id = form.scroll_id.data
file_name = form.file_name.data
query_filter = request.json.get('filter', {})
return_field_string = form.fields.data
if return_field_string:
return_fields = [x.strip() for x in return_field_string.split(',')]
else:
return_fields = query_filter.get('fields', [])
return_fields = [field['field'] for field in return_fields]
return_fields.extend(DEFAULT_SOURCE_FIELDS)
sketch_indices = {
t.searchindex.index_name
for t in sketch.timelines if t.get_status.status.lower() == 'ready'
}
if not query_filter:
query_filter = {}
indices = query_filter.get('indices', sketch_indices)
# If _all in indices then execute the query on all indices
if '_all' in indices:
indices = sketch_indices
# Make sure that the indices in the filter are part of the sketch.
# This will also remove any deleted timeline from the search result.
indices = get_validated_indices(indices, sketch_indices)
# Make sure we have a query string or star filter
if not (form.query.data, query_filter.get('star'),
query_filter.get('events'), query_dsl):
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'The request needs a query string/DSL and or a star filter.')
# Aggregate hit count per index.
index_stats_agg = {"indices": {"terms": {"field": "_index"}}}
if file_name:
file_object = io.BytesIO()
form_data = {
'created_at': datetime.datetime.utcnow().isoformat(),
'created_by': current_user.username,
'sketch': sketch_id,
'query': form.query.data,
'query_dsl': query_dsl,
'query_filter': query_filter,
'return_fields': return_fields,
}
with zipfile.ZipFile(file_object, mode='w') as zip_file:
zip_file.writestr('METADATA', data=json.dumps(form_data))
fh = export.query_to_filehandle(query_string=form.query.data,
query_dsl=query_dsl,
query_filter=query_filter,
indices=indices,
sketch=sketch,
datastore=self.datastore)
fh.seek(0)
zip_file.writestr('query_results.csv', fh.read())
file_object.seek(0)
return send_file(file_object,
mimetype='zip',
attachment_filename=file_name)
if scroll_id:
# pylint: disable=unexpected-keyword-arg
result = self.datastore.client.scroll(scroll_id=scroll_id,
scroll='1m')
else:
result = self.datastore.search(sketch_id,
form.query.data,
query_filter,
query_dsl,
indices,
aggregations=index_stats_agg,
return_fields=return_fields,
enable_scroll=enable_scroll)
# Get number of matching documents per index.
count_per_index = {}
try:
for bucket in result['aggregations']['indices']['buckets']:
key = bucket.get('key')
if key:
count_per_index[key] = bucket.get('doc_count')
except KeyError:
pass
# Get labels for each event that matches the sketch.
# Remove all other labels.
for event in result['hits']['hits']:
event['selected'] = False
event['_source']['label'] = []
try:
for label in event['_source']['timesketch_label']:
if sketch.id != label['sketch_id']:
continue
event['_source']['label'].append(label['name'])
del event['_source']['timesketch_label']
except KeyError:
pass
# Update or create user state view. This is used in the UI to let
# the user get back to the last state in the explore view.
view = View.get_or_create(user=current_user, sketch=sketch, name='')
view.query_string = form.query.data
view.query_filter = json.dumps(query_filter, ensure_ascii=False)
view.query_dsl = json.dumps(query_dsl, ensure_ascii=False)
db_session.add(view)
db_session.commit()
# Add metadata for the query result. This is used by the UI to
# render the event correctly and to display timing and hit count
# information.
tl_colors = {}
tl_names = {}
for timeline in sketch.timelines:
tl_colors[timeline.searchindex.index_name] = timeline.color
tl_names[timeline.searchindex.index_name] = timeline.name
meta = {
'es_time': result['took'],
'es_total_count': result['hits']['total'],
'timeline_colors': tl_colors,
'timeline_names': tl_names,
'count_per_index': count_per_index,
'scroll_id': result.get('_scroll_id', ''),
}
# Elasticsearch version 7.x returns total hits as a dictionary.
# TODO: Refactor when version 6.x has been deprecated.
if isinstance(meta['es_total_count'], dict):
meta['es_total_count'] = meta['es_total_count'].get('value', 0)
schema = {'meta': meta, 'objects': result['hits']['hits']}
return jsonify(schema)
def WriteHeader(self):
"""Sets up the Elasticsearch index and the Timesketch database object.
Creates the Elasticsearch index with Timesketch specific settings and the
Timesketch SearchIndex database object.
"""
# This cannot be static because we use the value of self._document_type
# from arguments.
mappings = {
'properties': {
'timesketch_label': {
'type': 'nested'
},
'datetime': {
'type': 'date'
}
}
}
# TODO: Remove once Elasticsearch v6.x is deprecated.
if self._GetClientMajorVersion() < 7:
mappings = {self._document_type: mappings}
# Get Elasticsearch host and port from Timesketch configuration.
with self._timesketch.app_context():
self._host = current_app.config['ELASTIC_HOST']
self._port = current_app.config['ELASTIC_PORT']
self._Connect()
self._CreateIndexIfNotExists(self._index_name, mappings)
user = None
if self._timeline_owner:
user = timesketch_user.User.query.filter_by(
username=self._timeline_owner).first()
if not user:
raise RuntimeError('Unknown Timesketch user: {0:s}'.format(
self._timeline_owner))
else:
logger.warning('Timeline will be visible to all Timesketch users')
with self._timesketch.app_context():
search_index = timesketch_sketch.SearchIndex.get_or_create(
name=self._timeline_name,
description=self._timeline_name,
user=user,
index_name=self._index_name)
# Grant the user read permission on the mapping object and set status.
# If user is None the timeline becomes visible to all users.
search_index.grant_permission(user=user, permission='read')
# In case we have a user grant additional permissions.
if user:
search_index.grant_permission(user=user, permission='write')
search_index.grant_permission(user=user, permission='delete')
# Let the Timesketch UI know that the timeline is processing.
search_index.set_status('processing')
# Save the mapping object to the Timesketch database.
timesketch_db_session.add(search_index)
timesketch_db_session.commit()
logger.debug('Adding events to Timesketch.')