def run(self, remove, group_name, user_name):
"""Add the user to the group."""
if not isinstance(group_name, six.text_type):
group_name = codecs.decode(group_name, 'utf-8')
if not isinstance(user_name, six.text_type):
user_name = codecs.decode(user_name, 'utf-8')
group = Group.query.filter_by(name=group_name).first()
user = User.query.filter_by(username=user_name).first()
# Add or remove user from group
if remove:
try:
user.groups.remove(group)
sys.stdout.write('{0:s} removed from group {1:s}\n'.format(
user_name, group_name))
db_session.commit()
except ValueError:
sys.stdout.write('{0:s} is not a member of group {1:s}\n'.
format(user_name, group_name))
else:
user.groups.append(group)
try:
db_session.commit()
sys.stdout.write('{0:s} added to group {1:s}\n'.format(
user_name, group_name))
except IntegrityError:
sys.stdout.write('{0:s} is already a member of group {1:s}\n'.
format(user_name, group_name))
def run_wrapper(self):
"""A wrapper method to run the analyzer.
This method is decorated to flush the bulk insert operation on the
datastore. This makes sure that all events are indexed at exit.
Returns:
Return value of the run method.
"""
result = self.run()
# Update the searchindex description with analyzer result.
# TODO: Don't overload the description field.
searchindex = SearchIndex.query.filter_by(
index_name=self.index_name).first()
# Some code paths set the description equals to the name. Remove that
# here to get a clean description with only analyzer results.
if searchindex.description == searchindex.name:
searchindex.description = ''
# Append the analyzer result.
if result:
searchindex.description = '{0:s}\n{1:s}'.format(
searchindex.description, result)
db_session.add(searchindex)
db_session.commit()
return result
def home():
"""Generates the home page view template.
Returns:
Template with context.
"""
form = HiddenNameDescriptionForm()
sketches = Sketch.all_with_acl().filter(
not_(Sketch.Status.status == 'deleted'),
Sketch.Status.parent).order_by(Sketch.updated_at.desc())
# Only render upload button if it is configured.
upload_enabled = current_app.config['UPLOAD_ENABLED']
# Handle form for creating a new sketch.
if form.validate_on_submit():
sketch = Sketch(
name=form.name.data,
description=form.description.data,
user=current_user)
sketch.status.append(sketch.Status(user=None, status='new'))
db_session.add(sketch)
db_session.commit()
# Give the requesting user permissions on the new sketch.
sketch.grant_permission(permission='read', user=current_user)
sketch.grant_permission(permission='write', user=current_user)
sketch.grant_permission(permission='delete', user=current_user)
return redirect(url_for('sketch_views.overview', sketch_id=sketch.id))
return render_template(
'home/home.html',
sketches=sketches,
form=form,
upload_enabled=upload_enabled)
def explore(sketch_id, view_id=None):
"""Generates the sketch explore view template.
Returns:
Template with context.
"""
sketch = Sketch.query.get_with_acl(sketch_id)
if view_id:
view = View.query.get(view_id)
else:
view = View.query.filter(
View.user == current_user,
View.name == u'',
View.sketch_id == sketch_id).order_by(
View.created_at.desc()).first()
if not view:
view = View(
user=current_user, name=u'', sketch=sketch, query_string=u'',
query_filter=u'{}')
db_session.add(view)
db_session.commit()
sketch_timelines = u','.join(
[t.searchindex.index_name for t in sketch.timelines])
view_form = SaveViewForm()
return render_template(
u'sketch/explore.html', sketch=sketch, view=view,
timelines=sketch_timelines, view_form=view_form)
def _set_timeline_status(index_name, status, error_msg=None):
"""Helper function to set status for searchindex and all related timelines.
Args:
index_name: Name of the datastore index.
status: Status to set.
error_msg: Error message.
"""
searchindex = SearchIndex.query.filter_by(index_name=index_name).first()
timelines = Timeline.query.filter_by(searchindex=searchindex).all()
# Set status
searchindex.set_status(status)
for timeline in timelines:
timeline.set_status(status)
db_session.add(timeline)
# Update description if there was a failure in ingestion
if error_msg and status == 'fail':
# TODO: Don't overload the description field.
searchindex.description = error_msg
# Commit changes to database
db_session.add(searchindex)
db_session.commit()
def add_view(self, view_name, analyzer_name, query_string=None,
query_dsl=None, query_filter=None):
"""Add saved view to the Sketch.
Args:
view_name: The name of the view.
analyzer_name: The name of the analyzer.
query_string: Elasticsearch query string.
query_dsl: Dictionary with Elasticsearch DSL query.
query_filter: Dictionary with Elasticsearch filters.
Raises:
ValueError: If both query_string an query_dsl are missing.
Returns: An instance of a SQLAlchemy View object.
"""
if not query_string or query_dsl:
raise ValueError('Both query_string and query_dsl are missing.')
if not query_filter:
query_filter = {'indices': '_all'}
name = '[{0:s}] {1:s}'.format(analyzer_name, view_name)
view = View.get_or_create(name=name, sketch=self.sql_sketch, user=None)
view.query_string = query_string
view.query_filter = view.validate_filter(query_filter)
view.query_dsl = query_dsl
view.searchtemplate = None
db_session.add(view)
db_session.commit()
return view
def timeline(sketch_id, timeline_id):
"""Generates the sketch timeline view template.
Returns:
Template with context.
"""
timeline_form = TimelineForm()
sketch = Sketch.query.get_with_acl(sketch_id)
sketch_timeline = Timeline.query.filter(
Timeline.id == timeline_id, Timeline.sketch == sketch).first()
if not sketch_timeline:
abort(HTTP_STATUS_CODE_NOT_FOUND)
if timeline_form.validate_on_submit():
if not sketch.has_permission(current_user, u'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN)
sketch_timeline.name = timeline_form.name.data
sketch_timeline.description = timeline_form.description.data
sketch_timeline.color = timeline_form.color.data
db_session.add(sketch_timeline)
db_session.commit()
return redirect(
url_for(u'sketch_views.timeline', sketch_id=sketch.id,
timeline_id=sketch_timeline.id))
return render_template(
u'sketch/timeline.html', sketch=sketch, timeline=sketch_timeline,
timeline_form=timeline_form)
def WriteHeader(self):
"""Sets up the Elasticsearch index and the Timesketch database object.
Creates the Elasticsearch index with Timesketch specific settings and the
Timesketch SearchIndex database object.
"""
# This cannot be static because we use the value of self._document_type
# from arguments.
mappings = {
self._document_type: {
'properties': {
'timesketch_label': {
'type': 'nested'
}
}
}
}
# Get Elasticsearch host and port from Timesketch configuration.
with self._timesketch.app_context():
self._host = current_app.config['ELASTIC_HOST']
self._port = current_app.config['ELASTIC_PORT']
self._Connect()
self._CreateIndexIfNotExists(self._index_name, mappings)
user = None
if self._timeline_owner:
user = timesketch_user.User.query.filter_by(
username=self._timeline_owner).first()
if not user:
raise RuntimeError(
'Unknown Timesketch user: {0:s}'.format(self._timeline_owner))
else:
logger.warning('Timeline will be visible to all Timesketch users')
with self._timesketch.app_context():
search_index = timesketch_sketch.SearchIndex.get_or_create(
name=self._timeline_name, description=self._timeline_name, user=user,
index_name=self._index_name)
# Grant the user read permission on the mapping object and set status.
# If user is None the timeline becomes visible to all users.
search_index.grant_permission(user=user, permission='read')
# In case we have a user grant additional permissions.
if user:
search_index.grant_permission(user=user, permission='write')
search_index.grant_permission(user=user, permission='delete')
# Let the Timesketch UI know that the timeline is processing.
search_index.set_status('processing')
# Save the mapping object to the Timesketch database.
timesketch_db_session.add(search_index)
timesketch_db_session.commit()
logger.debug('Adding events to Timesketch.')
def run(self, name):
"""Creates the group."""
if not isinstance(name, six.text_type):
name = codecs.decode(name, 'utf-8')
group = Group.get_or_create(name=name)
db_session.add(group)
db_session.commit()
sys.stdout.write('Group {0:s} created\n'.format(name))
def _commit_to_database(self, model):
"""Add object to the database session and commit.
Args:
model: Instance of timesketch.models.[model] object
"""
db_session.add(model)
db_session.commit()
def WriteHeader(self):
"""Setup the Elasticsearch index and the Timesketch database object.
Creates the Elasticsearch index with Timesketch specific settings and the
Timesketch SearchIndex database object.
"""
# This cannot be static because we use the value of self._doc_type from
# arguments.
_document_mapping = {
self._doc_type: {
u'properties': {
u'timesketch_label': {
u'type': u'nested'
}
}
}
}
# Get Elasticsearch host and port from Timesketch configuration.
with self._timesketch.app_context():
_host = current_app.config[u'ELASTIC_HOST']
_port = current_app.config[u'ELASTIC_PORT']
self._elastic = ElasticSearchHelper(
self._output_mediator, _host, _port, self._flush_interval,
self._index_name, _document_mapping, self._doc_type)
user = None
if self._username:
user = User.query.filter_by(username=self._username).first()
if not user:
raise RuntimeError(
u'Unknown Timesketch user: {0:s}'.format(self._username))
else:
logging.warning(u'Timeline will be visible to all Timesketch users')
with self._timesketch.app_context():
search_index = SearchIndex.get_or_create(
name=self._timeline_name, description=self._timeline_name, user=user,
index_name=self._index_name)
# Grant the user read permission on the mapping object and set status.
# If user is None the timeline becomes visible to all users.
search_index.grant_permission(user=user, permission=u'read')
# In case we have a user grant additional permissions.
if user:
search_index.grant_permission(user=user, permission=u'write')
search_index.grant_permission(user=user, permission=u'delete')
# Let the Timesketch UI know that the timeline is processing.
search_index.set_status(u'processing')
# Save the mapping object to the Timesketch database.
db_session.add(search_index)
db_session.commit()
logging.info(u'Adding events to Timesketch.')
def post(self, sketch_id):
"""Handles POST request to the resource.
Args:
sketch_id: Integer primary key for a sketch database model
Returns:
An annotation in JSON (instance of flask.wrappers.Response)
"""
form = EventAnnotationForm.build(request)
if form.validate_on_submit():
sketch = Sketch.query.get_with_acl(sketch_id)
indices = [t.searchindex.index_name for t in sketch.timelines]
annotation_type = form.annotation_type.data
searchindex_id = form.searchindex_id.data
searchindex = SearchIndex.query.get(searchindex_id)
event_id = form.event_id.data
if searchindex_id not in indices:
abort(HTTP_STATUS_CODE_BAD_REQUEST)
def _set_label(label, toggle=False):
"""Set label on the event in the datastore."""
self.datastore.set_label(
searchindex_id, event_id, sketch.id, current_user.id, label,
toggle=toggle)
# Get or create an event in the SQL database to have something to
# attach the annotation to.
event = Event.get_or_create(
sketch=sketch, searchindex=searchindex,
document_id=event_id)
# Add the annotation to the event object.
if u'comment' in annotation_type:
annotation = Event.Comment(
comment=form.annotation.data, user=current_user)
event.comments.append(annotation)
_set_label(u'__ts_comment')
elif u'label' in annotation_type:
annotation = Event.Label.get_or_create(
label=form.annotation.data, user=current_user)
if annotation not in event.labels:
event.labels.append(annotation)
toggle = False
if u'__ts_star' in form.annotation.data:
toggle = True
_set_label(form.annotation.data, toggle)
else:
abort(HTTP_STATUS_CODE_BAD_REQUEST)
# Save the event to the database
db_session.add(event)
db_session.commit()
return self.to_json(
annotation, status_code=HTTP_STATUS_CODE_CREATED)
return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def revoke_permission(self, user, permission):
"""Revoke permission to a user with the specific permission.
Args:
user: A user (Instance of timesketch.models.user.User)
permission: Permission as string (read, write or delete)
"""
for ace in self._get_ace(user, permission):
self.acl.remove(ace)
db_session.commit()
def overview(sketch_id):
"""Generates the sketch overview template.
Returns:
Template with context.
"""
sketch = Sketch.query.get_with_acl(sketch_id)
sketch_form = NameDescriptionForm()
permission_form = TogglePublic()
status_form = StatusForm()
trash_form = TrashForm()
# Edit sketch form POST
if sketch_form.validate_on_submit():
if not sketch.has_permission(current_user, u'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN)
sketch.name = sketch_form.name.data
sketch.description = sketch_form.description.data
db_session.commit()
return redirect(
url_for(u'sketch_views.overview', sketch_id=sketch.id))
# Toggle public/private form POST
if permission_form.validate_on_submit():
if not sketch.has_permission(current_user, u'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN)
if permission_form.permission.data == u'public':
sketch.grant_permission(user=None, permission=u'read')
else:
sketch.revoke_permission(user=None, permission=u'read')
db_session.commit()
return redirect(
url_for(u'sketch_views.overview', sketch_id=sketch.id))
# Change status form POST
if status_form.validate_on_submit():
if not sketch.has_permission(current_user, u'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN)
sketch.set_status(status=status_form.status.data)
return redirect(
url_for(u'sketch_views.overview', sketch_id=sketch.id))
# Trash form POST
if trash_form.validate_on_submit():
if not sketch.has_permission(current_user, u'delete'):
abort(HTTP_STATUS_CODE_FORBIDDEN)
sketch.set_status(status=u'deleted')
return redirect(
url_for(u'home_views.home'))
return render_template(
u'sketch/overview.html', sketch=sketch, sketch_form=sketch_form,
permission_form=permission_form, status_form=status_form,
trash_form=trash_form)
def set_status(self, status):
"""
Set status on object. Although this is a many-to-many relationship
this makes sure that the parent object only has one status set.
Args:
status: Name of the status
"""
for _status in self.status:
self.status.remove(_status)
self.status.append(self.Status(user=None, status=status))
db_session.commit()
def grant_permission(self, user, permission):
"""Grant permission to a user with the specific permission.
Args:
user: A user (Instance of timesketch.models.user.User)
permission: Permission as string (read, write or delete)
"""
if not self._get_ace(user, permission):
self.acl.append(
self.AccessControlEntry(
user=user, permission=permission))
db_session.commit()
def run(self, username, password):
"""Creates the user."""
if not password:
password = self.get_password_from_prompt()
if not isinstance(password, six.text_type):
password = codecs.decode(password, 'utf-8')
username = codecs.decode(username, 'utf-8')
user = User.get_or_create(username=username)
user.set_password(plaintext=password)
db_session.add(user)
db_session.commit()
sys.stdout.write('User {0:s} created/updated\n'.format(username))
def Close(self):
"""Closes the connection to TimeSketch Elasticsearch database.
Sends the remaining events for indexing and removes the processing status on
the Timesketch search index object.
"""
self._elastic.AddEvent(None, force_flush=True)
with self._timesketch.app_context():
search_index = SearchIndex.query.filter_by(
index_name=self._index_name).first()
search_index.status.remove(search_index.status[0])
db_session.add(search_index)
db_session.commit()
def post(self):
"""Handles POST request to the resource.
Returns:
A search index in JSON (instance of flask.wrappers.Response)
"""
form = forms.SearchIndexForm.build(request)
searchindex_name = form.searchindex_name.data
es_index_name = form.es_index_name.data
public = form.public.data
if not form.validate_on_submit():
abort(HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to validate form data')
searchindex = SearchIndex.query.filter_by(
index_name=es_index_name).first()
metadata = {'created': True}
if searchindex:
metadata['created'] = False
metadata['deleted'] = searchindex.get_status.status == 'deleted'
status_code = HTTP_STATUS_CODE_OK
else:
searchindex = SearchIndex.get_or_create(
name=searchindex_name,
description=searchindex_name,
user=current_user,
index_name=es_index_name)
searchindex.grant_permission(permission='read', user=current_user)
searchindex.grant_permission(permission='write', user=current_user)
searchindex.grant_permission(permission='delete',
user=current_user)
if public:
searchindex.grant_permission(permission='read', user=None)
datastore = self.datastore
if not datastore.client.indices.exists(index=es_index_name):
# Create the index in Elasticsearch
self.datastore.create_index(index_name=es_index_name,
doc_type='generic_event')
db_session.add(searchindex)
db_session.commit()
status_code = HTTP_STATUS_CODE_CREATED
return self.to_json(searchindex,
meta=metadata,
status_code=status_code)
def Close(self):
"""Closes the connection to TimeSketch Elasticsearch database.
Sends the remaining events for indexing and removes the processing status on
the Timesketch search index object.
"""
super(TimesketchOutputModule, self).Close()
with self._timesketch.app_context():
search_index = timesketch_sketch.SearchIndex.query.filter_by(
index_name=self._index_name).first()
search_index.status.remove(search_index.status[0])
timesketch_db_session.add(search_index)
timesketch_db_session.commit()
def timelines(sketch_id):
"""Generates the sketch explore view template.
Returns:
Template with context.
"""
sketch = Sketch.query.get_with_acl(sketch_id)
searchindices_in_sketch = [t.searchindex.id for t in sketch.timelines]
indices = SearchIndex.all_with_acl(current_user).order_by(
desc(SearchIndex.created_at)).filter(
not_(SearchIndex.id.in_(searchindices_in_sketch)))
upload_enabled = current_app.config[u'UPLOAD_ENABLED']
graphs_enabled = current_app.config[u'GRAPH_BACKEND_ENABLED']
try:
plaso_version = current_app.config[u'PLASO_VERSION']
except KeyError:
plaso_version = u'Unknown'
# Setup the form
form = AddTimelineForm()
form.timelines.choices = set((i.id, i.name) for i in indices.all())
# Create new timeline form POST
if form.validate_on_submit():
if not sketch.has_permission(current_user, u'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN)
for searchindex_id in form.timelines.data:
searchindex = SearchIndex.query.get_with_acl(searchindex_id)
if searchindex not in [t.searchindex for t in sketch.timelines]:
_timeline = Timeline(
name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=current_user,
searchindex=searchindex)
db_session.add(_timeline)
sketch.timelines.append(_timeline)
db_session.commit()
return redirect(
url_for(u'sketch_views.timelines', sketch_id=sketch.id))
return render_template(
u'sketch/timelines.html',
sketch=sketch,
timelines=indices.all(),
form=form,
upload_enabled=upload_enabled,
plaso_version=plaso_version,
graphs_enabled=graphs_enabled)
def home():
"""Generates the home page view template.
Returns:
Template with context.
"""
form = HiddenNameDescriptionForm()
sketches = Sketch.all_with_acl().filter(
not_(Sketch.Status.status == u'deleted'),
Sketch.Status.parent).order_by(Sketch.updated_at.desc())
query_filter = request.args.get(u'filter', u'')
query = request.args.get(u'q', u'')
# Only render upload button if it is configured.
upload_enabled = current_app.config[u'UPLOAD_ENABLED']
last_sketch = View.query.filter_by(user=current_user, name=u'').order_by(
View.updated_at.desc()).first()
if query_filter:
if query_filter == u'user':
sketches = sketches.filter(Sketch.user == current_user)
elif query_filter == u'shared':
sketches = sketches.filter(not_(Sketch.user == current_user))
# TODO: Figure out a better way to handle this.
if query:
if query.startswith(u'*'):
query = u''
else:
sketches = sketches.filter(Sketch.name.contains(query)).limit(100)
# Handle form for creating a new sketch.
if form.validate_on_submit():
sketch = Sketch(name=form.name.data,
description=form.description.data,
user=current_user)
sketch.status.append(sketch.Status(user=None, status=u'new'))
# Give the requesting user permissions on the new sketch.
sketch.grant_permission(current_user, u'read')
sketch.grant_permission(current_user, u'write')
sketch.grant_permission(current_user, u'delete')
db_session.add(sketch)
db_session.commit()
return redirect(url_for(u'sketch_views.overview', sketch_id=sketch.id))
return render_template(u'home/home.html',
sketches=sketches,
form=form,
query=query,
upload_enabled=upload_enabled,
last_sketch=last_sketch)
def Close(self):
"""Closes the connection to TimeSketch Elasticsearch database.
Sends the remaining events for indexing and removes the processing status on
the Timesketch search index object.
"""
super(TimesketchOutputModule, self).Close()
with self._timesketch.app_context():
search_index = timesketch_sketch.SearchIndex.query.filter_by(
index_name=self._index_name).first()
search_index.status.remove(search_index.status[0])
timesketch_db_session.add(search_index)
timesketch_db_session.commit()
def WriteHeader(self):
"""Sets up the Elasticsearch index and the Timesketch database object.
Creates the Elasticsearch index with Timesketch specific settings and the
Timesketch SearchIndex database object.
"""
# Get Elasticsearch host and port from Timesketch configuration.
with self._timesketch.app_context():
self._host = current_app.config['ELASTIC_HOST']
self._port = current_app.config['ELASTIC_PORT']
self._Connect()
self._CreateIndexIfNotExists(self._index_name, self._mappings)
user = None
if self._timeline_owner:
user = timesketch_user.User.query.filter_by(
username=self._timeline_owner).first()
if not user:
raise RuntimeError('Unknown Timesketch user: {0:s}'.format(
self._timeline_owner))
else:
logger.warning('Timeline will be visible to all Timesketch users')
with self._timesketch.app_context():
search_index = timesketch_sketch.SearchIndex.get_or_create(
name=self._timeline_name,
description=self._timeline_name,
user=user,
index_name=self._index_name)
# Grant the user read permission on the mapping object and set status.
# If user is None the timeline becomes visible to all users.
search_index.grant_permission(user=user, permission='read')
# In case we have a user grant additional permissions.
if user:
search_index.grant_permission(user=user, permission='write')
search_index.grant_permission(user=user, permission='delete')
# Let the Timesketch UI know that the timeline is processing.
search_index.set_status('processing')
# Save the mapping object to the Timesketch database.
timesketch_db_session.add(search_index)
timesketch_db_session.commit()
logger.debug('Adding events to Timesketch.')
def post(self, searchindex_id):
"""Handles POST request to the resource.
Returns:
A search index in JSON (instance of flask.wrappers.Response)
"""
form = request.json
if not form:
form = request.data
if not searchindex_id:
abort(HTTP_STATUS_CODE_BAD_REQUEST,
"Need to define a search index ID")
searchindex = SearchIndex.query.get_with_acl(searchindex_id)
if not searchindex:
abort(HTTP_STATUS_CODE_NOT_FOUND,
"No searchindex found with this ID.")
if not searchindex.has_permission(current_user, "write"):
abort(
HTTP_STATUS_CODE_FORBIDDEN,
("User does not have sufficient access rights to "
"edit the search index."),
)
if searchindex.get_status.status == "deleted":
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
"Search index is marked deleted, unable to continue.",
)
commit_to_db = False
new_status = form.get("status", "")
valid_status = ("ready", "fail", "processing", "timeout")
if new_status and new_status in valid_status:
searchindex.set_status(status=new_status)
commit_to_db = True
description = form.get("description", "")
if description:
searchindex.description = description
commit_to_db = True
if commit_to_db:
db_session.add(searchindex)
db_session.commit()
return self.to_json(searchindex)
def add_view(self,
view_name,
analyzer_name,
query_string=None,
query_dsl=None,
query_filter=None,
additional_fields=None):
"""Add saved view to the Sketch.
Args:
view_name: The name of the view.
analyzer_name: The name of the analyzer.
query_string: Elasticsearch query string.
query_dsl: Dictionary with Elasticsearch DSL query.
query_filter: Dictionary with Elasticsearch filters.
additional_fields: A list with field names to include in the
view output.
Raises:
ValueError: If both query_string an query_dsl are missing.
Returns: An instance of a SQLAlchemy View object.
"""
if not (query_string or query_dsl):
raise ValueError('Both query_string and query_dsl are missing.')
if not query_filter:
query_filter = {'indices': '_all'}
if additional_fields:
query_filter['fields'] = [{
'field': x.strip()
} for x in additional_fields]
description = 'analyzer: {0:s}'.format(analyzer_name)
view = View.get_or_create(name=view_name,
description=description,
sketch=self.sql_sketch,
user=None)
view.description = description
view.query_string = query_string
view.query_filter = view.validate_filter(query_filter)
view.query_dsl = query_dsl
view.searchtemplate = None
view.set_status(status='new')
db_session.add(view)
db_session.commit()
return view
def post(self, sketch_id, story_id):
"""Handles POST request to the resource.
Args:
sketch_id: Integer primary key for a sketch database model
story_id: Integer primary key for a story database model
Returns:
A view in JSON (instance of flask.wrappers.Response)
"""
sketch = Sketch.query.get_with_acl(sketch_id)
story = Story.query.get(story_id)
if not story:
msg = 'No Story found with this ID.'
abort(HTTP_STATUS_CODE_NOT_FOUND, msg)
if not sketch:
msg = 'No sketch found with this ID.'
abort(HTTP_STATUS_CODE_NOT_FOUND, msg)
if story.sketch_id != sketch.id:
abort(
HTTP_STATUS_CODE_NOT_FOUND,
'Sketch ID ({0:d}) does not match with the ID in '
'the story ({1:d})'.format(sketch.id, story.sketch_id))
if not sketch.has_permission(current_user, 'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN,
'User does not have write access controls on sketch.')
form = request.json
if not form:
form = request.data
if form and form.get('export_format'):
export_format = form.get('export_format')
return jsonify(story=self._export_story(
story=story, sketch_id=sketch_id, export_format=export_format))
story.title = form.get('title', '')
story.content = form.get('content', '[]')
db_session.add(story)
db_session.commit()
# Update the last activity of a sketch.
utils.update_sketch_last_activity(sketch)
return self.to_json(story, status_code=HTTP_STATUS_CODE_CREATED)
def WriteHeader(self):
"""Setup the Elasticsearch index and the Timesketch database object.
Creates the Elasticsearch index with Timesketch specific settings and the
Timesketch SearchIndex database object.
"""
# This cannot be static because we use the value of self._doc_type from
# arguments.
_document_mapping = {
self._doc_type: {
u'_timestamp': {
u'enabled': True,
u'path': u'datetime',
u'format': u'date_time_no_millis'
},
u'properties': {
u'timesketch_label': {
u'type': u'nested'
}
}
}
}
if not self._elastic_db.client.indices.exists(self._index_name):
try:
self._elastic_db.client.indices.create(
index=self._index_name,
body={u'mappings': _document_mapping})
except elastic_exceptions.ConnectionError as exception:
logging.error((
u'Unable to proceed, cannot connect to Timesketch backend '
u'with error: {0:s}.\nPlease verify connection.'
).format(exception))
raise RuntimeError(u'Unable to connect to Timesketch backend.')
with self._timesketch.app_context():
search_index = SearchIndex.get_or_create(
name=self._timeline_name,
description=self._timeline_name,
user=None,
index_name=self._index_name)
# Grant all users read permission on the mapping object and set status.
search_index.grant_permission(None, u'read')
search_index.set_status(u'processing')
# Save the mapping object to the Timesketch database.
db_session.add(search_index)
db_session.commit()
logging.info(u'Adding events to Timesketch..')
def explore(sketch_id, view_id=None):
"""Generates the sketch explore view template.
Returns:
Template with context.
"""
sketch = Sketch.query.get_with_acl(sketch_id)
sketch_timelines = [t.searchindex.index_name for t in sketch.timelines]
view_form = SaveViewForm()
# Get parameters from the GET query
url_query = request.args.get(u'q', u'')
url_time_start = request.args.get(u'time_start', None)
url_time_end = request.args.get(u'time_end', None)
if view_id:
view = View.query.get(view_id)
# Check that this view belongs to the sketch
if view.sketch_id != sketch.id:
abort(HTTP_STATUS_CODE_NOT_FOUND)
# Return 404 if view is deleted
if view.get_status.status == u'deleted':
return abort(HTTP_STATUS_CODE_NOT_FOUND)
else:
view = sketch.get_user_view(current_user)
if url_query:
view.query_string = url_query
query_filter = json.loads(view.query_filter)
query_filter[u'time_start'] = url_time_start
query_filter[u'time_end'] = url_time_end
view.query_filter = json.dumps(query_filter, ensure_ascii=False)
if not view:
query_filter = dict(indices=sketch_timelines)
view = View(user=current_user,
name=u'',
sketch=sketch,
query_string=u'',
query_filter=json.dumps(query_filter, ensure_ascii=False))
db_session.add(view)
db_session.commit()
return render_template(u'sketch/explore.html',
sketch=sketch,
view=view,
timelines=sketch_timelines,
view_form=view_form)
def timelines(sketch_id):
"""Generates the sketch explore view template.
Returns:
Template with context.
"""
TIMELINES_TO_SHOW = 20
sketch = Sketch.query.get_with_acl(sketch_id)
searchindices_in_sketch = [t.searchindex.id for t in sketch.timelines]
query = request.args.get(u'q', None)
indices = SearchIndex.all_with_acl(current_user).order_by(
desc(SearchIndex.created_at)).filter(
not_(SearchIndex.id.in_(searchindices_in_sketch)))
filtered = False
if query:
indices = indices.filter(SearchIndex.name.contains(query)).limit(500)
filtered = True
if not filtered:
indices = indices.limit(TIMELINES_TO_SHOW)
# Setup the form
form = AddTimelineForm()
form.timelines.choices = set((i.id, i.name) for i in indices.all())
# Create new timeline form POST
if form.validate_on_submit():
if not sketch.has_permission(current_user, u'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN)
for searchindex_id in form.timelines.data:
searchindex = SearchIndex.query.get_with_acl(searchindex_id)
if searchindex not in [t.searchindex for t in sketch.timelines]:
_timeline = Timeline(name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=current_user,
searchindex=searchindex)
db_session.add(_timeline)
sketch.timelines.append(_timeline)
db_session.commit()
return redirect(url_for(u'sketch_views.timelines',
sketch_id=sketch.id))
return render_template(u'sketch/timelines.html',
sketch=sketch,
timelines=indices.all(),
form=form,
filtered=filtered)
def add_aggregation(self,
name,
agg_name,
agg_params,
description='',
view_id=None,
chart_type=None,
label=''):
"""Add aggregation to the sketch.
Args:
name: the name of the aggregation run.
agg_name: the name of the aggregation class to run.
agg_params: a dictionary of the parameters for the aggregation.
description: description of the aggregation, visible in the UI,
this is optional.
view_id: optional ID of the view to attach the aggregation to.
chart_type: string representing the chart type.
label: string with a label to attach to the aggregation.
"""
if not agg_name:
raise ValueError('Aggregator name needs to be defined.')
if not agg_params:
raise ValueError('Aggregator parameters have to be defined.')
if view_id:
view = View.query.get(view_id)
else:
view = None
if chart_type:
agg_params['supported_charts'] = chart_type
agg_json = json.dumps(agg_params)
aggregation = Aggregation.get_or_create(name=name,
description=description,
agg_type=agg_name,
parameters=agg_json,
chart_type=chart_type,
user=None,
sketch=self.sql_sketch,
view=view)
if label:
aggregation.add_label(label)
db_session.add(aggregation)
db_session.commit()
return aggregation
def get(self, sketch_id, view_id):
"""Handles GET request to the resource.
Args:
sketch_id: Integer primary key for a sketch database model
view_id: Integer primary key for a view database model
Returns:
A view in JSON (instance of flask.wrappers.Response)
"""
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(
HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.')
view = View.query.get(view_id)
if not sketch.has_permission(current_user, 'read'):
abort(HTTP_STATUS_CODE_FORBIDDEN,
'User does not have read access controls on sketch.')
# Check that this view belongs to the sketch
if view.sketch_id != sketch.id:
abort(
HTTP_STATUS_CODE_NOT_FOUND,
'Sketch ID ({0:d}) does not match with the sketch ID '
'that is defined in the view ({1:d})'.format(
view.sketch_id, sketch.id))
# If this is a user state view, check that it
# belongs to the current_user
if view.name == '' and view.user != current_user:
abort(
HTTP_STATUS_CODE_FORBIDDEN,
'Unable to view a state view that belongs to a '
'different user.')
# Check if view has been deleted
if view.get_status.status == 'deleted':
meta = dict(deleted=True, name=view.name)
schema = dict(meta=meta, objects=[])
return jsonify(schema)
# Make sure we have all expected attributes in the query filter.
view.query_filter = view.validate_filter()
db_session.add(view)
db_session.commit()
return self.to_json(view)
def home():
"""Generates the home page view template.
Returns:
Template with context.
"""
form = HiddenNameDescriptionForm()
sketches = Sketch.all_with_acl().filter(
not_(Sketch.Status.status == u'deleted'),
Sketch.Status.parent).order_by(Sketch.updated_at.desc())
query_filter = request.args.get(u'filter', u'')
query = request.args.get(u'q', u'')
# Only render upload button if it is configured.
upload_enabled = current_app.config[u'UPLOAD_ENABLED']
last_sketch = View.query.filter_by(
user=current_user, name=u'').order_by(
View.updated_at.desc()).first()
if query_filter:
if query_filter == u'user':
sketches = sketches.filter(Sketch.user == current_user)
elif query_filter == u'shared':
sketches = sketches.filter(not_(Sketch.user == current_user))
# TODO: Figure out a better way to handle this.
if query:
if query.startswith(u'*'):
query = u''
else:
sketches = sketches.filter(Sketch.name.contains(query)).limit(100)
# Handle form for creating a new sketch.
if form.validate_on_submit():
sketch = Sketch(
name=form.name.data, description=form.description.data,
user=current_user)
sketch.status.append(sketch.Status(user=None, status=u'new'))
# Give the requesting user permissions on the new sketch.
sketch.grant_permission(current_user, u'read')
sketch.grant_permission(current_user, u'write')
sketch.grant_permission(current_user, u'delete')
db_session.add(sketch)
db_session.commit()
return redirect(url_for(u'sketch_views.overview', sketch_id=sketch.id))
return render_template(
u'home/home.html', sketches=sketches, form=form, query=query,
upload_enabled=upload_enabled, last_sketch=last_sketch)
def post(self, sketch_id):
"""Handles POST request to the resource.
Returns:
Graph in JSON (instance of flask.wrappers.Response)
"""
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.')
form = request.json
plugin_name = form.get('plugin')
graph_config = form.get('config')
refresh = form.get('refresh')
sketch_indices = [
timeline.searchindex.index_name
for timeline in sketch.active_timelines
]
cache = GraphCache.get_or_create(sketch=sketch,
graph_plugin=plugin_name)
# Refresh cache if timelines have been added/removed from the sketch.
if cache.graph_config:
cache_graph_config = json.loads(cache.graph_config)
if cache_graph_config:
cache_graph_config = json.loads(cache.graph_config)
cache_graph_filter = cache_graph_config.get('filter', {})
cache_filter_indices = cache_graph_filter.get('indices', [])
if set(sketch_indices) ^ set(cache_filter_indices):
refresh = True
if cache.graph_elements and not refresh:
return self.to_json(cache)
graph_class = manager.GraphManager.get_graph(plugin_name)
graph = graph_class(sketch=sketch)
cytoscape_json = graph.generate().to_cytoscape()
if cytoscape_json:
cache.graph_elements = json.dumps(cytoscape_json)
cache.graph_config = json.dumps(graph_config)
cache.update_modification_time()
db_session.add(cache)
db_session.commit()
return self.to_json(cache)
def run(self, username, remove):
"""Adds the admin bit to a user."""
user = User.query.filter_by(username=username).first()
if not user:
sys.stdout.write('User [{0:s}] does not exist.\n'.format(username))
return
user.admin = not remove
db_session.add(user)
db_session.commit()
if remove:
sys.stdout.write(
'User {0:s} is no longer an admin.\n'.format(username))
else:
sys.stdout.write('User {0:s} is now an admin.\n'.format(username))
def post(self):
"""Handles POST request to the resource.
Returns:
A view in JSON (instance of flask.wrappers.Response)
Raises:
ApiHTTPError
"""
UPLOAD_ENABLED = current_app.config[u'UPLOAD_ENABLED']
UPLOAD_FOLDER = current_app.config[u'UPLOAD_FOLDER']
form = UploadFileForm()
if form.validate_on_submit() and UPLOAD_ENABLED:
from timesketch.lib.tasks import run_plaso
file_storage = form.file.data
timeline_name = form.name.data
# We do not need a human readable filename or
# datastore index name, so we use UUIDs here.
filename = unicode(uuid.uuid4().hex)
index_name = unicode(uuid.uuid4().hex)
file_path = os.path.join(UPLOAD_FOLDER, filename)
file_storage.save(file_path)
search_index = SearchIndex.get_or_create(name=timeline_name,
description=timeline_name,
user=current_user,
index_name=index_name)
search_index.grant_permission(permission=u'read',
user=current_user)
search_index.grant_permission(permission=u'write',
user=current_user)
search_index.grant_permission(permission=u'delete',
user=current_user)
search_index.set_status(u'processing')
db_session.add(search_index)
db_session.commit()
run_plaso.apply_async((file_path, timeline_name, index_name),
task_id=index_name)
return self.to_json(search_index,
status_code=HTTP_STATUS_CODE_CREATED)
else:
raise ApiHTTPError(message=form.errors[u'file'][0],
status_code=HTTP_STATUS_CODE_BAD_REQUEST)
def delete(self, searchindex_id):
"""Handles DELETE request to the resource."""
searchindex = SearchIndex.query.get_with_acl(searchindex_id)
if not searchindex:
abort(HTTP_STATUS_CODE_NOT_FOUND,
"No searchindex found with this ID.")
if not searchindex.has_permission(current_user, "delete"):
abort(
HTTP_STATUS_CODE_FORBIDDEN,
("User does not have sufficient access rights to "
"delete the search index."),
)
if searchindex.get_status.status == "deleted":
abort(HTTP_STATUS_CODE_BAD_REQUEST,
"Search index already deleted.")
timelines = Timeline.query.filter_by(searchindex=searchindex).all()
sketches = [
t.sketch for t in timelines
if t.sketch and t.sketch.get_status.status != "deleted"
]
if sketches:
error_strings = ["WARNING: This timeline is in use by:"]
for sketch in sketches:
error_strings.append(" * {0:s}".format(sketch.name))
abort(HTTP_STATUS_CODE_FORBIDDEN, "\n".join(error_strings))
searchindex.set_status(status="deleted")
db_session.commit()
other_indexes = SearchIndex.query.filter_by(
index_name=searchindex.index_name).all()
if len(other_indexes) > 1:
logger.warning("Search index: {0:s} belongs to more than one "
"db entry.".format(searchindex.index_name))
return HTTP_STATUS_CODE_OK
try:
self.datastore.client.indices.close(index=searchindex.index_name)
except opensearchpy.NotFoundError:
logger.warning("Unable to close index: {0:s}, the index wasn't "
"found.".format(searchindex.index_name))
return HTTP_STATUS_CODE_OK
def post(self):
"""Handles POST request to the resource.
Returns:
A view in JSON (instance of flask.wrappers.Response)
Raises:
ApiHTTPError
"""
UPLOAD_ENABLED = current_app.config[u'UPLOAD_ENABLED']
UPLOAD_FOLDER = current_app.config[u'UPLOAD_FOLDER']
form = UploadFileForm()
if form.validate_on_submit() and UPLOAD_ENABLED:
from timesketch.lib.tasks import run_plaso
file_storage = form.file.data
timeline_name = form.name.data
# We do not need a human readable filename or
# datastore index name, so we use UUIDs here.
filename = unicode(uuid.uuid4().hex)
index_name = unicode(uuid.uuid4().hex)
file_path = os.path.join(UPLOAD_FOLDER, filename)
file_storage.save(file_path)
search_index = SearchIndex.get_or_create(
name=timeline_name, description=timeline_name,
user=current_user, index_name=index_name)
search_index.grant_permission(permission=u'read', user=current_user)
search_index.grant_permission(
permission=u'write', user=current_user)
search_index.grant_permission(
permission=u'delete', user=current_user)
search_index.set_status(u'processing')
db_session.add(search_index)
db_session.commit()
run_plaso.apply_async(
(file_path, timeline_name, index_name), task_id=index_name)
return self.to_json(
search_index, status_code=HTTP_STATUS_CODE_CREATED)
else:
raise ApiHTTPError(
message=form.errors[u'file'][0],
status_code=HTTP_STATUS_CODE_BAD_REQUEST)
def delete(self, sketch_id, timeline_id):
"""Handles DELETE request to the resource.
Args:
sketch_id: Integer primary key for a sketch database model
timeline_id: Integer primary key for a timeline database model
"""
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(
HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.')
timeline = Timeline.query.get(timeline_id)
# Check that this timeline belongs to the sketch
if timeline.sketch_id != sketch.id:
if not timeline:
msg = 'No timeline found with this ID.'
elif not sketch:
msg = 'No sketch found with this ID.'
else:
msg = (
'The sketch ID ({0:d}) does not match with the timeline '
'sketch ID ({1:d})'.format(sketch.id, timeline.sketch_id))
abort(HTTP_STATUS_CODE_NOT_FOUND, msg)
if not sketch.has_permission(user=current_user, permission='write'):
abort(
HTTP_STATUS_CODE_FORBIDDEN,
'The user does not have write permission on the sketch.')
not_delete_labels = current_app.config.get(
'LABELS_TO_PREVENT_DELETION', [])
for label in not_delete_labels:
if timeline.has_label(label):
abort(
HTTP_STATUS_CODE_FORBIDDEN,
'Timelines with label [{0:s}] cannot be deleted.'.format(
label))
sketch.timelines.remove(timeline)
db_session.commit()
# Update the last activity of a sketch.
utils.update_sketch_last_activity(sketch)
return HTTP_STATUS_CODE_OK
def add_story(self, title):
"""Add a story to the Sketch.
Args:
title: The name of the view.
Raises:
ValueError: If both query_string an query_dsl are missing.
Returns:
An instance of a Story object.
"""
story = SQLStory.get_or_create(
title=title, content='[]', sketch=self.sql_sketch, user=None)
db_session.add(story)
db_session.commit()
return Story(story)
def post(self, sketch_id):
"""Handles POST request to the resource."""
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(HTTP_STATUS_CODE_NOT_FOUND, "No sketch found with this ID.")
if not sketch.has_permission(current_user, "write"):
abort(
HTTP_STATUS_CODE_FORBIDDEN, "User does not have write access on sketch."
)
form = request.json
name = form.get("name")
description = form.get("description")
elements = form.get("elements")
graph_config = form.get("graph_config")
if graph_config:
if isinstance(graph_config, dict):
graph_json = json.dumps(graph_config)
elif isinstance(graph_config, str):
graph_json = graph_config
else:
graph_json = ""
logger.warning("Graph config not of the correct value, not saving.")
else:
graph_json = ""
graph = Graph(
user=current_user,
sketch=sketch,
name=str(name),
graph_config=graph_json,
description=description,
graph_elements=json.dumps(elements),
)
db_session.add(graph)
db_session.commit()
# Update the last activity of a sketch.
utils.update_sketch_last_activity(sketch)
return self.to_json(graph, status_code=HTTP_STATUS_CODE_CREATED)
def post(self, searchindex_id):
"""Handles POST request to the resource.
Returns:
A search index in JSON (instance of flask.wrappers.Response)
"""
form = request.json
if not form:
form = request.data
if not searchindex_id:
abort(HTTP_STATUS_CODE_BAD_REQUEST,
'Need to define a search index ID')
searchindex = SearchIndex.query.get_with_acl(searchindex_id)
if not searchindex:
abort(HTTP_STATUS_CODE_NOT_FOUND,
'No searchindex found with this ID.')
if not searchindex.has_permission(current_user, 'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN,
('User does not have sufficient access rights to '
'edit the search index.'))
if searchindex.get_status.status == 'deleted':
abort(HTTP_STATUS_CODE_BAD_REQUEST,
'Search index is marked deleted, unable to continue.')
commit_to_db = False
new_status = form.get('status', '')
valid_status = ('ready', 'fail', 'processing', 'timeout')
if new_status and new_status in valid_status:
searchindex.set_status(status=new_status)
commit_to_db = True
description = form.get('description', '')
if description:
searchindex.description = description
commit_to_db = True
if commit_to_db:
db_session.add(searchindex)
db_session.commit()
return self.to_json(searchindex)
def explore(sketch_id, view_id=None):
"""Generates the sketch explore view template.
Returns:
Template with context.
"""
sketch = Sketch.query.get_with_acl(sketch_id)
sketch_timelines = [t.searchindex.index_name for t in sketch.timelines]
view_form = SaveViewForm()
# Get parameters from the GET query
url_query = request.args.get(u'q', u'')
url_time_start = request.args.get(u'time_start', None)
url_time_end = request.args.get(u'time_end', None)
if view_id:
view = View.query.get(view_id)
# Check that this view belongs to the sketch
if view.sketch_id != sketch.id:
abort(HTTP_STATUS_CODE_NOT_FOUND)
# Return 404 if view is deleted
if view.get_status.status == u'deleted':
return abort(HTTP_STATUS_CODE_NOT_FOUND)
else:
view = sketch.get_user_view(current_user)
if url_query:
view.query_string = url_query
query_filter = json.loads(view.query_filter)
query_filter[u'time_start'] = url_time_start
query_filter[u'time_end'] = url_time_end
view.query_filter = json.dumps(query_filter, ensure_ascii=False)
if not view:
query_filter = dict(indices=sketch_timelines)
view = View(
user=current_user, name=u'', sketch=sketch, query_string=u'',
query_filter=json.dumps(query_filter, ensure_ascii=False))
db_session.add(view)
db_session.commit()
return render_template(
u'sketch/explore.html', sketch=sketch, view=view,
timelines=sketch_timelines, view_form=view_form)
def timelines(sketch_id):
"""Generates the sketch explore view template.
Returns:
Template with context.
"""
TIMELINES_TO_SHOW = 20
sketch = Sketch.query.get_with_acl(sketch_id)
searchindices_in_sketch = [t.searchindex.id for t in sketch.timelines]
query = request.args.get(u'q', None)
indices = SearchIndex.all_with_acl(
current_user).order_by(
desc(SearchIndex.created_at)).filter(
not_(SearchIndex.id.in_(searchindices_in_sketch)))
filtered = False
if query:
indices = indices.filter(SearchIndex.name.contains(query)).limit(500)
filtered = True
if not filtered:
indices = indices.limit(TIMELINES_TO_SHOW)
# Setup the form
form = AddTimelineForm()
form.timelines.choices = set((i.id, i.name) for i in indices.all())
# Create new timeline form POST
if form.validate_on_submit():
if not sketch.has_permission(current_user, u'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN)
for searchindex_id in form.timelines.data:
searchindex = SearchIndex.query.get_with_acl(searchindex_id)
if searchindex not in [t.searchindex for t in sketch.timelines]:
_timeline = Timeline(
name=searchindex.name, description=searchindex.description,
sketch=sketch, user=current_user, searchindex=searchindex)
db_session.add(_timeline)
sketch.timelines.append(_timeline)
db_session.commit()
return redirect(url_for(u'sketch_views.timelines', sketch_id=sketch.id))
return render_template(
u'sketch/timelines.html', sketch=sketch, timelines=indices.all(),
form=form, filtered=filtered)
def delete(self, searchindex_id):
"""Handles DELETE request to the resource."""
searchindex = SearchIndex.query.get_with_acl(searchindex_id)
if not searchindex:
abort(HTTP_STATUS_CODE_NOT_FOUND,
'No searchindex found with this ID.')
if not searchindex.has_permission(current_user, 'delete'):
abort(HTTP_STATUS_CODE_FORBIDDEN,
('User does not have sufficient access rights to '
'delete the search index.'))
if searchindex.get_status.status == 'deleted':
abort(HTTP_STATUS_CODE_BAD_REQUEST,
'Search index already deleted.')
timelines = Timeline.query.filter_by(searchindex=searchindex).all()
sketches = [
t.sketch for t in timelines
if t.sketch and t.sketch.get_status.status != 'deleted'
]
if sketches:
error_strings = ['WARNING: This timeline is in use by:']
for sketch in sketches:
error_strings.append(' * {0:s}'.format(sketch.name))
abort(HTTP_STATUS_CODE_FORBIDDEN, '\n'.join(error_strings))
searchindex.set_status(status='deleted')
db_session.commit()
other_indexes = SearchIndex.query.filter_by(
index_name=searchindex.index_name).all()
if len(other_indexes) > 1:
logger.warning('Search index: {0:s} belongs to more than one '
'db entry.'.format(searchindex.index_name))
return HTTP_STATUS_CODE_OK
try:
self.datastore.client.indices.close(index=searchindex.index_name)
except elasticsearch.NotFoundError:
logger.warning('Unable to close index: {0:s}, the index wasn\'t '
'found.'.format(searchindex.index_name))
return HTTP_STATUS_CODE_OK
def remove_group_member(group_name, username):
"""Remove a user from a group."""
group = Group.query.filter_by(name=group_name).first()
if not group:
print("No such group.")
return
user = User.query.filter_by(username=username).first()
if not user:
print("User does not exist.")
return
try:
user.groups.remove(group)
db_session.commit()
print("Removed user from group.")
except ValueError:
print("User is not a member of the group.")
def post(self, sketch_id):
"""Handles POST request to the resource.
Args:
sketch_id: Integer primary key for a sketch database model
Returns:
A view in JSON (instance of flask.wrappers.Response)
"""
form = StoryForm.build(request)
if form.validate_on_submit():
sketch = Sketch.query.get_with_acl(sketch_id)
story = Story(
title=u'', content=u'', sketch=sketch, user=current_user)
db_session.add(story)
db_session.commit()
return self.to_json(story, status_code=HTTP_STATUS_CODE_CREATED)
return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def add_group_member(group_name, username):
"""Add a user to a group."""
group = Group.query.filter_by(name=group_name).first()
if not group:
print("No such group.")
return
user = User.query.filter_by(username=username).first()
if not user:
print("User does not exist.")
return
try:
user.groups.append(group)
db_session.commit()
print("Added user to group.")
except IntegrityError:
print("User is already a member of the group.")
def post(self, sketch_id):
"""Handles POST request to the resource.
Args:
sketch_id: Integer primary key for a sketch database model
Returns:
A view in JSON (instance of flask.wrappers.Response)
"""
form = StoryForm.build(request)
if form.validate_on_submit():
sketch = Sketch.query.get_with_acl(sketch_id)
story = Story(
title=u'', content=u'', sketch=sketch, user=current_user)
db_session.add(story)
db_session.commit()
return self.to_json(story, status_code=HTTP_STATUS_CODE_CREATED)
return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def run_csv(source_file_path, timeline_name, index_name, username=None):
"""Create a Celery task for processing a CSV file.
Args:
source_file_path: Path to CSV file.
timeline_name: Name of the Timesketch timeline.
index_name: Name of the datastore index.
username: Username of the user who will own the timeline.
Returns:
Dictionary with count of processed events.
"""
flush_interval = 1000 # events to queue before bulk index
event_type = u'generic_event' # Document type for Elasticsearch
app = create_app()
# Log information to Celery
logging.info(u'Index name: %s', index_name)
logging.info(u'Timeline name: %s', timeline_name)
logging.info(u'Flush interval: %d', flush_interval)
logging.info(u'Document type: %s', event_type)
logging.info(u'Owner: %s', username)
es = ElasticsearchDataStore(
host=current_app.config[u'ELASTIC_HOST'],
port=current_app.config[u'ELASTIC_PORT'])
es.create_index(index_name=index_name, doc_type=event_type)
for event in read_and_validate_csv(source_file_path):
es.import_event(
flush_interval, index_name, event_type, event)
# Import the remaining events
total_events = es.import_event(flush_interval, index_name, event_type)
# We are done so let's remove the processing status flag
with app.app_context():
search_index = SearchIndex.query.filter_by(
index_name=index_name).first()
search_index.status.remove(search_index.status[0])
db_session.add(search_index)
db_session.commit()
return {u'Events processed': total_events}
def create_aggregation_from_form(sketch, form):
"""Creates an aggregation from form data.
Args:
sketch: Instance of timesketch.models.sketch.Sketch
form: Instance of timesketch.lib.forms.SaveAggregationForm
Returns:
An aggregation (instance of timesketch.models.sketch.Aggregation)
"""
# Default to user supplied data
name = form.get('name', '')
description = form.get('description', '')
agg_type = form.get('agg_type', '')
parameter_data = form.get('parameters', {})
parameters = json.dumps(parameter_data, ensure_ascii=False)
chart_type = form.get('chart_type', '')
view_id = form.get('view_id')
# Create the aggregation in the database
aggregation = Aggregation(
name=name,
description=description,
agg_type=agg_type,
parameters=parameters,
chart_type=chart_type,
user=current_user,
sketch=sketch,
view=view_id
)
labels = form.get('labels', '')
if labels:
for label in json.loads(labels):
if aggregation.has_label(label):
continue
aggregation.add_label(label)
db_session.add(aggregation)
db_session.commit()
return aggregation
def grant_permission(self, permission, user=None, group=None):
"""Grant permission to a user or group with the specific permission.
Args:
permission: Permission as string (read, write or delete)
user: A user (Instance of timesketch.models.user.User)
group: A group (Instance of timesketch.models.user.Group)
"""
# Grant permission to a group.
if group and not self._get_ace(permission, group=group):
self.acl.append(
self.AccessControlEntry(permission=permission, group=group))
db_session.commit()
return
# Grant permission to a user.
if not self._get_ace(permission, user=user, check_group=False):
self.acl.append(
self.AccessControlEntry(permission=permission, user=user))
db_session.commit()
def post(self):
"""Handles POST request to the resource.
Returns:
A sketch in JSON (instance of flask.wrappers.Response)
"""
form = NameDescriptionForm.build(request)
if form.validate_on_submit():
sketch = Sketch(
name=form.name.data, description=form.description.data,
user=current_user)
sketch.status.append(sketch.Status(user=None, status=u'new'))
# Give the requesting user permissions on the new sketch.
sketch.grant_permission(permission=u'read', user=current_user)
sketch.grant_permission(permission=u'write', user=current_user)
sketch.grant_permission(permission=u'delete', user=current_user)
db_session.add(sketch)
db_session.commit()
return self.to_json(sketch, status_code=HTTP_STATUS_CODE_CREATED)
return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def grant_permission(self, permission, user=None, group=None):
"""Grant permission to a user or group with the specific permission.
Args:
permission: Permission as string (read, write or delete)
user: A user (Instance of timesketch.models.user.User)
group: A group (Instance of timesketch.models.user.Group)
"""
# Grant permission to a group.
if group and not self._get_ace(permission, group=group):
self.acl.append(
self.AccessControlEntry(permission=permission, group=group))
db_session.commit()
return
# Grant permission to a user.
if not self._get_ace(permission, user=user, check_group=False):
self.acl.append(
self.AccessControlEntry(permission=permission, user=user))
db_session.commit()
def post(self):
"""Handles POST request to the resource.
Returns:
A sketch in JSON (instance of flask.wrappers.Response)
"""
form = NameDescriptionForm.build(request)
if form.validate_on_submit():
sketch = Sketch(name=form.name.data,
description=form.description.data,
user=current_user)
sketch.status.append(sketch.Status(user=None, status=u'new'))
# Give the requesting user permissions on the new sketch.
sketch.grant_permission(permission=u'read', user=current_user)
sketch.grant_permission(permission=u'write', user=current_user)
sketch.grant_permission(permission=u'delete', user=current_user)
db_session.add(sketch)
db_session.commit()
return self.to_json(sketch, status_code=HTTP_STATUS_CODE_CREATED)
return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def delete(self, sketch_id, timeline_id):
"""Handles DELETE request to the resource.
Args:
sketch_id: Integer primary key for a sketch database model
timeline_id: Integer primary key for a timeline database model
"""
sketch = Sketch.query.get_with_acl(sketch_id)
timeline = Timeline.query.get(timeline_id)
# Check that this timeline belongs to the sketch
if timeline.sketch_id != sketch.id:
abort(HTTP_STATUS_CODE_NOT_FOUND)
if not sketch.has_permission(user=current_user, permission=u'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN)
sketch.timelines.remove(timeline)
db_session.commit()
return HTTP_STATUS_CODE_OK
def post(self, sketch_id):
"""Handles POST request to the resource.
Args:
sketch_id: Integer primary key for a sketch database model
Returns:
A view in JSON (instance of flask.wrappers.Response)
"""
form = SaveViewForm.build(request)
if form.validate_on_submit():
sketch = Sketch.query.get_with_acl(sketch_id)
view = View(
name=form.name.data, sketch=sketch, user=current_user,
query_string=form.query.data,
query_filter=json.dumps(form.filter.data, ensure_ascii=False))
db_session.add(view)
db_session.commit()
return self.to_json(view, status_code=HTTP_STATUS_CODE_CREATED)
return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def WriteHeader(self):
"""Setup the Elasticsearch index and the Timesketch database object.
Creates the Elasticsearch index with Timesketch specific settings and the
Timesketch SearchIndex database object.
"""
# This cannot be static because we use the value of self._doc_type from
# arguments.
_document_mapping = {
self._doc_type: {
u'_timestamp': {
u'enabled': True,
u'path': u'datetime',
u'format': u'date_time_no_millis'
},
u'properties': {u'timesketch_label': {u'type': u'nested'}}
}
}
if not self._elastic_db.client.indices.exists(self._index_name):
try:
self._elastic_db.client.indices.create(
index=self._index_name, body={u'mappings': _document_mapping})
except elastic_exceptions.ConnectionError as exception:
logging.error((
u'Unable to proceed, cannot connect to Timesketch backend '
u'with error: {0:s}.\nPlease verify connection.').format(exception))
raise RuntimeError(u'Unable to connect to Timesketch backend.')
with self._timesketch.app_context():
search_index = SearchIndex.get_or_create(
name=self._timeline_name, description=self._timeline_name, user=None,
index_name=self._index_name)
# Grant all users read permission on the mapping object and set status.
search_index.grant_permission(None, u'read')
search_index.set_status(u'processing')
# Save the mapping object to the Timesketch database.
db_session.add(search_index)
db_session.commit()
logging.info(u'Adding events to Timesketch..')
def Close(self):
"""Closes the connection to TimeSketch Elasticsearch database.
Sends the remaining events for indexing and adds the timeline to Timesketch.
"""
self._FlushEventsToElasticsearch()
with self._timesketch.app_context():
# Get Timesketch user object, or None if user do not exist. This is a
# SQLAlchemy query against the Timesketch database.
user_query = User.query.filter_by(username=self._timeline_owner)
user = user_query.first()
search_index = SearchIndex(
name=self._timeline_name, description=self._timeline_name, user=user,
index_name=self._index_name)
# Grant all users read permission on the mapping object.
search_index.grant_permission(None, u'read')
# Save the mapping object to the Timesketch database.
db_session.add(search_index)
db_session.commit()
def post(self, sketch_id):
"""Handles POST request to the resource.
Returns:
A sketch in JSON (instance of flask.wrappers.Response)
Raises:
ApiHTTPError
"""
sketch = Sketch.query.get_with_acl(sketch_id)
searchindices_in_sketch = [t.searchindex.id for t in sketch.timelines]
indices = SearchIndex.all_with_acl(
current_user).order_by(
desc(SearchIndex.created_at)).filter(
not_(SearchIndex.id.in_(searchindices_in_sketch)))
add_timeline_form = AddTimelineForm.build(request)
add_timeline_form.timelines.choices = set(
(i.id, i.name) for i in indices.all())
if add_timeline_form.validate_on_submit():
if not sketch.has_permission(current_user, u'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN)
for searchindex_id in add_timeline_form.timelines.data:
searchindex = SearchIndex.query.get_with_acl(searchindex_id)
if searchindex not in [t.searchindex for t in sketch.timelines]:
_timeline = Timeline(
name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=current_user,
searchindex=searchindex)
db_session.add(_timeline)
sketch.timelines.append(_timeline)
db_session.commit()
return self.to_json(sketch, status_code=HTTP_STATUS_CODE_CREATED)
else:
raise ApiHTTPError(
message=add_timeline_form.errors,
status_code=HTTP_STATUS_CODE_BAD_REQUEST)
