def timelines(sketch_id):
"""Generates the sketch explore view template.
Returns:
Template with context.
"""
sketch = Sketch.query.get_with_acl(sketch_id)
searchindices_in_sketch = [t.searchindex.id for t in sketch.timelines]
indices = SearchIndex.all_with_acl(
current_user).order_by(
desc(SearchIndex.created_at)).filter(
not_(SearchIndex.id.in_(searchindices_in_sketch)))
# Setup the form
form = AddTimelineForm()
form.timelines.choices = set((i.id, i.name) for i in indices.all())
# Create new timeline form POST
if form.validate_on_submit():
if not sketch.has_permission(current_user, u'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN)
for searchindex_id in form.timelines.data:
searchindex = SearchIndex.query.get_with_acl(searchindex_id)
if searchindex not in [t.searchindex for t in sketch.timelines]:
_timeline = Timeline(
name=searchindex.name, description=searchindex.description,
sketch=sketch, user=current_user, searchindex=searchindex)
db_session.add(_timeline)
sketch.timelines.append(_timeline)
db_session.commit()
return redirect(url_for(u'sketch_views.timelines', sketch_id=sketch.id))
return render_template(
u'sketch/timelines.html', sketch=sketch, timelines=indices.all(),
form=form)
def timelines(sketch_id):
"""Generates the sketch explore view template.
Returns:
Template with context.
"""
sketch = Sketch.query.get_with_acl(sketch_id)
searchindices_in_sketch = [t.searchindex.id for t in sketch.timelines]
indices = SearchIndex.all_with_acl(current_user).order_by(
desc(SearchIndex.created_at)).filter(
not_(SearchIndex.id.in_(searchindices_in_sketch)))
upload_enabled = current_app.config[u'UPLOAD_ENABLED']
graphs_enabled = current_app.config[u'GRAPH_BACKEND_ENABLED']
try:
plaso_version = current_app.config[u'PLASO_VERSION']
except KeyError:
plaso_version = u'Unknown'
# Setup the form
form = AddTimelineForm()
form.timelines.choices = set((i.id, i.name) for i in indices.all())
# Create new timeline form POST
if form.validate_on_submit():
if not sketch.has_permission(current_user, u'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN)
for searchindex_id in form.timelines.data:
searchindex = SearchIndex.query.get_with_acl(searchindex_id)
if searchindex not in [t.searchindex for t in sketch.timelines]:
_timeline = Timeline(name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=current_user,
searchindex=searchindex)
db_session.add(_timeline)
sketch.timelines.append(_timeline)
db_session.commit()
# If enabled, run sketch analyzers when timeline is added.
# Import here to avoid circular imports.
from timesketch.lib import tasks
sketch_analyzer_group = tasks.build_sketch_analysis_pipeline(
sketch_id)
if sketch_analyzer_group:
pipeline = (tasks.run_sketch_init.s(
[searchindex.index_name]) | sketch_analyzer_group)
pipeline.apply_async(task_id=searchindex.index_name)
return redirect(url_for(u'sketch_views.timelines',
sketch_id=sketch.id))
return render_template(u'sketch/timelines.html',
sketch=sketch,
timelines=indices.all(),
form=form,
upload_enabled=upload_enabled,
plaso_version=plaso_version,
graphs_enabled=graphs_enabled)
def _create_timeline(self, name, sketch, searchindex, user):
"""Create a timeline in the database.
Args:
name: Name of the timeline (string)
sketch: A sketch (instance of timesketch.models.sketch.Sketch)
searchindex:
A searchindex (instance of timesketch.models.sketch.SearchIndex)
user: A user (instance of timesketch.models.user.User)
Returns:
A timeline (instance of timesketch.models.sketch.Timeline)
"""
timeline = Timeline(name=name,
description=name,
user=user,
sketch=sketch,
searchindex=searchindex,
color=self.COLOR_WHITE)
timeline.set_status(status='ready')
self._commit_to_database(timeline)
return timeline
def post(self, sketch_id):
"""Handles POST request to the resource.
Returns:
A sketch in JSON (instance of flask.wrappers.Response)
Raises:
ApiHTTPError
"""
sketch = Sketch.query.get_with_acl(sketch_id)
searchindices_in_sketch = [t.searchindex.id for t in sketch.timelines]
indices = SearchIndex.all_with_acl(
current_user).order_by(
desc(SearchIndex.created_at)).filter(
not_(SearchIndex.id.in_(searchindices_in_sketch)))
add_timeline_form = AddTimelineForm.build(request)
add_timeline_form.timelines.choices = set(
(i.id, i.name) for i in indices.all())
if add_timeline_form.validate_on_submit():
if not sketch.has_permission(current_user, u'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN)
for searchindex_id in add_timeline_form.timelines.data:
searchindex = SearchIndex.query.get_with_acl(searchindex_id)
if searchindex not in [t.searchindex for t in sketch.timelines]:
_timeline = Timeline(
name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=current_user,
searchindex=searchindex)
db_session.add(_timeline)
sketch.timelines.append(_timeline)
db_session.commit()
return self.to_json(sketch, status_code=HTTP_STATUS_CODE_CREATED)
else:
raise ApiHTTPError(
message=add_timeline_form.errors,
status_code=HTTP_STATUS_CODE_BAD_REQUEST)
def post(self, sketch_id):
"""Handles POST request to the resource.
Returns:
A sketch in JSON (instance of flask.wrappers.Response)
"""
sketch = Sketch.query.get_with_acl(sketch_id)
form = AddTimelineSimpleForm.build(request)
metadata = {u'created': True}
searchindex_id = form.timeline.data
searchindex = SearchIndex.query.get_with_acl(searchindex_id)
timeline_id = [
t.searchindex.id for t in sketch.timelines
if t.searchindex.id == searchindex_id
]
if form.validate_on_submit():
if not sketch.has_permission(current_user, u'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN)
if not timeline_id:
return_code = HTTP_STATUS_CODE_CREATED
timeline = Timeline(
name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=current_user,
searchindex=searchindex)
sketch.timelines.append(timeline)
db_session.add(timeline)
db_session.commit()
else:
metadata[u'created'] = False
return_code = HTTP_STATUS_CODE_OK
timeline = Timeline.query.get(timeline_id)
return self.to_json(
timeline, meta=metadata, status_code=return_code)
return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def post(self, sketch_id):
"""Handles POST request to the resource.
Returns:
A sketch in JSON (instance of flask.wrappers.Response)
"""
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(
HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.')
if not sketch.has_permission(current_user, 'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN,
'User does not have write access controls on sketch.')
form = request.json
if not form:
form = request.data
metadata = {'created': True}
searchindex_id = form.get('timeline', 0)
if isinstance(searchindex_id, str) and searchindex_id.isdigit():
searchindex_id = int(searchindex_id)
if not isinstance(searchindex_id, int):
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'The timeline (searchindex id) needs to be an integer.')
searchindex = SearchIndex.query.get_with_acl(searchindex_id)
if searchindex.get_status.status == 'deleted':
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'Unable to create a timeline using a deleted search index')
timeline_id = [
t.searchindex.id for t in sketch.timelines
if t.searchindex.id == searchindex_id
]
if not timeline_id:
return_code = HTTP_STATUS_CODE_CREATED
timeline_name = form.get('timeline_name', searchindex.name)
timeline = Timeline(
name=timeline_name,
description=searchindex.description,
sketch=sketch,
user=current_user,
searchindex=searchindex)
sketch.timelines.append(timeline)
labels_to_prevent_deletion = current_app.config.get(
'LABELS_TO_PREVENT_DELETION', [])
for label in sketch.get_labels:
if label not in labels_to_prevent_deletion:
continue
timeline.add_label(label)
searchindex.add_label(label)
# Set status to ready so the timeline can be queried.
timeline.set_status('ready')
db_session.add(timeline)
db_session.commit()
else:
metadata['created'] = False
return_code = HTTP_STATUS_CODE_OK
timeline = Timeline.query.get(timeline_id)
# Run sketch analyzers when timeline is added. Import here to avoid
# circular imports.
# pylint: disable=import-outside-toplevel
if current_app.config.get('AUTO_SKETCH_ANALYZERS'):
# pylint: disable=import-outside-toplevel
from timesketch.lib import tasks
sketch_analyzer_group, _ = tasks.build_sketch_analysis_pipeline(
sketch_id, searchindex_id, current_user.id,
timeline_id=timeline_id)
if sketch_analyzer_group:
pipeline = (tasks.run_sketch_init.s(
[searchindex.index_name]) | sketch_analyzer_group)
pipeline.apply_async()
# Update the last activity of a sketch.
utils.update_sketch_last_activity(sketch)
return self.to_json(
timeline, meta=metadata, status_code=return_code)
def post(self):
"""Handles POST request to the resource.
Returns:
A view in JSON (instance of flask.wrappers.Response)
"""
upload_enabled = current_app.config['UPLOAD_ENABLED']
if not upload_enabled:
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'Failed to create timeline, upload not enabled')
form = forms.CreateTimelineForm()
if not form.validate_on_submit():
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'Failed to create timeline, form data not validated')
sketch_id = form.sketch_id.data
timeline_name = form.name.data
sketch = None
if sketch_id:
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(
HTTP_STATUS_CODE_NOT_FOUND,
'No sketch found with this ID.')
# We do not need a human readable filename or
# datastore index name, so we use UUIDs here.
index_name = uuid.uuid4().hex
if not isinstance(index_name, six.text_type):
index_name = codecs.decode(index_name, 'utf-8')
# Create the search index in the Timesketch database
searchindex = SearchIndex.get_or_create(
name=timeline_name,
description=timeline_name,
user=current_user,
index_name=index_name)
searchindex.grant_permission(permission='read', user=current_user)
searchindex.grant_permission(permission='write', user=current_user)
searchindex.grant_permission(
permission='delete', user=current_user)
searchindex.set_status('processing')
db_session.add(searchindex)
db_session.commit()
timeline = None
if sketch and sketch.has_permission(current_user, 'write'):
timeline = Timeline(
name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=current_user,
searchindex=searchindex)
sketch.timelines.append(timeline)
db_session.add(timeline)
db_session.commit()
# Return Timeline if it was created.
# pylint: disable=no-else-return
if timeline:
return self.to_json(
timeline, status_code=HTTP_STATUS_CODE_CREATED)
# Update the last activity of a sketch.
utils.update_sketch_last_activity(sketch)
return self.to_json(
searchindex, status_code=HTTP_STATUS_CODE_CREATED)
def post(self, sketch_id):
"""Handles POST request to the resource.
Handler for /api/v1/sketches/:sketch_id/event/create/
Args:
sketch_id: Integer primary key for a sketch database model
Returns:
An annotation in JSON (instance of flask.wrappers.Response)
"""
sketch = Sketch.query.get_with_acl(sketch_id)
if not sketch:
abort(HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.')
if not sketch.has_permission(current_user, 'write'):
abort(HTTP_STATUS_CODE_FORBIDDEN,
'User does not have write access controls on sketch.')
form = request.json
if not form:
form = request.data
timeline_name = 'sketch specific timeline'
index_name_seed = 'timesketch_{0:d}'.format(sketch_id)
event_type = 'user_created_event'
date_string = form.get('date_string')
if not date_string:
date = datetime.datetime.utcnow().isoformat()
else:
# derive datetime from timestamp:
try:
date = dateutil.parser.parse(date_string)
except (dateutil.parser.ParserError, OverflowError) as e:
logger.error('Unable to convert date string', exc_info=True)
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'Unable to add event, not able to convert the date '
'string. Was it properly formatted? Error: '
'{0!s}'.format(e))
timestamp = int(time.mktime(date.utctimetuple())) * 1000000
timestamp += date.microsecond
event = {
'datetime': date_string,
'timestamp': timestamp,
'timestamp_desc': form.get('timestamp_desc', 'Event Happened'),
'message': form.get('message', 'No message string'),
}
attributes = form.get('attributes', {})
if not isinstance(attributes, dict):
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'Unable to add an event where the attributes are not a '
'dict object.')
event.update(attributes)
tag = form.get('tag', [])
if not isinstance(tag, list):
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'Unable to add an event where the tags are not a '
'list of strings.')
if tag and any([not isinstance(x, str) for x in tag]):
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'Unable to add an event where the tags are not a '
'list of strings.')
event['tag'] = tag
# We do not need a human readable filename or
# datastore index name, so we use UUIDs here.
index_name = hashlib.md5(index_name_seed.encode()).hexdigest()
if six.PY2:
index_name = codecs.decode(index_name, 'utf-8')
# Try to create index
try:
# Create the index in Elasticsearch (unless it already exists)
self.datastore.create_index(index_name=index_name,
doc_type=event_type)
# Create the search index in the Timesketch database
searchindex = SearchIndex.get_or_create(
name=timeline_name,
description='internal timeline for user-created events',
user=current_user,
index_name=index_name)
searchindex.grant_permission(permission='read', user=current_user)
searchindex.grant_permission(permission='write', user=current_user)
searchindex.grant_permission(permission='delete',
user=current_user)
searchindex.set_status('ready')
db_session.add(searchindex)
db_session.commit()
timeline = None
if sketch and sketch.has_permission(current_user, 'write'):
self.datastore.import_event(index_name,
event_type,
event,
flush_interval=1)
timeline = Timeline.get_or_create(
name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=current_user,
searchindex=searchindex)
if timeline not in sketch.timelines:
sketch.timelines.append(timeline)
timeline.set_status('ready')
db_session.add(timeline)
db_session.commit()
# Return Timeline if it was created.
# pylint: disable=no-else-return
if timeline:
return self.to_json(timeline,
status_code=HTTP_STATUS_CODE_CREATED)
else:
return self.to_json(searchindex,
status_code=HTTP_STATUS_CODE_CREATED)
# TODO: Can this be narrowed down, both in terms of the scope it
# applies to, as well as not to catch a generic exception.
except Exception as e: # pylint: disable=broad-except
abort(HTTP_STATUS_CODE_BAD_REQUEST,
'Failed to add event ({0!s})'.format(e))
def post(self, sketch_id=None):
"""Handles POST request to the resource.
Returns:
A view in JSON (instance of flask.wrappers.Response)
Raises:
ApiHTTPError
"""
UPLOAD_ENABLED = current_app.config[u'UPLOAD_ENABLED']
UPLOAD_FOLDER = current_app.config[u'UPLOAD_FOLDER']
sketch = None
if sketch_id:
sketch = Sketch.query.get_with_acl(sketch_id)
form = UploadFileForm()
if form.validate_on_submit() and UPLOAD_ENABLED:
from timesketch.lib.tasks import run_plaso
from timesketch.lib.tasks import run_csv
# Map the right task based on the file type
task_directory = {u'plaso': run_plaso, u'csv': run_csv}
file_storage = form.file.data
timeline_name = form.name.data
_, _extension = os.path.splitext(file_storage.filename)
file_extension = _extension.lstrip(u'.')
# Current user
username = current_user.username
# We do not need a human readable filename or
# datastore index name, so we use UUIDs here.
filename = unicode(uuid.uuid4().hex)
index_name = unicode(uuid.uuid4().hex)
file_path = os.path.join(UPLOAD_FOLDER, filename)
file_storage.save(file_path)
# Create the search index in the Timesketch database
searchindex = SearchIndex.get_or_create(name=timeline_name,
description=timeline_name,
user=current_user,
index_name=index_name)
searchindex.grant_permission(permission=u'read', user=current_user)
searchindex.grant_permission(permission=u'write',
user=current_user)
searchindex.grant_permission(permission=u'delete',
user=current_user)
searchindex.set_status(u'processing')
db_session.add(searchindex)
db_session.commit()
if sketch and sketch.has_permission(current_user, u'write'):
timeline = Timeline(name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=current_user,
searchindex=searchindex)
db_session.add(timeline)
sketch.timelines.append(timeline)
db_session.commit()
# Run the task in the background
task = task_directory.get(file_extension)
task.apply_async((file_path, timeline_name, index_name, username),
task_id=index_name)
return self.to_json(searchindex,
status_code=HTTP_STATUS_CODE_CREATED)
else:
raise ApiHTTPError(message=form.errors[u'file'][0],
status_code=HTTP_STATUS_CODE_BAD_REQUEST)
def run(self, file_path, sketch_id, username, timeline_name):
"""This is the run method."""
file_path = os.path.realpath(file_path)
file_path_no_extension, extension = os.path.splitext(file_path)
extension = extension.lstrip('.')
filename = os.path.basename(file_path_no_extension)
supported_extensions = ('plaso', 'csv', 'jsonl')
if not os.path.isfile(file_path):
sys.exit('No such file: {0:s}'.format(file_path))
if extension not in supported_extensions:
sys.exit('Extension {0:s} is not supported. '
'(supported extensions are: {1:s})'.format(
extension, ', '.join(supported_extensions)))
user = None
if not username:
username = pwd.getpwuid(os.stat(file_path).st_uid).pw_name
if not username == 'root':
if not isinstance(username, six.text_type):
username = codecs.decode(username, 'utf-8')
user = User.query.filter_by(username=username).first()
if not user:
sys.exit('Cannot determine user for file: {0:s}'.format(file_path))
sketch = None
# If filename starts with <number> then use that as sketch_id.
# E.g: 42_file_name.plaso means sketch_id is 42.
sketch_id_from_filename = filename.split('_')[0]
if not sketch_id and sketch_id_from_filename.isdigit():
sketch_id = sketch_id_from_filename
if sketch_id:
try:
sketch = Sketch.query.get_with_acl(sketch_id, user=user)
except Forbidden:
pass
if not timeline_name:
if timeline_name is None:
timeline_name = '{0:s}_timeline'.format(filename)
if not isinstance(timeline_name, six.text_type):
timeline_name = codecs.decode(timeline_name, 'utf-8')
timeline_name = timeline_name.replace('_', ' ')
# Remove sketch ID if present in the filename.
timeline_parts = timeline_name.split()
if timeline_parts[0].isdigit():
timeline_name = ' '.join(timeline_name.split()[1:])
if not sketch:
# Create a new sketch.
sketch_name = 'Sketch for: {0:s}'.format(timeline_name)
sketch = Sketch(name=sketch_name,
description=sketch_name,
user=user)
# Need to commit here to be able to set permissions later.
db_session.add(sketch)
db_session.commit()
sketch.grant_permission(permission='read', user=user)
sketch.grant_permission(permission='write', user=user)
sketch.grant_permission(permission='delete', user=user)
sketch.status.append(sketch.Status(user=None, status='new'))
db_session.add(sketch)
db_session.commit()
index_name = uuid.uuid4().hex
if not isinstance(index_name, six.text_type):
index_name = codecs.decode(index_name, 'utf-8')
searchindex = SearchIndex.get_or_create(name=timeline_name,
description=timeline_name,
user=user,
index_name=index_name)
searchindex.grant_permission(permission='read', user=user)
searchindex.grant_permission(permission='write', user=user)
searchindex.grant_permission(permission='delete', user=user)
searchindex.set_status('processing')
db_session.add(searchindex)
db_session.commit()
if sketch and sketch.has_permission(user, 'write'):
timeline = Timeline(name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=user,
searchindex=searchindex)
timeline.set_status('processing')
sketch.timelines.append(timeline)
db_session.add(timeline)
db_session.commit()
# Start Celery pipeline for indexing and analysis.
# Import here to avoid circular imports.
from timesketch.lib import tasks # pylint: disable=import-outside-toplevel
pipeline = tasks.build_index_pipeline(file_path=file_path,
events='',
timeline_name=timeline_name,
index_name=index_name,
file_extension=extension,
sketch_id=sketch.id)
pipeline.apply_async(task_id=index_name)
print('Imported {0:s} to sketch: {1:d} ({2:s})'.format(
file_path, sketch.id, sketch.name))
def _upload_and_index(
self, file_extension, timeline_name, index_name, sketch, form,
enable_stream, original_filename='', data_label='', file_path='',
events='', meta=None):
"""Creates a full pipeline for an uploaded file and returns the results.
Args:
file_extension: the extension of the uploaded file.
timeline_name: name the timeline will be stored under in the
datastore.
index_name: the Elastic index name for the timeline.
sketch: Instance of timesketch.models.sketch.Sketch
form: a dict with the configuration for the upload.
enable_stream: boolean indicating whether this is file is part of a
stream or not.
original_filename: Original filename from the upload.
data_label: Optional string with a data label for the search index.
file_path: the path to the file to be uploaded (optional).
events: a string with events to upload (optional).
meta: optional dict with additional meta fields that will be
included in the return.
Returns:
A timeline if created otherwise a search index in JSON (instance
of flask.wrappers.Response)
"""
searchindex = self._get_index(
name=timeline_name,
description=timeline_name,
sketch=sketch,
index_name=index_name,
data_label=data_label,
extension=file_extension)
if not searchindex:
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'We were unable to acquire a searchindex and therefore not '
'able to upload data, please try again. If this error persist '
'please create an issue on Github: https://github.com/'
'google/timesketch/issues/new/choose')
timelines = Timeline.query.filter_by(
name=timeline_name, sketch=sketch).all()
timeline = None
for timeline_ in timelines:
if timeline_.searchindex.index_name == searchindex.index_name:
timeline = timeline_
break
logger.error(
'There is a timeline in the sketch that has the same name '
'but is stored in a different index: name {0:s} attempting '
'index: {1:s} but found index {2:s} - retrying with a '
'different timeline name.'.format(
timeline_name, searchindex.index_name,
timeline_.searchindex.index_name))
timeline_name = '{0:s}_{1:s}'.format(
timeline_name, uuid.uuid4().hex[-5:])
return self._upload_and_index(
file_extension=file_extension, timeline_name=timeline_name,
index_name=searchindex.index_name, sketch=sketch, form=form,
enable_stream=enable_stream,
original_filename=original_filename, data_label=data_label,
file_path=file_path, events=events, meta=meta)
searchindex.set_status('processing')
if not timeline:
timeline = Timeline.get_or_create(
name=timeline_name,
description=timeline_name,
sketch=sketch,
user=current_user,
searchindex=searchindex)
if not timeline:
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'Unable to get or create a new Timeline object.')
# If the timeline already existed and has associated data sources
# then we don't want to set the status to processing.
if not timeline.datasources:
timeline.set_status('processing')
sketch.timelines.append(timeline)
labels_to_prevent_deletion = current_app.config.get(
'LABELS_TO_PREVENT_DELETION', [])
for sketch_label in sketch.get_labels:
if sketch_label not in labels_to_prevent_deletion:
continue
timeline.add_label(sketch_label)
searchindex.add_label(sketch_label)
file_size = form.get('total_file_size', 0)
datasource = DataSource(
timeline=timeline,
user=current_user,
provider=form.get('provider', 'N/A'),
context=form.get('context', 'N/A'),
file_on_disk=file_path,
file_size=int(file_size),
original_filename=original_filename,
data_label=data_label
)
timeline.datasources.append(datasource)
db_session.add(datasource)
db_session.add(timeline)
db_session.commit()
sketch_id = sketch.id
# Start Celery pipeline for indexing and analysis.
# Import here to avoid circular imports.
# pylint: disable=import-outside-toplevel
from timesketch.lib import tasks
pipeline = tasks.build_index_pipeline(
file_path=file_path, events=events, timeline_name=timeline_name,
index_name=searchindex.index_name, file_extension=file_extension,
sketch_id=sketch_id, only_index=enable_stream,
timeline_id=timeline.id)
task_id = uuid.uuid4().hex
pipeline.apply_async(task_id=task_id)
if meta is None:
meta = {}
meta['task_id'] = task_id
return self.to_json(
timeline, status_code=HTTP_STATUS_CODE_CREATED, meta=meta)
def _upload_and_index(self,
file_extension,
timeline_name,
index_name,
sketch,
enable_stream,
file_path='',
events='',
meta=None):
"""Creates a full pipeline for an uploaded file and returns the results.
Args:
file_extension: the extension of the uploaded file.
timeline_name: name the timeline will be stored under in the
datastore.
index_name: the Elastic index name for the timeline.
sketch: Instance of timesketch.models.sketch.Sketch
enable_stream: boolean indicating whether this is file is part of a
stream or not.
file_path: the path to the file to be uploaded (optional).
events: a string with events to upload (optional).
meta: optional dict with additional meta fields that will be
included in the return.
Returns:
A timeline if created otherwise a search index in JSON (instance
of flask.wrappers.Response)
"""
# Check if search index already exists.
searchindex = SearchIndex.query.filter_by(
name=timeline_name,
description=timeline_name,
user=current_user,
index_name=index_name).first()
timeline = None
if searchindex:
searchindex.set_status('processing')
timeline = Timeline.query.filter_by(
name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=current_user,
searchindex=searchindex).first()
else:
# Create the search index in the Timesketch database
searchindex = SearchIndex.get_or_create(name=timeline_name,
description='',
user=current_user,
index_name=index_name)
searchindex.grant_permission(permission='read', user=current_user)
searchindex.grant_permission(permission='write', user=current_user)
searchindex.grant_permission(permission='delete',
user=current_user)
searchindex.set_status('processing')
db_session.add(searchindex)
db_session.commit()
if sketch and sketch.has_permission(current_user, 'write'):
labels_to_prevent_deletion = current_app.config.get(
'LABELS_TO_PREVENT_DELETION', [])
timeline = Timeline(name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=current_user,
searchindex=searchindex)
timeline.set_status('processing')
sketch.timelines.append(timeline)
for label in sketch.get_labels:
if label not in labels_to_prevent_deletion:
continue
timeline.add_label(label)
searchindex.add_label(label)
db_session.add(timeline)
db_session.commit()
# Start Celery pipeline for indexing and analysis.
# Import here to avoid circular imports.
# pylint: disable=import-outside-toplevel
from timesketch.lib import tasks
pipeline = tasks.build_index_pipeline(file_path=file_path,
events=events,
timeline_name=timeline_name,
index_name=index_name,
file_extension=file_extension,
sketch_id=sketch.id,
only_index=enable_stream)
pipeline.apply_async()
# Return Timeline if it was created.
# pylint: disable=no-else-return
if timeline:
return self.to_json(timeline,
status_code=HTTP_STATUS_CODE_CREATED,
meta=meta)
return self.to_json(searchindex,
status_code=HTTP_STATUS_CODE_CREATED,
meta=meta)
def _upload_and_index(self,
file_extension,
timeline_name,
index_name,
sketch,
enable_stream,
data_label='',
file_path='',
events='',
meta=None):
"""Creates a full pipeline for an uploaded file and returns the results.
Args:
file_extension: the extension of the uploaded file.
timeline_name: name the timeline will be stored under in the
datastore.
index_name: the Elastic index name for the timeline.
sketch: Instance of timesketch.models.sketch.Sketch
enable_stream: boolean indicating whether this is file is part of a
stream or not.
data_label: Optional string with a data label for the search index.
file_path: the path to the file to be uploaded (optional).
events: a string with events to upload (optional).
meta: optional dict with additional meta fields that will be
included in the return.
Returns:
A timeline if created otherwise a search index in JSON (instance
of flask.wrappers.Response)
"""
searchindex = self._get_index(name=timeline_name,
description=timeline_name,
sketch=sketch,
index_name=index_name,
data_label=data_label,
extension=file_extension)
searchindex.set_status('processing')
timelines = Timeline.query.filter_by(name=timeline_name,
sketch=sketch).all()
timeline = None
for timeline_ in timelines:
if timeline_.searchindex.index_name == searchindex.index_name:
timeline = timeline_
break
abort(
HTTP_STATUS_CODE_BAD_REQUEST,
'There is a timeline in the sketch that has the same name '
'but is stored in a different index, check the data_label '
'on the uploaded data')
if not timeline:
timeline = Timeline.get_or_create(name=timeline_name,
description=timeline_name,
sketch=sketch,
user=current_user,
searchindex=searchindex)
if not timeline:
abort(HTTP_STATUS_CODE_BAD_REQUEST,
'Unable to get or create a new Timeline object.')
timeline.set_status('processing')
sketch.timelines.append(timeline)
labels_to_prevent_deletion = current_app.config.get(
'LABELS_TO_PREVENT_DELETION', [])
for sketch_label in sketch.get_labels:
if sketch_label not in labels_to_prevent_deletion:
continue
timeline.add_label(sketch_label)
searchindex.add_label(sketch_label)
db_session.add(timeline)
db_session.commit()
sketch_id = sketch.id
# Start Celery pipeline for indexing and analysis.
# Import here to avoid circular imports.
# pylint: disable=import-outside-toplevel
from timesketch.lib import tasks
pipeline = tasks.build_index_pipeline(
file_path=file_path,
events=events,
timeline_name=timeline_name,
index_name=searchindex.index_name,
file_extension=file_extension,
sketch_id=sketch_id,
only_index=enable_stream,
timeline_id=timeline.id)
pipeline.apply_async()
return self.to_json(timeline,
status_code=HTTP_STATUS_CODE_CREATED,
meta=meta)
def run(self, file_path, sketch_id, username, timeline_name):
"""This is the run method."""
file_path = os.path.realpath(file_path)
file_path_no_extension, extension = os.path.splitext(file_path)
extension = extension.lstrip('.')
filename = os.path.basename(file_path_no_extension)
supported_extensions = ('plaso', 'csv', 'jsonl')
if not os.path.isfile(file_path):
sys.exit('No such file: {0:s}'.format(file_path))
if extension not in supported_extensions:
sys.exit(
'Extension {0:s} is not supported. '
'(supported extensions are: {1:s})'.format(
extension, ', '.join(supported_extensions)))
user = None
if not username:
username = pwd.getpwuid(os.stat(file_path).st_uid).pw_name
if not username == 'root':
if not isinstance(username, six.text_type):
username = codecs.decode(username, 'utf-8')
user = User.query.filter_by(username=username).first()
if not user:
sys.exit('Cannot determine user for file: {0:s}'.format(file_path))
sketch = None
# If filename starts with <number> then use that as sketch_id.
# E.g: 42_file_name.plaso means sketch_id is 42.
sketch_id_from_filename = filename.split('_')[0]
if not sketch_id and sketch_id_from_filename.isdigit():
sketch_id = sketch_id_from_filename
if sketch_id:
try:
sketch = Sketch.query.get_with_acl(sketch_id, user=user)
except Forbidden:
pass
if not timeline_name:
if not isinstance(timeline_name, six.text_type):
timeline_name = codecs.decode(timeline_name, 'utf-8')
timeline_name = timeline_name.replace('_', ' ')
# Remove sketch ID if present in the filename.
timeline_parts = timeline_name.split()
if timeline_parts[0].isdigit():
timeline_name = ' '.join(timeline_name.split()[1:])
if not sketch:
# Create a new sketch.
sketch_name = 'Sketch for: {0:s}'.format(timeline_name)
sketch = Sketch(
name=sketch_name, description=sketch_name, user=user)
# Need to commit here to be able to set permissions later.
db_session.add(sketch)
db_session.commit()
sketch.grant_permission(permission='read', user=user)
sketch.grant_permission(permission='write', user=user)
sketch.grant_permission(permission='delete', user=user)
sketch.status.append(sketch.Status(user=None, status='new'))
db_session.add(sketch)
db_session.commit()
index_name = uuid.uuid4().hex
if not isinstance(index_name, six.text_type):
index_name = codecs.decode(index_name, 'utf-8')
searchindex = SearchIndex.get_or_create(
name=timeline_name,
description=timeline_name,
user=user,
index_name=index_name)
searchindex.grant_permission(permission='read', user=user)
searchindex.grant_permission(permission='write', user=user)
searchindex.grant_permission(permission='delete', user=user)
searchindex.set_status('processing')
db_session.add(searchindex)
db_session.commit()
if sketch and sketch.has_permission(user, 'write'):
timeline = Timeline(
name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=user,
searchindex=searchindex)
timeline.set_status('processing')
sketch.timelines.append(timeline)
db_session.add(timeline)
db_session.commit()
# Start Celery pipeline for indexing and analysis.
# Import here to avoid circular imports.
from timesketch.lib import tasks
pipeline = tasks.build_index_pipeline(
file_path, timeline_name, index_name, extension, sketch.id)
pipeline.apply_async(task_id=index_name)
print('Imported {0:s} to sketch: {1:d} ({2:s})'.format(
file_path, sketch.id, sketch.name))
def setup_sketch(timeline_name, index_name, username, sketch_id=None):
"""Use existing sketch or create a new sketch.
Args:
timeline_name: (str) Name of the Timeline
index_name: (str) Name of the index
username: (str) Who should own the timeline
sketch_id: (str) Optional sketch_id to add timeline to
Returns:
(tuple) sketch ID and timeline ID as integers
"""
with app.app_context():
user = User.get_or_create(username=username)
sketch = None
if sketch_id:
try:
sketch = Sketch.query.get_with_acl(sketch_id, user=user)
logger.info('Using existing sketch: {} ({})'.format(
sketch.name, sketch.id))
except Forbidden:
pass
if not (sketch or sketch_id):
# Create a new sketch.
sketch_name = 'Turbinia: {}'.format(timeline_name)
sketch = Sketch(name=sketch_name,
description=sketch_name,
user=user)
# Need to commit here to be able to set permissions later.
db_session.add(sketch)
db_session.commit()
sketch.grant_permission(permission='read', user=user)
sketch.grant_permission(permission='write', user=user)
sketch.grant_permission(permission='delete', user=user)
sketch.status.append(sketch.Status(user=None, status='new'))
db_session.add(sketch)
db_session.commit()
logger.info('Created new sketch: {} ({})'.format(
sketch.name, sketch.id))
searchindex = SearchIndex.get_or_create(
name=timeline_name,
description='Created by Turbinia.',
user=user,
index_name=index_name)
searchindex.grant_permission(permission='read', user=user)
searchindex.grant_permission(permission='write', user=user)
searchindex.grant_permission(permission='delete', user=user)
searchindex.set_status('processing')
db_session.add(searchindex)
db_session.commit()
timeline = Timeline(name=searchindex.name,
description=searchindex.description,
sketch=sketch,
user=user,
searchindex=searchindex)
# If the user doesn't have write access to the sketch then create the
# timeline but don't attach it to the sketch.
if not sketch.has_permission(user, 'write'):
timeline.sketch = None
else:
sketch.timelines.append(timeline)
db_session.add(timeline)
db_session.commit()
timeline.set_status('processing')
return sketch.id, timeline.id
