def timelines(sketch_id): """Generates the sketch explore view template. Returns: Template with context. """ sketch = Sketch.query.get_with_acl(sketch_id) searchindices_in_sketch = [t.searchindex.id for t in sketch.timelines] indices = SearchIndex.all_with_acl( current_user).order_by( desc(SearchIndex.created_at)).filter( not_(SearchIndex.id.in_(searchindices_in_sketch))) # Setup the form form = AddTimelineForm() form.timelines.choices = set((i.id, i.name) for i in indices.all()) # Create new timeline form POST if form.validate_on_submit(): if not sketch.has_permission(current_user, u'write'): abort(HTTP_STATUS_CODE_FORBIDDEN) for searchindex_id in form.timelines.data: searchindex = SearchIndex.query.get_with_acl(searchindex_id) if searchindex not in [t.searchindex for t in sketch.timelines]: _timeline = Timeline( name=searchindex.name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex) db_session.add(_timeline) sketch.timelines.append(_timeline) db_session.commit() return redirect(url_for(u'sketch_views.timelines', sketch_id=sketch.id)) return render_template( u'sketch/timelines.html', sketch=sketch, timelines=indices.all(), form=form)
def timelines(sketch_id): """Generates the sketch explore view template. Returns: Template with context. """ sketch = Sketch.query.get_with_acl(sketch_id) searchindices_in_sketch = [t.searchindex.id for t in sketch.timelines] indices = SearchIndex.all_with_acl(current_user).order_by( desc(SearchIndex.created_at)).filter( not_(SearchIndex.id.in_(searchindices_in_sketch))) upload_enabled = current_app.config[u'UPLOAD_ENABLED'] graphs_enabled = current_app.config[u'GRAPH_BACKEND_ENABLED'] try: plaso_version = current_app.config[u'PLASO_VERSION'] except KeyError: plaso_version = u'Unknown' # Setup the form form = AddTimelineForm() form.timelines.choices = set((i.id, i.name) for i in indices.all()) # Create new timeline form POST if form.validate_on_submit(): if not sketch.has_permission(current_user, u'write'): abort(HTTP_STATUS_CODE_FORBIDDEN) for searchindex_id in form.timelines.data: searchindex = SearchIndex.query.get_with_acl(searchindex_id) if searchindex not in [t.searchindex for t in sketch.timelines]: _timeline = Timeline(name=searchindex.name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex) db_session.add(_timeline) sketch.timelines.append(_timeline) db_session.commit() # If enabled, run sketch analyzers when timeline is added. # Import here to avoid circular imports. from timesketch.lib import tasks sketch_analyzer_group = tasks.build_sketch_analysis_pipeline( sketch_id) if sketch_analyzer_group: pipeline = (tasks.run_sketch_init.s( [searchindex.index_name]) | sketch_analyzer_group) pipeline.apply_async(task_id=searchindex.index_name) return redirect(url_for(u'sketch_views.timelines', sketch_id=sketch.id)) return render_template(u'sketch/timelines.html', sketch=sketch, timelines=indices.all(), form=form, upload_enabled=upload_enabled, plaso_version=plaso_version, graphs_enabled=graphs_enabled)
def _create_timeline(self, name, sketch, searchindex, user): """Create a timeline in the database. Args: name: Name of the timeline (string) sketch: A sketch (instance of timesketch.models.sketch.Sketch) searchindex: A searchindex (instance of timesketch.models.sketch.SearchIndex) user: A user (instance of timesketch.models.user.User) Returns: A timeline (instance of timesketch.models.sketch.Timeline) """ timeline = Timeline(name=name, description=name, user=user, sketch=sketch, searchindex=searchindex, color=self.COLOR_WHITE) timeline.set_status(status='ready') self._commit_to_database(timeline) return timeline
def post(self, sketch_id): """Handles POST request to the resource. Returns: A sketch in JSON (instance of flask.wrappers.Response) Raises: ApiHTTPError """ sketch = Sketch.query.get_with_acl(sketch_id) searchindices_in_sketch = [t.searchindex.id for t in sketch.timelines] indices = SearchIndex.all_with_acl( current_user).order_by( desc(SearchIndex.created_at)).filter( not_(SearchIndex.id.in_(searchindices_in_sketch))) add_timeline_form = AddTimelineForm.build(request) add_timeline_form.timelines.choices = set( (i.id, i.name) for i in indices.all()) if add_timeline_form.validate_on_submit(): if not sketch.has_permission(current_user, u'write'): abort(HTTP_STATUS_CODE_FORBIDDEN) for searchindex_id in add_timeline_form.timelines.data: searchindex = SearchIndex.query.get_with_acl(searchindex_id) if searchindex not in [t.searchindex for t in sketch.timelines]: _timeline = Timeline( name=searchindex.name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex) db_session.add(_timeline) sketch.timelines.append(_timeline) db_session.commit() return self.to_json(sketch, status_code=HTTP_STATUS_CODE_CREATED) else: raise ApiHTTPError( message=add_timeline_form.errors, status_code=HTTP_STATUS_CODE_BAD_REQUEST)
def post(self, sketch_id): """Handles POST request to the resource. Returns: A sketch in JSON (instance of flask.wrappers.Response) """ sketch = Sketch.query.get_with_acl(sketch_id) form = AddTimelineSimpleForm.build(request) metadata = {u'created': True} searchindex_id = form.timeline.data searchindex = SearchIndex.query.get_with_acl(searchindex_id) timeline_id = [ t.searchindex.id for t in sketch.timelines if t.searchindex.id == searchindex_id ] if form.validate_on_submit(): if not sketch.has_permission(current_user, u'write'): abort(HTTP_STATUS_CODE_FORBIDDEN) if not timeline_id: return_code = HTTP_STATUS_CODE_CREATED timeline = Timeline( name=searchindex.name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex) sketch.timelines.append(timeline) db_session.add(timeline) db_session.commit() else: metadata[u'created'] = False return_code = HTTP_STATUS_CODE_OK timeline = Timeline.query.get(timeline_id) return self.to_json( timeline, meta=metadata, status_code=return_code) return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def post(self, sketch_id): """Handles POST request to the resource. Returns: A sketch in JSON (instance of flask.wrappers.Response) """ sketch = Sketch.query.get_with_acl(sketch_id) if not sketch: abort( HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.') if not sketch.has_permission(current_user, 'write'): abort(HTTP_STATUS_CODE_FORBIDDEN, 'User does not have write access controls on sketch.') form = request.json if not form: form = request.data metadata = {'created': True} searchindex_id = form.get('timeline', 0) if isinstance(searchindex_id, str) and searchindex_id.isdigit(): searchindex_id = int(searchindex_id) if not isinstance(searchindex_id, int): abort( HTTP_STATUS_CODE_BAD_REQUEST, 'The timeline (searchindex id) needs to be an integer.') searchindex = SearchIndex.query.get_with_acl(searchindex_id) if searchindex.get_status.status == 'deleted': abort( HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to create a timeline using a deleted search index') timeline_id = [ t.searchindex.id for t in sketch.timelines if t.searchindex.id == searchindex_id ] if not timeline_id: return_code = HTTP_STATUS_CODE_CREATED timeline_name = form.get('timeline_name', searchindex.name) timeline = Timeline( name=timeline_name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex) sketch.timelines.append(timeline) labels_to_prevent_deletion = current_app.config.get( 'LABELS_TO_PREVENT_DELETION', []) for label in sketch.get_labels: if label not in labels_to_prevent_deletion: continue timeline.add_label(label) searchindex.add_label(label) # Set status to ready so the timeline can be queried. timeline.set_status('ready') db_session.add(timeline) db_session.commit() else: metadata['created'] = False return_code = HTTP_STATUS_CODE_OK timeline = Timeline.query.get(timeline_id) # Run sketch analyzers when timeline is added. Import here to avoid # circular imports. # pylint: disable=import-outside-toplevel if current_app.config.get('AUTO_SKETCH_ANALYZERS'): # pylint: disable=import-outside-toplevel from timesketch.lib import tasks sketch_analyzer_group, _ = tasks.build_sketch_analysis_pipeline( sketch_id, searchindex_id, current_user.id, timeline_id=timeline_id) if sketch_analyzer_group: pipeline = (tasks.run_sketch_init.s( [searchindex.index_name]) | sketch_analyzer_group) pipeline.apply_async() # Update the last activity of a sketch. utils.update_sketch_last_activity(sketch) return self.to_json( timeline, meta=metadata, status_code=return_code)
def post(self): """Handles POST request to the resource. Returns: A view in JSON (instance of flask.wrappers.Response) """ upload_enabled = current_app.config['UPLOAD_ENABLED'] if not upload_enabled: abort( HTTP_STATUS_CODE_BAD_REQUEST, 'Failed to create timeline, upload not enabled') form = forms.CreateTimelineForm() if not form.validate_on_submit(): abort( HTTP_STATUS_CODE_BAD_REQUEST, 'Failed to create timeline, form data not validated') sketch_id = form.sketch_id.data timeline_name = form.name.data sketch = None if sketch_id: sketch = Sketch.query.get_with_acl(sketch_id) if not sketch: abort( HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.') # We do not need a human readable filename or # datastore index name, so we use UUIDs here. index_name = uuid.uuid4().hex if not isinstance(index_name, six.text_type): index_name = codecs.decode(index_name, 'utf-8') # Create the search index in the Timesketch database searchindex = SearchIndex.get_or_create( name=timeline_name, description=timeline_name, user=current_user, index_name=index_name) searchindex.grant_permission(permission='read', user=current_user) searchindex.grant_permission(permission='write', user=current_user) searchindex.grant_permission( permission='delete', user=current_user) searchindex.set_status('processing') db_session.add(searchindex) db_session.commit() timeline = None if sketch and sketch.has_permission(current_user, 'write'): timeline = Timeline( name=searchindex.name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex) sketch.timelines.append(timeline) db_session.add(timeline) db_session.commit() # Return Timeline if it was created. # pylint: disable=no-else-return if timeline: return self.to_json( timeline, status_code=HTTP_STATUS_CODE_CREATED) # Update the last activity of a sketch. utils.update_sketch_last_activity(sketch) return self.to_json( searchindex, status_code=HTTP_STATUS_CODE_CREATED)
def post(self, sketch_id): """Handles POST request to the resource. Handler for /api/v1/sketches/:sketch_id/event/create/ Args: sketch_id: Integer primary key for a sketch database model Returns: An annotation in JSON (instance of flask.wrappers.Response) """ sketch = Sketch.query.get_with_acl(sketch_id) if not sketch: abort(HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.') if not sketch.has_permission(current_user, 'write'): abort(HTTP_STATUS_CODE_FORBIDDEN, 'User does not have write access controls on sketch.') form = request.json if not form: form = request.data timeline_name = 'sketch specific timeline' index_name_seed = 'timesketch_{0:d}'.format(sketch_id) event_type = 'user_created_event' date_string = form.get('date_string') if not date_string: date = datetime.datetime.utcnow().isoformat() else: # derive datetime from timestamp: try: date = dateutil.parser.parse(date_string) except (dateutil.parser.ParserError, OverflowError) as e: logger.error('Unable to convert date string', exc_info=True) abort( HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to add event, not able to convert the date ' 'string. Was it properly formatted? Error: ' '{0!s}'.format(e)) timestamp = int(time.mktime(date.utctimetuple())) * 1000000 timestamp += date.microsecond event = { 'datetime': date_string, 'timestamp': timestamp, 'timestamp_desc': form.get('timestamp_desc', 'Event Happened'), 'message': form.get('message', 'No message string'), } attributes = form.get('attributes', {}) if not isinstance(attributes, dict): abort( HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to add an event where the attributes are not a ' 'dict object.') event.update(attributes) tag = form.get('tag', []) if not isinstance(tag, list): abort( HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to add an event where the tags are not a ' 'list of strings.') if tag and any([not isinstance(x, str) for x in tag]): abort( HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to add an event where the tags are not a ' 'list of strings.') event['tag'] = tag # We do not need a human readable filename or # datastore index name, so we use UUIDs here. index_name = hashlib.md5(index_name_seed.encode()).hexdigest() if six.PY2: index_name = codecs.decode(index_name, 'utf-8') # Try to create index try: # Create the index in Elasticsearch (unless it already exists) self.datastore.create_index(index_name=index_name, doc_type=event_type) # Create the search index in the Timesketch database searchindex = SearchIndex.get_or_create( name=timeline_name, description='internal timeline for user-created events', user=current_user, index_name=index_name) searchindex.grant_permission(permission='read', user=current_user) searchindex.grant_permission(permission='write', user=current_user) searchindex.grant_permission(permission='delete', user=current_user) searchindex.set_status('ready') db_session.add(searchindex) db_session.commit() timeline = None if sketch and sketch.has_permission(current_user, 'write'): self.datastore.import_event(index_name, event_type, event, flush_interval=1) timeline = Timeline.get_or_create( name=searchindex.name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex) if timeline not in sketch.timelines: sketch.timelines.append(timeline) timeline.set_status('ready') db_session.add(timeline) db_session.commit() # Return Timeline if it was created. # pylint: disable=no-else-return if timeline: return self.to_json(timeline, status_code=HTTP_STATUS_CODE_CREATED) else: return self.to_json(searchindex, status_code=HTTP_STATUS_CODE_CREATED) # TODO: Can this be narrowed down, both in terms of the scope it # applies to, as well as not to catch a generic exception. except Exception as e: # pylint: disable=broad-except abort(HTTP_STATUS_CODE_BAD_REQUEST, 'Failed to add event ({0!s})'.format(e))
def post(self, sketch_id=None): """Handles POST request to the resource. Returns: A view in JSON (instance of flask.wrappers.Response) Raises: ApiHTTPError """ UPLOAD_ENABLED = current_app.config[u'UPLOAD_ENABLED'] UPLOAD_FOLDER = current_app.config[u'UPLOAD_FOLDER'] sketch = None if sketch_id: sketch = Sketch.query.get_with_acl(sketch_id) form = UploadFileForm() if form.validate_on_submit() and UPLOAD_ENABLED: from timesketch.lib.tasks import run_plaso from timesketch.lib.tasks import run_csv # Map the right task based on the file type task_directory = {u'plaso': run_plaso, u'csv': run_csv} file_storage = form.file.data timeline_name = form.name.data _, _extension = os.path.splitext(file_storage.filename) file_extension = _extension.lstrip(u'.') # Current user username = current_user.username # We do not need a human readable filename or # datastore index name, so we use UUIDs here. filename = unicode(uuid.uuid4().hex) index_name = unicode(uuid.uuid4().hex) file_path = os.path.join(UPLOAD_FOLDER, filename) file_storage.save(file_path) # Create the search index in the Timesketch database searchindex = SearchIndex.get_or_create(name=timeline_name, description=timeline_name, user=current_user, index_name=index_name) searchindex.grant_permission(permission=u'read', user=current_user) searchindex.grant_permission(permission=u'write', user=current_user) searchindex.grant_permission(permission=u'delete', user=current_user) searchindex.set_status(u'processing') db_session.add(searchindex) db_session.commit() if sketch and sketch.has_permission(current_user, u'write'): timeline = Timeline(name=searchindex.name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex) db_session.add(timeline) sketch.timelines.append(timeline) db_session.commit() # Run the task in the background task = task_directory.get(file_extension) task.apply_async((file_path, timeline_name, index_name, username), task_id=index_name) return self.to_json(searchindex, status_code=HTTP_STATUS_CODE_CREATED) else: raise ApiHTTPError(message=form.errors[u'file'][0], status_code=HTTP_STATUS_CODE_BAD_REQUEST)
def run(self, file_path, sketch_id, username, timeline_name): """This is the run method.""" file_path = os.path.realpath(file_path) file_path_no_extension, extension = os.path.splitext(file_path) extension = extension.lstrip('.') filename = os.path.basename(file_path_no_extension) supported_extensions = ('plaso', 'csv', 'jsonl') if not os.path.isfile(file_path): sys.exit('No such file: {0:s}'.format(file_path)) if extension not in supported_extensions: sys.exit('Extension {0:s} is not supported. ' '(supported extensions are: {1:s})'.format( extension, ', '.join(supported_extensions))) user = None if not username: username = pwd.getpwuid(os.stat(file_path).st_uid).pw_name if not username == 'root': if not isinstance(username, six.text_type): username = codecs.decode(username, 'utf-8') user = User.query.filter_by(username=username).first() if not user: sys.exit('Cannot determine user for file: {0:s}'.format(file_path)) sketch = None # If filename starts with <number> then use that as sketch_id. # E.g: 42_file_name.plaso means sketch_id is 42. sketch_id_from_filename = filename.split('_')[0] if not sketch_id and sketch_id_from_filename.isdigit(): sketch_id = sketch_id_from_filename if sketch_id: try: sketch = Sketch.query.get_with_acl(sketch_id, user=user) except Forbidden: pass if not timeline_name: if timeline_name is None: timeline_name = '{0:s}_timeline'.format(filename) if not isinstance(timeline_name, six.text_type): timeline_name = codecs.decode(timeline_name, 'utf-8') timeline_name = timeline_name.replace('_', ' ') # Remove sketch ID if present in the filename. timeline_parts = timeline_name.split() if timeline_parts[0].isdigit(): timeline_name = ' '.join(timeline_name.split()[1:]) if not sketch: # Create a new sketch. sketch_name = 'Sketch for: {0:s}'.format(timeline_name) sketch = Sketch(name=sketch_name, description=sketch_name, user=user) # Need to commit here to be able to set permissions later. db_session.add(sketch) db_session.commit() sketch.grant_permission(permission='read', user=user) sketch.grant_permission(permission='write', user=user) sketch.grant_permission(permission='delete', user=user) sketch.status.append(sketch.Status(user=None, status='new')) db_session.add(sketch) db_session.commit() index_name = uuid.uuid4().hex if not isinstance(index_name, six.text_type): index_name = codecs.decode(index_name, 'utf-8') searchindex = SearchIndex.get_or_create(name=timeline_name, description=timeline_name, user=user, index_name=index_name) searchindex.grant_permission(permission='read', user=user) searchindex.grant_permission(permission='write', user=user) searchindex.grant_permission(permission='delete', user=user) searchindex.set_status('processing') db_session.add(searchindex) db_session.commit() if sketch and sketch.has_permission(user, 'write'): timeline = Timeline(name=searchindex.name, description=searchindex.description, sketch=sketch, user=user, searchindex=searchindex) timeline.set_status('processing') sketch.timelines.append(timeline) db_session.add(timeline) db_session.commit() # Start Celery pipeline for indexing and analysis. # Import here to avoid circular imports. from timesketch.lib import tasks # pylint: disable=import-outside-toplevel pipeline = tasks.build_index_pipeline(file_path=file_path, events='', timeline_name=timeline_name, index_name=index_name, file_extension=extension, sketch_id=sketch.id) pipeline.apply_async(task_id=index_name) print('Imported {0:s} to sketch: {1:d} ({2:s})'.format( file_path, sketch.id, sketch.name))
def _upload_and_index( self, file_extension, timeline_name, index_name, sketch, form, enable_stream, original_filename='', data_label='', file_path='', events='', meta=None): """Creates a full pipeline for an uploaded file and returns the results. Args: file_extension: the extension of the uploaded file. timeline_name: name the timeline will be stored under in the datastore. index_name: the Elastic index name for the timeline. sketch: Instance of timesketch.models.sketch.Sketch form: a dict with the configuration for the upload. enable_stream: boolean indicating whether this is file is part of a stream or not. original_filename: Original filename from the upload. data_label: Optional string with a data label for the search index. file_path: the path to the file to be uploaded (optional). events: a string with events to upload (optional). meta: optional dict with additional meta fields that will be included in the return. Returns: A timeline if created otherwise a search index in JSON (instance of flask.wrappers.Response) """ searchindex = self._get_index( name=timeline_name, description=timeline_name, sketch=sketch, index_name=index_name, data_label=data_label, extension=file_extension) if not searchindex: abort( HTTP_STATUS_CODE_BAD_REQUEST, 'We were unable to acquire a searchindex and therefore not ' 'able to upload data, please try again. If this error persist ' 'please create an issue on Github: https://github.com/' 'google/timesketch/issues/new/choose') timelines = Timeline.query.filter_by( name=timeline_name, sketch=sketch).all() timeline = None for timeline_ in timelines: if timeline_.searchindex.index_name == searchindex.index_name: timeline = timeline_ break logger.error( 'There is a timeline in the sketch that has the same name ' 'but is stored in a different index: name {0:s} attempting ' 'index: {1:s} but found index {2:s} - retrying with a ' 'different timeline name.'.format( timeline_name, searchindex.index_name, timeline_.searchindex.index_name)) timeline_name = '{0:s}_{1:s}'.format( timeline_name, uuid.uuid4().hex[-5:]) return self._upload_and_index( file_extension=file_extension, timeline_name=timeline_name, index_name=searchindex.index_name, sketch=sketch, form=form, enable_stream=enable_stream, original_filename=original_filename, data_label=data_label, file_path=file_path, events=events, meta=meta) searchindex.set_status('processing') if not timeline: timeline = Timeline.get_or_create( name=timeline_name, description=timeline_name, sketch=sketch, user=current_user, searchindex=searchindex) if not timeline: abort( HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to get or create a new Timeline object.') # If the timeline already existed and has associated data sources # then we don't want to set the status to processing. if not timeline.datasources: timeline.set_status('processing') sketch.timelines.append(timeline) labels_to_prevent_deletion = current_app.config.get( 'LABELS_TO_PREVENT_DELETION', []) for sketch_label in sketch.get_labels: if sketch_label not in labels_to_prevent_deletion: continue timeline.add_label(sketch_label) searchindex.add_label(sketch_label) file_size = form.get('total_file_size', 0) datasource = DataSource( timeline=timeline, user=current_user, provider=form.get('provider', 'N/A'), context=form.get('context', 'N/A'), file_on_disk=file_path, file_size=int(file_size), original_filename=original_filename, data_label=data_label ) timeline.datasources.append(datasource) db_session.add(datasource) db_session.add(timeline) db_session.commit() sketch_id = sketch.id # Start Celery pipeline for indexing and analysis. # Import here to avoid circular imports. # pylint: disable=import-outside-toplevel from timesketch.lib import tasks pipeline = tasks.build_index_pipeline( file_path=file_path, events=events, timeline_name=timeline_name, index_name=searchindex.index_name, file_extension=file_extension, sketch_id=sketch_id, only_index=enable_stream, timeline_id=timeline.id) task_id = uuid.uuid4().hex pipeline.apply_async(task_id=task_id) if meta is None: meta = {} meta['task_id'] = task_id return self.to_json( timeline, status_code=HTTP_STATUS_CODE_CREATED, meta=meta)
def _upload_and_index(self, file_extension, timeline_name, index_name, sketch, enable_stream, file_path='', events='', meta=None): """Creates a full pipeline for an uploaded file and returns the results. Args: file_extension: the extension of the uploaded file. timeline_name: name the timeline will be stored under in the datastore. index_name: the Elastic index name for the timeline. sketch: Instance of timesketch.models.sketch.Sketch enable_stream: boolean indicating whether this is file is part of a stream or not. file_path: the path to the file to be uploaded (optional). events: a string with events to upload (optional). meta: optional dict with additional meta fields that will be included in the return. Returns: A timeline if created otherwise a search index in JSON (instance of flask.wrappers.Response) """ # Check if search index already exists. searchindex = SearchIndex.query.filter_by( name=timeline_name, description=timeline_name, user=current_user, index_name=index_name).first() timeline = None if searchindex: searchindex.set_status('processing') timeline = Timeline.query.filter_by( name=searchindex.name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex).first() else: # Create the search index in the Timesketch database searchindex = SearchIndex.get_or_create(name=timeline_name, description='', user=current_user, index_name=index_name) searchindex.grant_permission(permission='read', user=current_user) searchindex.grant_permission(permission='write', user=current_user) searchindex.grant_permission(permission='delete', user=current_user) searchindex.set_status('processing') db_session.add(searchindex) db_session.commit() if sketch and sketch.has_permission(current_user, 'write'): labels_to_prevent_deletion = current_app.config.get( 'LABELS_TO_PREVENT_DELETION', []) timeline = Timeline(name=searchindex.name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex) timeline.set_status('processing') sketch.timelines.append(timeline) for label in sketch.get_labels: if label not in labels_to_prevent_deletion: continue timeline.add_label(label) searchindex.add_label(label) db_session.add(timeline) db_session.commit() # Start Celery pipeline for indexing and analysis. # Import here to avoid circular imports. # pylint: disable=import-outside-toplevel from timesketch.lib import tasks pipeline = tasks.build_index_pipeline(file_path=file_path, events=events, timeline_name=timeline_name, index_name=index_name, file_extension=file_extension, sketch_id=sketch.id, only_index=enable_stream) pipeline.apply_async() # Return Timeline if it was created. # pylint: disable=no-else-return if timeline: return self.to_json(timeline, status_code=HTTP_STATUS_CODE_CREATED, meta=meta) return self.to_json(searchindex, status_code=HTTP_STATUS_CODE_CREATED, meta=meta)
def _upload_and_index(self, file_extension, timeline_name, index_name, sketch, enable_stream, data_label='', file_path='', events='', meta=None): """Creates a full pipeline for an uploaded file and returns the results. Args: file_extension: the extension of the uploaded file. timeline_name: name the timeline will be stored under in the datastore. index_name: the Elastic index name for the timeline. sketch: Instance of timesketch.models.sketch.Sketch enable_stream: boolean indicating whether this is file is part of a stream or not. data_label: Optional string with a data label for the search index. file_path: the path to the file to be uploaded (optional). events: a string with events to upload (optional). meta: optional dict with additional meta fields that will be included in the return. Returns: A timeline if created otherwise a search index in JSON (instance of flask.wrappers.Response) """ searchindex = self._get_index(name=timeline_name, description=timeline_name, sketch=sketch, index_name=index_name, data_label=data_label, extension=file_extension) searchindex.set_status('processing') timelines = Timeline.query.filter_by(name=timeline_name, sketch=sketch).all() timeline = None for timeline_ in timelines: if timeline_.searchindex.index_name == searchindex.index_name: timeline = timeline_ break abort( HTTP_STATUS_CODE_BAD_REQUEST, 'There is a timeline in the sketch that has the same name ' 'but is stored in a different index, check the data_label ' 'on the uploaded data') if not timeline: timeline = Timeline.get_or_create(name=timeline_name, description=timeline_name, sketch=sketch, user=current_user, searchindex=searchindex) if not timeline: abort(HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to get or create a new Timeline object.') timeline.set_status('processing') sketch.timelines.append(timeline) labels_to_prevent_deletion = current_app.config.get( 'LABELS_TO_PREVENT_DELETION', []) for sketch_label in sketch.get_labels: if sketch_label not in labels_to_prevent_deletion: continue timeline.add_label(sketch_label) searchindex.add_label(sketch_label) db_session.add(timeline) db_session.commit() sketch_id = sketch.id # Start Celery pipeline for indexing and analysis. # Import here to avoid circular imports. # pylint: disable=import-outside-toplevel from timesketch.lib import tasks pipeline = tasks.build_index_pipeline( file_path=file_path, events=events, timeline_name=timeline_name, index_name=searchindex.index_name, file_extension=file_extension, sketch_id=sketch_id, only_index=enable_stream, timeline_id=timeline.id) pipeline.apply_async() return self.to_json(timeline, status_code=HTTP_STATUS_CODE_CREATED, meta=meta)
def run(self, file_path, sketch_id, username, timeline_name): """This is the run method.""" file_path = os.path.realpath(file_path) file_path_no_extension, extension = os.path.splitext(file_path) extension = extension.lstrip('.') filename = os.path.basename(file_path_no_extension) supported_extensions = ('plaso', 'csv', 'jsonl') if not os.path.isfile(file_path): sys.exit('No such file: {0:s}'.format(file_path)) if extension not in supported_extensions: sys.exit( 'Extension {0:s} is not supported. ' '(supported extensions are: {1:s})'.format( extension, ', '.join(supported_extensions))) user = None if not username: username = pwd.getpwuid(os.stat(file_path).st_uid).pw_name if not username == 'root': if not isinstance(username, six.text_type): username = codecs.decode(username, 'utf-8') user = User.query.filter_by(username=username).first() if not user: sys.exit('Cannot determine user for file: {0:s}'.format(file_path)) sketch = None # If filename starts with <number> then use that as sketch_id. # E.g: 42_file_name.plaso means sketch_id is 42. sketch_id_from_filename = filename.split('_')[0] if not sketch_id and sketch_id_from_filename.isdigit(): sketch_id = sketch_id_from_filename if sketch_id: try: sketch = Sketch.query.get_with_acl(sketch_id, user=user) except Forbidden: pass if not timeline_name: if not isinstance(timeline_name, six.text_type): timeline_name = codecs.decode(timeline_name, 'utf-8') timeline_name = timeline_name.replace('_', ' ') # Remove sketch ID if present in the filename. timeline_parts = timeline_name.split() if timeline_parts[0].isdigit(): timeline_name = ' '.join(timeline_name.split()[1:]) if not sketch: # Create a new sketch. sketch_name = 'Sketch for: {0:s}'.format(timeline_name) sketch = Sketch( name=sketch_name, description=sketch_name, user=user) # Need to commit here to be able to set permissions later. db_session.add(sketch) db_session.commit() sketch.grant_permission(permission='read', user=user) sketch.grant_permission(permission='write', user=user) sketch.grant_permission(permission='delete', user=user) sketch.status.append(sketch.Status(user=None, status='new')) db_session.add(sketch) db_session.commit() index_name = uuid.uuid4().hex if not isinstance(index_name, six.text_type): index_name = codecs.decode(index_name, 'utf-8') searchindex = SearchIndex.get_or_create( name=timeline_name, description=timeline_name, user=user, index_name=index_name) searchindex.grant_permission(permission='read', user=user) searchindex.grant_permission(permission='write', user=user) searchindex.grant_permission(permission='delete', user=user) searchindex.set_status('processing') db_session.add(searchindex) db_session.commit() if sketch and sketch.has_permission(user, 'write'): timeline = Timeline( name=searchindex.name, description=searchindex.description, sketch=sketch, user=user, searchindex=searchindex) timeline.set_status('processing') sketch.timelines.append(timeline) db_session.add(timeline) db_session.commit() # Start Celery pipeline for indexing and analysis. # Import here to avoid circular imports. from timesketch.lib import tasks pipeline = tasks.build_index_pipeline( file_path, timeline_name, index_name, extension, sketch.id) pipeline.apply_async(task_id=index_name) print('Imported {0:s} to sketch: {1:d} ({2:s})'.format( file_path, sketch.id, sketch.name))
def setup_sketch(timeline_name, index_name, username, sketch_id=None): """Use existing sketch or create a new sketch. Args: timeline_name: (str) Name of the Timeline index_name: (str) Name of the index username: (str) Who should own the timeline sketch_id: (str) Optional sketch_id to add timeline to Returns: (tuple) sketch ID and timeline ID as integers """ with app.app_context(): user = User.get_or_create(username=username) sketch = None if sketch_id: try: sketch = Sketch.query.get_with_acl(sketch_id, user=user) logger.info('Using existing sketch: {} ({})'.format( sketch.name, sketch.id)) except Forbidden: pass if not (sketch or sketch_id): # Create a new sketch. sketch_name = 'Turbinia: {}'.format(timeline_name) sketch = Sketch(name=sketch_name, description=sketch_name, user=user) # Need to commit here to be able to set permissions later. db_session.add(sketch) db_session.commit() sketch.grant_permission(permission='read', user=user) sketch.grant_permission(permission='write', user=user) sketch.grant_permission(permission='delete', user=user) sketch.status.append(sketch.Status(user=None, status='new')) db_session.add(sketch) db_session.commit() logger.info('Created new sketch: {} ({})'.format( sketch.name, sketch.id)) searchindex = SearchIndex.get_or_create( name=timeline_name, description='Created by Turbinia.', user=user, index_name=index_name) searchindex.grant_permission(permission='read', user=user) searchindex.grant_permission(permission='write', user=user) searchindex.grant_permission(permission='delete', user=user) searchindex.set_status('processing') db_session.add(searchindex) db_session.commit() timeline = Timeline(name=searchindex.name, description=searchindex.description, sketch=sketch, user=user, searchindex=searchindex) # If the user doesn't have write access to the sketch then create the # timeline but don't attach it to the sketch. if not sketch.has_permission(user, 'write'): timeline.sketch = None else: sketch.timelines.append(timeline) db_session.add(timeline) db_session.commit() timeline.set_status('processing') return sketch.id, timeline.id