def home(): """Generates the home page view template. Returns: Template with context. """ form = HiddenNameDescriptionForm() sketches = Sketch.all_with_acl().filter( not_(Sketch.Status.status == 'deleted'), Sketch.Status.parent).order_by(Sketch.updated_at.desc()) # Only render upload button if it is configured. upload_enabled = current_app.config['UPLOAD_ENABLED'] # Handle form for creating a new sketch. if form.validate_on_submit(): sketch = Sketch( name=form.name.data, description=form.description.data, user=current_user) sketch.status.append(sketch.Status(user=None, status='new')) db_session.add(sketch) db_session.commit() # Give the requesting user permissions on the new sketch. sketch.grant_permission(permission='read', user=current_user) sketch.grant_permission(permission='write', user=current_user) sketch.grant_permission(permission='delete', user=current_user) return redirect(url_for('sketch_views.overview', sketch_id=sketch.id)) return render_template( 'home/home.html', sketches=sketches, form=form, upload_enabled=upload_enabled)
def run_wrapper(self): """A wrapper method to run the analyzer. This method is decorated to flush the bulk insert operation on the datastore. This makes sure that all events are indexed at exit. Returns: Return value of the run method. """ result = self.run() # Update the searchindex description with analyzer result. # TODO: Don't overload the description field. searchindex = SearchIndex.query.filter_by( index_name=self.index_name).first() # Some code paths set the description equals to the name. Remove that # here to get a clean description with only analyzer results. if searchindex.description == searchindex.name: searchindex.description = '' # Append the analyzer result. if result: searchindex.description = '{0:s}\n{1:s}'.format( searchindex.description, result) db_session.add(searchindex) db_session.commit() return result
def explore(sketch_id, view_id=None): """Generates the sketch explore view template. Returns: Template with context. """ sketch = Sketch.query.get_with_acl(sketch_id) if view_id: view = View.query.get(view_id) else: view = View.query.filter( View.user == current_user, View.name == u'', View.sketch_id == sketch_id).order_by( View.created_at.desc()).first() if not view: view = View( user=current_user, name=u'', sketch=sketch, query_string=u'', query_filter=u'{}') db_session.add(view) db_session.commit() sketch_timelines = u','.join( [t.searchindex.index_name for t in sketch.timelines]) view_form = SaveViewForm() return render_template( u'sketch/explore.html', sketch=sketch, view=view, timelines=sketch_timelines, view_form=view_form)
def _set_timeline_status(index_name, status, error_msg=None): """Helper function to set status for searchindex and all related timelines. Args: index_name: Name of the datastore index. status: Status to set. error_msg: Error message. """ searchindex = SearchIndex.query.filter_by(index_name=index_name).first() timelines = Timeline.query.filter_by(searchindex=searchindex).all() # Set status searchindex.set_status(status) for timeline in timelines: timeline.set_status(status) db_session.add(timeline) # Update description if there was a failure in ingestion if error_msg and status == 'fail': # TODO: Don't overload the description field. searchindex.description = error_msg # Commit changes to database db_session.add(searchindex) db_session.commit()
def add_view(self, view_name, analyzer_name, query_string=None, query_dsl=None, query_filter=None): """Add saved view to the Sketch. Args: view_name: The name of the view. analyzer_name: The name of the analyzer. query_string: Elasticsearch query string. query_dsl: Dictionary with Elasticsearch DSL query. query_filter: Dictionary with Elasticsearch filters. Raises: ValueError: If both query_string an query_dsl are missing. Returns: An instance of a SQLAlchemy View object. """ if not query_string or query_dsl: raise ValueError('Both query_string and query_dsl are missing.') if not query_filter: query_filter = {'indices': '_all'} name = '[{0:s}] {1:s}'.format(analyzer_name, view_name) view = View.get_or_create(name=name, sketch=self.sql_sketch, user=None) view.query_string = query_string view.query_filter = view.validate_filter(query_filter) view.query_dsl = query_dsl view.searchtemplate = None db_session.add(view) db_session.commit() return view
def timeline(sketch_id, timeline_id): """Generates the sketch timeline view template. Returns: Template with context. """ timeline_form = TimelineForm() sketch = Sketch.query.get_with_acl(sketch_id) sketch_timeline = Timeline.query.filter( Timeline.id == timeline_id, Timeline.sketch == sketch).first() if not sketch_timeline: abort(HTTP_STATUS_CODE_NOT_FOUND) if timeline_form.validate_on_submit(): if not sketch.has_permission(current_user, u'write'): abort(HTTP_STATUS_CODE_FORBIDDEN) sketch_timeline.name = timeline_form.name.data sketch_timeline.description = timeline_form.description.data sketch_timeline.color = timeline_form.color.data db_session.add(sketch_timeline) db_session.commit() return redirect( url_for(u'sketch_views.timeline', sketch_id=sketch.id, timeline_id=sketch_timeline.id)) return render_template( u'sketch/timeline.html', sketch=sketch, timeline=sketch_timeline, timeline_form=timeline_form)
def WriteHeader(self): """Sets up the Elasticsearch index and the Timesketch database object. Creates the Elasticsearch index with Timesketch specific settings and the Timesketch SearchIndex database object. """ # This cannot be static because we use the value of self._document_type # from arguments. mappings = { self._document_type: { 'properties': { 'timesketch_label': { 'type': 'nested' } } } } # Get Elasticsearch host and port from Timesketch configuration. with self._timesketch.app_context(): self._host = current_app.config['ELASTIC_HOST'] self._port = current_app.config['ELASTIC_PORT'] self._Connect() self._CreateIndexIfNotExists(self._index_name, mappings) user = None if self._timeline_owner: user = timesketch_user.User.query.filter_by( username=self._timeline_owner).first() if not user: raise RuntimeError( 'Unknown Timesketch user: {0:s}'.format(self._timeline_owner)) else: logger.warning('Timeline will be visible to all Timesketch users') with self._timesketch.app_context(): search_index = timesketch_sketch.SearchIndex.get_or_create( name=self._timeline_name, description=self._timeline_name, user=user, index_name=self._index_name) # Grant the user read permission on the mapping object and set status. # If user is None the timeline becomes visible to all users. search_index.grant_permission(user=user, permission='read') # In case we have a user grant additional permissions. if user: search_index.grant_permission(user=user, permission='write') search_index.grant_permission(user=user, permission='delete') # Let the Timesketch UI know that the timeline is processing. search_index.set_status('processing') # Save the mapping object to the Timesketch database. timesketch_db_session.add(search_index) timesketch_db_session.commit() logger.debug('Adding events to Timesketch.')
def run(self, name): """Creates the group.""" if not isinstance(name, six.text_type): name = codecs.decode(name, 'utf-8') group = Group.get_or_create(name=name) db_session.add(group) db_session.commit() sys.stdout.write('Group {0:s} created\n'.format(name))
def _commit_to_database(self, model): """Add object to the database session and commit. Args: model: Instance of timesketch.models.[model] object """ db_session.add(model) db_session.commit()
def post(self, sketch_id): """Handles POST request to the resource. Args: sketch_id: Integer primary key for a sketch database model Returns: An annotation in JSON (instance of flask.wrappers.Response) """ form = EventAnnotationForm.build(request) if form.validate_on_submit(): sketch = Sketch.query.get_with_acl(sketch_id) indices = [t.searchindex.index_name for t in sketch.timelines] annotation_type = form.annotation_type.data searchindex_id = form.searchindex_id.data searchindex = SearchIndex.query.get(searchindex_id) event_id = form.event_id.data if searchindex_id not in indices: abort(HTTP_STATUS_CODE_BAD_REQUEST) def _set_label(label, toggle=False): """Set label on the event in the datastore.""" self.datastore.set_label( searchindex_id, event_id, sketch.id, current_user.id, label, toggle=toggle) # Get or create an event in the SQL database to have something to # attach the annotation to. event = Event.get_or_create( sketch=sketch, searchindex=searchindex, document_id=event_id) # Add the annotation to the event object. if u'comment' in annotation_type: annotation = Event.Comment( comment=form.annotation.data, user=current_user) event.comments.append(annotation) _set_label(u'__ts_comment') elif u'label' in annotation_type: annotation = Event.Label.get_or_create( label=form.annotation.data, user=current_user) if annotation not in event.labels: event.labels.append(annotation) toggle = False if u'__ts_star' in form.annotation.data: toggle = True _set_label(form.annotation.data, toggle) else: abort(HTTP_STATUS_CODE_BAD_REQUEST) # Save the event to the database db_session.add(event) db_session.commit() return self.to_json( annotation, status_code=HTTP_STATUS_CODE_CREATED) return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def WriteHeader(self): """Setup the Elasticsearch index and the Timesketch database object. Creates the Elasticsearch index with Timesketch specific settings and the Timesketch SearchIndex database object. """ # This cannot be static because we use the value of self._doc_type from # arguments. _document_mapping = { self._doc_type: { u'properties': { u'timesketch_label': { u'type': u'nested' } } } } # Get Elasticsearch host and port from Timesketch configuration. with self._timesketch.app_context(): _host = current_app.config[u'ELASTIC_HOST'] _port = current_app.config[u'ELASTIC_PORT'] self._elastic = ElasticSearchHelper( self._output_mediator, _host, _port, self._flush_interval, self._index_name, _document_mapping, self._doc_type) user = None if self._username: user = User.query.filter_by(username=self._username).first() if not user: raise RuntimeError( u'Unknown Timesketch user: {0:s}'.format(self._username)) else: logging.warning(u'Timeline will be visible to all Timesketch users') with self._timesketch.app_context(): search_index = SearchIndex.get_or_create( name=self._timeline_name, description=self._timeline_name, user=user, index_name=self._index_name) # Grant the user read permission on the mapping object and set status. # If user is None the timeline becomes visible to all users. search_index.grant_permission(user=user, permission=u'read') # In case we have a user grant additional permissions. if user: search_index.grant_permission(user=user, permission=u'write') search_index.grant_permission(user=user, permission=u'delete') # Let the Timesketch UI know that the timeline is processing. search_index.set_status(u'processing') # Save the mapping object to the Timesketch database. db_session.add(search_index) db_session.commit() logging.info(u'Adding events to Timesketch.')
def run(self, username, password): """Creates the user.""" if not password: password = self.get_password_from_prompt() if not isinstance(password, six.text_type): password = codecs.decode(password, 'utf-8') username = codecs.decode(username, 'utf-8') user = User.get_or_create(username=username) user.set_password(plaintext=password) db_session.add(user) db_session.commit() sys.stdout.write('User {0:s} created/updated\n'.format(username))
def Close(self): """Closes the connection to TimeSketch Elasticsearch database. Sends the remaining events for indexing and removes the processing status on the Timesketch search index object. """ self._elastic.AddEvent(None, force_flush=True) with self._timesketch.app_context(): search_index = SearchIndex.query.filter_by( index_name=self._index_name).first() search_index.status.remove(search_index.status[0]) db_session.add(search_index) db_session.commit()
def Close(self): """Closes the connection to TimeSketch Elasticsearch database. Sends the remaining events for indexing and removes the processing status on the Timesketch search index object. """ super(TimesketchOutputModule, self).Close() with self._timesketch.app_context(): search_index = timesketch_sketch.SearchIndex.query.filter_by( index_name=self._index_name).first() search_index.status.remove(search_index.status[0]) timesketch_db_session.add(search_index) timesketch_db_session.commit()
def home(): """Generates the home page view template. Returns: Template with context. """ form = HiddenNameDescriptionForm() sketches = Sketch.all_with_acl().filter( not_(Sketch.Status.status == u'deleted'), Sketch.Status.parent).order_by(Sketch.updated_at.desc()) query_filter = request.args.get(u'filter', u'') query = request.args.get(u'q', u'') # Only render upload button if it is configured. upload_enabled = current_app.config[u'UPLOAD_ENABLED'] last_sketch = View.query.filter_by( user=current_user, name=u'').order_by( View.updated_at.desc()).first() if query_filter: if query_filter == u'user': sketches = sketches.filter(Sketch.user == current_user) elif query_filter == u'shared': sketches = sketches.filter(not_(Sketch.user == current_user)) # TODO: Figure out a better way to handle this. if query: if query.startswith(u'*'): query = u'' else: sketches = sketches.filter(Sketch.name.contains(query)).limit(100) # Handle form for creating a new sketch. if form.validate_on_submit(): sketch = Sketch( name=form.name.data, description=form.description.data, user=current_user) sketch.status.append(sketch.Status(user=None, status=u'new')) # Give the requesting user permissions on the new sketch. sketch.grant_permission(current_user, u'read') sketch.grant_permission(current_user, u'write') sketch.grant_permission(current_user, u'delete') db_session.add(sketch) db_session.commit() return redirect(url_for(u'sketch_views.overview', sketch_id=sketch.id)) return render_template( u'home/home.html', sketches=sketches, form=form, query=query, upload_enabled=upload_enabled, last_sketch=last_sketch)
def post(self): """Handles POST request to the resource. Returns: A view in JSON (instance of flask.wrappers.Response) Raises: ApiHTTPError """ UPLOAD_ENABLED = current_app.config[u'UPLOAD_ENABLED'] UPLOAD_FOLDER = current_app.config[u'UPLOAD_FOLDER'] form = UploadFileForm() if form.validate_on_submit() and UPLOAD_ENABLED: from timesketch.lib.tasks import run_plaso file_storage = form.file.data timeline_name = form.name.data # We do not need a human readable filename or # datastore index name, so we use UUIDs here. filename = unicode(uuid.uuid4().hex) index_name = unicode(uuid.uuid4().hex) file_path = os.path.join(UPLOAD_FOLDER, filename) file_storage.save(file_path) search_index = SearchIndex.get_or_create( name=timeline_name, description=timeline_name, user=current_user, index_name=index_name) search_index.grant_permission(permission=u'read', user=current_user) search_index.grant_permission( permission=u'write', user=current_user) search_index.grant_permission( permission=u'delete', user=current_user) search_index.set_status(u'processing') db_session.add(search_index) db_session.commit() run_plaso.apply_async( (file_path, timeline_name, index_name), task_id=index_name) return self.to_json( search_index, status_code=HTTP_STATUS_CODE_CREATED) else: raise ApiHTTPError( message=form.errors[u'file'][0], status_code=HTTP_STATUS_CODE_BAD_REQUEST)
def timelines(sketch_id): """Generates the sketch explore view template. Returns: Template with context. """ TIMELINES_TO_SHOW = 20 sketch = Sketch.query.get_with_acl(sketch_id) searchindices_in_sketch = [t.searchindex.id for t in sketch.timelines] query = request.args.get(u'q', None) indices = SearchIndex.all_with_acl( current_user).order_by( desc(SearchIndex.created_at)).filter( not_(SearchIndex.id.in_(searchindices_in_sketch))) filtered = False if query: indices = indices.filter(SearchIndex.name.contains(query)).limit(500) filtered = True if not filtered: indices = indices.limit(TIMELINES_TO_SHOW) # Setup the form form = AddTimelineForm() form.timelines.choices = set((i.id, i.name) for i in indices.all()) # Create new timeline form POST if form.validate_on_submit(): if not sketch.has_permission(current_user, u'write'): abort(HTTP_STATUS_CODE_FORBIDDEN) for searchindex_id in form.timelines.data: searchindex = SearchIndex.query.get_with_acl(searchindex_id) if searchindex not in [t.searchindex for t in sketch.timelines]: _timeline = Timeline( name=searchindex.name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex) db_session.add(_timeline) sketch.timelines.append(_timeline) db_session.commit() return redirect(url_for(u'sketch_views.timelines', sketch_id=sketch.id)) return render_template( u'sketch/timelines.html', sketch=sketch, timelines=indices.all(), form=form, filtered=filtered)
def explore(sketch_id, view_id=None): """Generates the sketch explore view template. Returns: Template with context. """ sketch = Sketch.query.get_with_acl(sketch_id) sketch_timelines = [t.searchindex.index_name for t in sketch.timelines] view_form = SaveViewForm() # Get parameters from the GET query url_query = request.args.get(u'q', u'') url_time_start = request.args.get(u'time_start', None) url_time_end = request.args.get(u'time_end', None) if view_id: view = View.query.get(view_id) # Check that this view belongs to the sketch if view.sketch_id != sketch.id: abort(HTTP_STATUS_CODE_NOT_FOUND) # Return 404 if view is deleted if view.get_status.status == u'deleted': return abort(HTTP_STATUS_CODE_NOT_FOUND) else: view = sketch.get_user_view(current_user) if url_query: view.query_string = url_query query_filter = json.loads(view.query_filter) query_filter[u'time_start'] = url_time_start query_filter[u'time_end'] = url_time_end view.query_filter = json.dumps(query_filter, ensure_ascii=False) if not view: query_filter = dict(indices=sketch_timelines) view = View( user=current_user, name=u'', sketch=sketch, query_string=u'', query_filter=json.dumps(query_filter, ensure_ascii=False)) db_session.add(view) db_session.commit() return render_template( u'sketch/explore.html', sketch=sketch, view=view, timelines=sketch_timelines, view_form=view_form)
def post(self, sketch_id): """Handles POST request to the resource. Args: sketch_id: Integer primary key for a sketch database model Returns: A view in JSON (instance of flask.wrappers.Response) """ form = StoryForm.build(request) if form.validate_on_submit(): sketch = Sketch.query.get_with_acl(sketch_id) story = Story( title=u'', content=u'', sketch=sketch, user=current_user) db_session.add(story) db_session.commit() return self.to_json(story, status_code=HTTP_STATUS_CODE_CREATED) return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def post(self): """Handles POST request to the resource. Returns: A sketch in JSON (instance of flask.wrappers.Response) """ form = NameDescriptionForm.build(request) if form.validate_on_submit(): sketch = Sketch( name=form.name.data, description=form.description.data, user=current_user) sketch.status.append(sketch.Status(user=None, status=u'new')) # Give the requesting user permissions on the new sketch. sketch.grant_permission(permission=u'read', user=current_user) sketch.grant_permission(permission=u'write', user=current_user) sketch.grant_permission(permission=u'delete', user=current_user) db_session.add(sketch) db_session.commit() return self.to_json(sketch, status_code=HTTP_STATUS_CODE_CREATED) return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def post(self, sketch_id): """Handles POST request to the resource. Args: sketch_id: Integer primary key for a sketch database model Returns: A view in JSON (instance of flask.wrappers.Response) """ form = SaveViewForm.build(request) if form.validate_on_submit(): sketch = Sketch.query.get_with_acl(sketch_id) view = View( name=form.name.data, sketch=sketch, user=current_user, query_string=form.query.data, query_filter=json.dumps(form.filter.data, ensure_ascii=False)) db_session.add(view) db_session.commit() return self.to_json(view, status_code=HTTP_STATUS_CODE_CREATED) return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def WriteHeader(self): """Setup the Elasticsearch index and the Timesketch database object. Creates the Elasticsearch index with Timesketch specific settings and the Timesketch SearchIndex database object. """ # This cannot be static because we use the value of self._doc_type from # arguments. _document_mapping = { self._doc_type: { u'_timestamp': { u'enabled': True, u'path': u'datetime', u'format': u'date_time_no_millis' }, u'properties': {u'timesketch_label': {u'type': u'nested'}} } } if not self._elastic_db.client.indices.exists(self._index_name): try: self._elastic_db.client.indices.create( index=self._index_name, body={u'mappings': _document_mapping}) except elastic_exceptions.ConnectionError as exception: logging.error(( u'Unable to proceed, cannot connect to Timesketch backend ' u'with error: {0:s}.\nPlease verify connection.').format(exception)) raise RuntimeError(u'Unable to connect to Timesketch backend.') with self._timesketch.app_context(): search_index = SearchIndex.get_or_create( name=self._timeline_name, description=self._timeline_name, user=None, index_name=self._index_name) # Grant all users read permission on the mapping object and set status. search_index.grant_permission(None, u'read') search_index.set_status(u'processing') # Save the mapping object to the Timesketch database. db_session.add(search_index) db_session.commit() logging.info(u'Adding events to Timesketch..')
def Close(self): """Closes the connection to TimeSketch Elasticsearch database. Sends the remaining events for indexing and adds the timeline to Timesketch. """ self._FlushEventsToElasticsearch() with self._timesketch.app_context(): # Get Timesketch user object, or None if user do not exist. This is a # SQLAlchemy query against the Timesketch database. user_query = User.query.filter_by(username=self._timeline_owner) user = user_query.first() search_index = SearchIndex( name=self._timeline_name, description=self._timeline_name, user=user, index_name=self._index_name) # Grant all users read permission on the mapping object. search_index.grant_permission(None, u'read') # Save the mapping object to the Timesketch database. db_session.add(search_index) db_session.commit()
def post(self, sketch_id): """Handles POST request to the resource. Returns: A sketch in JSON (instance of flask.wrappers.Response) Raises: ApiHTTPError """ sketch = Sketch.query.get_with_acl(sketch_id) searchindices_in_sketch = [t.searchindex.id for t in sketch.timelines] indices = SearchIndex.all_with_acl( current_user).order_by( desc(SearchIndex.created_at)).filter( not_(SearchIndex.id.in_(searchindices_in_sketch))) add_timeline_form = AddTimelineForm.build(request) add_timeline_form.timelines.choices = set( (i.id, i.name) for i in indices.all()) if add_timeline_form.validate_on_submit(): if not sketch.has_permission(current_user, u'write'): abort(HTTP_STATUS_CODE_FORBIDDEN) for searchindex_id in add_timeline_form.timelines.data: searchindex = SearchIndex.query.get_with_acl(searchindex_id) if searchindex not in [t.searchindex for t in sketch.timelines]: _timeline = Timeline( name=searchindex.name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex) db_session.add(_timeline) sketch.timelines.append(_timeline) db_session.commit() return self.to_json(sketch, status_code=HTTP_STATUS_CODE_CREATED) else: raise ApiHTTPError( message=add_timeline_form.errors, status_code=HTTP_STATUS_CODE_BAD_REQUEST)
def run(self, name, index, username): """Create the SearchIndex.""" es = ElasticsearchDataStore( host=current_app.config['ELASTIC_HOST'], port=current_app.config['ELASTIC_PORT']) user = User.query.filter_by(username=username).first() if not user: sys.stderr.write('User does not exist\n') sys.exit(1) if not es.client.indices.exists(index=index): sys.stderr.write('Index does not exist in the datastore\n') sys.exit(1) if SearchIndex.query.filter_by(name=name, index_name=index).first(): sys.stderr.write( 'Index with this name already exist in Timesketch\n') sys.exit(1) searchindex = SearchIndex( name=name, description=name, user=user, index_name=index) searchindex.grant_permission('read') db_session.add(searchindex) db_session.commit() sys.stdout.write('Search index {0:s} created\n'.format(name))
def add_comment(self, comment): """Add comment to event. Args: comment: Comment string. Raises: RuntimeError: if no sketch is present. """ if not self.sketch: raise RuntimeError('No sketch provided.') searchindex = SearchIndex.query.filter_by( index_name=self.index_name).first() db_event = SQLEvent.get_or_create( sketch=self.sketch.sql_sketch, searchindex=searchindex, document_id=self.event_id) comment = SQLEvent.Comment(comment=comment, user=None) db_event.comments.append(comment) db_session.add(db_event) db_session.commit() self.add_label(label='__ts_comment')
def explore(sketch_id, view_id=None): """Generates the sketch explore view template. Returns: Template with context. """ sketch = Sketch.query.get_with_acl(sketch_id) sketch_timelines = [t.searchindex.index_name for t in sketch.timelines] if view_id: view = View.query.get(view_id) else: view = sketch.get_user_view(current_user) if not view: query_filter = dict(indices=sketch_timelines) view = View( user=current_user, name=u'', sketch=sketch, query_string=u'', query_filter=json.dumps(query_filter, ensure_ascii=False)) db_session.add(view) db_session.commit() view_form = SaveViewForm() return render_template( u'sketch/explore.html', sketch=sketch, view=view, timelines=sketch_timelines, view_form=view_form)
def setup_sketch(timeline_name, index_name, username, sketch_id=None): """Use existing sketch or create a new sketch. Args: timeline_name: (str) Name of the Timeline index_name: (str) Name of the index username: (str) Who should own the timeline sketch_id: (str) Optional sketch_id to add timeline to Returns: (tuple) sketch ID and timeline ID as integers """ with app.app_context(): user = User.get_or_create(username=username) sketch = None if sketch_id: try: sketch = Sketch.query.get_with_acl(sketch_id, user=user) logger.info('Using existing sketch: {} ({})'.format( sketch.name, sketch.id)) except Forbidden: pass if not (sketch or sketch_id): # Create a new sketch. sketch_name = 'Turbinia: {}'.format(timeline_name) sketch = Sketch(name=sketch_name, description=sketch_name, user=user) # Need to commit here to be able to set permissions later. db_session.add(sketch) db_session.commit() sketch.grant_permission(permission='read', user=user) sketch.grant_permission(permission='write', user=user) sketch.grant_permission(permission='delete', user=user) sketch.status.append(sketch.Status(user=None, status='new')) db_session.add(sketch) db_session.commit() logger.info('Created new sketch: {} ({})'.format( sketch.name, sketch.id)) searchindex = SearchIndex.get_or_create( name=timeline_name, description='Created by Turbinia.', user=user, index_name=index_name) searchindex.grant_permission(permission='read', user=user) searchindex.grant_permission(permission='write', user=user) searchindex.grant_permission(permission='delete', user=user) searchindex.set_status('processing') db_session.add(searchindex) db_session.commit() timeline = Timeline(name=searchindex.name, description=searchindex.description, sketch=sketch, user=user, searchindex=searchindex) # If the user doesn't have write access to the sketch then create the # timeline but don't attach it to the sketch. if not sketch.has_permission(user, 'write'): timeline.sketch = None else: sketch.timelines.append(timeline) db_session.add(timeline) db_session.commit() timeline.set_status('processing') return sketch.id, timeline.id
def post(self, sketch_id, view_id): """Handles POST request to the resource. Args: sketch_id: Integer primary key for a sketch database model view_id: Integer primary key for a view database model Returns: A view in JSON (instance of flask.wrappers.Response) """ form = forms.SaveViewForm.build(request) if not form.validate_on_submit(): abort(HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to update view, not able to validate form data') sketch = Sketch.query.get_with_acl(sketch_id) if not sketch: abort(HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.') if not sketch.has_permission(current_user, 'write'): abort(HTTP_STATUS_CODE_FORBIDDEN, 'User does not have write access controls on sketch.') view = View.query.get(view_id) if not view: abort(HTTP_STATUS_CODE_NOT_FOUND, 'No view found with this ID.') if view.sketch.id != sketch.id: abort(HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to update view, view not attached to sketch.') view.query_string = form.query.data description = form.description.data if description: view.description = description query_filter = form.filter.data # Stripping potential pagination from views before saving it. if 'from' in query_filter: del query_filter['from'] view.query_filter = json.dumps(query_filter, ensure_ascii=False) view.query_dsl = json.dumps(form.dsl.data, ensure_ascii=False) name = form.name.data if name: view.name = name view.user = current_user view.sketch = sketch if form.dsl.data: view.query_string = '' db_session.add(view) db_session.commit() # Update the last activity of a sketch. utils.update_sketch_last_activity(sketch) return self.to_json(view, status_code=HTTP_STATUS_CODE_CREATED)
def create_view_from_form(sketch, form): """Creates a view from form data. Args: sketch: Instance of timesketch.models.sketch.Sketch form: Instance of timesketch.lib.forms.SaveViewForm Returns: A view (Instance of timesketch.models.sketch.View) """ # Default to user supplied data view_name = form.name.data query_string = form.query.data query_filter_dict = form.filter.data # Stripping potential pagination from views before saving it. if 'from' in query_filter_dict: del query_filter_dict['from'] query_filter = json.dumps(query_filter_dict, ensure_ascii=False) query_dsl = json.dumps(form.dsl.data, ensure_ascii=False) if isinstance(query_filter, tuple): query_filter = query_filter[0] # No search template by default (before we know if the user want to # create a template or use an existing template when creating the view) searchtemplate = None # Create view from a search template if form.from_searchtemplate_id.data: # Get the template from the datastore template_id = form.from_searchtemplate_id.data searchtemplate = SearchTemplate.query.get(template_id) # Copy values from the template view_name = searchtemplate.name query_string = searchtemplate.query_string query_filter = searchtemplate.query_filter query_dsl = searchtemplate.query_dsl # WTF form returns a tuple for the filter. This is not # compatible with SQLAlchemy. if isinstance(query_filter, tuple): query_filter = query_filter[0] if not view_name: abort(HTTP_STATUS_CODE_BAD_REQUEST, 'View name is missing.') # Create a new search template based on this view (only if requested by # the user). if form.new_searchtemplate.data: query_filter_dict = json.loads(query_filter) if query_filter_dict.get('indices', None): query_filter_dict['indices'] = '_all' query_filter = json.dumps(query_filter_dict, ensure_ascii=False) searchtemplate = SearchTemplate(name=view_name, user=current_user, query_string=query_string, query_filter=query_filter, query_dsl=query_dsl) db_session.add(searchtemplate) db_session.commit() # Create the view in the database view = View(name=view_name, sketch=sketch, user=current_user, query_string=query_string, query_filter=query_filter, query_dsl=query_dsl, searchtemplate=searchtemplate) db_session.add(view) db_session.commit() return view
def _upload_and_index(self, file_extension, timeline_name, index_name, sketch, enable_stream, data_label='', file_path='', events='', meta=None): """Creates a full pipeline for an uploaded file and returns the results. Args: file_extension: the extension of the uploaded file. timeline_name: name the timeline will be stored under in the datastore. index_name: the Elastic index name for the timeline. sketch: Instance of timesketch.models.sketch.Sketch enable_stream: boolean indicating whether this is file is part of a stream or not. data_label: Optional string with a data label for the search index. file_path: the path to the file to be uploaded (optional). events: a string with events to upload (optional). meta: optional dict with additional meta fields that will be included in the return. Returns: A timeline if created otherwise a search index in JSON (instance of flask.wrappers.Response) """ searchindex = self._get_index(name=timeline_name, description=timeline_name, sketch=sketch, index_name=index_name, data_label=data_label, extension=file_extension) searchindex.set_status('processing') timelines = Timeline.query.filter_by(name=timeline_name, sketch=sketch).all() timeline = None for timeline_ in timelines: if timeline_.searchindex.index_name == searchindex.index_name: timeline = timeline_ break abort( HTTP_STATUS_CODE_BAD_REQUEST, 'There is a timeline in the sketch that has the same name ' 'but is stored in a different index, check the data_label ' 'on the uploaded data') if not timeline: timeline = Timeline.get_or_create(name=timeline_name, description=timeline_name, sketch=sketch, user=current_user, searchindex=searchindex) if not timeline: abort(HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to get or create a new Timeline object.') timeline.set_status('processing') sketch.timelines.append(timeline) labels_to_prevent_deletion = current_app.config.get( 'LABELS_TO_PREVENT_DELETION', []) for sketch_label in sketch.get_labels: if sketch_label not in labels_to_prevent_deletion: continue timeline.add_label(sketch_label) searchindex.add_label(sketch_label) db_session.add(timeline) db_session.commit() sketch_id = sketch.id # Start Celery pipeline for indexing and analysis. # Import here to avoid circular imports. # pylint: disable=import-outside-toplevel from timesketch.lib import tasks pipeline = tasks.build_index_pipeline( file_path=file_path, events=events, timeline_name=timeline_name, index_name=searchindex.index_name, file_extension=file_extension, sketch_id=sketch_id, only_index=enable_stream, timeline_id=timeline.id) pipeline.apply_async() return self.to_json(timeline, status_code=HTTP_STATUS_CODE_CREATED, meta=meta)
def explore(sketch_id, view_id=None, searchtemplate_id=None): """Generates the sketch explore view template. Returns: Template with context. """ save_view = False # If the view should be saved to the database. sketch = Sketch.query.get_with_acl(sketch_id) sketch_timelines = [t.searchindex.index_name for t in sketch.timelines] view_form = SaveViewForm() graphs_enabled = current_app.config['GRAPH_BACKEND_ENABLED'] similarity_enabled = current_app.config.get('ENABLE_EXPERIMENTAL_UI') # Get parameters from the GET query url_query = request.args.get('q', '') url_time_start = request.args.get('time_start', None) url_time_end = request.args.get('time_end', None) url_index = request.args.get('index', None) url_size = request.args.get('size', None) if searchtemplate_id: searchtemplate = SearchTemplate.query.get(searchtemplate_id) view = sketch.get_user_view(current_user) if not view: view = View(user=current_user, name='', sketch=sketch) view.query_string = searchtemplate.query_string view.query_filter = searchtemplate.query_filter view.query_dsl = searchtemplate.query_dsl save_view = True elif view_id: view = View.query.get(view_id) # Check that this view belongs to the sketch if view.sketch_id != sketch.id: abort(HTTP_STATUS_CODE_NOT_FOUND) # Return 404 if view is deleted if view.get_status.status == 'deleted': return abort(HTTP_STATUS_CODE_NOT_FOUND) else: view = sketch.get_user_view(current_user) if not view: view = View( user=current_user, name='', sketch=sketch, query_string='*') view.query_filter = view.validate_filter( dict(indices=sketch_timelines)) save_view = True if url_query: view.query_string = url_query query_filter = json.loads(view.query_filter) query_filter['from'] = 0 # if we loaded from get, start at first event query_filter['time_start'] = url_time_start query_filter['time_end'] = url_time_end if url_index in sketch_timelines: query_filter['indices'] = [url_index] if url_size: query_filter['size'] = url_size view.query_filter = view.validate_filter(query_filter) view.query_dsl = None save_view = True if save_view: db_session.add(view) db_session.commit() return render_template( 'sketch/explore.html', sketch=sketch, view=view, named_view=view_id, timelines=sketch_timelines, view_form=view_form, searchtemplate_id=searchtemplate_id, graphs_enabled=graphs_enabled, similarity_enabled=similarity_enabled)
def post(self, sketch_id, timeline_id): """Handles GET request to the resource. Args: sketch_id: Integer primary key for a sketch database model timeline_id: Integer primary key for a timeline database model """ sketch = Sketch.query.get_with_acl(sketch_id) if not sketch: abort( HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.') timeline = Timeline.query.get(timeline_id) if not timeline: abort( HTTP_STATUS_CODE_NOT_FOUND, 'No timeline found with this ID.') # Check that this timeline belongs to the sketch if timeline.sketch_id != sketch.id: abort( HTTP_STATUS_CODE_NOT_FOUND, 'The sketch ID ({0:d}) does not match with the timeline ' 'sketch ID ({1:d})'.format(sketch.id, timeline.sketch_id)) if not sketch.has_permission(user=current_user, permission='write'): abort( HTTP_STATUS_CODE_FORBIDDEN, 'The user does not have write permission on the sketch.') form = forms.TimelineForm.build(request) if not form.validate_on_submit(): abort( HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to validate form data.') if form.labels.data: label_string = form.labels.data labels = json.loads(label_string) if not isinstance(labels, (list, tuple)): abort( HTTP_STATUS_CODE_BAD_REQUEST, ( 'Label needs to be a JSON string that ' 'converts to a list of strings.')) if not all([isinstance(x, str) for x in labels]): abort( HTTP_STATUS_CODE_BAD_REQUEST, ( 'Label needs to be a JSON string that ' 'converts to a list of strings (not all strings)')) label_action = form.label_action.data if label_action not in ('add', 'remove'): abort( HTTP_STATUS_CODE_BAD_REQUEST, 'Label action needs to be either add or remove.') changed = False if label_action == 'add': changes = [] for label in labels: changes.append( self._add_label(timeline=timeline, label=label)) changed = any(changes) elif label_action == 'remove': if not sketch.has_permission( user=current_user, permission='delete'): abort( HTTP_STATUS_CODE_FORBIDDEN, 'The user does not have delete permission on sketch.') changes = [] for label in labels: changes.append( self._remove_label(timeline=timeline, label=label)) changed = any(changes) if not changed: abort( HTTP_STATUS_CODE_BAD_REQUEST, 'Label [{0:s}] not {1:s}'.format( ', '.join(labels), label_action)) db_session.add(timeline) db_session.commit() return HTTP_STATUS_CODE_OK timeline.name = form.name.data timeline.description = form.description.data timeline.color = form.color.data db_session.add(timeline) db_session.commit() # Update the last activity of a sketch. utils.update_sketch_last_activity(sketch) return HTTP_STATUS_CODE_OK
def post(self, sketch_id): """Handles POST request to the resource. Handler for /api/v1/sketches/:sketch_id/explore/ Args: sketch_id: Integer primary key for a sketch database model Returns: JSON with list of matched events """ sketch = Sketch.query.get_with_acl(sketch_id) form = ExploreForm.build(request) if form.validate_on_submit(): query_dsl = form.dsl.data query_filter = form.filter.data sketch_indices = { t.searchindex.index_name for t in sketch.timelines } indices = query_filter.get(u'indices', sketch_indices) # If _all in indices then execute the query on all indices if u'_all' in indices: indices = sketch_indices # Make sure that the indices in the filter are part of the sketch. # This will also remove any deleted timeline from the search result. indices = get_validated_indices(indices, sketch_indices) # Make sure we have a query string or star filter if not (form.query.data, query_filter.get(u'star'), query_filter.get(u'events'), query_dsl): abort(HTTP_STATUS_CODE_BAD_REQUEST) result = self.datastore.search(sketch_id, form.query.data, query_filter, query_dsl, indices, aggregations=None, return_results=True) # Get labels for each event that matches the sketch. # Remove all other labels. for event in result[u'hits'][u'hits']: event[u'selected'] = False event[u'_source'][u'label'] = [] try: for label in event[u'_source'][u'timesketch_label']: if sketch.id != label[u'sketch_id']: continue event[u'_source'][u'label'].append(label[u'name']) del event[u'_source'][u'timesketch_label'] except KeyError: pass # Update or create user state view. This is used in the UI to let # the user get back to the last state in the explore view. view = View.get_or_create(user=current_user, sketch=sketch, name=u'') view.query_string = form.query.data view.query_filter = json.dumps(query_filter, ensure_ascii=False) view.query_dsl = json.dumps(query_dsl, ensure_ascii=False) db_session.add(view) db_session.commit() # Add metadata for the query result. This is used by the UI to # render the event correctly and to display timing and hit count # information. tl_colors = {} tl_names = {} for timeline in sketch.timelines: tl_colors[timeline.searchindex.index_name] = timeline.color tl_names[timeline.searchindex.index_name] = timeline.name meta = { u'es_time': result[u'took'], u'es_total_count': result[u'hits'][u'total'], u'timeline_colors': tl_colors, u'timeline_names': tl_names, } schema = {u'meta': meta, u'objects': result[u'hits'][u'hits']} return jsonify(schema) return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def post(self, sketch_id): """Handles POST request to the resource. Handler for /api/v1/sketches/:sketch_id/event/create/ Args: sketch_id: Integer primary key for a sketch database model Returns: An annotation in JSON (instance of flask.wrappers.Response) """ sketch = Sketch.query.get_with_acl(sketch_id) if not sketch: abort(HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.') if not sketch.has_permission(current_user, 'write'): abort(HTTP_STATUS_CODE_FORBIDDEN, 'User does not have write access controls on sketch.') form = request.json if not form: form = request.data timeline_name = 'sketch specific timeline' index_name_seed = 'timesketch_{0:d}'.format(sketch_id) event_type = 'user_created_event' date_string = form.get('date_string') if not date_string: date = datetime.datetime.utcnow().isoformat() else: # derive datetime from timestamp: try: date = dateutil.parser.parse(date_string) except (dateutil.parser.ParserError, OverflowError) as e: logger.error('Unable to convert date string', exc_info=True) abort( HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to add event, not able to convert the date ' 'string. Was it properly formatted? Error: ' '{0!s}'.format(e)) timestamp = int(time.mktime(date.utctimetuple())) * 1000000 timestamp += date.microsecond event = { 'datetime': date_string, 'timestamp': timestamp, 'timestamp_desc': form.get('timestamp_desc', 'Event Happened'), 'message': form.get('message', 'No message string'), } attributes = form.get('attributes', {}) if not isinstance(attributes, dict): abort( HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to add an event where the attributes are not a ' 'dict object.') event.update(attributes) tag = form.get('tag', []) if not isinstance(tag, list): abort( HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to add an event where the tags are not a ' 'list of strings.') if tag and any(not isinstance(x, str) for x in tag): abort( HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to add an event where the tags are not a ' 'list of strings.') event['tag'] = tag # We do not need a human readable filename or # datastore index name, so we use UUIDs here. index_name = hashlib.md5(index_name_seed.encode()).hexdigest() if six.PY2: index_name = codecs.decode(index_name, 'utf-8') # Try to create index timeline = None try: # Create the index in Elasticsearch (unless it already exists) self.datastore.create_index(index_name=index_name, doc_type=event_type) # Create the search index in the Timesketch database searchindex = SearchIndex.get_or_create( name=timeline_name, description='internal timeline for user-created events', user=current_user, index_name=index_name) searchindex.grant_permission(permission='read', user=current_user) searchindex.grant_permission(permission='write', user=current_user) searchindex.grant_permission(permission='delete', user=current_user) searchindex.set_status('ready') db_session.add(searchindex) db_session.commit() if sketch and sketch.has_permission(current_user, 'write'): self.datastore.import_event(index_name, event_type, event, flush_interval=1) timeline = Timeline.get_or_create( name=searchindex.name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex) if timeline not in sketch.timelines: sketch.timelines.append(timeline) timeline.set_status('ready') db_session.add(timeline) db_session.commit() # TODO: Can this be narrowed down, both in terms of the scope it # applies to, as well as not to catch a generic exception. except Exception as e: # pylint: disable=broad-except abort(HTTP_STATUS_CODE_BAD_REQUEST, 'Failed to add event ({0!s})'.format(e)) # Return Timeline if it was created. # pylint: disable=no-else-return if timeline: return self.to_json(timeline, status_code=HTTP_STATUS_CODE_CREATED) return self.to_json(searchindex, status_code=HTTP_STATUS_CODE_CREATED)
def _upload_and_index(self, file_extension, timeline_name, index_name, sketch, enable_stream, file_path='', events='', meta=None): """Creates a full pipeline for an uploaded file and returns the results. Args: file_extension: the extension of the uploaded file. timeline_name: name the timeline will be stored under in the datastore. index_name: the Elastic index name for the timeline. sketch: Instance of timesketch.models.sketch.Sketch enable_stream: boolean indicating whether this is file is part of a stream or not. file_path: the path to the file to be uploaded (optional). events: a string with events to upload (optional). meta: optional dict with additional meta fields that will be included in the return. Returns: A timeline if created otherwise a search index in JSON (instance of flask.wrappers.Response) """ # Check if search index already exists. searchindex = SearchIndex.query.filter_by( name=timeline_name, user=current_user, index_name=index_name).first() timeline = None if searchindex: searchindex.set_status('processing') timeline = Timeline.query.filter_by( name=searchindex.name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex).first() else: # Create the search index in the Timesketch database searchindex = SearchIndex.get_or_create(name=timeline_name, description='', user=current_user, index_name=index_name) searchindex.grant_permission(permission='read', user=current_user) searchindex.grant_permission(permission='write', user=current_user) searchindex.grant_permission(permission='delete', user=current_user) searchindex.set_status('processing') db_session.add(searchindex) db_session.commit() if sketch and sketch.has_permission(current_user, 'write'): labels_to_prevent_deletion = current_app.config.get( 'LABELS_TO_PREVENT_DELETION', []) timeline = Timeline(name=searchindex.name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex) timeline.set_status('processing') sketch.timelines.append(timeline) for label in sketch.get_labels: if label not in labels_to_prevent_deletion: continue timeline.add_label(label) searchindex.add_label(label) db_session.add(timeline) db_session.commit() # Start Celery pipeline for indexing and analysis. # Import here to avoid circular imports. # pylint: disable=import-outside-toplevel from timesketch.lib import tasks pipeline = tasks.build_index_pipeline(file_path=file_path, events=events, timeline_name=timeline_name, index_name=index_name, file_extension=file_extension, sketch_id=sketch.id, only_index=enable_stream) pipeline.apply_async() # Return Timeline if it was created. # pylint: disable=no-else-return if timeline: return self.to_json(timeline, status_code=HTTP_STATUS_CODE_CREATED, meta=meta) return self.to_json(searchindex, status_code=HTTP_STATUS_CODE_CREATED, meta=meta)
def _get_index(self, name, description, sketch, index_name='', data_label='', extension=''): """Returns a SearchIndex object to be used for uploads. Args: name: the name of the searchindex. description: the description of the searchindex. sketch: sketch object (instance of Sketch). index_name: optional index name, if supplied and if it exists then the index associated with that will be returned. data_label: optional label of the data, if supplied will be used to determine whether an already existing index can be used or a new one created. extension: optional file extension if a file is being uploaded, if supplied and no data label used, then the extension will be used as a data label. Returns: A SearchIndex object. """ if index_name: if not isinstance(index_name, str): index_name = codecs.decode(index_name, 'utf-8') searchindex = SearchIndex.query.filter_by( name=name, index_name=index_name).first() if searchindex and searchindex.has_permission(permission='write', user=current_user): return searchindex if extension and not data_label: data_label = extension if not data_label: data_label = 'generic' # Since CSV and JSON are basically the same label, we combine it here. if data_label in ('csv', 'json', 'jsonl'): data_label = 'csv_jsonl' indices = [t.searchindex for t in sketch.active_timelines] for index in indices: if index.has_label(data_label) and sketch.has_permission( permission='write', user=current_user): return index index_name = index_name or uuid.uuid4().hex searchindex = SearchIndex.get_or_create(name=name, index_name=index_name, description=description, user=current_user) searchindex.grant_permission(permission='read', user=current_user) searchindex.grant_permission(permission='write', user=current_user) searchindex.grant_permission(permission='delete', user=current_user) searchindex.set_status('processing') db_session.add(searchindex) db_session.commit() searchindex.add_label(data_label, user=current_user) return searchindex
def _upload_and_index(self, file_extension, timeline_name, index_name, sketch, form, enable_stream, original_filename='', data_label='', file_path='', events='', meta=None): """Creates a full pipeline for an uploaded file and returns the results. Args: file_extension: the extension of the uploaded file. timeline_name: name the timeline will be stored under in the datastore. index_name: the Elastic index name for the timeline. sketch: Instance of timesketch.models.sketch.Sketch form: a dict with the configuration for the upload. enable_stream: boolean indicating whether this is file is part of a stream or not. original_filename: Original filename from the upload. data_label: Optional string with a data label for the search index. file_path: the path to the file to be uploaded (optional). events: a string with events to upload (optional). meta: optional dict with additional meta fields that will be included in the return. Returns: A timeline if created otherwise a search index in JSON (instance of flask.wrappers.Response) """ searchindex = self._get_index(name=timeline_name, description=timeline_name, sketch=sketch, index_name=index_name, data_label=data_label, extension=file_extension) if not searchindex: abort( HTTP_STATUS_CODE_BAD_REQUEST, 'We were unable to acquire a searchindex and therefore not ' 'able to upload data, please try again. If this error persist ' 'please create an issue on Github: https://github.com/' 'google/timesketch/issues/new/choose') searchindex.set_status('processing') timelines = Timeline.query.filter_by(name=timeline_name, sketch=sketch).all() timeline = None for timeline_ in timelines: if timeline_.searchindex.index_name == searchindex.index_name: timeline = timeline_ break abort( HTTP_STATUS_CODE_BAD_REQUEST, 'There is a timeline in the sketch that has the same name ' 'but is stored in a different index, check the data_label ' 'on the uploaded data') if not timeline: timeline = Timeline.get_or_create(name=timeline_name, description=timeline_name, sketch=sketch, user=current_user, searchindex=searchindex) if not timeline: abort(HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to get or create a new Timeline object.') timeline.set_status('processing') sketch.timelines.append(timeline) labels_to_prevent_deletion = current_app.config.get( 'LABELS_TO_PREVENT_DELETION', []) for sketch_label in sketch.get_labels: if sketch_label not in labels_to_prevent_deletion: continue timeline.add_label(sketch_label) searchindex.add_label(sketch_label) file_size = form.get('total_file_size', 0) datasource = DataSource(timeline=timeline, user=current_user, provider=form.get('provider', 'N/A'), context=form.get('context', 'N/A'), file_on_disk=file_path, file_size=int(file_size), original_filename=original_filename, data_label=data_label) timeline.datasources.append(datasource) db_session.add(datasource) db_session.add(timeline) db_session.commit() sketch_id = sketch.id # Start Celery pipeline for indexing and analysis. # Import here to avoid circular imports. # pylint: disable=import-outside-toplevel from timesketch.lib import tasks pipeline = tasks.build_index_pipeline( file_path=file_path, events=events, timeline_name=timeline_name, index_name=searchindex.index_name, file_extension=file_extension, sketch_id=sketch_id, only_index=enable_stream, timeline_id=timeline.id) task_id = uuid.uuid4().hex pipeline.apply_async(task_id=task_id) if meta is None: meta = {} meta['task_id'] = task_id return self.to_json(timeline, status_code=HTTP_STATUS_CODE_CREATED, meta=meta)
def post(self, sketch_id): """Handles POST request to the resource. Returns: A sketch in JSON (instance of flask.wrappers.Response) """ sketch = Sketch.query.get_with_acl(sketch_id) if not sketch: abort( HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.') if not sketch.has_permission(current_user, 'write'): abort(HTTP_STATUS_CODE_FORBIDDEN, 'User does not have write access controls on sketch.') form = forms.AddTimelineSimpleForm.build(request) metadata = {'created': True} searchindex_id = form.timeline.data searchindex = SearchIndex.query.get_with_acl(searchindex_id) if searchindex.get_status.status == 'deleted': abort( HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to create a timeline using a deleted search index') timeline_id = [ t.searchindex.id for t in sketch.timelines if t.searchindex.id == searchindex_id ] if not form.validate_on_submit(): abort( HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to validate form data.') if not sketch.has_permission(current_user, 'write'): abort( HTTP_STATUS_CODE_FORBIDDEN, 'User does not have write access to the sketch.') if not timeline_id: return_code = HTTP_STATUS_CODE_CREATED timeline = Timeline( name=searchindex.name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex) sketch.timelines.append(timeline) labels_to_prevent_deletion = current_app.config.get( 'LABELS_TO_PREVENT_DELETION', []) for label in sketch.get_labels: if label not in labels_to_prevent_deletion: continue timeline.add_label(label) searchindex.add_label(label) # Set status to ready so the timeline can be queried. timeline.set_status('ready') db_session.add(timeline) db_session.commit() else: metadata['created'] = False return_code = HTTP_STATUS_CODE_OK timeline = Timeline.query.get(timeline_id) # Run sketch analyzers when timeline is added. Import here to avoid # circular imports. # pylint: disable=import-outside-toplevel if current_app.config.get('AUTO_SKETCH_ANALYZERS'): # pylint: disable=import-outside-toplevel from timesketch.lib import tasks sketch_analyzer_group, _ = tasks.build_sketch_analysis_pipeline( sketch_id, searchindex_id, current_user.id) if sketch_analyzer_group: pipeline = (tasks.run_sketch_init.s( [searchindex.index_name]) | sketch_analyzer_group) pipeline.apply_async() # Update the last activity of a sketch. utils.update_sketch_last_activity(sketch) return self.to_json( timeline, meta=metadata, status_code=return_code)
def post(self, sketch_id): """Handles POST request to the resource. Returns: Graph in JSON (instance of flask.wrappers.Response) """ sketch = Sketch.query.get_with_acl(sketch_id) if not sketch: abort(HTTP_STATUS_CODE_NOT_FOUND, "No sketch found with this ID.") form = request.json plugin_name = form.get("plugin") graph_config = form.get("config") refresh = form.get("refresh") timeline_ids = form.get("timeline_ids", None) if timeline_ids and not isinstance(timeline_ids, (list, tuple)): abort( HTTP_STATUS_CODE_BAD_REQUEST, "Timeline IDs if supplied need to be a list.", ) if timeline_ids and not all([isinstance(x, int) for x in timeline_ids]): abort( HTTP_STATUS_CODE_BAD_REQUEST, "Timeline IDs needs to be a list of integers.", ) sketch_indices = [ timeline.searchindex.index_name for timeline in sketch.active_timelines ] cache = GraphCache.get_or_create(sketch=sketch, graph_plugin=plugin_name) if graph_config: cache_config = graph_config else: cache_config = cache.graph_config if isinstance(cache_config, str): cache_config = json.loads(cache_config) # Refresh cache if timelines have been added/removed from the sketch. if cache_config: cache_graph_filter = cache_config.get("filter", {}) cache_filter_indices = cache_graph_filter.get("indices", []) if set(sketch_indices) ^ set(cache_filter_indices): refresh = True if cache.graph_elements and not refresh: return self.to_json(cache) graph_class = manager.GraphManager.get_graph(plugin_name) graph = graph_class(sketch=sketch, timeline_ids=timeline_ids) cytoscape_json = graph.generate().to_cytoscape() if cytoscape_json: cache.graph_elements = json.dumps(cytoscape_json) cache.graph_config = json.dumps(graph_config) cache.update_modification_time() db_session.add(cache) db_session.commit() # Update the last activity of a sketch. utils.update_sketch_last_activity(sketch) return self.to_json(cache)
def run(self, file_path, sketch_id, username, timeline_name): """This is the run method.""" file_path = os.path.realpath(file_path) file_path_no_extension, extension = os.path.splitext(file_path) extension = extension.lstrip('.') filename = os.path.basename(file_path_no_extension) supported_extensions = ('plaso', 'csv', 'jsonl') if not os.path.isfile(file_path): sys.exit('No such file: {0:s}'.format(file_path)) if extension not in supported_extensions: sys.exit('Extension {0:s} is not supported. ' '(supported extensions are: {1:s})'.format( extension, ', '.join(supported_extensions))) user = None if not username: username = pwd.getpwuid(os.stat(file_path).st_uid).pw_name if not username == 'root': if not isinstance(username, six.text_type): username = codecs.decode(username, 'utf-8') user = User.query.filter_by(username=username).first() if not user: sys.exit('Cannot determine user for file: {0:s}'.format(file_path)) sketch = None # If filename starts with <number> then use that as sketch_id. # E.g: 42_file_name.plaso means sketch_id is 42. sketch_id_from_filename = filename.split('_')[0] if not sketch_id and sketch_id_from_filename.isdigit(): sketch_id = sketch_id_from_filename if sketch_id: try: sketch = Sketch.query.get_with_acl(sketch_id, user=user) except Forbidden: pass if not timeline_name: if not isinstance(timeline_name, six.text_type): timeline_name = codecs.decode(timeline_name, 'utf-8') timeline_name = timeline_name.replace('_', ' ') # Remove sketch ID if present in the filename. timeline_parts = timeline_name.split() if timeline_parts[0].isdigit(): timeline_name = ' '.join(timeline_name.split()[1:]) if not sketch: # Create a new sketch. sketch_name = 'Sketch for: {0:s}'.format(timeline_name) sketch = Sketch(name=sketch_name, description=sketch_name, user=user) # Need to commit here to be able to set permissions later. db_session.add(sketch) db_session.commit() sketch.grant_permission(permission='read', user=user) sketch.grant_permission(permission='write', user=user) sketch.grant_permission(permission='delete', user=user) sketch.status.append(sketch.Status(user=None, status='new')) db_session.add(sketch) db_session.commit() index_name = uuid.uuid4().hex if not isinstance(index_name, six.text_type): index_name = codecs.decode(index_name, 'utf-8') searchindex = SearchIndex.get_or_create(name=timeline_name, description=timeline_name, user=user, index_name=index_name) searchindex.grant_permission(permission='read', user=user) searchindex.grant_permission(permission='write', user=user) searchindex.grant_permission(permission='delete', user=user) searchindex.set_status('processing') db_session.add(searchindex) db_session.commit() if sketch and sketch.has_permission(user, 'write'): timeline = Timeline(name=searchindex.name, description=searchindex.description, sketch=sketch, user=user, searchindex=searchindex) timeline.set_status('processing') sketch.timelines.append(timeline) db_session.add(timeline) db_session.commit() # Start Celery pipeline for indexing and analysis. # Import here to avoid circular imports. from timesketch.lib import tasks pipeline = tasks.build_index_pipeline(file_path, timeline_name, index_name, extension, sketch.id) pipeline.apply_async(task_id=index_name) print('Imported {0:s} to sketch: {1:d} ({2:s})'.format( file_path, sketch.id, sketch.name))
def run(self, import_location, export_location): """Export/Import search templates to/from file. Args: import_location: Path to the yaml file to import templates. export_location: Path to the yaml file to export templates. """ if export_location: search_templates = [] for search_template in SearchTemplate.query.all(): labels = [] for label in search_template.labels: if label.label.startswith('supported_os:'): labels.append(label.label.replace('supported_os:', '')) search_templates.append({ 'name': search_template.name, 'query_string': search_template.query_string, 'query_dsl': search_template.query_dsl, 'supported_os': labels }) with open(export_location, 'w') as fh: yaml.safe_dump(search_templates, stream=fh) if import_location: try: with open(import_location, 'rb') as fh: search_templates = yaml.safe_load(fh) except IOError as e: sys.stdout.write('Unable to open file: {0!s}\n'.format(e)) sys.exit(1) for search_template in search_templates: name = search_template['name'] query_string = search_template['query_string'] query_dsl = search_template['query_dsl'] # Skip search template if already exits. if SearchTemplate.query.filter_by(name=name).first(): continue imported_template = SearchTemplate(name=name, user=User(None), query_string=query_string, query_dsl=query_dsl) # Add supported_os labels. for supported_os in search_template['supported_os']: label_name = 'supported_os:{0:s}'.format(supported_os) label = SearchTemplate.Label.get_or_create( label=label_name, user=None) imported_template.labels.append(label) # Set flag to identify local vs import templates. remote_flag = SearchTemplate.Label.get_or_create( label='remote_template', user=None) imported_template.labels.append(remote_flag) db_session.add(imported_template) db_session.commit()
def post(self, sketch_id): """Handles POST request to the resource. Args: sketch_id: Integer primary key for a sketch database model Returns: An annotation in JSON (instance of flask.wrappers.Response) """ form = EventAnnotationForm.build(request) if form.validate_on_submit(): annotations = [] sketch = Sketch.query.get_with_acl(sketch_id) indices = [t.searchindex.index_name for t in sketch.timelines] annotation_type = form.annotation_type.data events = form.events.raw_data for _event in events: searchindex_id = _event[u'_index'] searchindex = SearchIndex.query.filter_by( index_name=searchindex_id).first() event_id = _event[u'_id'] event_type = _event[u'_type'] if searchindex_id not in indices: abort(HTTP_STATUS_CODE_BAD_REQUEST) # Get or create an event in the SQL database to have something # to attach the annotation to. event = Event.get_or_create(sketch=sketch, searchindex=searchindex, document_id=event_id) # Add the annotation to the event object. if u'comment' in annotation_type: annotation = Event.Comment(comment=form.annotation.data, user=current_user) event.comments.append(annotation) self.datastore.set_label(searchindex_id, event_id, event_type, sketch.id, current_user.id, u'__ts_comment', toggle=False) elif u'label' in annotation_type: annotation = Event.Label.get_or_create( label=form.annotation.data, user=current_user) if annotation not in event.labels: event.labels.append(annotation) toggle = False if u'__ts_star' or u'__ts_hidden' in form.annotation.data: toggle = True self.datastore.set_label(searchindex_id, event_id, event_type, sketch.id, current_user.id, form.annotation.data, toggle=toggle) else: abort(HTTP_STATUS_CODE_BAD_REQUEST) annotations.append(annotation) # Save the event to the database db_session.add(event) db_session.commit() return self.to_json(annotations, status_code=HTTP_STATUS_CODE_CREATED) return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def create_group(group_name): """Create a group.""" group = Group.get_or_create(name=group_name) db_session.add(group) db_session.commit() print(f"Group created: {group_name}")
def create_view_from_form(sketch, form): """Creates a view from form data. Args: sketch: Instance of timesketch.models.sketch.Sketch form: Instance of timesketch.lib.forms.SaveViewForm Returns: A view (Instance of timesketch.models.sketch.View) """ # Default to user supplied data view_name = form.name.data query_string = form.query.data query_filter = json.dumps(form.filter.data, ensure_ascii=False), query_dsl = json.dumps(form.dsl.data, ensure_ascii=False) # WTF forms turns the filter into a tuple for some reason. # pylint: disable=redefined-variable-type if isinstance(query_filter, tuple): query_filter = query_filter[0] # No search template by default (before we know if the user want to # create a template or use an existing template when creating the view) searchtemplate = None # Create view from a search template if form.from_searchtemplate_id.data: # Get the template from the datastore template_id = form.from_searchtemplate_id.data searchtemplate = SearchTemplate.query.get(template_id) # Copy values from the template view_name = searchtemplate.name query_string = searchtemplate.query_string query_filter = searchtemplate.query_filter, query_dsl = searchtemplate.query_dsl # WTF form returns a tuple for the filter. This is not # compatible with SQLAlchemy. if isinstance(query_filter, tuple): query_filter = query_filter[0] # Create a new search template based on this view (only if requested by # the user). if form.new_searchtemplate.data: query_filter_dict = json.loads(query_filter) if query_filter_dict.get(u'indices', None): query_filter_dict[u'indices'] = u'_all' # pylint: disable=redefined-variable-type query_filter = json.dumps(query_filter_dict, ensure_ascii=False) searchtemplate = SearchTemplate(name=view_name, user=current_user, query_string=query_string, query_filter=query_filter, query_dsl=query_dsl) db_session.add(searchtemplate) db_session.commit() # Create the view in the database view = View(name=view_name, sketch=sketch, user=current_user, query_string=query_string, query_filter=query_filter, query_dsl=query_dsl, searchtemplate=searchtemplate) db_session.add(view) db_session.commit() return view
def commit(self): """Commit changes to DB.""" self.group.orientation = self._orientation self.group.parameters = self._parameters db_session.add(self.group) db_session.commit()
def build_sketch_analysis_pipeline(sketch_id, searchindex_id, user_id, analyzer_names=None, analyzer_kwargs=None): """Build a pipeline for sketch analysis. If no analyzer_names is passed in then we assume auto analyzers should be run and get this list from the configuration. Parameters to the analyzers can be passed in to this function, otherwise they will be taken from the configuration. Either default kwargs for auto analyzers or defaults for manually run analyzers. Args: sketch_id (int): The ID of the sketch to analyze. searchindex_id (int): The ID of the searchindex to analyze. user_id (int): The ID of the user who started the analyzer. analyzer_names (list): List of analyzers to run. analyzer_kwargs (dict): Arguments to the analyzers. Returns: A tuple with a Celery group with analysis tasks or None if no analyzers are enabled and an analyzer session ID. """ tasks = [] if not analyzer_names: analyzer_names = current_app.config.get('AUTO_SKETCH_ANALYZERS', []) if not analyzer_kwargs: analyzer_kwargs = current_app.config.get( 'AUTO_SKETCH_ANALYZERS_KWARGS', {}) # Exit early if no sketch analyzers are configured to run. if not analyzer_names: return None, None if not analyzer_kwargs: analyzer_kwargs = current_app.config.get('ANALYZERS_DEFAULT_KWARGS', {}) if user_id: user = User.query.get(user_id) else: user = None sketch = Sketch.query.get(sketch_id) analysis_session = AnalysisSession(user, sketch) analyzers = manager.AnalysisManager.get_analyzers(analyzer_names) for analyzer_name, analyzer_cls in analyzers: if not analyzer_cls.IS_SKETCH_ANALYZER: continue kwargs = analyzer_kwargs.get(analyzer_name, {}) searchindex = SearchIndex.query.get(searchindex_id) timeline = Timeline.query.filter_by(sketch=sketch, searchindex=searchindex).first() analysis = Analysis(name=analyzer_name, description=analyzer_name, analyzer_name=analyzer_name, parameters=json.dumps(kwargs), user=user, sketch=sketch, timeline=timeline) analysis.set_status('PENDING') analysis_session.analyses.append(analysis) db_session.add(analysis) db_session.commit() tasks.append( run_sketch_analyzer.s(sketch_id, analysis.id, analyzer_name, **kwargs)) # Commit the analysis session to the database. db_session.add(analysis_session) db_session.commit() if current_app.config.get('ENABLE_EMAIL_NOTIFICATIONS'): tasks.append(run_email_result_task.s(sketch_id)) if not tasks: return None, None return chain(tasks), analysis_session.id
def post(self, sketch_id, group_id): """Handles POST request to the resource. Args: sketch_id: Integer primary key for a sketch database model. group_id: Integer primary key for an aggregation group database model. """ sketch = Sketch.query.get_with_acl(sketch_id) group = AggregationGroup.query.get(group_id) if not group: abort( HTTP_STATUS_CODE_NOT_FOUND, 'No Group found with this ID.') if not sketch: abort( HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.') # Check that this group belongs to the sketch if group.sketch_id != sketch.id: msg = ( 'The sketch ID ({0:d}) does not match with the aggregation ' 'group sketch ID ({1:d})'.format(sketch.id, group.sketch_id)) abort(HTTP_STATUS_CODE_FORBIDDEN, msg) if not sketch.has_permission(user=current_user, permission='write'): abort( HTTP_STATUS_CODE_FORBIDDEN, 'The user does not have write permission on the sketch.') form = request.json if not form: abort( HTTP_STATUS_CODE_BAD_REQUEST, 'No JSON data, unable to process request to create ' 'a new aggregation group.') group.name = form.get('name', group.name) group.description = form.get('description', group.description) group.parameters = form.get('parameters', group.parameters) group.orientation = form.get('orientation', group.orientation) group.user = current_user group.sketch = sketch agg_ids = json.loads(form.get('aggregations', group.aggregations)) aggregations = [] for agg_id in agg_ids: aggregation = Aggregation.query.get(agg_id) if not aggregation: abort( HTTP_STATUS_CODE_BAD_REQUEST, 'No aggregation found for ID: {0:d}'.format(agg_id)) aggregations.append(aggregation) group.aggregations = aggregations db_session.add(group) db_session.commit() return self.to_json(group, status_code=HTTP_STATUS_CODE_CREATED)
def timelines(sketch_id): """Generates the sketch explore view template. Returns: Template with context. """ sketch = Sketch.query.get_with_acl(sketch_id) searchindices_in_sketch = [t.searchindex.id for t in sketch.timelines] indices = SearchIndex.all_with_acl(current_user).order_by( desc(SearchIndex.created_at)).filter( not_(SearchIndex.id.in_(searchindices_in_sketch))) upload_enabled = current_app.config['UPLOAD_ENABLED'] graphs_enabled = current_app.config['GRAPH_BACKEND_ENABLED'] try: plaso_version = current_app.config['PLASO_VERSION'] except KeyError: plaso_version = 'Unknown' # Setup the form form = AddTimelineForm() form.timelines.choices = set((i.id, i.name) for i in indices.all()) # Create new timeline form POST if form.validate_on_submit(): if not sketch.has_permission(current_user, 'write'): abort(HTTP_STATUS_CODE_FORBIDDEN) for searchindex_id in form.timelines.data: searchindex = SearchIndex.query.get_with_acl(searchindex_id) if searchindex not in [t.searchindex for t in sketch.timelines]: _timeline = Timeline( name=searchindex.name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex) db_session.add(_timeline) sketch.timelines.append(_timeline) db_session.commit() # If enabled, run sketch analyzers when timeline is added. # Import here to avoid circular imports. from timesketch.lib import tasks sketch_analyzer_group = tasks.build_sketch_analysis_pipeline( sketch_id) if sketch_analyzer_group: pipeline = (tasks.run_sketch_init.s( [searchindex.index_name]) | sketch_analyzer_group) pipeline.apply_async(task_id=searchindex.index_name) return redirect( url_for('sketch_views.timelines', sketch_id=sketch.id)) return render_template( 'sketch/timelines.html', sketch=sketch, timelines=indices.all(), form=form, upload_enabled=upload_enabled, plaso_version=plaso_version, graphs_enabled=graphs_enabled)
def post(self, sketch_id, aggregation_id): """Handles POST request to the resource. Handler for /api/v1/sketches/:sketch_id/aggregation/:aggregation_id Args: sketch_id: Integer primary key for a sketch database model aggregation_id: Integer primary key for an aggregation database model """ form = request.json if not form: form = request.data if not form: abort( HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to validate form data.') sketch = Sketch.query.get_with_acl(sketch_id) if not sketch: abort( HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.') if not sketch.has_permission(current_user, 'write'): abort(HTTP_STATUS_CODE_FORBIDDEN, 'User does not have write access controls on sketch.') aggregation = Aggregation.query.get(aggregation_id) if not aggregation: abort( HTTP_STATUS_CODE_NOT_FOUND, 'No aggregation found with this ID.') if not sketch.has_permission(user=current_user, permission='write'): abort( HTTP_STATUS_CODE_FORBIDDEN, 'The user does not have write permission on the sketch.') aggregation.name = form.get('name', '') aggregation.description = form.get('description', '') aggregation.agg_type = form.get('agg_type', aggregation.agg_type) aggregation.chart_type = form.get('chart_type', aggregation.chart_type) aggregation.user = current_user aggregation.sketch = sketch labels = form.get('labels', '') if labels: for label in json.loads(labels): if aggregation.has_label(label): continue aggregation.add_label(label) if form.get('parameters'): aggregation.parameters = json.dumps( form.get('parameters'), ensure_ascii=False) if form.get('view_id'): aggregation.view = form.get('view_id') db_session.add(aggregation) db_session.commit() return self.to_json(aggregation, status_code=HTTP_STATUS_CODE_CREATED)
def explore(sketch_id, view_id=None, searchtemplate_id=None): """Generates the sketch explore view template. Returns: Template with context. """ save_view = False # If the view should be saved to the database. sketch = Sketch.query.get_with_acl(sketch_id) sketch_timelines = [t.searchindex.index_name for t in sketch.timelines] view_form = SaveViewForm() graphs_enabled = current_app.config['GRAPH_BACKEND_ENABLED'] similarity_enabled = current_app.config.get('ENABLE_EXPERIMENTAL_UI') # Get parameters from the GET query url_query = request.args.get('q', '') url_time_start = request.args.get('time_start', None) url_time_end = request.args.get('time_end', None) url_index = request.args.get('index', None) url_size = request.args.get('size', None) if searchtemplate_id: searchtemplate = SearchTemplate.query.get(searchtemplate_id) view = sketch.get_user_view(current_user) if not view: view = View(user=current_user, name='', sketch=sketch) view.query_string = searchtemplate.query_string view.query_filter = searchtemplate.query_filter view.query_dsl = searchtemplate.query_dsl save_view = True elif view_id: view = View.query.get(view_id) # Check that this view belongs to the sketch if view.sketch_id != sketch.id: abort(HTTP_STATUS_CODE_NOT_FOUND) # Return 404 if view is deleted if view.get_status.status == 'deleted': return abort(HTTP_STATUS_CODE_NOT_FOUND) else: view = sketch.get_user_view(current_user) if not view: view = View(user=current_user, name='', sketch=sketch, query_string='*') view.query_filter = view.validate_filter( dict(indices=sketch_timelines)) save_view = True if url_query: view.query_string = url_query query_filter = json.loads(view.query_filter) query_filter['from'] = 0 # if we loaded from get, start at first event query_filter['time_start'] = url_time_start query_filter['time_end'] = url_time_end if url_index in sketch_timelines: query_filter['indices'] = [url_index] if url_size: query_filter['size'] = url_size view.query_filter = view.validate_filter(query_filter) view.query_dsl = None save_view = True if save_view: db_session.add(view) db_session.commit() return render_template('sketch/explore.html', sketch=sketch, view=view, named_view=view_id, timelines=sketch_timelines, view_form=view_form, searchtemplate_id=searchtemplate_id, graphs_enabled=graphs_enabled, similarity_enabled=similarity_enabled)
def post(self): """Handles POST request to the resource. Returns: A view in JSON (instance of flask.wrappers.Response) """ upload_enabled = current_app.config['UPLOAD_ENABLED'] if not upload_enabled: abort( HTTP_STATUS_CODE_BAD_REQUEST, 'Failed to create timeline, upload not enabled') form = forms.CreateTimelineForm() if not form.validate_on_submit(): abort( HTTP_STATUS_CODE_BAD_REQUEST, 'Failed to create timeline, form data not validated') sketch_id = form.sketch_id.data timeline_name = form.name.data sketch = None if sketch_id: sketch = Sketch.query.get_with_acl(sketch_id) if not sketch: abort( HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.') # We do not need a human readable filename or # datastore index name, so we use UUIDs here. index_name = uuid.uuid4().hex if not isinstance(index_name, six.text_type): index_name = codecs.decode(index_name, 'utf-8') # Create the search index in the Timesketch database searchindex = SearchIndex.get_or_create( name=timeline_name, description=timeline_name, user=current_user, index_name=index_name) searchindex.grant_permission(permission='read', user=current_user) searchindex.grant_permission(permission='write', user=current_user) searchindex.grant_permission( permission='delete', user=current_user) searchindex.set_status('processing') db_session.add(searchindex) db_session.commit() timeline = None if sketch and sketch.has_permission(current_user, 'write'): timeline = Timeline( name=searchindex.name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex) sketch.timelines.append(timeline) db_session.add(timeline) db_session.commit() # Return Timeline if it was created. # pylint: disable=no-else-return if timeline: return self.to_json( timeline, status_code=HTTP_STATUS_CODE_CREATED) # Update the last activity of a sketch. utils.update_sketch_last_activity(sketch) return self.to_json( searchindex, status_code=HTTP_STATUS_CODE_CREATED)
def post(self, sketch_id): """Handles POST request to the resource. Handler for /api/v1/sketches/:sketch_id/explore/ Args: sketch_id: Integer primary key for a sketch database model Returns: JSON with list of matched events """ sketch = Sketch.query.get_with_acl(sketch_id) form = ExploreForm.build(request) if form.validate_on_submit(): query_filter = form.filter.data sketch_indices = [ t.searchindex.index_name for t in sketch.timelines] indices = query_filter.get(u'indices', sketch_indices) # Make sure that the indices in the filter are part of the sketch if set(indices) - set(sketch_indices): abort(HTTP_STATUS_CODE_BAD_REQUEST) # Make sure we have a query string or star filter if not form.query.data and not query_filter.get(u'star'): abort(HTTP_STATUS_CODE_BAD_REQUEST) result = self.datastore.search( sketch_id, form.query.data, query_filter, indices) # Get labels for each event that matches the sketch. # Remove all other labels. for event in result[u'hits'][u'hits']: event[u'selected'] = False event[u'_source'][u'label'] = [] try: for label in event[u'_source'][u'timesketch_label']: if sketch.id != label[u'sketch_id']: continue event[u'_source'][u'label'].append(label[u'name']) del event[u'_source'][u'timesketch_label'] except KeyError: pass # Update or create user state view. This is used in the UI to let # the user get back to the last state in the explore view. view = View.get_or_create( user=current_user, sketch=sketch, name=u'') view.query_string = form.query.data view.query_filter = json.dumps(query_filter) db_session.add(view) db_session.commit() # Add metadata for the query result. This is used by the UI to # render the event correctly and to display timing and hit count # information. tl_colors = {} tl_names = {} for timeline in sketch.timelines: tl_colors[timeline.searchindex.index_name] = timeline.color tl_names[timeline.searchindex.index_name] = timeline.name meta = { u'es_time': result[u'took'], u'es_total_count': result[u'hits'][u'total'], u'timeline_colors': tl_colors, u'timeline_names': tl_names } schema = { u'meta': meta, u'objects': result[u'hits'][u'hits'] } return jsonify(schema) return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def post(self, sketch_id): """Handles POST request to the resource. Args: sketch_id: Integer primary key for a sketch database model Returns: An annotation in JSON (instance of flask.wrappers.Response) """ form = forms.EventAnnotationForm.build(request) if not form.validate_on_submit(): abort(HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to validate form data.') annotations = [] sketch = Sketch.query.get_with_acl(sketch_id) if not sketch: abort(HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.') if not sketch.has_permission(current_user, 'write'): abort(HTTP_STATUS_CODE_FORBIDDEN, 'User does not have write access controls on sketch.') indices = [ t.searchindex.index_name for t in sketch.timelines if t.get_status.status.lower() == 'ready' ] annotation_type = form.annotation_type.data events = form.events.raw_data for _event in events: searchindex_id = _event['_index'] searchindex = SearchIndex.query.filter_by( index_name=searchindex_id).first() event_id = _event['_id'] event_type = _event['_type'] if searchindex_id not in indices: abort( HTTP_STATUS_CODE_BAD_REQUEST, 'Search index ID ({0!s}) does not belong to the list ' 'of indices'.format(searchindex_id)) # Get or create an event in the SQL database to have something # to attach the annotation to. event = Event.get_or_create(sketch=sketch, searchindex=searchindex, document_id=event_id) # Add the annotation to the event object. if 'comment' in annotation_type: annotation = Event.Comment(comment=form.annotation.data, user=current_user) event.comments.append(annotation) self.datastore.set_label(searchindex_id, event_id, event_type, sketch.id, current_user.id, '__ts_comment', toggle=False) elif 'label' in annotation_type: annotation = Event.Label.get_or_create( label=form.annotation.data, user=current_user) if annotation not in event.labels: event.labels.append(annotation) toggle = False if '__ts_star' in form.annotation.data: toggle = True if '__ts_hidden' in form.annotation.data: toggle = True if form.remove.data: toggle = True self.datastore.set_label(searchindex_id, event_id, event_type, sketch.id, current_user.id, form.annotation.data, toggle=toggle) else: abort( HTTP_STATUS_CODE_BAD_REQUEST, 'Annotation type needs to be either label or comment, ' 'not {0!s}'.format(annotation_type)) annotations.append(annotation) # Save the event to the database db_session.add(event) db_session.commit() return self.to_json(annotations, status_code=HTTP_STATUS_CODE_CREATED)
def WriteHeader(self): """Setup the Elasticsearch index and the Timesketch database object. Creates the Elasticsearch index with Timesketch specific settings and the Timesketch SearchIndex database object. """ # This cannot be static because we use the value of self._doc_type from # arguments. _document_mapping = { self._doc_type: { u'properties': { u'timesketch_label': { u'type': u'nested' } } } } # Get Elasticsearch host and port from Timesketch configuration. with self._timesketch.app_context(): _host = current_app.config[u'ELASTIC_HOST'] _port = current_app.config[u'ELASTIC_PORT'] self._elastic = ElasticSearchHelper(self._output_mediator, _host, _port, self._flush_interval, self._index_name, _document_mapping, self._doc_type) user = None if self._username: user = User.query.filter_by(username=self._username).first() if not user: raise RuntimeError(u'Unknown Timesketch user: {0:s}'.format( self._username)) else: logging.warning( u'Timeline will be visible to all Timesketch users') with self._timesketch.app_context(): search_index = SearchIndex.get_or_create( name=self._timeline_name, description=self._timeline_name, user=user, index_name=self._index_name) # Grant the user read permission on the mapping object and set status. # If user is None the timeline becomes visible to all users. search_index.grant_permission(user=user, permission=u'read') # In case we have a user grant additional permissions. if user: search_index.grant_permission(user=user, permission=u'write') search_index.grant_permission(user=user, permission=u'delete') # Let the Timesketch UI know that the timeline is processing. search_index.set_status(u'processing') # Save the mapping object to the Timesketch database. db_session.add(search_index) db_session.commit() logging.info(u'Adding events to Timesketch.')
def post(self): """Handles POST request to the resource. Returns: A view in JSON (instance of flask.wrappers.Response) Raises: ApiHTTPError """ UPLOAD_ENABLED = current_app.config[u'UPLOAD_ENABLED'] UPLOAD_FOLDER = current_app.config[u'UPLOAD_FOLDER'] form = UploadFileForm() if form.validate_on_submit() and UPLOAD_ENABLED: from timesketch.lib.tasks import run_plaso from timesketch.lib.tasks import run_csv # Map the right task based on the file type task_directory = {u'plaso': run_plaso, u'csv': run_csv} sketch_id = form.sketch_id.data file_storage = form.file.data _filename, _extension = os.path.splitext(file_storage.filename) file_extension = _extension.lstrip(u'.') timeline_name = form.name.data or _filename.rstrip(u'.') sketch = None if sketch_id: sketch = Sketch.query.get_with_acl(sketch_id) # Current user username = current_user.username # We do not need a human readable filename or # datastore index name, so we use UUIDs here. filename = unicode(uuid.uuid4().hex) index_name = unicode(uuid.uuid4().hex) file_path = os.path.join(UPLOAD_FOLDER, filename) file_storage.save(file_path) # Create the search index in the Timesketch database searchindex = SearchIndex.get_or_create(name=timeline_name, description=timeline_name, user=current_user, index_name=index_name) searchindex.grant_permission(permission=u'read', user=current_user) searchindex.grant_permission(permission=u'write', user=current_user) searchindex.grant_permission(permission=u'delete', user=current_user) searchindex.set_status(u'processing') db_session.add(searchindex) db_session.commit() if sketch and sketch.has_permission(current_user, u'write'): timeline = Timeline(name=searchindex.name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex) db_session.add(timeline) sketch.timelines.append(timeline) db_session.commit() # Run the task in the background task = task_directory.get(file_extension) task.apply_async((file_path, timeline_name, index_name, username), task_id=index_name) return self.to_json(searchindex, status_code=HTTP_STATUS_CODE_CREATED) else: raise ApiHTTPError(message=form.errors[u'file'][0], status_code=HTTP_STATUS_CODE_BAD_REQUEST)
def explore(sketch_id, view_id=None): """Generates the sketch explore view template. Returns: Template with context. """ save_view = False # If the view should be saved to the database. sketch = Sketch.query.get_with_acl(sketch_id) sketch_timelines = [t.searchindex.index_name for t in sketch.timelines] view_form = SaveViewForm() # Get parameters from the GET query url_query = request.args.get(u'q', u'') url_time_start = request.args.get(u'time_start', None) url_time_end = request.args.get(u'time_end', None) url_index = request.args.get(u'index', None) url_limit = request.args.get(u'limit', None) if view_id: view = View.query.get(view_id) # Check that this view belongs to the sketch if view.sketch_id != sketch.id: abort(HTTP_STATUS_CODE_NOT_FOUND) # Return 404 if view is deleted if view.get_status.status == u'deleted': return abort(HTTP_STATUS_CODE_NOT_FOUND) else: view = sketch.get_user_view(current_user) if not view: query_filter = view.validate_filter(dict(indices=sketch_timelines)) view = View(user=current_user, name=u'', sketch=sketch, query_string=u'*', query_filter=query_filter) save_view = True if url_query: view.query_string = url_query query_filter = json.loads(view.query_filter) query_filter[u'time_start'] = url_time_start query_filter[u'time_end'] = url_time_end if url_index in sketch_timelines: query_filter[u'indices'] = [url_index] if url_limit: query_filter[u'limit'] = url_limit view.query_filter = view.validate_filter(query_filter) save_view = True if save_view: db_session.add(view) db_session.commit() return render_template(u'sketch/explore.html', sketch=sketch, view=view, named_view=view_id, timelines=sketch_timelines, view_form=view_form)
def post(self, sketch_id): """Handles POST request to the resource. Returns: A HTTP 200 if the attribute is successfully added or modified. """ sketch = Sketch.query.get_with_acl(sketch_id) if not sketch: abort(HTTP_STATUS_CODE_NOT_FOUND, "No sketch found with this ID.") if not sketch.has_permission(current_user, "write"): return abort( HTTP_STATUS_CODE_FORBIDDEN, "User does not have write permission on the sketch.", ) form = request.json if not form: form = request.data if not form: return abort( HTTP_STATUS_CODE_FORBIDDEN, "Unable to add or modify an attribute from a " "sketch without any data submitted.", ) for check in ["name", "ontology"]: error_message = self._validate_form_entry(form, check) if error_message: return abort(HTTP_STATUS_CODE_BAD_REQUEST, error_message) name = form.get("name") ontology = form.get("ontology", "text") ontology_def = ontology_lib.ONTOLOGY ontology_dict = ontology_def.get(ontology, {}) cast_as_string = ontology_dict.get("cast_as", "str") values = form.get("values") if not values: return abort(HTTP_STATUS_CODE_BAD_REQUEST, "Missing values from the request.") if not isinstance(values, (list, tuple)): return abort(HTTP_STATUS_CODE_BAD_REQUEST, "Values needs to be a list.") value_strings = [ ontology_lib.OntologyManager.encode_value(x, cast_as_string) for x in values ] if any([not isinstance(x, str) for x in value_strings]): return abort( HTTP_STATUS_CODE_BAD_REQUEST, "All values needs to be stored as strings.", ) attribute = None message = "" update_attribute = False for attribute in sketch.attributes: if (attribute.name == name) and (attribute.ontology == ontology): message = "Attribute Updated" update_attribute = True break if update_attribute: _ = AttributeValue.query.filter_by(attribute=attribute).delete() else: attribute = Attribute(user=current_user, sketch=sketch, name=name, ontology=ontology) db_session.add(attribute) db_session.commit() for value in value_strings: attribute_value = AttributeValue(user=current_user, attribute=attribute, value=value) attribute.values.append(attribute_value) db_session.add(attribute_value) db_session.commit() db_session.add(attribute) db_session.commit() return_data = { "name": name, "ontology": ontology, "cast_as": cast_as_string, } response = None if message: return_data["action"] = "update" response = jsonify(return_data) response.status_code = HTTP_STATUS_CODE_OK else: return_data["action"] = "create" response = jsonify(return_data) response.status_code = HTTP_STATUS_CODE_CREATED return response
def post(self, sketch_id): """Handles POST request to the resource. Handler for /api/v1/sketches/:sketch_id/explore/ Args: sketch_id: Integer primary key for a sketch database model Returns: JSON with list of matched events """ sketch = Sketch.query.get_with_acl(sketch_id) if not sketch: abort(HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.') if not sketch.has_permission(current_user, 'read'): abort(HTTP_STATUS_CODE_FORBIDDEN, 'User does not have read access controls on sketch.') if sketch.get_status.status == 'archived': abort(HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to query on an archived sketch.') form = forms.ExploreForm.build(request) if not form.validate_on_submit(): abort(HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to explore data, unable to validate form data') # TODO: Remove form and use json instead. query_dsl = form.dsl.data enable_scroll = form.enable_scroll.data scroll_id = form.scroll_id.data file_name = form.file_name.data query_filter = request.json.get('filter', {}) return_field_string = form.fields.data if return_field_string: return_fields = [x.strip() for x in return_field_string.split(',')] else: return_fields = query_filter.get('fields', []) return_fields = [field['field'] for field in return_fields] return_fields.extend(DEFAULT_SOURCE_FIELDS) sketch_indices = { t.searchindex.index_name for t in sketch.timelines if t.get_status.status.lower() == 'ready' } if not query_filter: query_filter = {} indices = query_filter.get('indices', sketch_indices) # If _all in indices then execute the query on all indices if '_all' in indices: indices = sketch_indices # Make sure that the indices in the filter are part of the sketch. # This will also remove any deleted timeline from the search result. indices = get_validated_indices(indices, sketch_indices) # Make sure we have a query string or star filter if not (form.query.data, query_filter.get('star'), query_filter.get('events'), query_dsl): abort( HTTP_STATUS_CODE_BAD_REQUEST, 'The request needs a query string/DSL and or a star filter.') # Aggregate hit count per index. index_stats_agg = {"indices": {"terms": {"field": "_index"}}} if file_name: file_object = io.BytesIO() form_data = { 'created_at': datetime.datetime.utcnow().isoformat(), 'created_by': current_user.username, 'sketch': sketch_id, 'query': form.query.data, 'query_dsl': query_dsl, 'query_filter': query_filter, 'return_fields': return_fields, } with zipfile.ZipFile(file_object, mode='w') as zip_file: zip_file.writestr('METADATA', data=json.dumps(form_data)) fh = export.query_to_filehandle(query_string=form.query.data, query_dsl=query_dsl, query_filter=query_filter, indices=indices, sketch=sketch, datastore=self.datastore) fh.seek(0) zip_file.writestr('query_results.csv', fh.read()) file_object.seek(0) return send_file(file_object, mimetype='zip', attachment_filename=file_name) if scroll_id: # pylint: disable=unexpected-keyword-arg result = self.datastore.client.scroll(scroll_id=scroll_id, scroll='1m') else: result = self.datastore.search(sketch_id, form.query.data, query_filter, query_dsl, indices, aggregations=index_stats_agg, return_fields=return_fields, enable_scroll=enable_scroll) # Get number of matching documents per index. count_per_index = {} try: for bucket in result['aggregations']['indices']['buckets']: key = bucket.get('key') if key: count_per_index[key] = bucket.get('doc_count') except KeyError: pass # Get labels for each event that matches the sketch. # Remove all other labels. for event in result['hits']['hits']: event['selected'] = False event['_source']['label'] = [] try: for label in event['_source']['timesketch_label']: if sketch.id != label['sketch_id']: continue event['_source']['label'].append(label['name']) del event['_source']['timesketch_label'] except KeyError: pass # Update or create user state view. This is used in the UI to let # the user get back to the last state in the explore view. view = View.get_or_create(user=current_user, sketch=sketch, name='') view.query_string = form.query.data view.query_filter = json.dumps(query_filter, ensure_ascii=False) view.query_dsl = json.dumps(query_dsl, ensure_ascii=False) db_session.add(view) db_session.commit() # Add metadata for the query result. This is used by the UI to # render the event correctly and to display timing and hit count # information. tl_colors = {} tl_names = {} for timeline in sketch.timelines: tl_colors[timeline.searchindex.index_name] = timeline.color tl_names[timeline.searchindex.index_name] = timeline.name meta = { 'es_time': result['took'], 'es_total_count': result['hits']['total'], 'timeline_colors': tl_colors, 'timeline_names': tl_names, 'count_per_index': count_per_index, 'scroll_id': result.get('_scroll_id', ''), } # Elasticsearch version 7.x returns total hits as a dictionary. # TODO: Refactor when version 6.x has been deprecated. if isinstance(meta['es_total_count'], dict): meta['es_total_count'] = meta['es_total_count'].get('value', 0) schema = {'meta': meta, 'objects': result['hits']['hits']} return jsonify(schema)
def WriteHeader(self): """Sets up the Elasticsearch index and the Timesketch database object. Creates the Elasticsearch index with Timesketch specific settings and the Timesketch SearchIndex database object. """ # This cannot be static because we use the value of self._document_type # from arguments. mappings = { 'properties': { 'timesketch_label': { 'type': 'nested' }, 'datetime': { 'type': 'date' } } } # TODO: Remove once Elasticsearch v6.x is deprecated. if self._GetClientMajorVersion() < 7: mappings = {self._document_type: mappings} # Get Elasticsearch host and port from Timesketch configuration. with self._timesketch.app_context(): self._host = current_app.config['ELASTIC_HOST'] self._port = current_app.config['ELASTIC_PORT'] self._Connect() self._CreateIndexIfNotExists(self._index_name, mappings) user = None if self._timeline_owner: user = timesketch_user.User.query.filter_by( username=self._timeline_owner).first() if not user: raise RuntimeError('Unknown Timesketch user: {0:s}'.format( self._timeline_owner)) else: logger.warning('Timeline will be visible to all Timesketch users') with self._timesketch.app_context(): search_index = timesketch_sketch.SearchIndex.get_or_create( name=self._timeline_name, description=self._timeline_name, user=user, index_name=self._index_name) # Grant the user read permission on the mapping object and set status. # If user is None the timeline becomes visible to all users. search_index.grant_permission(user=user, permission='read') # In case we have a user grant additional permissions. if user: search_index.grant_permission(user=user, permission='write') search_index.grant_permission(user=user, permission='delete') # Let the Timesketch UI know that the timeline is processing. search_index.set_status('processing') # Save the mapping object to the Timesketch database. timesketch_db_session.add(search_index) timesketch_db_session.commit() logger.debug('Adding events to Timesketch.')