def WriteHeader(self): """Setup the Elasticsearch index and the Timesketch database object. Creates the Elasticsearch index with Timesketch specific settings and the Timesketch SearchIndex database object. """ # This cannot be static because we use the value of self._doc_type from # arguments. _document_mapping = { self._doc_type: { u'properties': { u'timesketch_label': { u'type': u'nested' } } } } # Get Elasticsearch host and port from Timesketch configuration. with self._timesketch.app_context(): _host = current_app.config[u'ELASTIC_HOST'] _port = current_app.config[u'ELASTIC_PORT'] self._elastic = ElasticSearchHelper( self._output_mediator, _host, _port, self._flush_interval, self._index_name, _document_mapping, self._doc_type) user = None if self._username: user = User.query.filter_by(username=self._username).first() if not user: raise RuntimeError( u'Unknown Timesketch user: {0:s}'.format(self._username)) else: logging.warning(u'Timeline will be visible to all Timesketch users') with self._timesketch.app_context(): search_index = SearchIndex.get_or_create( name=self._timeline_name, description=self._timeline_name, user=user, index_name=self._index_name) # Grant the user read permission on the mapping object and set status. # If user is None the timeline becomes visible to all users. search_index.grant_permission(user=user, permission=u'read') # In case we have a user grant additional permissions. if user: search_index.grant_permission(user=user, permission=u'write') search_index.grant_permission(user=user, permission=u'delete') # Let the Timesketch UI know that the timeline is processing. search_index.set_status(u'processing') # Save the mapping object to the Timesketch database. db_session.add(search_index) db_session.commit() logging.info(u'Adding events to Timesketch.')
def _create_searchindex(self, name, user, acl=False): """Create a searchindex in the database. Args: name: Name of the searchindex (string) user: A user (instance of timesketch.models.user.User) acl: Boolean value to decide if ACL permissions should be set Returns: A searchindex (instance of timesketch.models.sketch.SearchIndex) """ searchindex = SearchIndex( name=name, description=name, index_name=name, user=user) if acl: for permission in [u'read', u'write', u'delete']: searchindex.grant_permission(user=user, permission=permission) self._commit_to_database(searchindex) return searchindex
def Close(self): """Closes the connection to TimeSketch Elasticsearch database. Sends the remaining events for indexing and adds the timeline to Timesketch. """ self._FlushEventsToElasticsearch() with self._timesketch.app_context(): # Get Timesketch user object, or None if user do not exist. This is a # SQLAlchemy query against the Timesketch database. user_query = User.query.filter_by(username=self._timeline_owner) user = user_query.first() search_index = SearchIndex( name=self._timeline_name, description=self._timeline_name, user=user, index_name=self._index_name) # Grant all users read permission on the mapping object. search_index.grant_permission(None, u'read') # Save the mapping object to the Timesketch database. db_session.add(search_index) db_session.commit()
def run(self, name, index, username): """Create the SearchIndex.""" es = ElasticsearchDataStore( host=current_app.config['ELASTIC_HOST'], port=current_app.config['ELASTIC_PORT']) user = User.query.filter_by(username=username).first() if not user: sys.stderr.write('User does not exist\n') sys.exit(1) if not es.client.indices.exists(index=index): sys.stderr.write('Index does not exist in the datastore\n') sys.exit(1) if SearchIndex.query.filter_by(name=name, index_name=index).first(): sys.stderr.write( 'Index with this name already exist in Timesketch\n') sys.exit(1) searchindex = SearchIndex( name=name, description=name, user=user, index_name=index) searchindex.grant_permission('read') db_session.add(searchindex) db_session.commit() sys.stdout.write('Search index {0:s} created\n'.format(name))
def post(self): """Handles POST request to the resource. Returns: A search index in JSON (instance of flask.wrappers.Response) """ form = forms.SearchIndexForm.build(request) searchindex_name = form.searchindex_name.data es_index_name = form.es_index_name.data public = form.public.data if not form.validate_on_submit(): abort(HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to validate form data') searchindex = SearchIndex.query.filter_by( index_name=es_index_name).first() metadata = {'created': True} if searchindex: metadata['created'] = False metadata['deleted'] = searchindex.get_status.status == 'deleted' status_code = HTTP_STATUS_CODE_OK else: searchindex = SearchIndex.get_or_create( name=searchindex_name, description=searchindex_name, user=current_user, index_name=es_index_name) searchindex.grant_permission(permission='read', user=current_user) searchindex.grant_permission(permission='write', user=current_user) searchindex.grant_permission(permission='delete', user=current_user) if public: searchindex.grant_permission(permission='read', user=None) datastore = self.datastore if not datastore.client.indices.exists(index=es_index_name): # Create the index in Elasticsearch self.datastore.create_index(index_name=es_index_name, doc_type='generic_event') db_session.add(searchindex) db_session.commit() status_code = HTTP_STATUS_CODE_CREATED return self.to_json(searchindex, meta=metadata, status_code=status_code)
def post(self): """Handles POST request to the resource. Returns: A view in JSON (instance of flask.wrappers.Response) Raises: ApiHTTPError """ UPLOAD_ENABLED = current_app.config[u'UPLOAD_ENABLED'] UPLOAD_FOLDER = current_app.config[u'UPLOAD_FOLDER'] form = UploadFileForm() if form.validate_on_submit() and UPLOAD_ENABLED: from timesketch.lib.tasks import run_plaso file_storage = form.file.data timeline_name = form.name.data # We do not need a human readable filename or # datastore index name, so we use UUIDs here. filename = unicode(uuid.uuid4().hex) index_name = unicode(uuid.uuid4().hex) file_path = os.path.join(UPLOAD_FOLDER, filename) file_storage.save(file_path) search_index = SearchIndex.get_or_create( name=timeline_name, description=timeline_name, user=current_user, index_name=index_name) search_index.grant_permission(permission=u'read', user=current_user) search_index.grant_permission( permission=u'write', user=current_user) search_index.grant_permission( permission=u'delete', user=current_user) search_index.set_status(u'processing') db_session.add(search_index) db_session.commit() run_plaso.apply_async( (file_path, timeline_name, index_name), task_id=index_name) return self.to_json( search_index, status_code=HTTP_STATUS_CODE_CREATED) else: raise ApiHTTPError( message=form.errors[u'file'][0], status_code=HTTP_STATUS_CODE_BAD_REQUEST)
def timelines(sketch_id): """Generates the sketch explore view template. Returns: Template with context. """ TIMELINES_TO_SHOW = 20 sketch = Sketch.query.get_with_acl(sketch_id) searchindices_in_sketch = [t.searchindex.id for t in sketch.timelines] query = request.args.get(u'q', None) indices = SearchIndex.all_with_acl( current_user).order_by( desc(SearchIndex.created_at)).filter( not_(SearchIndex.id.in_(searchindices_in_sketch))) filtered = False if query: indices = indices.filter(SearchIndex.name.contains(query)).limit(500) filtered = True if not filtered: indices = indices.limit(TIMELINES_TO_SHOW) # Setup the form form = AddTimelineForm() form.timelines.choices = set((i.id, i.name) for i in indices.all()) # Create new timeline form POST if form.validate_on_submit(): if not sketch.has_permission(current_user, u'write'): abort(HTTP_STATUS_CODE_FORBIDDEN) for searchindex_id in form.timelines.data: searchindex = SearchIndex.query.get_with_acl(searchindex_id) if searchindex not in [t.searchindex for t in sketch.timelines]: _timeline = Timeline( name=searchindex.name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex) db_session.add(_timeline) sketch.timelines.append(_timeline) db_session.commit() return redirect(url_for(u'sketch_views.timelines', sketch_id=sketch.id)) return render_template( u'sketch/timelines.html', sketch=sketch, timelines=indices.all(), form=form, filtered=filtered)
def WriteHeader(self): """Setup the Elasticsearch index and the Timesketch database object. Creates the Elasticsearch index with Timesketch specific settings and the Timesketch SearchIndex database object. """ # This cannot be static because we use the value of self._doc_type from # arguments. _document_mapping = { self._doc_type: { u'_timestamp': { u'enabled': True, u'path': u'datetime', u'format': u'date_time_no_millis' }, u'properties': {u'timesketch_label': {u'type': u'nested'}} } } if not self._elastic_db.client.indices.exists(self._index_name): try: self._elastic_db.client.indices.create( index=self._index_name, body={u'mappings': _document_mapping}) except elastic_exceptions.ConnectionError as exception: logging.error(( u'Unable to proceed, cannot connect to Timesketch backend ' u'with error: {0:s}.\nPlease verify connection.').format(exception)) raise RuntimeError(u'Unable to connect to Timesketch backend.') with self._timesketch.app_context(): search_index = SearchIndex.get_or_create( name=self._timeline_name, description=self._timeline_name, user=None, index_name=self._index_name) # Grant all users read permission on the mapping object and set status. search_index.grant_permission(None, u'read') search_index.set_status(u'processing') # Save the mapping object to the Timesketch database. db_session.add(search_index) db_session.commit() logging.info(u'Adding events to Timesketch..')
def post(self, sketch_id): """Handles POST request to the resource. Returns: A sketch in JSON (instance of flask.wrappers.Response) Raises: ApiHTTPError """ sketch = Sketch.query.get_with_acl(sketch_id) searchindices_in_sketch = [t.searchindex.id for t in sketch.timelines] indices = SearchIndex.all_with_acl( current_user).order_by( desc(SearchIndex.created_at)).filter( not_(SearchIndex.id.in_(searchindices_in_sketch))) add_timeline_form = AddTimelineForm.build(request) add_timeline_form.timelines.choices = set( (i.id, i.name) for i in indices.all()) if add_timeline_form.validate_on_submit(): if not sketch.has_permission(current_user, u'write'): abort(HTTP_STATUS_CODE_FORBIDDEN) for searchindex_id in add_timeline_form.timelines.data: searchindex = SearchIndex.query.get_with_acl(searchindex_id) if searchindex not in [t.searchindex for t in sketch.timelines]: _timeline = Timeline( name=searchindex.name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex) db_session.add(_timeline) sketch.timelines.append(_timeline) db_session.commit() return self.to_json(sketch, status_code=HTTP_STATUS_CODE_CREATED) else: raise ApiHTTPError( message=add_timeline_form.errors, status_code=HTTP_STATUS_CODE_BAD_REQUEST)
def post(self): """Handles POST request to the resource. Returns: A search index in JSON (instance of flask.wrappers.Response) """ form = SearchIndexForm.build(request) timeline_name = form.timeline_name.data index_name = form.index_name.data public = form.public.data if form.validate_on_submit(): searchindex = SearchIndex.query.filter_by( index_name=index_name).first() if not searchindex: searchindex = SearchIndex.get_or_create( name=timeline_name, description=timeline_name, user=current_user, index_name=index_name) searchindex.grant_permission(permission=u'read', user=current_user) if public: searchindex.grant_permission(permission=u'read', user=None) # Create the index in Elasticsearch self.datastore.create_index(index_name=index_name, doc_type=u'generic_event') db_session.add(searchindex) db_session.commit() return self.to_json(searchindex, status_code=HTTP_STATUS_CODE_CREATED) return abort(HTTP_STATUS_CODE_BAD_REQUEST)
def timelines(sketch_id): """Generates the sketch explore view template. Returns: Template with context. """ sketch = Sketch.query.get_with_acl(sketch_id) searchindices_in_sketch = [t.searchindex.id for t in sketch.timelines] indices = SearchIndex.all_with_acl(current_user).order_by( desc(SearchIndex.created_at)).filter( not_(SearchIndex.id.in_(searchindices_in_sketch))) upload_enabled = current_app.config['UPLOAD_ENABLED'] graphs_enabled = current_app.config['GRAPH_BACKEND_ENABLED'] try: plaso_version = current_app.config['PLASO_VERSION'] except KeyError: plaso_version = 'Unknown' # Setup the form form = AddTimelineForm() form.timelines.choices = set((i.id, i.name) for i in indices.all()) # Create new timeline form POST if form.validate_on_submit(): if not sketch.has_permission(current_user, 'write'): abort(HTTP_STATUS_CODE_FORBIDDEN) for searchindex_id in form.timelines.data: searchindex = SearchIndex.query.get_with_acl(searchindex_id) if searchindex not in [t.searchindex for t in sketch.timelines]: _timeline = Timeline( name=searchindex.name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex) db_session.add(_timeline) sketch.timelines.append(_timeline) db_session.commit() # If enabled, run sketch analyzers when timeline is added. # Import here to avoid circular imports. from timesketch.lib import tasks sketch_analyzer_group = tasks.build_sketch_analysis_pipeline( sketch_id) if sketch_analyzer_group: pipeline = (tasks.run_sketch_init.s( [searchindex.index_name]) | sketch_analyzer_group) pipeline.apply_async(task_id=searchindex.index_name) return redirect( url_for('sketch_views.timelines', sketch_id=sketch.id)) return render_template( 'sketch/timelines.html', sketch=sketch, timelines=indices.all(), form=form, upload_enabled=upload_enabled, plaso_version=plaso_version, graphs_enabled=graphs_enabled)
def run(self, file_path, sketch_id, username, timeline_name): """This is the run method.""" file_path = os.path.realpath(file_path) file_path_no_extension, extension = os.path.splitext(file_path) extension = extension.lstrip('.') filename = os.path.basename(file_path_no_extension) supported_extensions = ('plaso', 'csv', 'jsonl') if not os.path.isfile(file_path): sys.exit('No such file: {0:s}'.format(file_path)) if extension not in supported_extensions: sys.exit( 'Extension {0:s} is not supported. ' '(supported extensions are: {1:s})'.format( extension, ', '.join(supported_extensions))) user = None if not username: username = pwd.getpwuid(os.stat(file_path).st_uid).pw_name if not username == 'root': if not isinstance(username, six.text_type): username = codecs.decode(username, 'utf-8') user = User.query.filter_by(username=username).first() if not user: sys.exit('Cannot determine user for file: {0:s}'.format(file_path)) sketch = None # If filename starts with <number> then use that as sketch_id. # E.g: 42_file_name.plaso means sketch_id is 42. sketch_id_from_filename = filename.split('_')[0] if not sketch_id and sketch_id_from_filename.isdigit(): sketch_id = sketch_id_from_filename if sketch_id: try: sketch = Sketch.query.get_with_acl(sketch_id, user=user) except Forbidden: pass if not timeline_name: if not isinstance(timeline_name, six.text_type): timeline_name = codecs.decode(timeline_name, 'utf-8') timeline_name = timeline_name.replace('_', ' ') # Remove sketch ID if present in the filename. timeline_parts = timeline_name.split() if timeline_parts[0].isdigit(): timeline_name = ' '.join(timeline_name.split()[1:]) if not sketch: # Create a new sketch. sketch_name = 'Sketch for: {0:s}'.format(timeline_name) sketch = Sketch( name=sketch_name, description=sketch_name, user=user) # Need to commit here to be able to set permissions later. db_session.add(sketch) db_session.commit() sketch.grant_permission(permission='read', user=user) sketch.grant_permission(permission='write', user=user) sketch.grant_permission(permission='delete', user=user) sketch.status.append(sketch.Status(user=None, status='new')) db_session.add(sketch) db_session.commit() index_name = uuid.uuid4().hex if not isinstance(index_name, six.text_type): index_name = codecs.decode(index_name, 'utf-8') searchindex = SearchIndex.get_or_create( name=timeline_name, description=timeline_name, user=user, index_name=index_name) searchindex.grant_permission(permission='read', user=user) searchindex.grant_permission(permission='write', user=user) searchindex.grant_permission(permission='delete', user=user) searchindex.set_status('processing') db_session.add(searchindex) db_session.commit() if sketch and sketch.has_permission(user, 'write'): timeline = Timeline( name=searchindex.name, description=searchindex.description, sketch=sketch, user=user, searchindex=searchindex) timeline.set_status('processing') sketch.timelines.append(timeline) db_session.add(timeline) db_session.commit() # Start Celery pipeline for indexing and analysis. # Import here to avoid circular imports. from timesketch.lib import tasks pipeline = tasks.build_index_pipeline( file_path, timeline_name, index_name, extension, sketch.id) pipeline.apply_async(task_id=index_name) print('Imported {0:s} to sketch: {1:d} ({2:s})'.format( file_path, sketch.id, sketch.name))
def setup_sketch(timeline_name, index_name, username, sketch_id=None): """Use existing sketch or create a new sketch. Args: timeline_name: (str) Name of the Timeline index_name: (str) Name of the index username: (str) Who should own the timeline sketch_id: (str) Optional sketch_id to add timeline to Returns: (tuple) sketch ID and timeline ID as integers """ with app.app_context(): user = User.get_or_create(username=username) sketch = None if sketch_id: try: sketch = Sketch.query.get_with_acl(sketch_id, user=user) logger.info('Using existing sketch: {} ({})'.format( sketch.name, sketch.id)) except Forbidden: pass if not (sketch or sketch_id): # Create a new sketch. sketch_name = 'Turbinia: {}'.format(timeline_name) sketch = Sketch(name=sketch_name, description=sketch_name, user=user) # Need to commit here to be able to set permissions later. db_session.add(sketch) db_session.commit() sketch.grant_permission(permission='read', user=user) sketch.grant_permission(permission='write', user=user) sketch.grant_permission(permission='delete', user=user) sketch.status.append(sketch.Status(user=None, status='new')) db_session.add(sketch) db_session.commit() logger.info('Created new sketch: {} ({})'.format( sketch.name, sketch.id)) searchindex = SearchIndex.get_or_create( name=timeline_name, description='Created by Turbinia.', user=user, index_name=index_name) searchindex.grant_permission(permission='read', user=user) searchindex.grant_permission(permission='write', user=user) searchindex.grant_permission(permission='delete', user=user) searchindex.set_status('processing') db_session.add(searchindex) db_session.commit() timeline = Timeline(name=searchindex.name, description=searchindex.description, sketch=sketch, user=user, searchindex=searchindex) # If the user doesn't have write access to the sketch then create the # timeline but don't attach it to the sketch. if not sketch.has_permission(user, 'write'): timeline.sketch = None else: sketch.timelines.append(timeline) db_session.add(timeline) db_session.commit() timeline.set_status('processing') return sketch.id, timeline.id