def _create_timeline(self, name, sketch, searchindex, user): """Create a timeline in the database. Args: name: Name of the timeline (string) sketch: A sketch (instance of timesketch.models.sketch.Sketch) searchindex: A searchindex (instance of timesketch.models.sketch.SearchIndex) user: A user (instance of timesketch.models.user.User) Returns: A timeline (instance of timesketch.models.sketch.Timeline) """ timeline = Timeline(name=name, description=name, user=user, sketch=sketch, searchindex=searchindex, color=self.COLOR_WHITE) timeline.set_status(status='ready') self._commit_to_database(timeline) return timeline
def post(self, sketch_id): """Handles POST request to the resource. Returns: A sketch in JSON (instance of flask.wrappers.Response) """ sketch = Sketch.query.get_with_acl(sketch_id) if not sketch: abort( HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.') if not sketch.has_permission(current_user, 'write'): abort(HTTP_STATUS_CODE_FORBIDDEN, 'User does not have write access controls on sketch.') form = request.json if not form: form = request.data metadata = {'created': True} searchindex_id = form.get('timeline', 0) if isinstance(searchindex_id, str) and searchindex_id.isdigit(): searchindex_id = int(searchindex_id) if not isinstance(searchindex_id, int): abort( HTTP_STATUS_CODE_BAD_REQUEST, 'The timeline (searchindex id) needs to be an integer.') searchindex = SearchIndex.query.get_with_acl(searchindex_id) if searchindex.get_status.status == 'deleted': abort( HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to create a timeline using a deleted search index') timeline_id = [ t.searchindex.id for t in sketch.timelines if t.searchindex.id == searchindex_id ] if not timeline_id: return_code = HTTP_STATUS_CODE_CREATED timeline_name = form.get('timeline_name', searchindex.name) timeline = Timeline( name=timeline_name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex) sketch.timelines.append(timeline) labels_to_prevent_deletion = current_app.config.get( 'LABELS_TO_PREVENT_DELETION', []) for label in sketch.get_labels: if label not in labels_to_prevent_deletion: continue timeline.add_label(label) searchindex.add_label(label) # Set status to ready so the timeline can be queried. timeline.set_status('ready') db_session.add(timeline) db_session.commit() else: metadata['created'] = False return_code = HTTP_STATUS_CODE_OK timeline = Timeline.query.get(timeline_id) # Run sketch analyzers when timeline is added. Import here to avoid # circular imports. # pylint: disable=import-outside-toplevel if current_app.config.get('AUTO_SKETCH_ANALYZERS'): # pylint: disable=import-outside-toplevel from timesketch.lib import tasks sketch_analyzer_group, _ = tasks.build_sketch_analysis_pipeline( sketch_id, searchindex_id, current_user.id, timeline_id=timeline_id) if sketch_analyzer_group: pipeline = (tasks.run_sketch_init.s( [searchindex.index_name]) | sketch_analyzer_group) pipeline.apply_async() # Update the last activity of a sketch. utils.update_sketch_last_activity(sketch) return self.to_json( timeline, meta=metadata, status_code=return_code)
def run(self, file_path, sketch_id, username, timeline_name): """This is the run method.""" file_path = os.path.realpath(file_path) file_path_no_extension, extension = os.path.splitext(file_path) extension = extension.lstrip('.') filename = os.path.basename(file_path_no_extension) supported_extensions = ('plaso', 'csv', 'jsonl') if not os.path.isfile(file_path): sys.exit('No such file: {0:s}'.format(file_path)) if extension not in supported_extensions: sys.exit('Extension {0:s} is not supported. ' '(supported extensions are: {1:s})'.format( extension, ', '.join(supported_extensions))) user = None if not username: username = pwd.getpwuid(os.stat(file_path).st_uid).pw_name if not username == 'root': if not isinstance(username, six.text_type): username = codecs.decode(username, 'utf-8') user = User.query.filter_by(username=username).first() if not user: sys.exit('Cannot determine user for file: {0:s}'.format(file_path)) sketch = None # If filename starts with <number> then use that as sketch_id. # E.g: 42_file_name.plaso means sketch_id is 42. sketch_id_from_filename = filename.split('_')[0] if not sketch_id and sketch_id_from_filename.isdigit(): sketch_id = sketch_id_from_filename if sketch_id: try: sketch = Sketch.query.get_with_acl(sketch_id, user=user) except Forbidden: pass if not timeline_name: if timeline_name is None: timeline_name = '{0:s}_timeline'.format(filename) if not isinstance(timeline_name, six.text_type): timeline_name = codecs.decode(timeline_name, 'utf-8') timeline_name = timeline_name.replace('_', ' ') # Remove sketch ID if present in the filename. timeline_parts = timeline_name.split() if timeline_parts[0].isdigit(): timeline_name = ' '.join(timeline_name.split()[1:]) if not sketch: # Create a new sketch. sketch_name = 'Sketch for: {0:s}'.format(timeline_name) sketch = Sketch(name=sketch_name, description=sketch_name, user=user) # Need to commit here to be able to set permissions later. db_session.add(sketch) db_session.commit() sketch.grant_permission(permission='read', user=user) sketch.grant_permission(permission='write', user=user) sketch.grant_permission(permission='delete', user=user) sketch.status.append(sketch.Status(user=None, status='new')) db_session.add(sketch) db_session.commit() index_name = uuid.uuid4().hex if not isinstance(index_name, six.text_type): index_name = codecs.decode(index_name, 'utf-8') searchindex = SearchIndex.get_or_create(name=timeline_name, description=timeline_name, user=user, index_name=index_name) searchindex.grant_permission(permission='read', user=user) searchindex.grant_permission(permission='write', user=user) searchindex.grant_permission(permission='delete', user=user) searchindex.set_status('processing') db_session.add(searchindex) db_session.commit() if sketch and sketch.has_permission(user, 'write'): timeline = Timeline(name=searchindex.name, description=searchindex.description, sketch=sketch, user=user, searchindex=searchindex) timeline.set_status('processing') sketch.timelines.append(timeline) db_session.add(timeline) db_session.commit() # Start Celery pipeline for indexing and analysis. # Import here to avoid circular imports. from timesketch.lib import tasks # pylint: disable=import-outside-toplevel pipeline = tasks.build_index_pipeline(file_path=file_path, events='', timeline_name=timeline_name, index_name=index_name, file_extension=extension, sketch_id=sketch.id) pipeline.apply_async(task_id=index_name) print('Imported {0:s} to sketch: {1:d} ({2:s})'.format( file_path, sketch.id, sketch.name))
def _upload_and_index(self, file_extension, timeline_name, index_name, sketch, enable_stream, file_path='', events='', meta=None): """Creates a full pipeline for an uploaded file and returns the results. Args: file_extension: the extension of the uploaded file. timeline_name: name the timeline will be stored under in the datastore. index_name: the Elastic index name for the timeline. sketch: Instance of timesketch.models.sketch.Sketch enable_stream: boolean indicating whether this is file is part of a stream or not. file_path: the path to the file to be uploaded (optional). events: a string with events to upload (optional). meta: optional dict with additional meta fields that will be included in the return. Returns: A timeline if created otherwise a search index in JSON (instance of flask.wrappers.Response) """ # Check if search index already exists. searchindex = SearchIndex.query.filter_by( name=timeline_name, description=timeline_name, user=current_user, index_name=index_name).first() timeline = None if searchindex: searchindex.set_status('processing') timeline = Timeline.query.filter_by( name=searchindex.name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex).first() else: # Create the search index in the Timesketch database searchindex = SearchIndex.get_or_create(name=timeline_name, description='', user=current_user, index_name=index_name) searchindex.grant_permission(permission='read', user=current_user) searchindex.grant_permission(permission='write', user=current_user) searchindex.grant_permission(permission='delete', user=current_user) searchindex.set_status('processing') db_session.add(searchindex) db_session.commit() if sketch and sketch.has_permission(current_user, 'write'): labels_to_prevent_deletion = current_app.config.get( 'LABELS_TO_PREVENT_DELETION', []) timeline = Timeline(name=searchindex.name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex) timeline.set_status('processing') sketch.timelines.append(timeline) for label in sketch.get_labels: if label not in labels_to_prevent_deletion: continue timeline.add_label(label) searchindex.add_label(label) db_session.add(timeline) db_session.commit() # Start Celery pipeline for indexing and analysis. # Import here to avoid circular imports. # pylint: disable=import-outside-toplevel from timesketch.lib import tasks pipeline = tasks.build_index_pipeline(file_path=file_path, events=events, timeline_name=timeline_name, index_name=index_name, file_extension=file_extension, sketch_id=sketch.id, only_index=enable_stream) pipeline.apply_async() # Return Timeline if it was created. # pylint: disable=no-else-return if timeline: return self.to_json(timeline, status_code=HTTP_STATUS_CODE_CREATED, meta=meta) return self.to_json(searchindex, status_code=HTTP_STATUS_CODE_CREATED, meta=meta)
def run(self, file_path, sketch_id, username, timeline_name): """This is the run method.""" file_path = os.path.realpath(file_path) file_path_no_extension, extension = os.path.splitext(file_path) extension = extension.lstrip('.') filename = os.path.basename(file_path_no_extension) supported_extensions = ('plaso', 'csv', 'jsonl') if not os.path.isfile(file_path): sys.exit('No such file: {0:s}'.format(file_path)) if extension not in supported_extensions: sys.exit( 'Extension {0:s} is not supported. ' '(supported extensions are: {1:s})'.format( extension, ', '.join(supported_extensions))) user = None if not username: username = pwd.getpwuid(os.stat(file_path).st_uid).pw_name if not username == 'root': if not isinstance(username, six.text_type): username = codecs.decode(username, 'utf-8') user = User.query.filter_by(username=username).first() if not user: sys.exit('Cannot determine user for file: {0:s}'.format(file_path)) sketch = None # If filename starts with <number> then use that as sketch_id. # E.g: 42_file_name.plaso means sketch_id is 42. sketch_id_from_filename = filename.split('_')[0] if not sketch_id and sketch_id_from_filename.isdigit(): sketch_id = sketch_id_from_filename if sketch_id: try: sketch = Sketch.query.get_with_acl(sketch_id, user=user) except Forbidden: pass if not timeline_name: if not isinstance(timeline_name, six.text_type): timeline_name = codecs.decode(timeline_name, 'utf-8') timeline_name = timeline_name.replace('_', ' ') # Remove sketch ID if present in the filename. timeline_parts = timeline_name.split() if timeline_parts[0].isdigit(): timeline_name = ' '.join(timeline_name.split()[1:]) if not sketch: # Create a new sketch. sketch_name = 'Sketch for: {0:s}'.format(timeline_name) sketch = Sketch( name=sketch_name, description=sketch_name, user=user) # Need to commit here to be able to set permissions later. db_session.add(sketch) db_session.commit() sketch.grant_permission(permission='read', user=user) sketch.grant_permission(permission='write', user=user) sketch.grant_permission(permission='delete', user=user) sketch.status.append(sketch.Status(user=None, status='new')) db_session.add(sketch) db_session.commit() index_name = uuid.uuid4().hex if not isinstance(index_name, six.text_type): index_name = codecs.decode(index_name, 'utf-8') searchindex = SearchIndex.get_or_create( name=timeline_name, description=timeline_name, user=user, index_name=index_name) searchindex.grant_permission(permission='read', user=user) searchindex.grant_permission(permission='write', user=user) searchindex.grant_permission(permission='delete', user=user) searchindex.set_status('processing') db_session.add(searchindex) db_session.commit() if sketch and sketch.has_permission(user, 'write'): timeline = Timeline( name=searchindex.name, description=searchindex.description, sketch=sketch, user=user, searchindex=searchindex) timeline.set_status('processing') sketch.timelines.append(timeline) db_session.add(timeline) db_session.commit() # Start Celery pipeline for indexing and analysis. # Import here to avoid circular imports. from timesketch.lib import tasks pipeline = tasks.build_index_pipeline( file_path, timeline_name, index_name, extension, sketch.id) pipeline.apply_async(task_id=index_name) print('Imported {0:s} to sketch: {1:d} ({2:s})'.format( file_path, sketch.id, sketch.name))
def setup_sketch(timeline_name, index_name, username, sketch_id=None): """Use existing sketch or create a new sketch. Args: timeline_name: (str) Name of the Timeline index_name: (str) Name of the index username: (str) Who should own the timeline sketch_id: (str) Optional sketch_id to add timeline to Returns: (tuple) sketch ID and timeline ID as integers """ with app.app_context(): user = User.get_or_create(username=username) sketch = None if sketch_id: try: sketch = Sketch.query.get_with_acl(sketch_id, user=user) logger.info('Using existing sketch: {} ({})'.format( sketch.name, sketch.id)) except Forbidden: pass if not (sketch or sketch_id): # Create a new sketch. sketch_name = 'Turbinia: {}'.format(timeline_name) sketch = Sketch(name=sketch_name, description=sketch_name, user=user) # Need to commit here to be able to set permissions later. db_session.add(sketch) db_session.commit() sketch.grant_permission(permission='read', user=user) sketch.grant_permission(permission='write', user=user) sketch.grant_permission(permission='delete', user=user) sketch.status.append(sketch.Status(user=None, status='new')) db_session.add(sketch) db_session.commit() logger.info('Created new sketch: {} ({})'.format( sketch.name, sketch.id)) searchindex = SearchIndex.get_or_create( name=timeline_name, description='Created by Turbinia.', user=user, index_name=index_name) searchindex.grant_permission(permission='read', user=user) searchindex.grant_permission(permission='write', user=user) searchindex.grant_permission(permission='delete', user=user) searchindex.set_status('processing') db_session.add(searchindex) db_session.commit() timeline = Timeline(name=searchindex.name, description=searchindex.description, sketch=sketch, user=user, searchindex=searchindex) # If the user doesn't have write access to the sketch then create the # timeline but don't attach it to the sketch. if not sketch.has_permission(user, 'write'): timeline.sketch = None else: sketch.timelines.append(timeline) db_session.add(timeline) db_session.commit() timeline.set_status('processing') return sketch.id, timeline.id
def post(self): """Handles POST request to the resource. Returns: A view in JSON (instance of flask.wrappers.Response) Raises: ApiHTTPError """ upload_enabled = current_app.config[u'UPLOAD_ENABLED'] upload_folder = current_app.config[u'UPLOAD_FOLDER'] form = UploadFileForm() if form.validate_on_submit() and upload_enabled: from timesketch.lib.tasks import run_plaso from timesketch.lib.tasks import run_csv_jsonl # Map the right task based on the file type task_directory = {u'plaso': run_plaso, u'csv': run_csv_jsonl, u'jsonl': run_csv_jsonl} sketch_id = form.sketch_id.data file_storage = form.file.data _filename, _extension = os.path.splitext(file_storage.filename) file_extension = _extension.lstrip(u'.') timeline_name = form.name.data or _filename.rstrip(u'.') sketch = None if sketch_id: sketch = Sketch.query.get_with_acl(sketch_id) # Current user username = current_user.username # We do not need a human readable filename or # datastore index name, so we use UUIDs here. filename = unicode(uuid.uuid4().hex) index_name = unicode(uuid.uuid4().hex) file_path = os.path.join(upload_folder, filename) file_storage.save(file_path) # Create the search index in the Timesketch database searchindex = SearchIndex.get_or_create( name=timeline_name, description=timeline_name, user=current_user, index_name=index_name) searchindex.grant_permission(permission=u'read', user=current_user) searchindex.grant_permission(permission=u'write', user=current_user) searchindex.grant_permission( permission=u'delete', user=current_user) searchindex.set_status(u'processing') db_session.add(searchindex) db_session.commit() timeline = None if sketch and sketch.has_permission(current_user, u'write'): timeline = Timeline( name=searchindex.name, description=searchindex.description, sketch=sketch, user=current_user, searchindex=searchindex) timeline.set_status(u'processing') sketch.timelines.append(timeline) db_session.add(timeline) db_session.commit() # Run the task in the background task = task_directory.get(file_extension) task.apply_async( ( file_path, timeline_name, index_name, file_extension, username ), task_id=index_name ) # Return Timeline if it was created. # pylint: disable=no-else-return if timeline: return self.to_json( timeline, status_code=HTTP_STATUS_CODE_CREATED) else: return self.to_json( searchindex, status_code=HTTP_STATUS_CODE_CREATED) else: raise ApiHTTPError( message=form.errors[u'file'][0], status_code=HTTP_STATUS_CODE_BAD_REQUEST)