def __init__(self, output_mediator): """Initializes the output module object. Args: output_mediator: The output mediator object (instance of OutputMediator). """ super(TimesketchOutputModule, self).__init__(output_mediator) self._counter = Counter() self._doc_type = u'plaso_event' self._events = [] self._flush_interval = None self._index_name = None self._timesketch = timesketch.create_app() # TODO: Support reading in server and port information and set this using # options. with self._timesketch.app_context(): self._elastic_db = ElasticSearchDataStore( host=current_app.config[u'ELASTIC_HOST'], port=current_app.config[u'ELASTIC_PORT']) hostname = self._output_mediator.GetStoredHostname() if hostname: logging.info(u'Hostname: {0:s}'.format(hostname)) self._timeline_name = hostname else: self._timeline_name = None
def __init__(self, output_mediator): """Initializes an output module object. Args: output_mediator (OutputMediator): mediates interactions between output modules and other components, such as storage and dfvfs. """ super(TimesketchOutputModule, self).__init__(output_mediator) self._output_mediator = output_mediator self._host = None self._port = None self._flush_interval = None self._index_name = None self._doc_type = None self._username = None self._mapping = None self._elastic = None self._timesketch = timesketch.create_app() hostname = self._output_mediator.GetStoredHostname() if hostname: logging.info(u'Hostname: {0:s}'.format(hostname)) self._timeline_name = hostname else: self._timeline_name = None
def __init__(self, store, formatter_mediator, filehandle=sys.stdout, config=None, filter_use=None): """Initializes the log output formatter object. Args: store: A storage file object (instance of StorageFile). formatter_mediator: The formatter mediator object (instance of FormatterMediator). filehandle: Optional file-like object that can be written to. The default is sys.stdout. config: Optional configuration object, containing config information. The default is None. filter_use: Optional filter object (instance of FilterObject). The default is None. """ super(TimesketchOutput, self).__init__(store, formatter_mediator, filehandle=filehandle, config=config, filter_use=filter_use) # Get Elasticsearch config from Timesketch. self._timesketch = timesketch.create_app() with self._timesketch.app_context(): self._elastic_db = ElasticSearchDataStore( host=current_app.config[u'ELASTIC_HOST'], port=current_app.config[u'ELASTIC_PORT']) self._counter = Counter() self._doc_type = u'plaso_event' self._events = [] self._flush_interval = getattr(config, u'flush_interval') self._index_name = getattr(config, u'index') self._timeline_name = getattr(config, u'name') self._timeline_owner = getattr(config, u'owner') self._timing_start = datetime.now() # Try to get the hostname from the pre-processing object. hostname = None if self.store: for info in self.store.GetStorageInformation(): if hasattr(info, u'hostname'): hostname = info.hostname logging.info(u'Hostname: {0:s}'.format(hostname)) # Make sure we have a name for the timeline. Prompt the user if not. if not self._timeline_name: if hostname: self._timeline_name = hostname else: # This should not be handles in this module. # TODO: Move this to CLI code when available. self._timeline_name = raw_input(u'Timeline name: ') logging.info(u'Timeline name: {0:s}'.format(self._timeline_name)) logging.info(u'Index: {0:s}'.format(self._index_name))
def create_app(self): """Setup the Flask application. Returns: Flask application (instance of flask.app.Flask) """ app = create_app(TestConfig) return app
def __init__(self, output_mediator): """Initializes a Timesketch output module. Args: output_mediator (OutputMediator): mediates interactions between output modules and other components, such as storage and dfvfs. """ hostname = output_mediator.GetStoredHostname() if hostname: logger.debug('Hostname: {0:s}'.format(hostname)) super(TimesketchOutputModule, self).__init__(output_mediator) self._timeline_name = hostname self._timeline_owner = None self._timesketch = timesketch.create_app()
def run_csv(source_file_path, timeline_name, index_name, username=None): """Create a Celery task for processing a CSV file. Args: source_file_path: Path to CSV file. timeline_name: Name of the Timesketch timeline. index_name: Name of the datastore index. username: Username of the user who will own the timeline. Returns: Dictionary with count of processed events. """ flush_interval = 1000 # events to queue before bulk index event_type = u'generic_event' # Document type for Elasticsearch app = create_app() # Log information to Celery logging.info(u'Index name: %s', index_name) logging.info(u'Timeline name: %s', timeline_name) logging.info(u'Flush interval: %d', flush_interval) logging.info(u'Document type: %s', event_type) logging.info(u'Owner: %s', username) es = ElasticsearchDataStore( host=current_app.config[u'ELASTIC_HOST'], port=current_app.config[u'ELASTIC_PORT']) es.create_index(index_name=index_name, doc_type=event_type) for event in read_and_validate_csv(source_file_path): es.import_event( flush_interval, index_name, event_type, event) # Import the remaining events total_events = es.import_event(flush_interval, index_name, event_type) # We are done so let's remove the processing status flag with app.app_context(): search_index = SearchIndex.query.filter_by( index_name=index_name).first() search_index.status.remove(search_index.status[0]) db_session.add(search_index) db_session.commit() return {u'Events processed': total_events}
def __init__(self, output_mediator, **kwargs): """Initializes the output module object. Args: output_mediator: The output mediator object (instance of OutputMediator). """ super(TimesketchOutputModule, self).__init__(output_mediator, **kwargs) # Get Elasticsearch config from Timesketch. self._timesketch = timesketch.create_app() with self._timesketch.app_context(): self._elastic_db = ElasticSearchDataStore( host=current_app.config[u'ELASTIC_HOST'], port=current_app.config[u'ELASTIC_PORT']) self._counter = Counter() self._doc_type = u'plaso_event' self._events = [] self._flush_interval = self._output_mediator.GetConfigurationValue( u'flush_interval') self._index_name = self._output_mediator.GetConfigurationValue( u'index') self._timeline_name = self._output_mediator.GetConfigurationValue( u'name') self._timeline_owner = self._output_mediator.GetConfigurationValue( u'owner') hostname = self._output_mediator.GetStoredHostname() if hostname: logging.info(u'Hostname: {0:s}'.format(hostname)) # Make sure we have a name for the timeline. Prompt the user if not. if not self._timeline_name: if hostname: self._timeline_name = hostname else: # This should not be handles in this module. # TODO: Move this to CLI code when available. self._timeline_name = raw_input(u'Timeline name: ') logging.info(u'Timeline name: {0:s}'.format(self._timeline_name)) logging.info(u'Index: {0:s}'.format(self._index_name))
def __init__(self, output_mediator, **kwargs): """Initializes the output module object. Args: output_mediator: The output mediator object (instance of OutputMediator). """ super(TimesketchOutputModule, self).__init__(output_mediator, **kwargs) # Get Elasticsearch config from Timesketch. self._timesketch = timesketch.create_app() with self._timesketch.app_context(): self._elastic_db = ElasticSearchDataStore( host=current_app.config[u'ELASTIC_HOST'], port=current_app.config[u'ELASTIC_PORT']) self._counter = Counter() self._doc_type = u'plaso_event' self._events = [] self._flush_interval = self._output_mediator.GetConfigurationValue( u'flush_interval') self._index_name = self._output_mediator.GetConfigurationValue(u'index') self._timeline_name = self._output_mediator.GetConfigurationValue(u'name') self._timeline_owner = self._output_mediator.GetConfigurationValue(u'owner') hostname = self._output_mediator.GetStoredHostname() if hostname: logging.info(u'Hostname: {0:s}'.format(hostname)) # Make sure we have a name for the timeline. Prompt the user if not. if not self._timeline_name: if hostname: self._timeline_name = hostname else: # This should not be handles in this module. # TODO: Move this to CLI code when available. self._timeline_name = raw_input(u'Timeline name: ') logging.info(u'Timeline name: {0:s}'.format(self._timeline_name)) logging.info(u'Index: {0:s}'.format(self._index_name))
from celery import group from celery import chain from flask import current_app from timesketch import create_app from timesketch import create_celery_app from timesketch.lib.analyzers import manager from timesketch.lib.datastores.elastic import ElasticsearchDataStore from timesketch.lib.utils import read_and_validate_csv from timesketch.lib.utils import read_and_validate_jsonl from timesketch.models import db_session from timesketch.models.sketch import SearchIndex from timesketch.models.sketch import Timeline celery = create_celery_app() flask_app = create_app() class SqlAlchemyTask(celery.Task): """An abstract task that runs on task completion.""" abstract = True def after_return(self, *args, **kwargs): """Close the database session on task completion.""" db_session.remove() super(SqlAlchemyTask, self).after_return(*args, **kwargs) def _set_timeline_status(index_name, status, error_msg=None): """Helper function to set status for searchindex and all related timelines.
result = [] for event in events: src_ws = event[0] svc_name = event[1] start_type = event[2] image_path = event[3] image_path_short = image_path.strip().strip('\\').split('\\')[-1] timestamp = event[4] es_index_name = event[5] es_id = event[6] src_ws = src_ws.split('.')[0].upper() result.append({ 'src': src_ws, 'svc_name': svc_name, 'start_type': start_type, 'image_path': image_path, 'image_path_short': image_path_short, 'timestamp': timestamp, 'es_index_name': es_index_name, 'es_query': '_index:{} AND _id:{}'.format(es_index_name, es_id) }) return result if __name__ == "__main__": from timesketch import create_app with create_app().app_context(): main()
"""This module is for creating the app for a WSGI server. Example with Gunicorn: gunicorn -b 127.0.0.1:80 --log-file - --timeout 120 timesketch.wsgi:application Example configuration for Apache with mod_wsgi (a2enmod mod_wsgi): <VirtualHost *:443> ServerAdmin root@localhost SSLEngine On SSLCertificateFile /etc/apache2/cert.crt SSLCertificateKeyFile /etc/apache2/cert.key WSGIScriptAlias / /path/to/this/file/wsgi.py </VirtualHost> """ # If you installed Timesketch in a virtualenv you need to activate it. # This needs to be before any imports in order to import from the virtualenv. #activate_virtualenv = '/path/to/your/virtualenv/bin/activate_this.py' #execfile(activate_virtualenv, dict(__file__=activate_virtualenv)) from timesketch import create_app from timesketch.models import db_session application = create_app() # pylint: disable=unused-argument @application.teardown_appcontext def shutdown_session(exception=None): """Remove the database session after every request or app shutdown.""" db_session.remove()
Example with Gunicorn: gunicorn -b 127.0.0.1:80 --log-file - --timeout 120 timesketch.wsgi:application Example configuration for Apache with mod_wsgi (a2enmod mod_wsgi): <VirtualHost *:443> ServerAdmin root@localhost SSLEngine On SSLCertificateFile /etc/apache2/cert.crt SSLCertificateKeyFile /etc/apache2/cert.key WSGIScriptAlias / /path/to/this/file/wsgi.py </VirtualHost> """ # If you installed Timesketch in a virtualenv you need to activate it. # This needs to be before any imports in order to import from the virtualenv. #activate_virtualenv = '/path/to/your/virtualenv/bin/activate_this.py' #execfile(activate_virtualenv, dict(__file__=activate_virtualenv)) from timesketch import create_app from timesketch.models import db_session application = create_app() # pylint: disable=unused-argument @application.teardown_appcontext def shutdown_session(exception=None): """Remove the database session after every request or app shutdown.""" db_session.remove()
file_extension='plaso', sketch_id=sketch_id) pipeline.apply_async() logger.info('File sent for indexing: {}'.format(gcs_base_filename)) if __name__ == '__main__': parser = argparse.ArgumentParser(description='GCS importer') parser.add_argument('--project', help='Google Cloud Project ID') parser.add_argument('--bucket', help='Google Cloud Storage bucket to monitor') parser.add_argument('--subscription', help='Google Cloud PubSub subscription') parser.add_argument('--output', default='/tmp', help='Directory for downloads') args = parser.parse_args() # Create flask app app = create_app() # Setup Google Cloud Pub/Sub subscriber = pubsub_v1.SubscriberClient() subscription_path = subscriber.subscription_path(args.project, args.subscription) subscriber.subscribe(subscription_path, callback=callback) logger.info('Listening on PubSub queue: {}'.format(args.subscription)) while True: time.sleep(10)