def __init__(self, output_mediator):
"""Initializes the output module object.
Args:
output_mediator: The output mediator object (instance of OutputMediator).
"""
super(TimesketchOutputModule, self).__init__(output_mediator)
self._counter = Counter()
self._doc_type = u'plaso_event'
self._events = []
self._flush_interval = None
self._index_name = None
self._timesketch = timesketch.create_app()
# TODO: Support reading in server and port information and set this using
# options.
with self._timesketch.app_context():
self._elastic_db = ElasticSearchDataStore(
host=current_app.config[u'ELASTIC_HOST'],
port=current_app.config[u'ELASTIC_PORT'])
hostname = self._output_mediator.GetStoredHostname()
if hostname:
logging.info(u'Hostname: {0:s}'.format(hostname))
self._timeline_name = hostname
else:
self._timeline_name = None
def __init__(self, output_mediator):
"""Initializes an output module object.
Args:
output_mediator (OutputMediator): mediates interactions between output
modules and other components, such as storage and dfvfs.
"""
super(TimesketchOutputModule, self).__init__(output_mediator)
self._output_mediator = output_mediator
self._host = None
self._port = None
self._flush_interval = None
self._index_name = None
self._doc_type = None
self._username = None
self._mapping = None
self._elastic = None
self._timesketch = timesketch.create_app()
hostname = self._output_mediator.GetStoredHostname()
if hostname:
logging.info(u'Hostname: {0:s}'.format(hostname))
self._timeline_name = hostname
else:
self._timeline_name = None
def __init__(self, output_mediator):
"""Initializes the output module object.
Args:
output_mediator: The output mediator object (instance of OutputMediator).
"""
super(TimesketchOutputModule, self).__init__(output_mediator)
self._counter = Counter()
self._doc_type = u'plaso_event'
self._events = []
self._flush_interval = None
self._index_name = None
self._timesketch = timesketch.create_app()
# TODO: Support reading in server and port information and set this using
# options.
with self._timesketch.app_context():
self._elastic_db = ElasticSearchDataStore(
host=current_app.config[u'ELASTIC_HOST'],
port=current_app.config[u'ELASTIC_PORT'])
hostname = self._output_mediator.GetStoredHostname()
if hostname:
logging.info(u'Hostname: {0:s}'.format(hostname))
self._timeline_name = hostname
else:
self._timeline_name = None
def __init__(self,
store,
formatter_mediator,
filehandle=sys.stdout,
config=None,
filter_use=None):
"""Initializes the log output formatter object.
Args:
store: A storage file object (instance of StorageFile).
formatter_mediator: The formatter mediator object (instance of
FormatterMediator).
filehandle: Optional file-like object that can be written to.
The default is sys.stdout.
config: Optional configuration object, containing config information.
The default is None.
filter_use: Optional filter object (instance of FilterObject).
The default is None.
"""
super(TimesketchOutput, self).__init__(store,
formatter_mediator,
filehandle=filehandle,
config=config,
filter_use=filter_use)
# Get Elasticsearch config from Timesketch.
self._timesketch = timesketch.create_app()
with self._timesketch.app_context():
self._elastic_db = ElasticSearchDataStore(
host=current_app.config[u'ELASTIC_HOST'],
port=current_app.config[u'ELASTIC_PORT'])
self._counter = Counter()
self._doc_type = u'plaso_event'
self._events = []
self._flush_interval = getattr(config, u'flush_interval')
self._index_name = getattr(config, u'index')
self._timeline_name = getattr(config, u'name')
self._timeline_owner = getattr(config, u'owner')
self._timing_start = datetime.now()
# Try to get the hostname from the pre-processing object.
hostname = None
if self.store:
for info in self.store.GetStorageInformation():
if hasattr(info, u'hostname'):
hostname = info.hostname
logging.info(u'Hostname: {0:s}'.format(hostname))
# Make sure we have a name for the timeline. Prompt the user if not.
if not self._timeline_name:
if hostname:
self._timeline_name = hostname
else:
# This should not be handles in this module.
# TODO: Move this to CLI code when available.
self._timeline_name = raw_input(u'Timeline name: ')
logging.info(u'Timeline name: {0:s}'.format(self._timeline_name))
logging.info(u'Index: {0:s}'.format(self._index_name))
def create_app(self):
"""Setup the Flask application.
Returns:
Flask application (instance of flask.app.Flask)
"""
app = create_app(TestConfig)
return app
def __init__(self, output_mediator):
"""Initializes an output module object.
Args:
output_mediator (OutputMediator): mediates interactions between output
modules and other components, such as storage and dfvfs.
"""
super(TimesketchOutputModule, self).__init__(output_mediator)
self._output_mediator = output_mediator
self._host = None
self._port = None
self._flush_interval = None
self._index_name = None
self._doc_type = None
self._username = None
self._mapping = None
self._elastic = None
self._timesketch = timesketch.create_app()
hostname = self._output_mediator.GetStoredHostname()
if hostname:
logging.info(u'Hostname: {0:s}'.format(hostname))
self._timeline_name = hostname
else:
self._timeline_name = None
def create_app(self):
"""Setup the Flask application.
Returns:
Flask application (instance of flask.app.Flask)
"""
app = create_app(TestConfig)
return app
def __init__(self, output_mediator):
"""Initializes a Timesketch output module.
Args:
output_mediator (OutputMediator): mediates interactions between output
modules and other components, such as storage and dfvfs.
"""
hostname = output_mediator.GetStoredHostname()
if hostname:
logger.debug('Hostname: {0:s}'.format(hostname))
super(TimesketchOutputModule, self).__init__(output_mediator)
self._timeline_name = hostname
self._timeline_owner = None
self._timesketch = timesketch.create_app()
def __init__(self, output_mediator):
"""Initializes a Timesketch output module.
Args:
output_mediator (OutputMediator): mediates interactions between output
modules and other components, such as storage and dfvfs.
"""
hostname = output_mediator.GetStoredHostname()
if hostname:
logger.debug('Hostname: {0:s}'.format(hostname))
super(TimesketchOutputModule, self).__init__(output_mediator)
self._timeline_name = hostname
self._timeline_owner = None
self._timesketch = timesketch.create_app()
def run_csv(source_file_path, timeline_name, index_name, username=None):
"""Create a Celery task for processing a CSV file.
Args:
source_file_path: Path to CSV file.
timeline_name: Name of the Timesketch timeline.
index_name: Name of the datastore index.
username: Username of the user who will own the timeline.
Returns:
Dictionary with count of processed events.
"""
flush_interval = 1000 # events to queue before bulk index
event_type = u'generic_event' # Document type for Elasticsearch
app = create_app()
# Log information to Celery
logging.info(u'Index name: %s', index_name)
logging.info(u'Timeline name: %s', timeline_name)
logging.info(u'Flush interval: %d', flush_interval)
logging.info(u'Document type: %s', event_type)
logging.info(u'Owner: %s', username)
es = ElasticsearchDataStore(
host=current_app.config[u'ELASTIC_HOST'],
port=current_app.config[u'ELASTIC_PORT'])
es.create_index(index_name=index_name, doc_type=event_type)
for event in read_and_validate_csv(source_file_path):
es.import_event(
flush_interval, index_name, event_type, event)
# Import the remaining events
total_events = es.import_event(flush_interval, index_name, event_type)
# We are done so let's remove the processing status flag
with app.app_context():
search_index = SearchIndex.query.filter_by(
index_name=index_name).first()
search_index.status.remove(search_index.status[0])
db_session.add(search_index)
db_session.commit()
return {u'Events processed': total_events}
def __init__(self, output_mediator, **kwargs):
"""Initializes the output module object.
Args:
output_mediator: The output mediator object (instance of OutputMediator).
"""
super(TimesketchOutputModule, self).__init__(output_mediator, **kwargs)
# Get Elasticsearch config from Timesketch.
self._timesketch = timesketch.create_app()
with self._timesketch.app_context():
self._elastic_db = ElasticSearchDataStore(
host=current_app.config[u'ELASTIC_HOST'],
port=current_app.config[u'ELASTIC_PORT'])
self._counter = Counter()
self._doc_type = u'plaso_event'
self._events = []
self._flush_interval = self._output_mediator.GetConfigurationValue(
u'flush_interval')
self._index_name = self._output_mediator.GetConfigurationValue(
u'index')
self._timeline_name = self._output_mediator.GetConfigurationValue(
u'name')
self._timeline_owner = self._output_mediator.GetConfigurationValue(
u'owner')
hostname = self._output_mediator.GetStoredHostname()
if hostname:
logging.info(u'Hostname: {0:s}'.format(hostname))
# Make sure we have a name for the timeline. Prompt the user if not.
if not self._timeline_name:
if hostname:
self._timeline_name = hostname
else:
# This should not be handles in this module.
# TODO: Move this to CLI code when available.
self._timeline_name = raw_input(u'Timeline name: ')
logging.info(u'Timeline name: {0:s}'.format(self._timeline_name))
logging.info(u'Index: {0:s}'.format(self._index_name))
def __init__(self, output_mediator, **kwargs):
"""Initializes the output module object.
Args:
output_mediator: The output mediator object (instance of OutputMediator).
"""
super(TimesketchOutputModule, self).__init__(output_mediator, **kwargs)
# Get Elasticsearch config from Timesketch.
self._timesketch = timesketch.create_app()
with self._timesketch.app_context():
self._elastic_db = ElasticSearchDataStore(
host=current_app.config[u'ELASTIC_HOST'],
port=current_app.config[u'ELASTIC_PORT'])
self._counter = Counter()
self._doc_type = u'plaso_event'
self._events = []
self._flush_interval = self._output_mediator.GetConfigurationValue(
u'flush_interval')
self._index_name = self._output_mediator.GetConfigurationValue(u'index')
self._timeline_name = self._output_mediator.GetConfigurationValue(u'name')
self._timeline_owner = self._output_mediator.GetConfigurationValue(u'owner')
hostname = self._output_mediator.GetStoredHostname()
if hostname:
logging.info(u'Hostname: {0:s}'.format(hostname))
# Make sure we have a name for the timeline. Prompt the user if not.
if not self._timeline_name:
if hostname:
self._timeline_name = hostname
else:
# This should not be handles in this module.
# TODO: Move this to CLI code when available.
self._timeline_name = raw_input(u'Timeline name: ')
logging.info(u'Timeline name: {0:s}'.format(self._timeline_name))
logging.info(u'Index: {0:s}'.format(self._index_name))
from celery import group
from celery import chain
from flask import current_app
from timesketch import create_app
from timesketch import create_celery_app
from timesketch.lib.analyzers import manager
from timesketch.lib.datastores.elastic import ElasticsearchDataStore
from timesketch.lib.utils import read_and_validate_csv
from timesketch.lib.utils import read_and_validate_jsonl
from timesketch.models import db_session
from timesketch.models.sketch import SearchIndex
from timesketch.models.sketch import Timeline
celery = create_celery_app()
flask_app = create_app()
class SqlAlchemyTask(celery.Task):
"""An abstract task that runs on task completion."""
abstract = True
def after_return(self, *args, **kwargs):
"""Close the database session on task completion."""
db_session.remove()
super(SqlAlchemyTask, self).after_return(*args, **kwargs)
def _set_timeline_status(index_name, status, error_msg=None):
"""Helper function to set status for searchindex and all related timelines.
result = []
for event in events:
src_ws = event[0]
svc_name = event[1]
start_type = event[2]
image_path = event[3]
image_path_short = image_path.strip().strip('\\').split('\\')[-1]
timestamp = event[4]
es_index_name = event[5]
es_id = event[6]
src_ws = src_ws.split('.')[0].upper()
result.append({
'src': src_ws,
'svc_name': svc_name,
'start_type': start_type,
'image_path': image_path,
'image_path_short': image_path_short,
'timestamp': timestamp,
'es_index_name': es_index_name,
'es_query': '_index:{} AND _id:{}'.format(es_index_name, es_id)
})
return result
if __name__ == "__main__":
from timesketch import create_app
with create_app().app_context():
main()
"""This module is for creating the app for a WSGI server.
Example with Gunicorn:
gunicorn -b 127.0.0.1:80 --log-file - --timeout 120 timesketch.wsgi:application
Example configuration for Apache with mod_wsgi (a2enmod mod_wsgi):
<VirtualHost *:443>
ServerAdmin root@localhost
SSLEngine On
SSLCertificateFile /etc/apache2/cert.crt
SSLCertificateKeyFile /etc/apache2/cert.key
WSGIScriptAlias / /path/to/this/file/wsgi.py
</VirtualHost>
"""
# If you installed Timesketch in a virtualenv you need to activate it.
# This needs to be before any imports in order to import from the virtualenv.
#activate_virtualenv = '/path/to/your/virtualenv/bin/activate_this.py'
#execfile(activate_virtualenv, dict(__file__=activate_virtualenv))
from timesketch import create_app
from timesketch.models import db_session
application = create_app()
# pylint: disable=unused-argument
@application.teardown_appcontext
def shutdown_session(exception=None):
"""Remove the database session after every request or app shutdown."""
db_session.remove()
Example with Gunicorn:
gunicorn -b 127.0.0.1:80 --log-file - --timeout 120 timesketch.wsgi:application
Example configuration for Apache with mod_wsgi (a2enmod mod_wsgi):
<VirtualHost *:443>
ServerAdmin root@localhost
SSLEngine On
SSLCertificateFile /etc/apache2/cert.crt
SSLCertificateKeyFile /etc/apache2/cert.key
WSGIScriptAlias / /path/to/this/file/wsgi.py
</VirtualHost>
"""
# If you installed Timesketch in a virtualenv you need to activate it.
# This needs to be before any imports in order to import from the virtualenv.
#activate_virtualenv = '/path/to/your/virtualenv/bin/activate_this.py'
#execfile(activate_virtualenv, dict(__file__=activate_virtualenv))
from timesketch import create_app
from timesketch.models import db_session
application = create_app()
# pylint: disable=unused-argument
@application.teardown_appcontext
def shutdown_session(exception=None):
"""Remove the database session after every request or app shutdown."""
db_session.remove()
file_extension='plaso',
sketch_id=sketch_id)
pipeline.apply_async()
logger.info('File sent for indexing: {}'.format(gcs_base_filename))
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='GCS importer')
parser.add_argument('--project', help='Google Cloud Project ID')
parser.add_argument('--bucket',
help='Google Cloud Storage bucket to monitor')
parser.add_argument('--subscription',
help='Google Cloud PubSub subscription')
parser.add_argument('--output',
default='/tmp',
help='Directory for downloads')
args = parser.parse_args()
# Create flask app
app = create_app()
# Setup Google Cloud Pub/Sub
subscriber = pubsub_v1.SubscriberClient()
subscription_path = subscriber.subscription_path(args.project,
args.subscription)
subscriber.subscribe(subscription_path, callback=callback)
logger.info('Listening on PubSub queue: {}'.format(args.subscription))
while True:
time.sleep(10)
