def import_timeline(self, filename):
"""Import a Plaso, CSV or JSONL file.
Args:
filename (str): Filename of the file to be imported.
Raises:
TimeoutError if import takes too long.
"""
file_path = os.path.join(TEST_DATA_DIR, filename)
print('Importing: {0:s}'.format(file_path))
with importer.ImportStreamer() as streamer:
streamer.set_sketch(self.sketch)
streamer.set_timeline_name(file_path)
streamer.add_file(file_path)
timeline = streamer.timeline
# Poll the timeline status and wait for the timeline to be ready
max_time_seconds = 600 # Timeout after 10min
sleep_time_seconds = 5 # Sleep between API calls
max_retries = max_time_seconds / sleep_time_seconds
retry_count = 0
while True:
if retry_count >= max_retries:
raise TimeoutError
_ = timeline.lazyload_data(refresh_cache=True)
status = timeline.status
# TODO: Do something with other statuses? (e.g. failed)
if status == 'ready':
break
retry_count += 1
time.sleep(sleep_time_seconds)
def Process(self):
"""Executes a Timesketch export."""
if not self.timesketch_api:
message = 'Could not connect to Timesketch server'
self.ModuleError(message, critical=True)
sketch = self.state.GetFromCache('timesketch_sketch')
if not sketch:
sketch = self.timesketch_api.get_sketch(self.sketch_id)
recipe_name = self.state.recipe.get('name', 'no_recipe')
input_names = []
for file_container in self.state.GetContainers(containers.File):
description = file_container.name
if not description:
continue
name = description.rpartition('.')[0]
name = name.replace(' ', '_').replace('-', '_')
input_names.append(name)
if input_names:
timeline_name = '{0:s}_{1:s}'.format(
recipe_name, '_'.join(input_names))
else:
timeline_name = recipe_name
with importer.ImportStreamer() as streamer:
streamer.set_sketch(sketch)
streamer.set_timeline_name(timeline_name)
for file_container in self.state.GetContainers(containers.File):
path = file_container.path
streamer.add_file(path)
api_root = sketch.api.api_root
host_url = api_root.partition('api/v1')[0]
sketch_url = '{0:s}sketches/{1:d}/'.format(host_url, sketch.id)
message = 'Your Timesketch URL is: {0:s}'.format(sketch_url)
container = containers.Report(
module_name='TimesketchExporter',
text=message,
text_format='markdown')
self.state.StoreContainer(container)
for analyzer in self._analyzers:
results = sketch.run_analyzer(
analyzer_name=analyzer, timeline_name=timeline_name)
if not results:
self.logger.info('Analyzer [{0:s}] not able to run on {1:s}'.format(
analyzer, timeline_name))
continue
session_id = results.id
if not session_id:
self.logger.info(
'Analyzer [{0:s}] didn\'t provide any session data'.format(
analyzer))
continue
self.logger.info('Analyzer: {0:s} is running, session ID: {1:d}'.format(
analyzer, session_id))
self.logger.info(results.status_string)
def upload_file(
my_sketch: sketch.Sketch, config_dict: Dict[str, any],
file_path: str) -> str:
"""Uploads a file to Timesketch.
Args:
my_sketch (sketch.Sketch): a sketch object to point to the sketch the
data will be imported to.
config_dict (dict): dict with settings for the importer.
file_path (str): the path to the file to upload.
Returns:
A string with results (whether successful or not).
"""
if not my_sketch or not hasattr(my_sketch, 'id'):
return 'Sketch needs to be set'
_, _, file_extension = file_path.rpartition('.')
if file_extension.lower() not in ('plaso', 'mans', 'csv', 'jsonl'):
return (
'File needs to have one of the following extensions: '
'.plaso, .mans, .csv, '
'.jsonl (not {0:s})').format(file_extension.lower())
with importer.ImportStreamer() as streamer:
streamer.set_sketch(my_sketch)
format_string = config_dict.get('message_format_string')
if format_string:
streamer.set_message_format_string(format_string)
timeline_name = config_dict.get('timeline_name')
if timeline_name:
streamer.set_timeline_name(timeline_name)
index_name = config_dict.get('index_name')
if index_name:
streamer.set_index_name(index_name)
time_desc = config_dict.get('timestamp_description')
if time_desc:
streamer.set_timestamp_description(time_desc)
log_config_file = config_dict.get('log_config_file', '')
streamer.set_config_file(log_config_file)
entry_threshold = config_dict.get('entry_threshold')
if entry_threshold:
streamer.set_entry_threshold(entry_threshold)
size_threshold = config_dict.get('size_threshold')
if size_threshold:
streamer.set_filesize_threshold(size_threshold)
streamer.add_file(file_path)
return 'File got successfully uploaded to sketch: {0:d}'.format(
my_sketch.id)
def importer(ctx, name, timeout, file_path):
"""Import timeline.
Args:
ctx: Click CLI context object.
name: Name of the timeline to create.
timeout: Seconds to wait for indexing.
file_path: File path to the file to import.
"""
sketch = ctx.obj.sketch
if not name:
name = click.format_filename(file_path, shorten=True)
timeline = None
with import_client.ImportStreamer() as streamer:
click.echo("Uploading to server .. ", nl=False)
streamer.set_sketch(sketch)
streamer.set_timeline_name(name)
streamer.set_provider("Timesketch CLI client")
# TODO: Consider using the whole command as upload context instead
# of the file path.
streamer.set_upload_context(file_path)
streamer.add_file(file_path)
timeline = streamer.timeline
if not timeline:
click.echo("Error creating timeline, please try again.")
sys.exit(1)
click.echo("Done")
# Poll the timeline status and wait for the timeline to be ready
click.echo("Indexing .. ", nl=False)
max_time_seconds = timeout
sleep_time_seconds = 5 # Sleep between API calls
max_retries = max_time_seconds / sleep_time_seconds
retry_count = 0
while True:
if retry_count >= max_retries:
click.echo(
(
"WARNING: The command timed out before indexing finished. "
"The timeline will continue to be indexed in the background"
)
)
break
status = timeline.status
# TODO: Do something with other statuses? (e.g. failed)
if status == "ready":
click.echo("Done")
break
retry_count += 1
time.sleep(sleep_time_seconds)
click.echo(f"Timeline imported: {timeline.name}")
def timesketch_upload_file(data: Text, name: Optional[Text] = ''):
"""Upload a file to Timesketch.
Args:
data (str): Path to the file that's about to be uploaded.
name (str): Name of the timeline.
"""
if not os.path.isfile(data):
print('File [{0:s}] does not exist.'.format(data))
return
connect()
state_obj = state.state()
sketch = state_obj.get_from_cache('timesketch_sketch')
result = None
if not name:
name = data.split(os.sep)[-1]
first, _, last = name.rpartition('.')
name = first or last
name = name.replace(' ', '_').replace('-', '_')
timeline = None
with importer.ImportStreamer() as streamer:
streamer.set_sketch(sketch)
streamer.set_timeline_name(name)
# Set the file size to 20Mb before the file is split.
streamer.set_filesize_threshold(20971520)
streamer.add_file(data)
# Force a flush.
streamer.flush()
result = streamer.response
timeline = streamer.timeline
if not result:
print('Unable to upload data.')
return
if not timeline.name:
print('Unable to get a timeline.')
return
print(f'Timeline: {timeline.name}{timeline.description}\n'
f'Status: {timeline.status}')
def timesketch_upload_file(data: Text, name: Optional[Text] = ''):
"""Upload a file to Timesketch.
Args:
data (str): Path to the file that's about to be uploaded.
name (str): Name of the timeline.
"""
if not os.path.isfile(data):
print('File [{0:s}] does not exist.'.format(data))
return
connect()
state_obj = state.state()
sketch = state_obj.get_from_cache('timesketch_sketch')
result = None
if not name:
name = data.split(os.sep)[-1]
first, _, last = name.rpartition('.')
name = first or last
name = name.replace(' ', '_').replace('-', '_')
with importer.ImportStreamer() as streamer:
streamer.set_sketch(sketch)
streamer.set_timeline_name(name)
# Set the file size to 20Mb before the file is split.
streamer.set_filesize_threshold(20971520)
streamer.add_file(data)
result = streamer.response
if not result:
print('Unable to upload data.')
return
for timesketch_object in result.get('objects', []):
if not timesketch_object:
continue
print('Timeline: {0:s}\nStatus: {1:s}'.format(
timesketch_object.get('description'), ','.join(
[x.get('status') for x in timesketch_object.get('status')])))
def to_timesketch(self):
"""
to_timesketch: push dataframe to timesketch
"""
import_helper = helper.ImportHelper()
with importer.ImportStreamer() as streamer:
streamer.set_sketch(self.sketch)
streamer.set_provider("MansToEs")
streamer.set_config_helper(import_helper)
streamer.set_timeline_name(self.timeline_name)
for file in glob(self.folder_path + "/tmp__*.json"):
df = pd.read_json(file,
orient="records",
lines=True,
dtype=False)
filetype = file.split("tmp___")[-1].split(".")[0].split("_")[0]
streamer.set_upload_context(filetype)
streamer.add_data_frame(df, part_of_iter=True)
if self.exd_alerts:
for alert in self.exd_alerts:
streamer.set_upload_context("EXD alerts")
streamer.add_dict(alert)
logging.debug("[MAIN] Bulk timesketch push [✔]")
def timesketch_upload_data(data: pd.DataFrame,
name: Optional[Text] = '',
format_message_string: Optional[Text] = ''):
"""Upload a data frame to TimeSketch.
Args:
data (pandas.core.frame.DataFrame): the DataFrame to upload.
name (str): the name used for the timeline in Timesketch.
format_message_string (str): formatting string for the message column of
the data frame, eg: "{src_ip:s} to {dst_ip:s}, {bytes:d} bytes
transferred"'
Raises:
ValueError: if the dataframe cannot be uploaded to Timesketch or the
data is invalid.
"""
if not isinstance(data, pd.DataFrame):
raise ValueError(
('The data attribute is not a pandas DataFrame, please use curly '
'braces to expand variables.'))
if not name:
name = 'unknown_timeline'
connect()
state_obj = state.state()
sketch = state_obj.get_from_cache('timesketch_sketch')
if not sketch:
raise ValueError('Unable to upload data frame, need to set sketch.')
result = None
import_helper = helper.ImportHelper()
timeline = None
with importer.ImportStreamer() as streamer:
streamer.set_sketch(sketch)
if 'data_type' not in data:
data_type = utils.ask_question('What is the value of [data_type]?',
input_type=str)
if data_type:
streamer.set_data_type(data_type)
else:
data_types = data.data_type.unique()
data_type = data_types[0]
columns = list(data.columns)
streamer.set_config_helper(import_helper)
import_helper.configure_streamer(streamer,
data_type=data_type,
columns=columns)
if format_message_string:
streamer.set_message_format_string(format_message_string)
streamer.set_timeline_name(name)
streamer.add_data_frame(data)
streamer.flush()
result = streamer.response
timeline = streamer.timeline
if not result:
print('Unable to upload data.')
return
if not timeline.name:
print('Unable to import the timeline.')
return
print(
f'Timeline: [{timeline.id}]{timeline.name} - {timeline.description}\n'
f'Status: {timeline.status}')
def upload_file(my_sketch: sketch.Sketch, config_dict: Dict[str, any],
file_path: str) -> str:
"""Uploads a file to Timesketch.
Args:
my_sketch (sketch.Sketch): a sketch object to point to the sketch the
data will be imported to.
config_dict (dict): dict with settings for the importer.
file_path (str): the path to the file to upload.
Returns:
A tuple with the timeline object (timeline.Timeline) or None if not
able to upload the timeline as well as the celery task identification
for the indexing.
"""
if not my_sketch or not hasattr(my_sketch, 'id'):
return 'Sketch needs to be set'
_, _, file_extension = file_path.rpartition('.')
if file_extension.lower() not in ('plaso', 'csv', 'jsonl'):
return ('File needs to have one of the following extensions: '
'.plaso, .csv, '
'.jsonl (not {0:s})').format(file_extension.lower())
import_helper = helper.ImportHelper()
import_helper.add_config_dict(config_dict)
log_config_file = config_dict.get('log_config_file', '')
if log_config_file:
import_helper.add_config(log_config_file)
timeline = None
task_id = ''
logger.info('About to upload file.')
with importer.ImportStreamer() as streamer:
streamer.set_sketch(my_sketch)
streamer.set_config_helper(import_helper)
streamer.set_provider('CLI importer tool')
format_string = config_dict.get('message_format_string')
if format_string:
streamer.set_message_format_string(format_string)
timeline_name = config_dict.get('timeline_name')
if timeline_name:
streamer.set_timeline_name(timeline_name)
index_name = config_dict.get('index_name')
if index_name:
streamer.set_index_name(index_name)
time_desc = config_dict.get('timestamp_description')
if time_desc:
streamer.set_timestamp_description(time_desc)
entry_threshold = config_dict.get('entry_threshold')
if entry_threshold:
streamer.set_entry_threshold(entry_threshold)
size_threshold = config_dict.get('size_threshold')
if size_threshold:
streamer.set_filesize_threshold(size_threshold)
data_label = config_dict.get('data_label')
if data_label:
streamer.set_data_label(data_label)
context = config_dict.get('context')
if context:
streamer.set_upload_context(context)
else:
streamer.set_upload_context(' '.join(sys.argv))
streamer.add_file(file_path)
# Force a flush.
streamer.flush()
timeline = streamer.timeline
task_id = streamer.celery_task_id
logger.info('File upload completed.')
return timeline, task_id
