def Process(self):
"""Executes a Timesketch export."""
if not self.timesketch_api:
message = 'Could not connect to Timesketch server'
self.ModuleError(message, critical=True)
sketch = self.state.GetFromCache('timesketch_sketch')
if not sketch:
sketch = self.timesketch_api.get_sketch(self.sketch_id)
recipe_name = self.state.recipe.get('name', 'no_recipe')
input_names = []
for file_container in self.state.GetContainers(containers.File):
description = file_container.name
if not description:
continue
name = description.rpartition('.')[0]
name = name.replace(' ', '_').replace('-', '_')
input_names.append(name)
if input_names:
timeline_name = '{0:s}_{1:s}'.format(
recipe_name, '_'.join(input_names))
else:
timeline_name = recipe_name
with importer.ImportStreamer() as streamer:
streamer.set_sketch(sketch)
streamer.set_timeline_name(timeline_name)
for file_container in self.state.GetContainers(containers.File):
path = file_container.path
streamer.add_file(path)
api_root = sketch.api.api_root
host_url = api_root.partition('api/v1')[0]
sketch_url = '{0:s}sketches/{1:d}/'.format(host_url, sketch.id)
message = 'Your Timesketch URL is: {0:s}'.format(sketch_url)
container = containers.Report(
module_name='TimesketchExporter',
text=message,
text_format='markdown')
self.state.StoreContainer(container)
for analyzer in self._analyzers:
results = sketch.run_analyzer(
analyzer_name=analyzer, timeline_name=timeline_name)
if not results:
self.logger.info('Analyzer [{0:s}] not able to run on {1:s}'.format(
analyzer, timeline_name))
continue
session_id = results.id
if not session_id:
self.logger.info(
'Analyzer [{0:s}] didn\'t provide any session data'.format(
analyzer))
continue
self.logger.info('Analyzer: {0:s} is running, session ID: {1:d}'.format(
analyzer, session_id))
self.logger.info(results.status_string)
def testGetContainer(self):
"""Tests that containers can be retrieved."""
test_state = state.DFTimewolfState(config.Config)
test_report = containers.Report(module_name='foo', text='bar')
test_state.StoreContainer(test_report)
reports = test_state.GetContainers(containers.Report)
self.assertEqual(len(reports), 1)
self.assertIsInstance(reports[0], containers.Report)
def testStoreContainer(self):
"""Tests that containers are stored correctly."""
test_state = state.DFTimewolfState(config.Config)
test_report = containers.Report(module_name='foo', text='bar')
test_state.StoreContainer(test_report)
self.assertEqual(len(test_state.store), 1)
self.assertIn('report', test_state.store)
self.assertEqual(len(test_state.store['report']), 1)
self.assertIsInstance(test_state.store['report'][0], containers.Report)
def testStreamingCallback(self, mock_callback):
"""Tests that registered callbacks are appropriately called."""
test_state = state.DFTimewolfState(config.Config)
test_state.LoadRecipe(test_recipe.contents)
test_state.SetupModules()
# DummyModule1 has registered a StreamingConsumer
report = containers.Report(module_name='testing', text='asd')
test_state.StreamContainer(report)
mock_callback.assert_called_with(report)
def testGetAttributeNames(self):
"""Tests the GetAttributeNames function."""
attribute_container = containers.Report(module_name='name', text='text')
expected_attribute_names = [
'attributes', 'module_name', 'text', 'text_format']
attribute_names = sorted(attribute_container.GetAttributeNames())
self.assertEqual(attribute_names, expected_attribute_names)
def _ProcessStories(self, stories):
"""Extracts story content from a list of stories and saves as a report.
The function runs through all saved stories in a sketch and adds a
formatted version of the story as a report container.
Args:
stories (list): a list of Story objects (timesketch_api.story.Story).
"""
for story in stories:
if self._formatter.FORMAT == 'html':
story_string = story.to_html()
elif self._formatter.FORMAT == 'markdown':
story_string = story.to_markdown()
else:
story_string = story.to_export_format(self._formatter.FORMAT)
self.state.StoreContainer(containers.Report(
module_name='TimesketchEnhancer',
text_format=self._formatter.FORMAT,
text=story_string))
def Process(self):
"""Process files with Turbinia."""
log_file_path = os.path.join(self._output_path, 'turbinia.log')
print('Turbinia log file: {0:s}'.format(log_file_path))
if self.state.input and not self.disk_name:
_, disk = self.state.input[0]
self.disk_name = disk.name
print('Using disk {0:s} from previous collector'.format(
self.disk_name))
evidence_ = evidence.GoogleCloudDisk(disk_name=self.disk_name,
project=self.project,
zone=self.turbinia_zone)
try:
evidence_.validate()
except TurbiniaException as exception:
self.state.AddError(exception, critical=True)
return
request = TurbiniaRequest(requester=getpass.getuser())
request.evidence.append(evidence_)
if self.sketch_id:
request.recipe['sketch_id'] = self.sketch_id
if not self.run_all_jobs:
request.recipe['jobs_blacklist'] = ['StringsJob']
# Get threat intelligence data from any modules that have stored some.
# In this case, observables is a list of containers.ThreatIntelligence
# objects.
threatintel = self.state.GetContainers(containers.ThreatIntelligence)
if threatintel:
print(
'Sending {0:d} threatintel to Turbinia GrepWorkers...'.format(
len(threatintel)))
indicators = [item.indicator for item in threatintel]
request.recipe['filter_patterns'] = indicators
request_dict = {
'instance': self.instance,
'project': self.project,
'region': self.turbinia_region,
'request_id': request.request_id
}
try:
print('Creating Turbinia request {0:s} with Evidence {1!s}'.format(
request.request_id, evidence_.name))
self.client.send_request(request)
print('Waiting for Turbinia request {0:s} to complete'.format(
request.request_id))
self.client.wait_for_request(**request_dict)
task_data = self.client.get_task_data(**request_dict)
except TurbiniaException as exception:
# TODO: determine if exception should be converted into a string as
# elsewhere in the codebase.
self.state.AddError(exception, critical=True)
return
message = self.client.format_task_status(**request_dict,
full_report=True)
short_message = self.client.format_task_status(**request_dict)
print(short_message)
# Store the message for consumption by any reporting modules.
report = containers.Report(module_name='TurbiniaProcessor',
text=message,
text_format='markdown')
self.state.StoreContainer(report)
local_paths, gs_paths = self._DeterminePaths(task_data)
if not local_paths and not gs_paths:
self.state.AddError(
'No interesting files found in Turbinia output.',
critical=True)
return
timeline_label = '{0:s}-{1:s}'.format(self.project, self.disk_name)
# Any local files that exist we can add immediately to the output
all_local_paths = [(timeline_label, p) for p in local_paths
if os.path.exists(p)]
downloaded_gs_paths = self._DownloadFilesFromGCS(
timeline_label, gs_paths)
all_local_paths.extend(downloaded_gs_paths)
if not all_local_paths:
self.state.AddError('No interesting files could be found.',
critical=True)
self.state.output = all_local_paths
for _, path in all_local_paths:
if path.endswith('BinaryExtractorTask.tar.gz'):
self.state.StoreContainer(
containers.ThreatIntelligence(
name='BinaryExtractorResults',
indicator=None,
path=path))
if path.endswith('hashes.json'):
self.state.StoreContainer(
containers.ThreatIntelligence(name='ImageExportHashes',
indicator=None,
path=path))
def Process(self):
"""Process files with Turbinia."""
log_file_path = os.path.join(self._output_path, 'turbinia.log')
print('Turbinia log file: {0:s}'.format(log_file_path))
if self.state.input and not self.disk_name:
_, disk = self.state.input[0]
self.disk_name = disk.name
print('Using disk {0:s} from previous collector'.format(
self.disk_name))
evidence_ = evidence.GoogleCloudDisk(disk_name=self.disk_name,
project=self.project,
zone=self.turbinia_zone)
try:
evidence_.validate()
except TurbiniaException as exception:
self.state.AddError(exception, critical=True)
return
request = TurbiniaRequest(requester=getpass.getuser())
request.evidence.append(evidence_)
if self.sketch_id:
request.recipe['sketch_id'] = self.sketch_id
if not self.run_all_jobs:
request.recipe['jobs_blacklist'] = ['StringsJob']
# Get threat intelligence data from any modules that have stored some.
# In this case, observables is a list of containers.ThreatIntelligence
# objects.
threatintel = self.state.GetContainers(containers.ThreatIntelligence)
if threatintel:
print(
'Sending {0:d} threatintel to Turbinia GrepWorkers...'.format(
len(threatintel)))
indicators = [item.indicator for item in threatintel]
request.recipe['filter_patterns'] = indicators
request_dict = {
'instance': self.instance,
'project': self.project,
'region': self.turbinia_region,
'request_id': request.request_id
}
try:
print('Creating Turbinia request {0:s} with Evidence {1!s}'.format(
request.request_id, evidence_.name))
self.client.send_request(request)
print('Waiting for Turbinia request {0:s} to complete'.format(
request.request_id))
self.client.wait_for_request(**request_dict)
task_data = self.client.get_task_data(**request_dict)
except TurbiniaException as exception:
# TODO: determine if exception should be converted into a string as
# elsewhere in the codebase.
self.state.AddError(exception, critical=True)
return
message = self.client.format_task_status(**request_dict,
full_report=True)
short_message = self.client.format_task_status(**request_dict)
print(short_message)
# Store the message for consumption by any reporting modules.
report = containers.Report(module_name='TurbiniaProcessor',
text=message,
text_format='markdown')
self.state.StoreContainer(report)
# This finds all .plaso files in the Turbinia output, and determines if they
# are local or remote (it's possible this will be running against a local
# instance of Turbinia).
local_paths = []
gs_paths = []
timeline_label = '{0:s}-{1:s}'.format(self.project, self.disk_name)
for task in task_data:
# saved_paths may be set to None
for path in task.get('saved_paths') or []:
if path.startswith('/') and path.endswith('.plaso'):
local_paths.append(path)
if path.startswith('gs://') and path.endswith('.plaso'):
gs_paths.append(path)
if not local_paths and not gs_paths:
self.state.AddError('No .plaso files found in Turbinia output.',
critical=True)
return
# Any local .plaso files that exist we can add immediately to the output
self.state.output = [(timeline_label, p) for p in local_paths
if os.path.exists(p)]
# For files remote in GCS we copy each plaso file back from GCS and then add
# to output paths
# TODO: Externalize fetching files from GCS buckets to a different module.
for path in gs_paths:
local_path = None
try:
output_writer = output_manager.GCSOutputWriter(
path, local_output_dir=self._output_path)
local_path = output_writer.copy_from(path)
except TurbiniaException as exception:
# TODO: determine if exception should be converted into a string as
# elsewhere in the codebase.
self.state.AddError(exception, critical=True)
return
if local_path:
self.state.output.append((timeline_label, local_path))
if not self.state.output:
self.state.AddError('No .plaso files could be found.',
critical=True)
def TurbiniaProcess(self, evidence_):
"""Creates and sends a Turbinia processing request.
Args:
evidence_(turbinia.evidence.Evidence): The evience to proecess
Returns:
list[dict]: The Turbinia task data
"""
try:
evidence_.validate()
except TurbiniaException as exception:
self.ModuleError(str(exception), critical=True)
request = TurbiniaRequest(requester=getpass.getuser())
request.evidence.append(evidence_)
if self.sketch_id:
request.recipe['sketch_id'] = self.sketch_id
if not self.run_all_jobs:
# TODO(aarontp): Remove once the release with
# https://github.com/google/turbinia/pull/554 is live.
request.recipe['jobs_blacklist'] = [
'StringsJob', 'BinaryExtractorJob', 'BulkExtractorJob', 'PhotorecJob']
request.recipe['jobs_denylist'] = [
'StringsJob', 'BinaryExtractorJob', 'BulkExtractorJob', 'PhotorecJob']
# Get threat intelligence data from any modules that have stored some.
# In this case, observables is a list of containers.ThreatIntelligence
# objects.
threatintel = self.state.GetContainers(containers.ThreatIntelligence)
if threatintel:
self.logger.info(
'Sending {0:d} threatintel to Turbinia GrepWorkers...'.format(
len(threatintel)))
indicators = [item.indicator for item in threatintel]
request.recipe['filter_patterns'] = indicators
request_dict = {
'instance': self.instance,
'project': self.project,
'region': self.turbinia_region,
'request_id': request.request_id
}
task_data = None
try:
self.logger.info(
'Creating Turbinia request {0:s} with Evidence {1!s}'.format(
request.request_id, evidence_.name))
self.client.send_request(request)
self.logger.info('Waiting for Turbinia request {0:s} to complete'.format(
request.request_id))
self.client.wait_for_request(**request_dict)
task_data = self.client.get_task_data(**request_dict)
except TurbiniaException as exception:
# TODO: determine if exception should be converted into a string as
# elsewhere in the codebase.
self.ModuleError(str(exception), critical=True)
message = self.client.format_task_status(full_report=True, **request_dict)
short_message = self.client.format_task_status(**request_dict)
self.logger.info(short_message)
# Store the message for consumption by any reporting modules.
report = containers.Report(
module_name='TurbiniaProcessor', text=message, text_format='markdown')
self.state.StoreContainer(report)
return task_data
def Process(self):
"""Executes a Timesketch enhancer module."""
if not self._wait_for_analyzers:
self.logger.warning(
'Not waiting for analyzers to run, skipping enhancer.')
return
if not self.timesketch_api:
message = 'Could not connect to Timesketch server'
self.ModuleError(message, critical=True)
sketch = self.state.GetFromCache('timesketch_sketch')
if not sketch:
message = (
'Sketch not found in cache, maybe the previous module was unable '
'to connect to Timesketch or unable to connect to and/or create '
'a sketch.')
self.ModuleError(message, critical=True)
self.logger.info('Waiting for analyzers to complete their run.')
summary_lines = [self._formatter.Heading('Timesketch Run', level=1)]
summary_lines.append(self._formatter.Paragraph(
'This is a summary of actions taken by Timesketch '
'during its run.'))
summary_lines.append(self._formatter.Paragraph(
'To visit the sketch, click {0:s}'.format(self._formatter.Link(
url=self._GetSketchURL(sketch), text='here'))))
summary_lines.append(self._formatter.Paragraph(
'Here is an overview of actions taken:'))
self._WaitForAnalyzers(sketch)
# Force a refresh of sketch data.
_ = sketch.lazyload_data(refresh_cache=True)
summary_lines.append(self._formatter.IndentStart())
saved_searches = sketch.list_saved_searches()
self._ProcessSavedSearches(saved_searches)
sketch_url = self._GetSketchURL(sketch)
search_string = self._GenerateSavedSearchString(
saved_searches, sketch_url)
formatted_string = ''
if search_string:
formatted_string = self._formatter.IndentText(
'The following saved searches were discovered:\n'
'{0:s}{1:s}{2:s}'.format(
self._formatter.IndentStart(),
search_string,
self._formatter.IndentEnd()))
else:
formatted_string = self._formatter.IndentText(
'Analyzers didn\'t save any searches.')
summary_lines.append(formatted_string)
aggregations = sketch.list_aggregations(
exclude_labels=['informational'])
self._ProcessAggregations(aggregations)
aggregation_string = self._GenerateAggregationString(aggregations)
if aggregation_string:
formatted_string = self._formatter.IndentText(
'The following aggregations were discovered:'
'\n{0:s}{1:s}{2:s}'.format(
self._formatter.IndentStart(),
aggregation_string,
self._formatter.IndentEnd()))
else:
formatted_string = self._formatter.IndentText(
'No aggregations were generated by analyzers.')
summary_lines.append(formatted_string)
stories = sketch.list_stories()
if self._include_stories:
self._ProcessStories(stories)
story_string = self._GenerateStoryString(stories, sketch_url)
if story_string:
formatted_string = self._formatter.IndentText(
'The following stories were generated:\n{0:s}{1:s}{2:s}'.format(
self._formatter.IndentStart(),
story_string,
self._formatter.IndentEnd()))
else:
formatted_string = self._formatter.IndentText(
'No stories were generated by analyzers.')
summary_lines.append(formatted_string)
summary_lines.append(self._formatter.IndentEnd())
analyzer_results = sketch.get_analyzer_status(as_sessions=True)
if analyzer_results:
line_string = self._formatter.Line()
summary_lines.append(line_string)
paragraph = self._formatter.Paragraph(
'Information from analyzer run:')
summary_lines.append(paragraph)
indent = self._formatter.IndentStart()
summary_lines.append(indent)
completed_ids = set()
for result in analyzer_results:
if result.id in completed_ids:
continue
if result.log:
log_text = self._formatter.IndentText(
'Logs: {0:s}'.format(result.log), level=2)
else:
log_text = ''
formatted_string = self._formatter.IndentText(
'ID: {0:d}\n{1:s}{2:s}\n{3:s}{4:s}'.format(
result.id,
self._formatter.IndentStart(),
'\n'.join([self._formatter.IndentText(
x.strip(), level=2) for x in result.results.split('\n')]),
log_text,
self._formatter.IndentEnd()
)
)
summary_lines.append(formatted_string)
completed_ids.add(result.id)
summary_lines.append(self._formatter.IndentEnd())
report_attributes = [{'update_comment': True}]
self.state.StoreContainer(containers.Report(
module_name='TimesketchEnhancer', text_format=self._formatter.FORMAT,
text='\n'.join(summary_lines), attributes=report_attributes))
self.logger.info('Analyzer reports generated')
