def testGetContainer(self):
"""Tests that containers can be retrieved."""
test_state = state.DFTimewolfState(config.Config)
dummy_report = containers.Report(module_name='foo', text='bar')
test_state.store_container(dummy_report)
reports = test_state.get_containers(containers.Report)
self.assertEqual(len(reports), 1)
self.assertIsInstance(reports[0], containers.Report)
def test_get_attribute_names(self):
"""Tests the get_attribute_names function."""
attribute_container = containers.Report(module_name='name', text='text')
expected_attribute_names = ['attributes', 'module_name', 'text']
attribute_names = sorted(attribute_container.get_attribute_names())
self.assertEqual(attribute_names, expected_attribute_names)
def testStoreContainer(self):
"""Tests that containers are stored correctly."""
test_state = state.DFTimewolfState(config.Config)
test_state.store_container(
containers.Report(module_name='foo', text='bar'))
self.assertEqual(len(test_state.store), 1)
self.assertIn('report', test_state.store)
self.assertEqual(len(test_state.store['report']), 1)
self.assertIsInstance(test_state.store['report'][0], containers.Report)
def Process(self):
"""Process files with Turbinia."""
log_file_path = os.path.join(self._output_path, 'turbinia.log')
print('Turbinia log file: {0:s}'.format(log_file_path))
if self.state.input and not self.disk_name:
_, disk = self.state.input[0]
self.disk_name = disk.name
print('Using disk {0:s} from previous collector'.format(self.disk_name))
evidence_ = evidence.GoogleCloudDisk(
disk_name=self.disk_name, project=self.project, zone=self.turbinia_zone)
try:
evidence_.validate()
except TurbiniaException as exception:
self.state.AddError(exception, critical=True)
return
request = TurbiniaRequest()
request.evidence.append(evidence_)
# Get threat intelligence data from any modules that have stored some.
# In this case, observables is a list of containers.ThreatIntelligence
# objects.
threatintel = self.state.GetContainers(containers.ThreatIntelligence)
if threatintel:
print('Sending {0:d} threatintel to Turbinia GrepWorkers...'.format(
len(threatintel)))
indicators = [item.indicator for item in threatintel]
request.recipe['filter_patterns'] = indicators
request_dict = {
'instance': self.instance,
'project': self.project,
'region': self.turbinia_region,
'request_id': request.request_id
}
try:
print('Creating Turbinia request {0:s} with Evidence {1!s}'.format(
request.request_id, evidence_.name))
self.client.send_request(request)
print('Waiting for Turbinia request {0:s} to complete'.format(
request.request_id))
self.client.wait_for_request(**request_dict)
task_data = self.client.get_task_data(**request_dict)
except TurbiniaException as exception:
# TODO: determine if exception should be converted into a string as
# elsewhere in the codebase.
self.state.AddError(exception, critical=True)
return
# Turbinia run complete, build a human-readable message of results.
message = 'Completed {0:d} Turbinia tasks\n'.format(len(task_data))
for task in task_data:
message += '{0!s} ({1!s}): {2!s}\n'.format(
task.get('name'),
task.get('id'),
task.get('status', 'No task status'))
# saved_paths may be set to None
for path in task.get('saved_paths') or []:
if path.endswith('worker-log.txt'):
continue
if path.endswith('{0!s}.log'.format(task.get('id'))):
continue
if path.startswith('/'):
continue
message += ' {0:s}\n'.format(path)
print(message)
# Store the message for consumption by any reporting modules.
report = containers.Report(module_name='TurbiniaProcessor', text=message)
self.state.StoreContainer(report)
# This finds all .plaso files in the Turbinia output, and determines if they
# are local or remote (it's possible this will be running against a local
# instance of Turbinia).
local_paths = []
gs_paths = []
timeline_label = '{0:s}-{1:s}'.format(self.project, self.disk_name)
for task in task_data:
# saved_paths may be set to None
for path in task.get('saved_paths') or []:
if path.startswith('/') and path.endswith('.plaso'):
local_paths.append(path)
if path.startswith('gs://') and path.endswith('.plaso'):
gs_paths.append(path)
if not local_paths and not gs_paths:
self.state.AddError(
'No .plaso files found in Turbinia output.', critical=True)
return
# Any local .plaso files that exist we can add immediately to the output
self.state.output = [
(timeline_label, p) for p in local_paths if os.path.exists(p)]
# For files remote in GCS we copy each plaso file back from GCS and then add
# to output paths
# TODO: Externalize fetching files from GCS buckets to a different module.
for path in gs_paths:
local_path = None
try:
output_writer = output_manager.GCSOutputWriter(
path, local_output_dir=self._output_path)
local_path = output_writer.copy_from(path)
except TurbiniaException as exception:
# TODO: determine if exception should be converted into a string as
# elsewhere in the codebase.
self.state.AddError(exception, critical=True)
return
if local_path:
self.state.output.append((timeline_label, local_path))
if not self.state.output:
self.state.AddError('No .plaso files could be found.', critical=True)