def _AnalyzeHadoopAppRoot(self, collected_artifacts, output_dir):
"""Runs a naive AppRoot files parsing method.
This extracts strings from the saved task file, and searches for usual
post-compromise suspicious patterns.
TODO: properly parse the Proto. Some documentation can be found over there:
https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23.7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
Args:
collected_artifacts(list(str)): a list of paths to extracted files
output_dir(str): The base directory the artfacts are in.
Returns:
Tuple(
list(str): The report data as a list of lines
report_priority(int): The priority of the report (0 - 100)
summary(str): A summary of the report (used for task status)
)
"""
report = []
evil_commands = []
strings_count = 0
priority = Priority.MEDIUM
summary = ''
for filepath in collected_artifacts:
relpath = os.path.relpath(filepath, output_dir)
command = 'strings -a "{0:s}"'.format(filepath)
log.debug('Running command [{0:s}]'.format(command))
proc = subprocess.Popen(
command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
strings_output, _ = proc.communicate()
strings_output = codecs.decode(strings_output, 'utf-8')
for line in strings_output.splitlines():
strings_count += 1
if (line.find('curl') >= 0) or (line.find('wget') >= 0):
evil_commands.append((relpath, line))
if evil_commands:
msg = 'Found suspicious commands!'
report.append(fmt.heading4(fmt.bold(msg)))
summary = msg
priority = Priority.CRITICAL
else:
msg = 'Did not find any suspicious commands.'
report.append(fmt.heading4(msg))
summary = msg
for filepath, command in evil_commands:
report.append(fmt.bullet(fmt.bold('Command:')))
report.append(fmt.code(command))
report.append('Found in file:')
report.append(fmt.code(filepath))
msg = 'Extracted {0:d} strings from {1:d} file(s)'.format(
strings_count, len(collected_artifacts))
report.append(fmt.bullet(msg))
return (report, priority, summary)
def format_task_detail(self, task, show_files=False):
"""Formats a single task in detail.
Args:
task (dict): The task to format data for
show_files (bool): Whether we want to print out log file paths
Returns:
list: Formatted task data
"""
report = []
saved_paths = task.get('saved_paths') or []
status = task.get('status') or 'No task status'
report.append(fmt.heading2(task.get('name')))
line = '{0:s} {1:s}'.format(fmt.bold('Status:'), status)
report.append(fmt.bullet(line))
report.append(fmt.bullet('Task Id: {0:s}'.format(task.get('id'))))
report.append(
fmt.bullet('Executed on worker {0:s}'.format(
task.get('worker_name'))))
if task.get('report_data'):
report.append('')
report.append(fmt.heading3('Task Reported Data'))
report.extend(task.get('report_data').splitlines())
if show_files:
report.append('')
report.append(fmt.heading3('Saved Task Files:'))
for path in saved_paths:
report.append(fmt.bullet(fmt.code(path)))
report.append('')
return report
def testFormatting(self):
"""Test text formatting."""
self.assertEqual('**testing**', fmt.bold(self.test_string))
self.assertEqual('# testing', fmt.heading1(self.test_string))
self.assertEqual('## testing', fmt.heading2(self.test_string))
self.assertEqual('### testing', fmt.heading3(self.test_string))
self.assertEqual('#### testing', fmt.heading4(self.test_string))
self.assertEqual('##### testing', fmt.heading5(self.test_string))
self.assertEqual('* testing', fmt.bullet(self.test_string))
self.assertEqual(' * testing',
fmt.bullet(self.test_string, level=3))
self.assertEqual('`testing`', fmt.code(self.test_string))
def format_task(self, task, show_files=False):
"""Formats a single task in short form.
Args:
task (dict): The task to format data for
show_files (bool): Whether we want to print out log file paths
Returns:
list: Formatted task data
"""
report = []
saved_paths = task.get('saved_paths') or []
status = task.get('status') or 'No task status'
report.append(fmt.bullet('{0:s}: {1:s}'.format(task.get('name'), status)))
if show_files:
for path in saved_paths:
report.append(fmt.bullet(fmt.code(path), level=2))
report.append('')
return report
def format_request_status(
self, instance, project, region, days=0, all_fields=False):
"""Formats the recent history for Turbinia Requests.
Args:
instance (string): The Turbinia instance name (by default the same as the
INSTANCE_ID in the config).
project (string): The name of the project.
region (string): The name of the zone to execute in.
days (int): The number of days we want history for.
all_fields (bool): Include all fields for the Request, which includes,
saved file paths.
Returns:
String of Request status
"""
# Set number of days to retrieve data
num_days = 7
if days != 0:
num_days = days
task_results = self.get_task_data(instance, project, region, days=num_days)
if not task_results:
return ''
# Sort task_results by last updated timestamp.
task_results = sorted(
task_results, key=itemgetter('last_update'), reverse=True)
# Create dictionary of request_id: {saved_paths, last_update, requester,
# task_id}
request_dict = {}
for result in task_results:
request_id = result.get('request_id')
saved_paths = result.get('saved_paths')
if request_id not in request_dict:
saved_paths = set(saved_paths) if saved_paths else set()
request_dict[request_id] = {}
request_dict[request_id]['saved_paths'] = saved_paths
request_dict[request_id]['last_update'] = result.get('last_update')
request_dict[request_id]['requester'] = result.get('requester')
request_dict[request_id]['task_id'] = set([result.get('id')])
else:
if saved_paths:
request_dict[request_id]['saved_paths'].update(saved_paths)
request_dict[request_id]['task_id'].update([result.get('id')])
# Generate report header
report = []
report.append(
fmt.heading1(
'Turbinia report for Requests made within {0:d} days'.format(
num_days)))
report.append(
fmt.bullet(
'{0:d} requests were made within this timeframe.'.format(
len(request_dict.keys()))))
# Print report data for Requests
for request_id, values in request_dict.items():
report.append('')
report.append(fmt.heading2('Request ID: {0:s}'.format(request_id)))
report.append(
fmt.bullet(
'Last Update: {0:s}'.format(
values['last_update'].strftime(DATETIME_FORMAT))))
report.append(fmt.bullet('Requester: {0:s}'.format(values['requester'])))
report.append(
fmt.bullet('Task Count: {0:d}'.format(len(values['task_id']))))
if all_fields:
report.append(fmt.bullet('Associated Evidence:'))
# Append all saved paths in request
for path in sorted(values['saved_paths']):
report.append(fmt.bullet(fmt.code(path), level=2))
report.append('')
return '\n'.join(report)
