def testFormatting(self):
"""Test text formatting."""
self.assertEqual('**testing**', fmt.bold(self.test_string))
self.assertEqual('# testing', fmt.heading1(self.test_string))
self.assertEqual('## testing', fmt.heading2(self.test_string))
self.assertEqual('### testing', fmt.heading3(self.test_string))
self.assertEqual('#### testing', fmt.heading4(self.test_string))
self.assertEqual('##### testing', fmt.heading5(self.test_string))
self.assertEqual('* testing', fmt.bullet(self.test_string))
self.assertEqual(' * testing',
fmt.bullet(self.test_string, level=3))
self.assertEqual('`testing`', fmt.code(self.test_string))
def format_task_status(self,
instance,
project,
region,
days=0,
task_id=None,
request_id=None,
user=None,
all_fields=False,
full_report=False,
priority_filter=Priority.HIGH):
"""Formats the recent history for Turbinia Tasks.
Args:
instance (string): The Turbinia instance name (by default the same as the
INSTANCE_ID in the config).
project (string): The name of the project.
region (string): The name of the zone to execute in.
days (int): The number of days we want history for.
task_id (string): The Id of the task.
request_id (string): The Id of the request we want tasks for.
user (string): The user of the request we want tasks for.
all_fields (bool): Include all fields for the task, including task,
request ids and saved file paths.
full_report (bool): Generate a full markdown report instead of just a
summary.
priority_filter (int): Output only a summary for Tasks with a value
greater than the priority_filter.
Returns:
String of task status
"""
if user and days == 0:
days = 1000
task_results = self.get_task_data(instance, project, region, days,
task_id, request_id, user)
if not task_results:
return ''
# Sort all tasks by the report_priority so that tasks with a higher
# priority are listed first in the report.
for result in task_results:
# 0 is a valid value, so checking against specific values
if result.get('report_priority') in (None, ''):
result['report_priority'] = Priority.LOW
task_results = sorted(task_results, key=itemgetter('report_priority'))
num_results = len(task_results)
if not num_results:
msg = 'No Turbinia Tasks found.'
log.info(msg)
return '\n{0:s}'.format(msg)
# Build up data
report = []
requester = task_results[0].get('requester')
request_id = task_results[0].get('request_id')
success_types = ['Successful', 'Failed', 'Scheduled or Running']
success_values = [True, False, None]
# Reverse mapping values to types
success_map = dict(zip(success_values, success_types))
task_map = defaultdict(list)
success_types.insert(0, 'High Priority')
for task in task_results:
if task.get('report_priority') <= priority_filter:
task_map['High Priority'].append(task)
else:
task_map[success_map[task.get('successful')]].append(task)
# Generate report header
report.append('\n')
report.append(fmt.heading1('Turbinia report {0:s}'.format(request_id)))
report.append(
fmt.bullet('Processed {0:d} Tasks for user {1:s}'.format(
num_results, requester)))
# Print report data for tasks
for success_type in success_types:
report.append('')
report.append(fmt.heading1('{0:s} Tasks'.format(success_type)))
if not task_map[success_type]:
report.append(fmt.bullet('None'))
for task in task_map[success_type]:
if full_report and success_type == success_types[0]:
report.extend(
self.format_task_detail(task, show_files=all_fields))
else:
report.extend(self.format_task(task,
show_files=all_fields))
return '\n'.join(report)
def format_request_status(
self, instance, project, region, days=0, all_fields=False):
"""Formats the recent history for Turbinia Requests.
Args:
instance (string): The Turbinia instance name (by default the same as the
INSTANCE_ID in the config).
project (string): The name of the project.
region (string): The name of the zone to execute in.
days (int): The number of days we want history for.
all_fields (bool): Include all fields for the Request, which includes,
saved file paths.
Returns:
String of Request status
"""
# Set number of days to retrieve data
num_days = 7
if days != 0:
num_days = days
task_results = self.get_task_data(instance, project, region, days=num_days)
if not task_results:
return ''
# Sort task_results by last updated timestamp.
task_results = sorted(
task_results, key=itemgetter('last_update'), reverse=True)
# Create dictionary of request_id: {saved_paths, last_update, requester,
# task_id}
request_dict = {}
for result in task_results:
request_id = result.get('request_id')
saved_paths = result.get('saved_paths')
if request_id not in request_dict:
saved_paths = set(saved_paths) if saved_paths else set()
request_dict[request_id] = {}
request_dict[request_id]['saved_paths'] = saved_paths
request_dict[request_id]['last_update'] = result.get('last_update')
request_dict[request_id]['requester'] = result.get('requester')
request_dict[request_id]['task_id'] = set([result.get('id')])
else:
if saved_paths:
request_dict[request_id]['saved_paths'].update(saved_paths)
request_dict[request_id]['task_id'].update([result.get('id')])
# Generate report header
report = []
report.append(
fmt.heading1(
'Turbinia report for Requests made within {0:d} days'.format(
num_days)))
report.append(
fmt.bullet(
'{0:d} requests were made within this timeframe.'.format(
len(request_dict.keys()))))
# Print report data for Requests
for request_id, values in request_dict.items():
report.append('')
report.append(fmt.heading2('Request ID: {0:s}'.format(request_id)))
report.append(
fmt.bullet(
'Last Update: {0:s}'.format(
values['last_update'].strftime(DATETIME_FORMAT))))
report.append(fmt.bullet('Requester: {0:s}'.format(values['requester'])))
report.append(
fmt.bullet('Task Count: {0:d}'.format(len(values['task_id']))))
if all_fields:
report.append(fmt.bullet('Associated Evidence:'))
# Append all saved paths in request
for path in sorted(values['saved_paths']):
report.append(fmt.bullet(fmt.code(path), level=2))
report.append('')
return '\n'.join(report)
def format_worker_status(
self, instance, project, region, days=0, all_fields=False):
"""Formats the recent history for Turbinia Workers.
Args:
instance (string): The Turbinia instance name (by default the same as the
INSTANCE_ID in the config).
project (string): The name of the project.
region (string): The name of the zone to execute in.
days (int): The number of days we want history for.
all_fields (bool): Include historical Task information for the worker.
Returns:
String of Request status
"""
# Set number of days to retrieve data
num_days = 7
if days != 0:
num_days = days
task_results = self.get_task_data(instance, project, region, days=num_days)
if not task_results:
return ''
# Sort task_results by last updated timestamp.
task_results = sorted(
task_results, key=itemgetter('last_update'), reverse=True)
# Create dictionary of worker_node: {{task_id, task_update,
# task_name, task_status}}
workers_dict = {}
scheduled_counter = 0
for result in task_results:
worker_node = result.get('worker_name')
status = result.get('status')
status = status if status else 'No task status'
if worker_node and worker_node not in workers_dict:
workers_dict[worker_node] = []
if worker_node:
task_dict = {}
task_dict['task_id'] = result.get('id')
task_dict['last_update'] = result.get('last_update')
task_dict['task_name'] = result.get('name')
task_dict['status'] = status
# Check status for anything that is running.
if 'running' in status:
run_time = (datetime.now() -
result.get('last_update')).total_seconds()
run_time = timedelta(seconds=run_time)
task_dict['run_time'] = run_time
else:
run_time = result.get('run_time')
task_dict['run_time'] = run_time if run_time else 'No run time.'
workers_dict[worker_node].append(task_dict)
else:
# Track scheduled/unassigned Tasks for reporting.
scheduled_counter += 1
# Generate report header
report = []
report.append(
fmt.heading1(
'Turbinia report for Worker activity within {0:d} days'.format(
num_days)))
report.append(
fmt.bullet('{0:d} Worker(s) found.'.format(len(workers_dict.keys()))))
report.append(
fmt.bullet(
'{0:d} Task(s) unassigned or scheduled and pending Worker assignment.'
.format(scheduled_counter)))
for worker_node, tasks in workers_dict.items():
report.append('')
report.append(fmt.heading2('Worker Node: {0:s}'.format(worker_node)))
# Append the statuses chronologically
run_status, queued_status, other_status = [], [], []
for task in tasks:
if 'running' in task['status']:
run_status.extend(self.format_worker_task(task))
elif 'queued' in task['status']:
queued_status.extend(self.format_worker_task(task))
else:
other_status.extend(self.format_worker_task(task))
# Add each of the status lists back to report list
not_found = [fmt.bullet('No Tasks found.')]
report.append(fmt.heading3('Running Tasks'))
report.extend(run_status if run_status else not_found)
report.append('')
report.append(fmt.heading3('Queued Tasks'))
report.extend(queued_status if queued_status else not_found)
# Add Historical Tasks
if all_fields:
report.append('')
report.append(fmt.heading3('Finished Tasks'))
report.extend(other_status if other_status else not_found)
return '\n'.join(report)
def format_task_status(self,
instance,
project,
region,
days=0,
task_id=None,
request_id=None,
group_id=None,
user=None,
all_fields=False,
full_report=False,
priority_filter=Priority.HIGH,
output_json=False,
report=None):
"""Formats the recent history for Turbinia Tasks.
Args:
instance (string): The Turbinia instance name (by default the same as the
INSTANCE_ID in the config).
project (string): The name of the project.
region (string): The name of the zone to execute in.
days (int): The number of days we want history for.
task_id (string): The Id of the task.
request_id (string): The Id of the request we want tasks for.
group_id (string): Group Id of the requests.
user (string): The user of the request we want tasks for.
all_fields (bool): Include all fields for the task, including task,
request ids and saved file paths.
full_report (bool): Generate a full markdown report instead of just a
summary.
priority_filter (int): Output only a summary for Tasks with a value
greater than the priority_filter.
output_json (bool): Whether to return JSON output.
report (string): Status report that will be returned.
Returns:
String of task status in JSON or human readable format.
"""
if user and days == 0:
days = 1000
task_results = self.get_task_data(instance,
project,
region,
days,
task_id,
request_id,
group_id,
user,
output_json=output_json)
if not task_results:
return ''
if output_json:
return task_results
# Sort all tasks by the report_priority so that tasks with a higher
# priority are listed first in the report.
for result in task_results:
# 0 is a valid value, so checking against specific values
if result.get('report_priority') in (None, ''):
result['report_priority'] = Priority.LOW
task_results = sorted(task_results, key=itemgetter('report_priority'))
num_results = len(task_results)
if not num_results:
msg = 'No Turbinia Tasks found.'
log.info(msg)
return '\n{0:s}'.format(msg)
# Build up data
if report is None:
report = []
success_types = ['Successful', 'Failed', 'Scheduled or Running']
success_values = [True, False, None]
# Reverse mapping values to types
success_map = dict(zip(success_values, success_types))
# This is used for group ID status
requests = defaultdict(dict)
requester = task_results[0].get('requester')
request_id = task_results[0].get('request_id')
task_map = defaultdict(list)
success_types.insert(0, 'High Priority')
for task in task_results:
if task.get('request_id') not in requests:
requests[task.get('request_id')] = {
'Successful': 0,
'Failed': 0,
'Scheduled or Running': 0
}
requests[task.get('request_id')][success_map[task.get(
'successful')]] += 1
if task.get('report_priority') <= priority_filter:
task_map['High Priority'].append(task)
else:
task_map[success_map[task.get('successful')]].append(task)
if group_id:
report.append('\n')
report.append(
fmt.heading1(
'Turbinia report for group ID {0:s}'.format(group_id)))
for request_id, success_counts in requests.items():
report.append(
fmt.bullet(
'Request Id {0:s} with {1:d} successful, {2:d} failed, and {3:d} running tasks.'
.format(request_id, success_counts['Successful'],
success_counts['Failed'],
success_counts['Scheduled or Running'])))
if full_report:
self.format_task_status(instance,
project,
region,
days=0,
task_id=None,
request_id=request_id,
user=user,
all_fields=all_fields,
full_report=full_report,
priority_filter=priority_filter,
output_json=output_json,
report=report)
return '\n'.join(report)
# Generate report header
report.append('\n')
report.append(fmt.heading1('Turbinia report {0:s}'.format(request_id)))
report.append(
fmt.bullet('Processed {0:d} Tasks for user {1:s}'.format(
num_results, requester)))
# Print report data for tasks
for success_type in success_types:
report.append('')
report.append(fmt.heading1('{0:s} Tasks'.format(success_type)))
if not task_map[success_type]:
report.append(fmt.bullet('None'))
task_counter = defaultdict(int)
for task in task_map[success_type]:
if full_report and success_type == success_types[0]:
report.extend(
self.format_task_detail(task, show_files=all_fields))
elif success_type == success_types[2]:
report.extend(self.format_task(task,
show_files=all_fields))
else:
task_counter['\n'.join(
self.format_task(task, show_files=all_fields))] += 1
if len(task_counter):
for k, v in task_counter.items():
if v == 1:
report.append(k)
else:
report.append('{0:s} x {1:d}'.format(k, v))
return '\n'.join(report)
