def testFormatting(self): """Test text formatting.""" self.assertEqual('**testing**', fmt.bold(self.test_string)) self.assertEqual('# testing', fmt.heading1(self.test_string)) self.assertEqual('## testing', fmt.heading2(self.test_string)) self.assertEqual('### testing', fmt.heading3(self.test_string)) self.assertEqual('#### testing', fmt.heading4(self.test_string)) self.assertEqual('##### testing', fmt.heading5(self.test_string)) self.assertEqual('* testing', fmt.bullet(self.test_string)) self.assertEqual(' * testing', fmt.bullet(self.test_string, level=3)) self.assertEqual('`testing`', fmt.code(self.test_string))
def format_task_status(self, instance, project, region, days=0, task_id=None, request_id=None, user=None, all_fields=False, full_report=False, priority_filter=Priority.HIGH): """Formats the recent history for Turbinia Tasks. Args: instance (string): The Turbinia instance name (by default the same as the INSTANCE_ID in the config). project (string): The name of the project. region (string): The name of the zone to execute in. days (int): The number of days we want history for. task_id (string): The Id of the task. request_id (string): The Id of the request we want tasks for. user (string): The user of the request we want tasks for. all_fields (bool): Include all fields for the task, including task, request ids and saved file paths. full_report (bool): Generate a full markdown report instead of just a summary. priority_filter (int): Output only a summary for Tasks with a value greater than the priority_filter. Returns: String of task status """ if user and days == 0: days = 1000 task_results = self.get_task_data(instance, project, region, days, task_id, request_id, user) if not task_results: return '' # Sort all tasks by the report_priority so that tasks with a higher # priority are listed first in the report. for result in task_results: # 0 is a valid value, so checking against specific values if result.get('report_priority') in (None, ''): result['report_priority'] = Priority.LOW task_results = sorted(task_results, key=itemgetter('report_priority')) num_results = len(task_results) if not num_results: msg = 'No Turbinia Tasks found.' log.info(msg) return '\n{0:s}'.format(msg) # Build up data report = [] requester = task_results[0].get('requester') request_id = task_results[0].get('request_id') success_types = ['Successful', 'Failed', 'Scheduled or Running'] success_values = [True, False, None] # Reverse mapping values to types success_map = dict(zip(success_values, success_types)) task_map = defaultdict(list) success_types.insert(0, 'High Priority') for task in task_results: if task.get('report_priority') <= priority_filter: task_map['High Priority'].append(task) else: task_map[success_map[task.get('successful')]].append(task) # Generate report header report.append('\n') report.append(fmt.heading1('Turbinia report {0:s}'.format(request_id))) report.append( fmt.bullet('Processed {0:d} Tasks for user {1:s}'.format( num_results, requester))) # Print report data for tasks for success_type in success_types: report.append('') report.append(fmt.heading1('{0:s} Tasks'.format(success_type))) if not task_map[success_type]: report.append(fmt.bullet('None')) for task in task_map[success_type]: if full_report and success_type == success_types[0]: report.extend( self.format_task_detail(task, show_files=all_fields)) else: report.extend(self.format_task(task, show_files=all_fields)) return '\n'.join(report)
def format_request_status( self, instance, project, region, days=0, all_fields=False): """Formats the recent history for Turbinia Requests. Args: instance (string): The Turbinia instance name (by default the same as the INSTANCE_ID in the config). project (string): The name of the project. region (string): The name of the zone to execute in. days (int): The number of days we want history for. all_fields (bool): Include all fields for the Request, which includes, saved file paths. Returns: String of Request status """ # Set number of days to retrieve data num_days = 7 if days != 0: num_days = days task_results = self.get_task_data(instance, project, region, days=num_days) if not task_results: return '' # Sort task_results by last updated timestamp. task_results = sorted( task_results, key=itemgetter('last_update'), reverse=True) # Create dictionary of request_id: {saved_paths, last_update, requester, # task_id} request_dict = {} for result in task_results: request_id = result.get('request_id') saved_paths = result.get('saved_paths') if request_id not in request_dict: saved_paths = set(saved_paths) if saved_paths else set() request_dict[request_id] = {} request_dict[request_id]['saved_paths'] = saved_paths request_dict[request_id]['last_update'] = result.get('last_update') request_dict[request_id]['requester'] = result.get('requester') request_dict[request_id]['task_id'] = set([result.get('id')]) else: if saved_paths: request_dict[request_id]['saved_paths'].update(saved_paths) request_dict[request_id]['task_id'].update([result.get('id')]) # Generate report header report = [] report.append( fmt.heading1( 'Turbinia report for Requests made within {0:d} days'.format( num_days))) report.append( fmt.bullet( '{0:d} requests were made within this timeframe.'.format( len(request_dict.keys())))) # Print report data for Requests for request_id, values in request_dict.items(): report.append('') report.append(fmt.heading2('Request ID: {0:s}'.format(request_id))) report.append( fmt.bullet( 'Last Update: {0:s}'.format( values['last_update'].strftime(DATETIME_FORMAT)))) report.append(fmt.bullet('Requester: {0:s}'.format(values['requester']))) report.append( fmt.bullet('Task Count: {0:d}'.format(len(values['task_id'])))) if all_fields: report.append(fmt.bullet('Associated Evidence:')) # Append all saved paths in request for path in sorted(values['saved_paths']): report.append(fmt.bullet(fmt.code(path), level=2)) report.append('') return '\n'.join(report)
def format_worker_status( self, instance, project, region, days=0, all_fields=False): """Formats the recent history for Turbinia Workers. Args: instance (string): The Turbinia instance name (by default the same as the INSTANCE_ID in the config). project (string): The name of the project. region (string): The name of the zone to execute in. days (int): The number of days we want history for. all_fields (bool): Include historical Task information for the worker. Returns: String of Request status """ # Set number of days to retrieve data num_days = 7 if days != 0: num_days = days task_results = self.get_task_data(instance, project, region, days=num_days) if not task_results: return '' # Sort task_results by last updated timestamp. task_results = sorted( task_results, key=itemgetter('last_update'), reverse=True) # Create dictionary of worker_node: {{task_id, task_update, # task_name, task_status}} workers_dict = {} scheduled_counter = 0 for result in task_results: worker_node = result.get('worker_name') status = result.get('status') status = status if status else 'No task status' if worker_node and worker_node not in workers_dict: workers_dict[worker_node] = [] if worker_node: task_dict = {} task_dict['task_id'] = result.get('id') task_dict['last_update'] = result.get('last_update') task_dict['task_name'] = result.get('name') task_dict['status'] = status # Check status for anything that is running. if 'running' in status: run_time = (datetime.now() - result.get('last_update')).total_seconds() run_time = timedelta(seconds=run_time) task_dict['run_time'] = run_time else: run_time = result.get('run_time') task_dict['run_time'] = run_time if run_time else 'No run time.' workers_dict[worker_node].append(task_dict) else: # Track scheduled/unassigned Tasks for reporting. scheduled_counter += 1 # Generate report header report = [] report.append( fmt.heading1( 'Turbinia report for Worker activity within {0:d} days'.format( num_days))) report.append( fmt.bullet('{0:d} Worker(s) found.'.format(len(workers_dict.keys())))) report.append( fmt.bullet( '{0:d} Task(s) unassigned or scheduled and pending Worker assignment.' .format(scheduled_counter))) for worker_node, tasks in workers_dict.items(): report.append('') report.append(fmt.heading2('Worker Node: {0:s}'.format(worker_node))) # Append the statuses chronologically run_status, queued_status, other_status = [], [], [] for task in tasks: if 'running' in task['status']: run_status.extend(self.format_worker_task(task)) elif 'queued' in task['status']: queued_status.extend(self.format_worker_task(task)) else: other_status.extend(self.format_worker_task(task)) # Add each of the status lists back to report list not_found = [fmt.bullet('No Tasks found.')] report.append(fmt.heading3('Running Tasks')) report.extend(run_status if run_status else not_found) report.append('') report.append(fmt.heading3('Queued Tasks')) report.extend(queued_status if queued_status else not_found) # Add Historical Tasks if all_fields: report.append('') report.append(fmt.heading3('Finished Tasks')) report.extend(other_status if other_status else not_found) return '\n'.join(report)
def format_task_status(self, instance, project, region, days=0, task_id=None, request_id=None, group_id=None, user=None, all_fields=False, full_report=False, priority_filter=Priority.HIGH, output_json=False, report=None): """Formats the recent history for Turbinia Tasks. Args: instance (string): The Turbinia instance name (by default the same as the INSTANCE_ID in the config). project (string): The name of the project. region (string): The name of the zone to execute in. days (int): The number of days we want history for. task_id (string): The Id of the task. request_id (string): The Id of the request we want tasks for. group_id (string): Group Id of the requests. user (string): The user of the request we want tasks for. all_fields (bool): Include all fields for the task, including task, request ids and saved file paths. full_report (bool): Generate a full markdown report instead of just a summary. priority_filter (int): Output only a summary for Tasks with a value greater than the priority_filter. output_json (bool): Whether to return JSON output. report (string): Status report that will be returned. Returns: String of task status in JSON or human readable format. """ if user and days == 0: days = 1000 task_results = self.get_task_data(instance, project, region, days, task_id, request_id, group_id, user, output_json=output_json) if not task_results: return '' if output_json: return task_results # Sort all tasks by the report_priority so that tasks with a higher # priority are listed first in the report. for result in task_results: # 0 is a valid value, so checking against specific values if result.get('report_priority') in (None, ''): result['report_priority'] = Priority.LOW task_results = sorted(task_results, key=itemgetter('report_priority')) num_results = len(task_results) if not num_results: msg = 'No Turbinia Tasks found.' log.info(msg) return '\n{0:s}'.format(msg) # Build up data if report is None: report = [] success_types = ['Successful', 'Failed', 'Scheduled or Running'] success_values = [True, False, None] # Reverse mapping values to types success_map = dict(zip(success_values, success_types)) # This is used for group ID status requests = defaultdict(dict) requester = task_results[0].get('requester') request_id = task_results[0].get('request_id') task_map = defaultdict(list) success_types.insert(0, 'High Priority') for task in task_results: if task.get('request_id') not in requests: requests[task.get('request_id')] = { 'Successful': 0, 'Failed': 0, 'Scheduled or Running': 0 } requests[task.get('request_id')][success_map[task.get( 'successful')]] += 1 if task.get('report_priority') <= priority_filter: task_map['High Priority'].append(task) else: task_map[success_map[task.get('successful')]].append(task) if group_id: report.append('\n') report.append( fmt.heading1( 'Turbinia report for group ID {0:s}'.format(group_id))) for request_id, success_counts in requests.items(): report.append( fmt.bullet( 'Request Id {0:s} with {1:d} successful, {2:d} failed, and {3:d} running tasks.' .format(request_id, success_counts['Successful'], success_counts['Failed'], success_counts['Scheduled or Running']))) if full_report: self.format_task_status(instance, project, region, days=0, task_id=None, request_id=request_id, user=user, all_fields=all_fields, full_report=full_report, priority_filter=priority_filter, output_json=output_json, report=report) return '\n'.join(report) # Generate report header report.append('\n') report.append(fmt.heading1('Turbinia report {0:s}'.format(request_id))) report.append( fmt.bullet('Processed {0:d} Tasks for user {1:s}'.format( num_results, requester))) # Print report data for tasks for success_type in success_types: report.append('') report.append(fmt.heading1('{0:s} Tasks'.format(success_type))) if not task_map[success_type]: report.append(fmt.bullet('None')) task_counter = defaultdict(int) for task in task_map[success_type]: if full_report and success_type == success_types[0]: report.extend( self.format_task_detail(task, show_files=all_fields)) elif success_type == success_types[2]: report.extend(self.format_task(task, show_files=all_fields)) else: task_counter['\n'.join( self.format_task(task, show_files=all_fields))] += 1 if len(task_counter): for k, v in task_counter.items(): if v == 1: report.append(k) else: report.append('{0:s} x {1:d}'.format(k, v)) return '\n'.join(report)