def _AnalyzeHadoopAppRoot(self, collected_artifacts, output_dir): """Runs a naive AppRoot files parsing method. This extracts strings from the saved task file, and searches for usual post-compromise suspicious patterns. TODO: properly parse the Proto. Some documentation can be found over there: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23.7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto Args: collected_artifacts(list(str)): a list of paths to extracted files output_dir(str): The base directory the artfacts are in. Returns: Tuple( list(str): The report data as a list of lines report_priority(int): The priority of the report (0 - 100) summary(str): A summary of the report (used for task status) ) """ report = [] evil_commands = [] strings_count = 0 priority = Priority.MEDIUM summary = '' for filepath in collected_artifacts: relpath = os.path.relpath(filepath, output_dir) command = 'strings -a "{0:s}"'.format(filepath) log.debug('Running command [{0:s}]'.format(command)) proc = subprocess.Popen( command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) strings_output, _ = proc.communicate() strings_output = codecs.decode(strings_output, 'utf-8') for line in strings_output.splitlines(): strings_count += 1 if (line.find('curl') >= 0) or (line.find('wget') >= 0): evil_commands.append((relpath, line)) if evil_commands: msg = 'Found suspicious commands!' report.append(fmt.heading4(fmt.bold(msg))) summary = msg priority = Priority.CRITICAL else: msg = 'Did not find any suspicious commands.' report.append(fmt.heading4(msg)) summary = msg for filepath, command in evil_commands: report.append(fmt.bullet(fmt.bold('Command:'))) report.append(fmt.code(command)) report.append('Found in file:') report.append(fmt.code(filepath)) msg = 'Extracted {0:d} strings from {1:d} file(s)'.format( strings_count, len(collected_artifacts)) report.append(fmt.bullet(msg)) return (report, priority, summary)
def format_task_detail(self, task, show_files=False): """Formats a single task in detail. Args: task (dict): The task to format data for show_files (bool): Whether we want to print out log file paths Returns: list: Formatted task data """ report = [] saved_paths = task.get('saved_paths') or [] status = task.get('status') or 'No task status' report.append(fmt.heading2(task.get('name'))) line = '{0:s} {1:s}'.format(fmt.bold('Status:'), status) report.append(fmt.bullet(line)) report.append(fmt.bullet('Task Id: {0:s}'.format(task.get('id')))) report.append( fmt.bullet('Executed on worker {0:s}'.format( task.get('worker_name')))) if task.get('report_data'): report.append('') report.append(fmt.heading3('Task Reported Data')) report.extend(task.get('report_data').splitlines()) if show_files: report.append('') report.append(fmt.heading3('Saved Task Files:')) for path in saved_paths: report.append(fmt.bullet(fmt.code(path))) report.append('') return report
def testFormatting(self): """Test text formatting.""" self.assertEqual('**testing**', fmt.bold(self.test_string)) self.assertEqual('# testing', fmt.heading1(self.test_string)) self.assertEqual('## testing', fmt.heading2(self.test_string)) self.assertEqual('### testing', fmt.heading3(self.test_string)) self.assertEqual('#### testing', fmt.heading4(self.test_string)) self.assertEqual('##### testing', fmt.heading5(self.test_string)) self.assertEqual('* testing', fmt.bullet(self.test_string)) self.assertEqual(' * testing', fmt.bullet(self.test_string, level=3)) self.assertEqual('`testing`', fmt.code(self.test_string))
def format_task(self, task, show_files=False): """Formats a single task in short form. Args: task (dict): The task to format data for show_files (bool): Whether we want to print out log file paths Returns: list: Formatted task data """ report = [] saved_paths = task.get('saved_paths') or [] status = task.get('status') or 'No task status' report.append(fmt.bullet('{0:s}: {1:s}'.format(task.get('name'), status))) if show_files: for path in saved_paths: report.append(fmt.bullet(fmt.code(path), level=2)) report.append('') return report
def format_request_status( self, instance, project, region, days=0, all_fields=False): """Formats the recent history for Turbinia Requests. Args: instance (string): The Turbinia instance name (by default the same as the INSTANCE_ID in the config). project (string): The name of the project. region (string): The name of the zone to execute in. days (int): The number of days we want history for. all_fields (bool): Include all fields for the Request, which includes, saved file paths. Returns: String of Request status """ # Set number of days to retrieve data num_days = 7 if days != 0: num_days = days task_results = self.get_task_data(instance, project, region, days=num_days) if not task_results: return '' # Sort task_results by last updated timestamp. task_results = sorted( task_results, key=itemgetter('last_update'), reverse=True) # Create dictionary of request_id: {saved_paths, last_update, requester, # task_id} request_dict = {} for result in task_results: request_id = result.get('request_id') saved_paths = result.get('saved_paths') if request_id not in request_dict: saved_paths = set(saved_paths) if saved_paths else set() request_dict[request_id] = {} request_dict[request_id]['saved_paths'] = saved_paths request_dict[request_id]['last_update'] = result.get('last_update') request_dict[request_id]['requester'] = result.get('requester') request_dict[request_id]['task_id'] = set([result.get('id')]) else: if saved_paths: request_dict[request_id]['saved_paths'].update(saved_paths) request_dict[request_id]['task_id'].update([result.get('id')]) # Generate report header report = [] report.append( fmt.heading1( 'Turbinia report for Requests made within {0:d} days'.format( num_days))) report.append( fmt.bullet( '{0:d} requests were made within this timeframe.'.format( len(request_dict.keys())))) # Print report data for Requests for request_id, values in request_dict.items(): report.append('') report.append(fmt.heading2('Request ID: {0:s}'.format(request_id))) report.append( fmt.bullet( 'Last Update: {0:s}'.format( values['last_update'].strftime(DATETIME_FORMAT)))) report.append(fmt.bullet('Requester: {0:s}'.format(values['requester']))) report.append( fmt.bullet('Task Count: {0:d}'.format(len(values['task_id'])))) if all_fields: report.append(fmt.bullet('Associated Evidence:')) # Append all saved paths in request for path in sorted(values['saved_paths']): report.append(fmt.bullet(fmt.code(path), level=2)) report.append('') return '\n'.join(report)