class BaseIntegrator(object): # Subclasses override these attributes TOOL_NAME = 'External tool' DEFAULT_MAPPING_FILE = None AVAILABLE_IMPORTERS = [] # Subclasses must fill this list VALID_IMPORT_BEHAVIOUR = ['replace', 'replace-scanner', 'combine'] # An internal map of possible verification and acceptable status meanings VALID_VERIFICATION_MAP = {'pass': ['TODO', 'DONE'], 'partial': ['TODO', 'DONE'], 'fail': ['TODO']} def __init__(self, config, tool_name, supported_input_types={}): self.findings = [] self.mapping = {} self.report_id = "Not specified" self.config = config self.emit = self.config.emit self.behaviour = 'replace' self.weakness_map_identifier = 'ID' # default XML attribute with weakness identifier self.weakness_title = {} self.confidence = {} self.task_mapping = None self.taskstatuses = {} self.plugin = PlugInExperience(self.config) self.supported_input_types = supported_input_types if supported_input_types and 'file' in supported_input_types: # 'report_file' is required if the tool only supports files report_file_mandatory = len(supported_input_types.keys()) == 1 self.config.opts.add( "report_file", "Common separated list of %s Report Files" % tool_name.capitalize(), "x", default=None if report_file_mandatory else '' ) self.config.opts.add( "report_type", "%s Report Type: %s|auto" % (tool_name.capitalize(), ', '.join(supported_input_types['file'])), default="auto" ) self.config.opts.add( "mapping_file", "Task ID -> Weakness mapping (default=%s)" % self.get_default_mapping_file(), "m", '' ) self.config.opts.add( "import_behaviour", "One of the following: %s" % ', '.join(self.VALID_IMPORT_BEHAVIOUR), default="replace" ) self.config.opts.add( 'task_status_mapping', 'Update task status based on verification. Provide a mapping of (%s) to a task status slug' '(JSON encoded dictionary of strings)' % ', '.join(self.VALID_VERIFICATION_MAP.keys()), default='' ) self.config.opts.add("flaws_only", "Only update tasks identified having flaws. (True | False)", "z", "False") self.config.opts.add("trial_run", "Trial run only: (True | False)", "t", "False") def initialize(self): """ This is a post init initialization. It needs to be called as the first function after configuration is processed (usually first call inside handler of the module) """ self.config.process_boolean_config('flaws_only') self.config.process_boolean_config('trial_run') self.config.process_json_str_dict('task_status_mapping') if self.config['import_behaviour'] in self.VALID_IMPORT_BEHAVIOUR: self.behaviour = self.config['import_behaviour'] else: raise UsageError('Invalid import_behaviour %s' % self.config['import_behaviour']) if self.config['task_status_mapping']: # Get the available system task statuses and their meanings self._setup_taskstatuses() # Validate the mapping against the available system statuses # Sanity check the mapping # - pass, partial may mark tasks with a status having meaning TODO or DONE only # - fail may mark tasks with a status having meaning TODO only. for verification, status_name in self.config['task_status_mapping'].iteritems(): if verification not in self.VALID_VERIFICATION_MAP: raise UsageError('Invalid task_status_mapping verification %s' % verification) if status_name not in self.taskstatuses: raise UsageError('Invalid task_status_mapping status "%s" for verification "%s"' % (status_name, verification)) if self.taskstatuses[status_name]['meaning'] not in self.VALID_VERIFICATION_MAP[verification]: raise UsageError('Unexpected task_status_mapping status "%s" for verification "%s"' % (status_name, verification)) # Validate the report_type config. If report_type is not auto, we will process only # the specified report_type, else we process all supported file types. if 'file' in self.supported_input_types: if self.config['report_type'] in self.supported_input_types['file']: self.supported_input_types['file'] = [self.config['report_type']] elif self.config['report_type'] != 'auto': raise UsageError('Invalid report_type %s' % self.config['report_type']) self.process_report_file_config() def get_default_mapping_file(self): return get_media_file_path(os.path.join('analysis', self.DEFAULT_MAPPING_FILE)) def _setup_taskstatuses(self): statuses = self.plugin.get_taskstatuses() for status in statuses: self.taskstatuses[status['slug']] = status def detect_importer(self, report_file): for item in self.AVAILABLE_IMPORTERS: if item['importer'].can_parse_file(report_file): return item['importer'] return None @staticmethod def _get_file_extension(file_path): return os.path.splitext(file_path)[1][1:] @abstractmethod def parse_report_file(self, report_file, report_type): """ Returns the raw findings and the report id for a single report file """ return [], None def set_tool_name(self, tool_name): self.TOOL_NAME = tool_name def parse(self): _raw_findings = [] _report_ids = [] for report_file in self.config['report_file']: if self.config['report_type'] == 'auto': if not isinstance(report_file, basestring): raise UsageError("On auto-detect mode, the file name needs to be specified.") report_type = self._get_file_extension(report_file) else: report_type = self.config['report_type'] raw_findings, report_id = self.parse_report_file(report_file, report_type) _raw_findings.extend(raw_findings) if report_id: _report_ids.append(report_id) self.findings = _raw_findings if _report_ids: self.report_id = ', '.join(_report_ids) else: self.report_id = "Not specified" self.emit.info("Report ID not found in report: Using default.") def process_report_file_config(self): """ If report files contains a directory path, find all possible files in that folder """ if not self.config['report_file']: raise UsageError("Missing configuration option 'report_file'") if not isinstance(self.config['report_file'], basestring): # Should be a file object self.config['report_file'] = [self.config['report_file']] else: processed_report_files = [] for file_path in self.config['report_file'].split(','): file_path = file_path.strip() file_name, file_ext = os.path.splitext(file_path) file_ext = file_ext[1:] if file_ext in self.supported_input_types['file']: processed_report_files.extend(glob.glob(file_path)) elif re.search('[*?]', file_ext): # Run the glob and filter out unsupported file types processed_report_files.extend([f for f in glob.iglob(file_path) if self._get_file_extension(f) in self.supported_input_types['file']]) elif not file_ext: # Glob using our supported file types if os.path.isdir(file_path): _base_path = file_path + '/*' else: _base_path = file_name for file_type in self.supported_input_types['file']: processed_report_files.extend(glob.glob('%s.%s' % (_base_path, file_type))) else: raise UsageError('%s does not match any supported file type(s): %s' % (file_path, self.supported_input_types['file'])) if not processed_report_files: raise UsageError("Did not find any report files. Check if 'report_file' is configured properly.") else: self.config['report_file'] = processed_report_files def load_mapping_from_xml(self): self.task_mapping = Mapping(self.weakness_map_identifier) mapping_file = self.config['mapping_file'] or self.get_default_mapping_file() self.task_mapping.load_mapping(mapping_file) if not self.task_mapping.count(): raise IntegrationError("No base mapping was found in file '%s'" % self.config['mapping_file']) def generate_findings(self): return [] def unique_findings(self): """ Return a map (task_id=> *flaw) based on list of findings (weakness) Where flaw is defined as: flaw[weaknesses] flaw[related_tasks] """ unique_findings = {'nomap': []} for finding in self.generate_findings(): weakness_id = finding['weakness_id'] mapped_tasks = self.task_mapping.get_tasks(weakness_id) if not mapped_tasks: unique_findings['nomap'].append(weakness_id) continue for mapped_task in mapped_tasks: if mapped_task['id'] in unique_findings: flaws = unique_findings[mapped_task['id']] else: flaws = {'weaknesses': []} flaws['weaknesses'].append(finding) flaws['related_tasks'] = mapped_tasks unique_findings[mapped_task['id']] = flaws return unique_findings def task_exists_in_project_tasks(self, task_id, project_tasks): """ Return True if task_id is present in the array of project_tasks, False otherwise task_id is of the form [^\d]+\d+ project_tasks is an array of maps. Each map contains a key 'id' with a corresponding integer value """ for task in project_tasks: task_search = re.search('^(\d+)-([^\d]+\d+)$', task['id']) if task_search: project_task_id = task_search.group(2) if project_task_id == task_id: return True return False def import_findings(self): stats_failures_added = 0 stats_api_errors = 0 stats_total_skips = 0 stats_total_skips_findings = 0 stats_total_flaws_found = 0 import_start_datetime = datetime.now() logger.info("Integration underway for: %s" % self.report_id) logger.info("Mapped SD application/project: %s/%s" % (self.config['sde_application'], self.config['sde_project'])) if self.config['trial_run']: logger.info("Trial run only. No changes will be made") else: ret = self.plugin.create_analysis_session(self.report_id, self.TOOL_NAME) project_analysis_note_ref = ret['id'] task_list = self.plugin.get_task_list() logger.debug("Retrieved %d tasks from %s/%s/%s" % ( len(task_list), self.config['sde_businessunit'], self.config['sde_application'], self.config['sde_project'])) unique_findings = self.unique_findings() missing_weakness_map = unique_findings['nomap'] del unique_findings['nomap'] task_ids = sorted(unique_findings.iterkeys()) # Remap a finding to a different task (T193) if it maps to a task not found in the project for task_id in task_ids: finding = unique_findings[task_id] if not self.task_exists_in_project_tasks(task_id, task_list): logger.debug("Task %s not found in project tasks" % task_id) mapped_tasks = self.task_mapping.get_tasks("*") if not mapped_tasks: continue new_task = mapped_tasks[0] # use the first one if task_id == new_task['id']: continue logger.info("Task %s was not found in the project, mapping it to the default task %s." % (task_id, new_task['id'])) if new_task['id'] not in unique_findings: unique_findings[new_task['id']] = finding else: for weakness in finding['weaknesses']: unique_findings[new_task['id']]['weaknesses'].append(weakness) del unique_findings[task_id] task_ids = sorted(unique_findings.iterkeys()) # Update the tasks' verification status for failure for task_id in task_ids: finding = unique_findings[task_id] stats_total_flaws_found += len(finding['weaknesses']) if not self.task_exists_in_project_tasks(task_id, task_list): logger.error("Task %s was not found in the project, skipping %d findings." % (task_id, len(finding['weaknesses']))) stats_total_skips += 1 stats_total_skips_findings += len(finding['weaknesses']) continue analysis_findings = [] last_weakness = None weakness_finding = {} # Build the list of structured findings needed by the API for weakness in sorted(finding['weaknesses']): if 'description' in weakness: weakness_description = weakness['description'] else: weakness_description = self.task_mapping.get_title_for_weakness(weakness['weakness_id']) if not weakness_description: weakness_description = weakness['weakness_id'] if last_weakness != weakness_description: if len(weakness_finding.items()) > 0: analysis_findings.append(weakness_finding) weakness_finding = {} weakness_finding['count'] = 0 mapped_weakness = self.task_mapping.get_weakness(weakness['weakness_id']) if mapped_weakness: cwe = mapped_weakness['cwe'] if cwe: weakness_finding['cwe'] = cwe weakness_finding['desc'] = weakness_description last_weakness = weakness_description if 'count' in weakness: weakness_finding['count'] += weakness['count'] else: weakness_finding['count'] += 1 if len(finding.items()) > 0: analysis_findings.append(weakness_finding) task = self.task_mapping.get_task(task_id) if task: finding_confidence = task['confidence'] else: finding_confidence = 'low' # Send the finding details, if any, to the API if not self.config['trial_run']: try: ret = self.plugin.add_analysis_note(task_id, project_analysis_note_ref, finding_confidence, analysis_findings, self.behaviour, self.config['task_status_mapping']) logger.debug("Marked %s as FAILURE with %s confidence" % (task_id, finding_confidence)) stats_failures_added += 1 except APIError as e: logger.exception("Unable to mark %s as FAILURE - Reason: %s" % (task_id, str(e))) self.emit.error("API Error: Unable to mark %s as FAILURE. Skipping ..." % (task_id)) stats_api_errors += 1 stats_passes_added = 0 affected_tasks = [] noflaw_tasks = [] # Sift through the SDE tasks for any non-failures for task in task_list: task_search = re.search('^(\d+)-([^\d]+\d+)$', task['id']) if task_search: task_id = task_search.group(2) # Skip certain tasks unless they are explicitly mapped if not self.task_mapping.contains_task(task_id): continue # The tool found a weakness that maps to the task if unique_findings.has_key(task_id): affected_tasks.append(task_id) continue # The tool found nothing related to the task noflaw_tasks.append(task_id) # Mark non-failures as PASS if not self.config['flaws_only']: for task_id in noflaw_tasks: task = self.task_mapping.get_task(task_id) if task: finding_confidence = task['confidence'] else: finding_confidence = 'low' successful_verification_update = False if self.config['trial_run']: successful_verification_update = True else: analysis_findings = [] try: self.plugin.add_analysis_note(task_id, project_analysis_note_ref, finding_confidence, analysis_findings, self.behaviour, self.config['task_status_mapping']) except APIError as e: logger.exception("Unable to mark %s as PASS - Reason: %s" % (task_id, str(e))) self.emit.error("API Error: Unable to mark %s as PASS. Skipping ..." % (task_id)) stats_api_errors += 1 else: successful_verification_update = True if successful_verification_update: logger.info("Marked %s as PASS with %s confidence" % (task_id, finding_confidence)) stats_passes_added += 1 if missing_weakness_map: self.emit.error("Could not map %s flaws" % (len(missing_weakness_map)), err_type='unmapped_weakness', weakness_list=missing_weakness_map) else: self.emit.info("All flaws successfully mapped to tasks.") results = {} results['total_flaws_found'] = (stats_total_flaws_found, 'Total Flaw Types Found') results['tasks_marked_fail'] = (stats_failures_added, 'Number of Tasks marked as FAILED') results['tasks_without_findings'] = (noflaw_tasks, 'Number of Tasks in the project without any flaws') if stats_total_skips: results['skipped_flaws'] = ( stats_total_skips_findings, 'Number of flaws skipped because the related task was not found in the project' ) results['skipped_tasks'] = (stats_total_skips, 'Number of tasks with flaws not found in project') # We queue the information to be sent along the close emit self.emit.queue(results=results) return IntegrationResult(import_start_datetime=import_start_datetime, import_finish_datetime=datetime.now(), affected_tasks=affected_tasks, noflaw_tasks=noflaw_tasks, error_count=stats_api_errors, error_weaknesses_unmapped=len(missing_weakness_map))