class BaseIntegrator(object):
# Subclasses override these attributes
TOOL_NAME = 'External tool'
DEFAULT_MAPPING_FILE = None
AVAILABLE_IMPORTERS = [] # Subclasses must fill this list
VALID_IMPORT_BEHAVIOUR = ['replace', 'replace-scanner', 'combine']
# An internal map of possible verification and acceptable status meanings
VALID_VERIFICATION_MAP = {'pass': ['TODO', 'DONE'], 'partial': ['TODO', 'DONE'], 'fail': ['TODO']}
def __init__(self, config, tool_name, supported_input_types={}):
self.findings = []
self.mapping = {}
self.report_id = "Not specified"
self.config = config
self.emit = self.config.emit
self.behaviour = 'replace'
self.weakness_map_identifier = 'ID' # default XML attribute with weakness identifier
self.weakness_title = {}
self.confidence = {}
self.task_mapping = None
self.taskstatuses = {}
self.plugin = PlugInExperience(self.config)
self.supported_input_types = supported_input_types
if supported_input_types and 'file' in supported_input_types:
# 'report_file' is required if the tool only supports files
report_file_mandatory = len(supported_input_types.keys()) == 1
self.config.opts.add(
"report_file",
"Common separated list of %s Report Files" % tool_name.capitalize(),
"x", default=None if report_file_mandatory else ''
)
self.config.opts.add(
"report_type",
"%s Report Type: %s|auto" % (tool_name.capitalize(), ', '.join(supported_input_types['file'])),
default="auto"
)
self.config.opts.add(
"mapping_file",
"Task ID -> Weakness mapping (default=%s)" % self.get_default_mapping_file(),
"m", ''
)
self.config.opts.add(
"import_behaviour",
"One of the following: %s" % ', '.join(self.VALID_IMPORT_BEHAVIOUR),
default="replace"
)
self.config.opts.add(
'task_status_mapping',
'Update task status based on verification. Provide a mapping of (%s) to a task status slug'
'(JSON encoded dictionary of strings)' % ', '.join(self.VALID_VERIFICATION_MAP.keys()),
default=''
)
self.config.opts.add("flaws_only", "Only update tasks identified having flaws. (True | False)", "z", "False")
self.config.opts.add("trial_run", "Trial run only: (True | False)", "t", "False")
def initialize(self):
"""
This is a post init initialization. It needs to be called as the first
function after configuration is processed (usually first call inside handler of
the module)
"""
self.config.process_boolean_config('flaws_only')
self.config.process_boolean_config('trial_run')
self.config.process_json_str_dict('task_status_mapping')
if self.config['import_behaviour'] in self.VALID_IMPORT_BEHAVIOUR:
self.behaviour = self.config['import_behaviour']
else:
raise UsageError('Invalid import_behaviour %s' % self.config['import_behaviour'])
if self.config['task_status_mapping']:
# Get the available system task statuses and their meanings
self._setup_taskstatuses()
# Validate the mapping against the available system statuses
# Sanity check the mapping
# - pass, partial may mark tasks with a status having meaning TODO or DONE only
# - fail may mark tasks with a status having meaning TODO only.
for verification, status_name in self.config['task_status_mapping'].iteritems():
if verification not in self.VALID_VERIFICATION_MAP:
raise UsageError('Invalid task_status_mapping verification %s' % verification)
if status_name not in self.taskstatuses:
raise UsageError('Invalid task_status_mapping status "%s" for verification "%s"' %
(status_name, verification))
if self.taskstatuses[status_name]['meaning'] not in self.VALID_VERIFICATION_MAP[verification]:
raise UsageError('Unexpected task_status_mapping status "%s" for verification "%s"' %
(status_name, verification))
# Validate the report_type config. If report_type is not auto, we will process only
# the specified report_type, else we process all supported file types.
if 'file' in self.supported_input_types:
if self.config['report_type'] in self.supported_input_types['file']:
self.supported_input_types['file'] = [self.config['report_type']]
elif self.config['report_type'] != 'auto':
raise UsageError('Invalid report_type %s' % self.config['report_type'])
self.process_report_file_config()
def get_default_mapping_file(self):
return get_media_file_path(os.path.join('analysis', self.DEFAULT_MAPPING_FILE))
def _setup_taskstatuses(self):
statuses = self.plugin.get_taskstatuses()
for status in statuses:
self.taskstatuses[status['slug']] = status
def detect_importer(self, report_file):
for item in self.AVAILABLE_IMPORTERS:
if item['importer'].can_parse_file(report_file):
return item['importer']
return None
@staticmethod
def _get_file_extension(file_path):
return os.path.splitext(file_path)[1][1:]
@abstractmethod
def parse_report_file(self, report_file, report_type):
""" Returns the raw findings and the report id for a single report file """
return [], None
def set_tool_name(self, tool_name):
self.TOOL_NAME = tool_name
def parse(self):
_raw_findings = []
_report_ids = []
for report_file in self.config['report_file']:
if self.config['report_type'] == 'auto':
if not isinstance(report_file, basestring):
raise UsageError("On auto-detect mode, the file name needs to be specified.")
report_type = self._get_file_extension(report_file)
else:
report_type = self.config['report_type']
raw_findings, report_id = self.parse_report_file(report_file, report_type)
_raw_findings.extend(raw_findings)
if report_id:
_report_ids.append(report_id)
self.findings = _raw_findings
if _report_ids:
self.report_id = ', '.join(_report_ids)
else:
self.report_id = "Not specified"
self.emit.info("Report ID not found in report: Using default.")
def process_report_file_config(self):
"""
If report files contains a directory path, find all possible files in that folder
"""
if not self.config['report_file']:
raise UsageError("Missing configuration option 'report_file'")
if not isinstance(self.config['report_file'], basestring):
# Should be a file object
self.config['report_file'] = [self.config['report_file']]
else:
processed_report_files = []
for file_path in self.config['report_file'].split(','):
file_path = file_path.strip()
file_name, file_ext = os.path.splitext(file_path)
file_ext = file_ext[1:]
if file_ext in self.supported_input_types['file']:
processed_report_files.extend(glob.glob(file_path))
elif re.search('[*?]', file_ext):
# Run the glob and filter out unsupported file types
processed_report_files.extend([f for f in glob.iglob(file_path)
if self._get_file_extension(f) in self.supported_input_types['file']])
elif not file_ext:
# Glob using our supported file types
if os.path.isdir(file_path):
_base_path = file_path + '/*'
else:
_base_path = file_name
for file_type in self.supported_input_types['file']:
processed_report_files.extend(glob.glob('%s.%s' % (_base_path, file_type)))
else:
raise UsageError('%s does not match any supported file type(s): %s' %
(file_path, self.supported_input_types['file']))
if not processed_report_files:
raise UsageError("Did not find any report files. Check if 'report_file' is configured properly.")
else:
self.config['report_file'] = processed_report_files
def load_mapping_from_xml(self):
self.task_mapping = Mapping(self.weakness_map_identifier)
mapping_file = self.config['mapping_file'] or self.get_default_mapping_file()
self.task_mapping.load_mapping(mapping_file)
if not self.task_mapping.count():
raise IntegrationError("No base mapping was found in file '%s'" % self.config['mapping_file'])
def generate_findings(self):
return []
def unique_findings(self):
"""
Return a map (task_id=> *flaw) based on list of findings (weakness)
Where flaw is defined as:
flaw[weaknesses]
flaw[related_tasks]
"""
unique_findings = {'nomap': []}
for finding in self.generate_findings():
weakness_id = finding['weakness_id']
mapped_tasks = self.task_mapping.get_tasks(weakness_id)
if not mapped_tasks:
unique_findings['nomap'].append(weakness_id)
continue
for mapped_task in mapped_tasks:
if mapped_task['id'] in unique_findings:
flaws = unique_findings[mapped_task['id']]
else:
flaws = {'weaknesses': []}
flaws['weaknesses'].append(finding)
flaws['related_tasks'] = mapped_tasks
unique_findings[mapped_task['id']] = flaws
return unique_findings
def task_exists_in_project_tasks(self, task_id, project_tasks):
"""
Return True if task_id is present in the array of project_tasks, False otherwise
task_id is of the form [^\d]+\d+
project_tasks is an array of maps. Each map contains a key 'id' with a corresponding integer value
"""
for task in project_tasks:
task_search = re.search('^(\d+)-([^\d]+\d+)$', task['id'])
if task_search:
project_task_id = task_search.group(2)
if project_task_id == task_id:
return True
return False
def import_findings(self):
stats_failures_added = 0
stats_api_errors = 0
stats_total_skips = 0
stats_total_skips_findings = 0
stats_total_flaws_found = 0
import_start_datetime = datetime.now()
logger.info("Integration underway for: %s" % self.report_id)
logger.info("Mapped SD application/project: %s/%s" %
(self.config['sde_application'], self.config['sde_project']))
if self.config['trial_run']:
logger.info("Trial run only. No changes will be made")
else:
ret = self.plugin.create_analysis_session(self.report_id, self.TOOL_NAME)
project_analysis_note_ref = ret['id']
task_list = self.plugin.get_task_list()
logger.debug("Retrieved %d tasks from %s/%s/%s" % (
len(task_list),
self.config['sde_businessunit'],
self.config['sde_application'],
self.config['sde_project']))
unique_findings = self.unique_findings()
missing_weakness_map = unique_findings['nomap']
del unique_findings['nomap']
task_ids = sorted(unique_findings.iterkeys())
# Remap a finding to a different task (T193) if it maps to a task not found in the project
for task_id in task_ids:
finding = unique_findings[task_id]
if not self.task_exists_in_project_tasks(task_id, task_list):
logger.debug("Task %s not found in project tasks" % task_id)
mapped_tasks = self.task_mapping.get_tasks("*")
if not mapped_tasks:
continue
new_task = mapped_tasks[0] # use the first one
if task_id == new_task['id']:
continue
logger.info("Task %s was not found in the project, mapping it to the default task %s." %
(task_id, new_task['id']))
if new_task['id'] not in unique_findings:
unique_findings[new_task['id']] = finding
else:
for weakness in finding['weaknesses']:
unique_findings[new_task['id']]['weaknesses'].append(weakness)
del unique_findings[task_id]
task_ids = sorted(unique_findings.iterkeys())
# Update the tasks' verification status for failure
for task_id in task_ids:
finding = unique_findings[task_id]
stats_total_flaws_found += len(finding['weaknesses'])
if not self.task_exists_in_project_tasks(task_id, task_list):
logger.error("Task %s was not found in the project, skipping %d findings." %
(task_id, len(finding['weaknesses'])))
stats_total_skips += 1
stats_total_skips_findings += len(finding['weaknesses'])
continue
analysis_findings = []
last_weakness = None
weakness_finding = {}
# Build the list of structured findings needed by the API
for weakness in sorted(finding['weaknesses']):
if 'description' in weakness:
weakness_description = weakness['description']
else:
weakness_description = self.task_mapping.get_title_for_weakness(weakness['weakness_id'])
if not weakness_description:
weakness_description = weakness['weakness_id']
if last_weakness != weakness_description:
if len(weakness_finding.items()) > 0:
analysis_findings.append(weakness_finding)
weakness_finding = {}
weakness_finding['count'] = 0
mapped_weakness = self.task_mapping.get_weakness(weakness['weakness_id'])
if mapped_weakness:
cwe = mapped_weakness['cwe']
if cwe:
weakness_finding['cwe'] = cwe
weakness_finding['desc'] = weakness_description
last_weakness = weakness_description
if 'count' in weakness:
weakness_finding['count'] += weakness['count']
else:
weakness_finding['count'] += 1
if len(finding.items()) > 0:
analysis_findings.append(weakness_finding)
task = self.task_mapping.get_task(task_id)
if task:
finding_confidence = task['confidence']
else:
finding_confidence = 'low'
# Send the finding details, if any, to the API
if not self.config['trial_run']:
try:
ret = self.plugin.add_analysis_note(task_id, project_analysis_note_ref,
finding_confidence, analysis_findings,
self.behaviour, self.config['task_status_mapping'])
logger.debug("Marked %s as FAILURE with %s confidence" % (task_id, finding_confidence))
stats_failures_added += 1
except APIError as e:
logger.exception("Unable to mark %s as FAILURE - Reason: %s" % (task_id, str(e)))
self.emit.error("API Error: Unable to mark %s as FAILURE. Skipping ..." % (task_id))
stats_api_errors += 1
stats_passes_added = 0
affected_tasks = []
noflaw_tasks = []
# Sift through the SDE tasks for any non-failures
for task in task_list:
task_search = re.search('^(\d+)-([^\d]+\d+)$', task['id'])
if task_search:
task_id = task_search.group(2)
# Skip certain tasks unless they are explicitly mapped
if not self.task_mapping.contains_task(task_id):
continue
# The tool found a weakness that maps to the task
if unique_findings.has_key(task_id):
affected_tasks.append(task_id)
continue
# The tool found nothing related to the task
noflaw_tasks.append(task_id)
# Mark non-failures as PASS
if not self.config['flaws_only']:
for task_id in noflaw_tasks:
task = self.task_mapping.get_task(task_id)
if task:
finding_confidence = task['confidence']
else:
finding_confidence = 'low'
successful_verification_update = False
if self.config['trial_run']:
successful_verification_update = True
else:
analysis_findings = []
try:
self.plugin.add_analysis_note(task_id, project_analysis_note_ref, finding_confidence,
analysis_findings, self.behaviour,
self.config['task_status_mapping'])
except APIError as e:
logger.exception("Unable to mark %s as PASS - Reason: %s" % (task_id, str(e)))
self.emit.error("API Error: Unable to mark %s as PASS. Skipping ..." % (task_id))
stats_api_errors += 1
else:
successful_verification_update = True
if successful_verification_update:
logger.info("Marked %s as PASS with %s confidence" % (task_id, finding_confidence))
stats_passes_added += 1
if missing_weakness_map:
self.emit.error("Could not map %s flaws" % (len(missing_weakness_map)),
err_type='unmapped_weakness',
weakness_list=missing_weakness_map)
else:
self.emit.info("All flaws successfully mapped to tasks.")
results = {}
results['total_flaws_found'] = (stats_total_flaws_found, 'Total Flaw Types Found')
results['tasks_marked_fail'] = (stats_failures_added, 'Number of Tasks marked as FAILED')
results['tasks_without_findings'] = (noflaw_tasks, 'Number of Tasks in the project without any flaws')
if stats_total_skips:
results['skipped_flaws'] = (
stats_total_skips_findings,
'Number of flaws skipped because the related task was not found in the project'
)
results['skipped_tasks'] = (stats_total_skips, 'Number of tasks with flaws not found in project')
# We queue the information to be sent along the close emit
self.emit.queue(results=results)
return IntegrationResult(import_start_datetime=import_start_datetime,
import_finish_datetime=datetime.now(),
affected_tasks=affected_tasks,
noflaw_tasks=noflaw_tasks,
error_count=stats_api_errors,
error_weaknesses_unmapped=len(missing_weakness_map))
