class Command(BaseCommand): name = 'lint' help = 'SDE Lint tool scans project file and displays tasks that match the context of each file.' conf_syntax = 'target1 [target2 ...]' conf_help = 'target(s) are the directory/file to be scanned.' def configure(self): self.scanner = Scanner(self.config) self.plugin = PlugInExperience(self.config) def process_args(self): if self.args: targets = self.args else: targets = self.config['args'] err_reason = self.scanner.set_targets(targets) if err_reason: raise UsageError(err_reason) return True def handle(self): content = self.plugin.get_compiled_task_list() self.scanner.set_content(content) self.scanner.scan()
def __init__(self, config, alm_plugin): """ Initialization of the Connector Keyword arguments: sde_plugin -- An SD Elements Plugin configuration object alm_plugin -- A plugin to connect to the ALM tool """ self.config = config self.ignored_tasks = [] self.sde_plugin = PlugInExperience(self.config) self.alm_plugin = alm_plugin self._add_alm_config_options() self.emit = self.config.emit self.has_migrated = False self.migration_required = False
def __init__(self, config, tool_name, supported_input_types={}): self.findings = [] self.mapping = {} self.report_id = "Not specified" self.config = config self.emit = self.config.emit self.behaviour = 'replace' self.weakness_map_identifier = 'ID' # default XML attribute with weakness identifier self.weakness_title = {} self.confidence = {} self.task_mapping = None self.taskstatuses = {} self.plugin = PlugInExperience(self.config) self.supported_input_types = supported_input_types if supported_input_types and 'file' in supported_input_types: # 'report_file' is required if the tool only supports files report_file_mandatory = len(supported_input_types.keys()) == 1 self.config.opts.add( "report_file", "Common separated list of %s Report Files" % tool_name.capitalize(), "x", default=None if report_file_mandatory else '' ) self.config.opts.add( "report_type", "%s Report Type: %s|auto" % (tool_name.capitalize(), ', '.join(supported_input_types['file'])), default="auto" ) self.config.opts.add( "mapping_file", "Task ID -> Weakness mapping (default=%s)" % self.get_default_mapping_file(), "m", '' ) self.config.opts.add( "import_behaviour", "One of the following: %s" % ', '.join(self.VALID_IMPORT_BEHAVIOUR), default="replace" ) self.config.opts.add( 'task_status_mapping', 'Update task status based on verification. Provide a mapping of (%s) to a task status slug' '(JSON encoded dictionary of strings)' % ', '.join(self.VALID_VERIFICATION_MAP.keys()), default='' ) self.config.opts.add("flaws_only", "Only update tasks identified having flaws. (True | False)", "z", "False") self.config.opts.add("trial_run", "Trial run only: (True | False)", "t", "False")
class Command(BaseCommand): help = 'Search and replace text in task notes' sde_plugin = None def configure(self): self.sde_plugin = PlugInExperience(self.config) self.config.opts.add('search_string', "Search string to find in a task note") self.config.opts.add('replace_string', "Replacement string") def sde_connect(self): if not self.sde_plugin: raise Error('Requires initialization') try: self.sde_plugin.connect() except APIError as err: raise Error('Unable to connect to SD Elements. Please review URL, id,' ' and password in configuration file. Reason: %s' % (str(err))) def handle(self): if not self.config['search_string']: raise UsageError('Missing value for search_string') if not self.config['replace_string']: raise UsageError('Missing value for replace_string') self.sde_connect() tasks = self.sde_plugin.get_task_list() for task in tasks: task_notes = self.sde_plugin.get_task_notes(AlmConnector._extract_task_id(task['id'])) if 'ide' in task_notes: for note in task_notes['ide']: new_note_text = re.sub(self.config['search_string'], self.config['replace_string'], note['text']) if new_note_text != note['text']: self.sde_plugin.update_task_ide_note(note['id'], new_note_text) if 'text' in task_notes: for note in task_notes['text']: new_note_text = re.sub(self.config['search_string'], self.config['replace_string'], note['text']) if new_note_text != note['text']: self.sde_plugin.update_task_text_note(note['id'], new_note_text) return True
def configure(self): self.scanner = Scanner(self.config) self.plugin = PlugInExperience(self.config)
class BaseIntegrator(object): # Subclasses override these attributes TOOL_NAME = 'External tool' DEFAULT_MAPPING_FILE = None AVAILABLE_IMPORTERS = [] # Subclasses must fill this list VALID_IMPORT_BEHAVIOUR = ['replace', 'replace-scanner', 'combine'] # An internal map of possible verification and acceptable status meanings VALID_VERIFICATION_MAP = {'pass': ['TODO', 'DONE'], 'partial': ['TODO', 'DONE'], 'fail': ['TODO']} def __init__(self, config, tool_name, supported_input_types={}): self.findings = [] self.mapping = {} self.report_id = "Not specified" self.config = config self.emit = self.config.emit self.behaviour = 'replace' self.weakness_map_identifier = 'ID' # default XML attribute with weakness identifier self.weakness_title = {} self.confidence = {} self.task_mapping = None self.taskstatuses = {} self.plugin = PlugInExperience(self.config) self.supported_input_types = supported_input_types if supported_input_types and 'file' in supported_input_types: # 'report_file' is required if the tool only supports files report_file_mandatory = len(supported_input_types.keys()) == 1 self.config.opts.add( "report_file", "Common separated list of %s Report Files" % tool_name.capitalize(), "x", default=None if report_file_mandatory else '' ) self.config.opts.add( "report_type", "%s Report Type: %s|auto" % (tool_name.capitalize(), ', '.join(supported_input_types['file'])), default="auto" ) self.config.opts.add( "mapping_file", "Task ID -> Weakness mapping (default=%s)" % self.get_default_mapping_file(), "m", '' ) self.config.opts.add( "import_behaviour", "One of the following: %s" % ', '.join(self.VALID_IMPORT_BEHAVIOUR), default="replace" ) self.config.opts.add( 'task_status_mapping', 'Update task status based on verification. Provide a mapping of (%s) to a task status slug' '(JSON encoded dictionary of strings)' % ', '.join(self.VALID_VERIFICATION_MAP.keys()), default='' ) self.config.opts.add("flaws_only", "Only update tasks identified having flaws. (True | False)", "z", "False") self.config.opts.add("trial_run", "Trial run only: (True | False)", "t", "False") def initialize(self): """ This is a post init initialization. It needs to be called as the first function after configuration is processed (usually first call inside handler of the module) """ self.config.process_boolean_config('flaws_only') self.config.process_boolean_config('trial_run') self.config.process_json_str_dict('task_status_mapping') if self.config['import_behaviour'] in self.VALID_IMPORT_BEHAVIOUR: self.behaviour = self.config['import_behaviour'] else: raise UsageError('Invalid import_behaviour %s' % self.config['import_behaviour']) if self.config['task_status_mapping']: # Get the available system task statuses and their meanings self._setup_taskstatuses() # Validate the mapping against the available system statuses # Sanity check the mapping # - pass, partial may mark tasks with a status having meaning TODO or DONE only # - fail may mark tasks with a status having meaning TODO only. for verification, status_name in self.config['task_status_mapping'].iteritems(): if verification not in self.VALID_VERIFICATION_MAP: raise UsageError('Invalid task_status_mapping verification %s' % verification) if status_name not in self.taskstatuses: raise UsageError('Invalid task_status_mapping status "%s" for verification "%s"' % (status_name, verification)) if self.taskstatuses[status_name]['meaning'] not in self.VALID_VERIFICATION_MAP[verification]: raise UsageError('Unexpected task_status_mapping status "%s" for verification "%s"' % (status_name, verification)) # Validate the report_type config. If report_type is not auto, we will process only # the specified report_type, else we process all supported file types. if 'file' in self.supported_input_types: if self.config['report_type'] in self.supported_input_types['file']: self.supported_input_types['file'] = [self.config['report_type']] elif self.config['report_type'] != 'auto': raise UsageError('Invalid report_type %s' % self.config['report_type']) self.process_report_file_config() def get_default_mapping_file(self): return get_media_file_path(os.path.join('analysis', self.DEFAULT_MAPPING_FILE)) def _setup_taskstatuses(self): statuses = self.plugin.get_taskstatuses() for status in statuses: self.taskstatuses[status['slug']] = status def detect_importer(self, report_file): for item in self.AVAILABLE_IMPORTERS: if item['importer'].can_parse_file(report_file): return item['importer'] return None @staticmethod def _get_file_extension(file_path): return os.path.splitext(file_path)[1][1:] @abstractmethod def parse_report_file(self, report_file, report_type): """ Returns the raw findings and the report id for a single report file """ return [], None def set_tool_name(self, tool_name): self.TOOL_NAME = tool_name def parse(self): _raw_findings = [] _report_ids = [] for report_file in self.config['report_file']: if self.config['report_type'] == 'auto': if not isinstance(report_file, basestring): raise UsageError("On auto-detect mode, the file name needs to be specified.") report_type = self._get_file_extension(report_file) else: report_type = self.config['report_type'] raw_findings, report_id = self.parse_report_file(report_file, report_type) _raw_findings.extend(raw_findings) if report_id: _report_ids.append(report_id) self.findings = _raw_findings if _report_ids: self.report_id = ', '.join(_report_ids) else: self.report_id = "Not specified" self.emit.info("Report ID not found in report: Using default.") def process_report_file_config(self): """ If report files contains a directory path, find all possible files in that folder """ if not self.config['report_file']: raise UsageError("Missing configuration option 'report_file'") if not isinstance(self.config['report_file'], basestring): # Should be a file object self.config['report_file'] = [self.config['report_file']] else: processed_report_files = [] for file_path in self.config['report_file'].split(','): file_path = file_path.strip() file_name, file_ext = os.path.splitext(file_path) file_ext = file_ext[1:] if file_ext in self.supported_input_types['file']: processed_report_files.extend(glob.glob(file_path)) elif re.search('[*?]', file_ext): # Run the glob and filter out unsupported file types processed_report_files.extend([f for f in glob.iglob(file_path) if self._get_file_extension(f) in self.supported_input_types['file']]) elif not file_ext: # Glob using our supported file types if os.path.isdir(file_path): _base_path = file_path + '/*' else: _base_path = file_name for file_type in self.supported_input_types['file']: processed_report_files.extend(glob.glob('%s.%s' % (_base_path, file_type))) else: raise UsageError('%s does not match any supported file type(s): %s' % (file_path, self.supported_input_types['file'])) if not processed_report_files: raise UsageError("Did not find any report files. Check if 'report_file' is configured properly.") else: self.config['report_file'] = processed_report_files def load_mapping_from_xml(self): self.task_mapping = Mapping(self.weakness_map_identifier) mapping_file = self.config['mapping_file'] or self.get_default_mapping_file() self.task_mapping.load_mapping(mapping_file) if not self.task_mapping.count(): raise IntegrationError("No base mapping was found in file '%s'" % self.config['mapping_file']) def generate_findings(self): return [] def unique_findings(self): """ Return a map (task_id=> *flaw) based on list of findings (weakness) Where flaw is defined as: flaw[weaknesses] flaw[related_tasks] """ unique_findings = {'nomap': []} for finding in self.generate_findings(): weakness_id = finding['weakness_id'] mapped_tasks = self.task_mapping.get_tasks(weakness_id) if not mapped_tasks: unique_findings['nomap'].append(weakness_id) continue for mapped_task in mapped_tasks: if mapped_task['id'] in unique_findings: flaws = unique_findings[mapped_task['id']] else: flaws = {'weaknesses': []} flaws['weaknesses'].append(finding) flaws['related_tasks'] = mapped_tasks unique_findings[mapped_task['id']] = flaws return unique_findings def task_exists_in_project_tasks(self, task_id, project_tasks): """ Return True if task_id is present in the array of project_tasks, False otherwise task_id is of the form [^\d]+\d+ project_tasks is an array of maps. Each map contains a key 'id' with a corresponding integer value """ for task in project_tasks: task_search = re.search('^(\d+)-([^\d]+\d+)$', task['id']) if task_search: project_task_id = task_search.group(2) if project_task_id == task_id: return True return False def import_findings(self): stats_failures_added = 0 stats_api_errors = 0 stats_total_skips = 0 stats_total_skips_findings = 0 stats_total_flaws_found = 0 import_start_datetime = datetime.now() logger.info("Integration underway for: %s" % self.report_id) logger.info("Mapped SD application/project: %s/%s" % (self.config['sde_application'], self.config['sde_project'])) if self.config['trial_run']: logger.info("Trial run only. No changes will be made") else: ret = self.plugin.create_analysis_session(self.report_id, self.TOOL_NAME) project_analysis_note_ref = ret['id'] task_list = self.plugin.get_task_list() logger.debug("Retrieved %d tasks from %s/%s/%s" % ( len(task_list), self.config['sde_businessunit'], self.config['sde_application'], self.config['sde_project'])) unique_findings = self.unique_findings() missing_weakness_map = unique_findings['nomap'] del unique_findings['nomap'] task_ids = sorted(unique_findings.iterkeys()) # Remap a finding to a different task (T193) if it maps to a task not found in the project for task_id in task_ids: finding = unique_findings[task_id] if not self.task_exists_in_project_tasks(task_id, task_list): logger.debug("Task %s not found in project tasks" % task_id) mapped_tasks = self.task_mapping.get_tasks("*") if not mapped_tasks: continue new_task = mapped_tasks[0] # use the first one if task_id == new_task['id']: continue logger.info("Task %s was not found in the project, mapping it to the default task %s." % (task_id, new_task['id'])) if new_task['id'] not in unique_findings: unique_findings[new_task['id']] = finding else: for weakness in finding['weaknesses']: unique_findings[new_task['id']]['weaknesses'].append(weakness) del unique_findings[task_id] task_ids = sorted(unique_findings.iterkeys()) # Update the tasks' verification status for failure for task_id in task_ids: finding = unique_findings[task_id] stats_total_flaws_found += len(finding['weaknesses']) if not self.task_exists_in_project_tasks(task_id, task_list): logger.error("Task %s was not found in the project, skipping %d findings." % (task_id, len(finding['weaknesses']))) stats_total_skips += 1 stats_total_skips_findings += len(finding['weaknesses']) continue analysis_findings = [] last_weakness = None weakness_finding = {} # Build the list of structured findings needed by the API for weakness in sorted(finding['weaknesses']): if 'description' in weakness: weakness_description = weakness['description'] else: weakness_description = self.task_mapping.get_title_for_weakness(weakness['weakness_id']) if not weakness_description: weakness_description = weakness['weakness_id'] if last_weakness != weakness_description: if len(weakness_finding.items()) > 0: analysis_findings.append(weakness_finding) weakness_finding = {} weakness_finding['count'] = 0 mapped_weakness = self.task_mapping.get_weakness(weakness['weakness_id']) if mapped_weakness: cwe = mapped_weakness['cwe'] if cwe: weakness_finding['cwe'] = cwe weakness_finding['desc'] = weakness_description last_weakness = weakness_description if 'count' in weakness: weakness_finding['count'] += weakness['count'] else: weakness_finding['count'] += 1 if len(finding.items()) > 0: analysis_findings.append(weakness_finding) task = self.task_mapping.get_task(task_id) if task: finding_confidence = task['confidence'] else: finding_confidence = 'low' # Send the finding details, if any, to the API if not self.config['trial_run']: try: ret = self.plugin.add_analysis_note(task_id, project_analysis_note_ref, finding_confidence, analysis_findings, self.behaviour, self.config['task_status_mapping']) logger.debug("Marked %s as FAILURE with %s confidence" % (task_id, finding_confidence)) stats_failures_added += 1 except APIError as e: logger.exception("Unable to mark %s as FAILURE - Reason: %s" % (task_id, str(e))) self.emit.error("API Error: Unable to mark %s as FAILURE. Skipping ..." % (task_id)) stats_api_errors += 1 stats_passes_added = 0 affected_tasks = [] noflaw_tasks = [] # Sift through the SDE tasks for any non-failures for task in task_list: task_search = re.search('^(\d+)-([^\d]+\d+)$', task['id']) if task_search: task_id = task_search.group(2) # Skip certain tasks unless they are explicitly mapped if not self.task_mapping.contains_task(task_id): continue # The tool found a weakness that maps to the task if unique_findings.has_key(task_id): affected_tasks.append(task_id) continue # The tool found nothing related to the task noflaw_tasks.append(task_id) # Mark non-failures as PASS if not self.config['flaws_only']: for task_id in noflaw_tasks: task = self.task_mapping.get_task(task_id) if task: finding_confidence = task['confidence'] else: finding_confidence = 'low' successful_verification_update = False if self.config['trial_run']: successful_verification_update = True else: analysis_findings = [] try: self.plugin.add_analysis_note(task_id, project_analysis_note_ref, finding_confidence, analysis_findings, self.behaviour, self.config['task_status_mapping']) except APIError as e: logger.exception("Unable to mark %s as PASS - Reason: %s" % (task_id, str(e))) self.emit.error("API Error: Unable to mark %s as PASS. Skipping ..." % (task_id)) stats_api_errors += 1 else: successful_verification_update = True if successful_verification_update: logger.info("Marked %s as PASS with %s confidence" % (task_id, finding_confidence)) stats_passes_added += 1 if missing_weakness_map: self.emit.error("Could not map %s flaws" % (len(missing_weakness_map)), err_type='unmapped_weakness', weakness_list=missing_weakness_map) else: self.emit.info("All flaws successfully mapped to tasks.") results = {} results['total_flaws_found'] = (stats_total_flaws_found, 'Total Flaw Types Found') results['tasks_marked_fail'] = (stats_failures_added, 'Number of Tasks marked as FAILED') results['tasks_without_findings'] = (noflaw_tasks, 'Number of Tasks in the project without any flaws') if stats_total_skips: results['skipped_flaws'] = ( stats_total_skips_findings, 'Number of flaws skipped because the related task was not found in the project' ) results['skipped_tasks'] = (stats_total_skips, 'Number of tasks with flaws not found in project') # We queue the information to be sent along the close emit self.emit.queue(results=results) return IntegrationResult(import_start_datetime=import_start_datetime, import_finish_datetime=datetime.now(), affected_tasks=affected_tasks, noflaw_tasks=noflaw_tasks, error_count=stats_api_errors, error_weaknesses_unmapped=len(missing_weakness_map))
def configure(self): self.sde_plugin = PlugInExperience(self.config) self.config.opts.add('search_string', "Search string to find in a task note") self.config.opts.add('replace_string', "Replacement string")
class AlmConnector(object): """ Abstract base class for connectors to Application Lifecycle Management tools such as JIRA, Team Foundation Server, Rally, etc. """ # This needs to be overwritten alm_name = 'ALM Module' ALM_PRIORITY_MAP = 'alm_priority_map' VERIFICATION_STATUSES = ['none', 'partial', 'pass', 'fail'] TEST_OPTIONS = ['server', 'project', 'settings'] STANDARD_STATUS_LIST = ['TODO', 'DONE', 'NA'] FIELD_OPTIONS = ['task_id', 'title', 'context', 'application', 'project'] DEFAULT_TITLE_FORMAT = '${task_id} ${title}' default_issue_template = {} default_priority_map = None feature_custom_lookup = False # Assume the connector does not support custom lookups supports_references = False # By default alm connections don't support task references sync_titles_only = False user_friendly_name_map = None # This is an abstract base class __metaclass__ = abc.ABCMeta def __init__(self, config, alm_plugin): """ Initialization of the Connector Keyword arguments: sde_plugin -- An SD Elements Plugin configuration object alm_plugin -- A plugin to connect to the ALM tool """ self.config = config self.ignored_tasks = [] self.sde_plugin = PlugInExperience(self.config) self.alm_plugin = alm_plugin self._add_alm_config_options() self.emit = self.config.emit self.has_migrated = False self.migration_required = False def _add_alm_config_options(self): """ Adds ALM config options to the config file""" self.config.opts.add('alm_phases', 'Phases to sync ' '(comma separated list, e.g. requirements,testing)', default='requirements,architecture-design,development') self.config.opts.add('sde_statuses_in_scope', 'SDE statuses for adding to %s ' '(comma separated %s)' % (self.alm_name, ','.join(AlmConnector.STANDARD_STATUS_LIST)), default='TODO') self.config.opts.add('sde_min_priority', 'Minimum SDE priority in scope', default='7') self.config.opts.add('sde_tags_filter', 'Filter project tasks by tag (tag1, tag2)', default='') self.config.opts.add('sde_verification_filter', 'Filter project tasks by verification statuses. Valid statuses ' 'are: ' + ', '.join(AlmConnector.VERIFICATION_STATUSES), default='') self.config.opts.add('how_tos_in_scope', 'Whether or not HowTos should be included', default='False') self.config.opts.add('selected_tasks', 'Optionally limit the sync to certain tasks ' '(comma separated, e.g. T12,T13). Note: Overrides other selections.', default='') self.config.opts.add('alm_project', 'Project in %s' % self.alm_name, default='') self.config.opts.add('conflict_policy', 'Conflict policy to use', default='alm') self.config.opts.add('alm_context', 'Add additional context to issues created in %s' % self.alm_name, default='') self.config.opts.add('start_fresh', 'Delete any matching issues in %s' % self.alm_name, default='False') self.config.opts.add('show_progress', 'Show progress', default='False') self.config.opts.add('alm_title_format', 'Task title format in %s. May be composed of: %s' % ( self.alm_name, ','.join(AlmConnector.FIELD_OPTIONS)), default=AlmConnector.DEFAULT_TITLE_FORMAT) self.config.opts.add('test_alm', 'Test Alm "server", "project" or "settings" configuration only', default='') self.config.opts.add('alm_standard_workflow', 'Standard workflow in ALM?', default='True') self.config.opts.add('alm_custom_fields', 'Customized fields to include when creating a task in %s ' '(JSON encoded dictionary of strings)' % self.alm_name, default='') self.config.opts.add('alm_reference_context', 'Context to use to track external task references', default='') if self.feature_custom_lookup: self.config.opts.add('alm_custom_lookup_fields', 'Custom fields and values to use when finding a task in %s ' '(JSON encoded dictionary of strings)' % self.alm_name, default='') if self.default_priority_map: self.config.opts.add(self.ALM_PRIORITY_MAP, 'Customized map from priority in SDE to %s ' '(JSON encoded dictionary of strings)' % self.alm_name, default='') def initialize(self): """ Verify that the configuration options are set properly """ # Note: This will consider space as empty due to strip # We do this before checking if the config is non-empty later self.config.process_list_config('selected_tasks') for task in self.config['selected_tasks']: if not RE_TASK_IDS.match(task): raise UsageError('Invalid Task ID: %s' % task) self.config.process_list_config('sde_verification_filter') for verification_status in self.config['sde_verification_filter']: if verification_status not in AlmConnector.VERIFICATION_STATUSES: raise UsageError('Invalid status specified in sde_verification_filter: %s' % verification_status) if not self.config['selected_tasks']: self.config.process_list_config('alm_phases') if not self.config['alm_phases']: raise AlmException('Missing alm_phases in configuration') self.config.process_list_config('sde_statuses_in_scope') if not self.config['sde_statuses_in_scope']: raise AlmException('Missing the SD Elements statuses in scope') for status in self.config['sde_statuses_in_scope']: if status not in AlmConnector.STANDARD_STATUS_LIST: raise AlmException('Invalid status specified in sde_statuses_in_scope') self.config.process_list_config('sde_tags_filter') if (not self.config['conflict_policy'] or not (self.config['conflict_policy'] == 'alm' or self.config['conflict_policy'] == 'sde' or self.config['conflict_policy'] == 'timestamp')): raise AlmException('Missing or incorrect conflict_policy ' 'in configuration. Valid values are ' 'alm, sde, or timestamp.') if self.config['conflict_policy'] == 'timestamp' and not self.alm_supports_timestamp_sync(): raise AlmException('ALM does not support the timestamp conflict policy. Please use sde or alm') if self.config['sde_min_priority'] is not None: bad_priority_msg = 'Incorrect sde_min_priority specified in configuration. Valid values are > 0 ' bad_priority_msg += 'and <= 10' try: self.config['sde_min_priority'] = int(self.config['sde_min_priority']) except: raise AlmException(bad_priority_msg) if self.config['sde_min_priority'] < 1 or self.config['sde_min_priority'] > 10: raise AlmException(bad_priority_msg) else: self.config['sde_min_priority'] = 1 if self.config['test_alm'] and self.config['test_alm'] not in AlmConnector.TEST_OPTIONS: raise AlmException('Incorrect test_alm configuration setting. ' 'Valid values are: %s' % ','.join(AlmConnector.TEST_OPTIONS)) self.config.process_boolean_config('start_fresh') self.config.process_boolean_config('show_progress') self.config.process_boolean_config('how_tos_in_scope') self.config.process_boolean_config('alm_standard_workflow') self.config.process_json_dict('alm_custom_fields') if self.feature_custom_lookup: self.config.process_json_dict('alm_custom_lookup_fields') if self.config['start_fresh'] and not self.alm_supports_delete(): raise AlmException('Incorrect start_fresh configuration: task deletion is not supported.') self.alm_plugin.post_conf_init() if self.ALM_PRIORITY_MAP in self.config: self.config.process_json_str_dict(self.ALM_PRIORITY_MAP) if not self.config[self.ALM_PRIORITY_MAP]: self.config[self.ALM_PRIORITY_MAP] = self.default_priority_map self._validate_alm_priority_map() matches = re.findall('\$\{?([a-zA-Z_]+)\}?', self.config['alm_title_format']) if not matches: raise AlmException('Incorrect alm_title_format configuration') if 'title' not in matches: raise AlmException('Incorrect alm_title_format configuration. Missing ${title}') if 'task_id' not in matches: raise AlmException('Incorrect alm_title_format configuration. Missing ${task_id}') if 'context' in matches and not self.config['alm_context']: raise AlmException('Missing alm_context in configuration') for match in matches: if match not in AlmConnector.FIELD_OPTIONS: raise AlmException('Incorrect alm_title_format configuration. Invalid field: ${%s}' % match) logger.info('*** AlmConnector initialized ***') def _transform_value(self, value, mapping): """ Search for all macros in value and substitute them with their values """ if isinstance(value, list): new_list = [] for v in value: new_v = self._transform_value(v, mapping) if isinstance(new_v, list): new_list.extend(new_v) else: new_list.append(self._transform_value(v, mapping)) return new_list elif isinstance(value, basestring): macros = re.findall('\$\{?([a-zA-Z_]+)\}?', value) for required_macro in macros: if required_macro not in mapping or not mapping[required_macro]: raise AlmException("Missing value for configuration macro ${%s}" % required_macro) # the first non string macro that this finds will be returned # hence taking over the entire field if not isinstance(mapping[required_macro], basestring): return mapping[required_macro] return Template(value).substitute(mapping).strip() else: raise TypeError('Unsupported type, cannot transform value: %s' % value) def transform_config_value(self, key, mapping): """ Perform macro substitutions on configuration - key - Configuration key - mapping - dictionary of macro->substitution values """ for field, value in self.config[key].iteritems(): self.config[key][field] = self._transform_value(value, mapping) def alm_connect(self): self.alm_connect_server() if self.config['test_alm'] == 'server': return self.alm_connect_project() if self.config['test_alm'] == 'project': return self.alm_validate_configurations() @abstractmethod def alm_connect_server(self): """ Sets up a connection to the ALM tool. Raises an AlmException on encountering an error """ pass def alm_validate_configurations(self): """ Validates alm-specific configurations Raises an AlmException on encountering an error """ pass @abstractmethod def alm_connect_project(self): """ Sets up a connection to the ALM tool. Raises an AlmException on encountering an error """ pass def alm_get_task_by_reference(self, task, reference): """ Returns an AlmTask that represents the value of this SD Elements task in the ALM, or None if the task doesn't exist Raises an AlmException on encountering an error Keyword arguments: task -- An SDE task reference -- A task reference """ return None def alm_get_task_legacy(self, task): """ Returns an AlmTask that represents the value of this SD Elements task in the ALM, or None if the task doesn't exist Raises an AlmException on encountering an error Keyword arguments: task -- An SDE task """ return None def alm_get_task(self, task): """ Returns an AlmTask that represents the value of this SD Elements task in the ALM, or None if the task doesn't exist Raises an AlmException on encountering an error Keyword arguments: task -- An SDE task """ reference = self._get_matching_reference(task) if self.supports_references: if reference: logger.debug('Looking up task %s via its reference' % task['task_id']) return self.alm_get_task_by_reference(task, reference) elif self.has_migrated: return None logger.debug('Looking up task %s via legacy lookup' % task['task_id']) return self.alm_get_task_legacy(task) @abstractmethod def alm_add_task(self, task): """ Adds SD Elements task to the ALM tool. Returns a string representing the task in the ALM tool, or None if that's not possible. This string will be added to a note for the task. Raises an AlmException on encountering an error. Keyword arguments: task -- An SDE task """ pass def alm_supports_timestamp_sync(self): """ Returns True if we can retrieve the timestamp of the last status change on a Task. """ return False def alm_supports_delete(self): """ Returns True if Task Delete is supported by the ALM """ delete_method = getattr(self, "alm_remove_task", None) if delete_method: return callable(delete_method) return False @abstractmethod def alm_update_task_status(self, task, status): """ Updates the specified task in the ALM tool with a new status Raises an AlmException on encountering an error Keyword arguments: task -- An AlmTask representing the task to be updated status -- A string specifying the new status. Either 'DONE', 'TODO', or 'NA' """ pass @abstractmethod def alm_disconnect(self): """ Attempt to disconnect from ALM, if necessary Raises an AlmException on encountering an error """ pass def sde_connect(self): """ Connects to SD Elements server specified in plugin object Raises an AlmException on encountering an error """ if not self.sde_plugin: raise AlmException('Requires initialization') try: self.sde_plugin.connect() except APIError as err: raise AlmException('Unable to connect to SD Elements. Please review URL, id,' ' and password in configuration file. Reason: %s' % (str(err))) self.sde_validate_configuration() self.init_statuses() def init_statuses(self): # Set up the STATUS constant by making an api call statuses = self.sde_plugin.get_taskstatuses() done_list = [] todo_list = [] na_list = [] for status in statuses: setattr(STATUS, status['slug'].encode('utf-8'), status['id']) if status['meaning'] == 'DONE': done_list.append(status['id']) elif status['meaning'] == 'NA': na_list.append(status['id']) else: todo_list.append(status['id']) setattr(STATUS, 'DONE_SET', set(done_list)) setattr(STATUS, 'TODO_SET', set(todo_list)) setattr(STATUS, 'NA_SET', set(na_list)) def sde_validate_configuration(self): """ Validate selected phases, if applicable """ if not self.config['selected_tasks']: result = self.sde_plugin.get_phases() if not result: raise AlmException('Unable to retrieve phases from SD Elements') all_phases_slugs = [phase['slug'] for phase in result] for selected_phase in self.config['alm_phases']: if selected_phase not in all_phases_slugs: raise AlmException('Incorrect alm_phase configuration: %s is not a valid phase' % selected_phase) def is_sde_connected(self): """ Returns true if currently connected to SD Elements""" if not self.sde_plugin: return False return self.sde_plugin.connected def _validate_alm_priority_map(self): """ Validate a priority mapping dictionary. The mapping specifies which value to use in another system based on the SD Elements task's priority numeric value. Priorities are numeric values from the range 1 to 10. This method ensures that: 1. Keys represent a single priority {'10':'Critical'} or a range of priorities {'7-10':'High'} 2. Priorities 1 to 10 are represented exactly once in the dictionary keys 3. Mappings containing a range of priorities {'7-10':'High'} must have their values ordered from low to high. Valid example: {'1-3': 'Low', '4-6': 'Medium', '7-10': 'High'} All SD Elements tasks with priority 1 to 3 are to be mapped to the value "Low" in the other system. All SD Elements tasks with priority 4 to 6 are to be mapped to the value "Medium" in the other system. All SD Elements tasks with priority 7 to 10 are to be mapped to the value "High" in the other system. Invalid examples: {'1-3': 'Low', '4-6': 'Medium', '7-9': 'High'} {'1-3': 'Low', '4-6': 'Medium', '6-10': 'High'} {'3-1': 'Low', '6-4': 'Medium', '10-7': 'High'} """ if self.ALM_PRIORITY_MAP not in self.config: return priority_set = set() for key, value in self.config[self.ALM_PRIORITY_MAP].iteritems(): if not RE_MAP_RANGE_KEY.match(key): raise AlmException('Unable to process %s (not a JSON dictionary). ' 'Reason: Invalid range key %s' % (self.ALM_PRIORITY_MAP, key)) if '-' in key: lrange, hrange = key.split('-') lrange = int(lrange) hrange = int(hrange) if lrange >= hrange: raise AlmException('Invalid %s entry %s => %s: Priority %d should be less than %d' % (self.ALM_PRIORITY_MAP, key, value, lrange, hrange)) for mapped_priority in range(lrange, hrange + 1): if mapped_priority in priority_set: raise AlmException('Invalid %s entry %s => %s: Priority %d is duplicated' % (self.ALM_PRIORITY_MAP, key, value, mapped_priority)) priority_set.add(mapped_priority) else: key_value = int(key) if key_value in priority_set: raise AlmException('Invalid %s entry %s => %s: Priority %d is duplicated' % (self.ALM_PRIORITY_MAP, key, value, key_value)) priority_set.add(key_value) for mapped_priority in xrange(1, 11): if mapped_priority not in priority_set: raise AlmException('Invalid %s: missing a value mapping for priority %d' % (self.ALM_PRIORITY_MAP, mapped_priority)) @staticmethod def _extract_task_id(full_task_id): """ Extract the task id e.g. "CT213" from the full project_id-task_id string e.g. "123-CT213" """ task_id = None task_search = re.search('^(\d+)-([^\d]+\d+)$', full_task_id) if task_search: task_id = task_search.group(2) return task_id @staticmethod def get_alm_task_title(config, task, fixed=False): """ Return the user-defined formatted fixed or full alm title for an SDE task. The fixed title can be used to uniquely identify an issue inside the ALM, allowing one ALM project to sync with more than one instance of the same SDE task """ task_id = AlmConnector._extract_task_id(task['id']) # Remove the task id from the title to get the actual title title = re.sub('^%s:\s+' % task_id, '', task['title']) mapping = { 'task_id': '%s:' % task_id, # Force a suffix to avoid finding "T222" when searching with "T2" 'context': config['alm_context'], 'application': config['sde_application'], 'project': config['sde_project'], 'title': title, } if fixed: mapping['title'] = '' return Template(config['alm_title_format']).substitute(mapping).strip() def sde_get_project(self): """ Returns a single project from SD Elements with the given project_id Raises an AlmException on encountering an error """ if not self.sde_plugin: raise AlmException('Requires initialization') try: return self.sde_plugin.get_project() except APIError as err: logger.error(err) raise AlmException('Unable to get project from SD Elements. Please ensure' ' the project is valid and that the user has' ' sufficient permission to access the project. Reason: %s' % (str(err))) def sde_get_tasks(self, accepted=True): """ Gets all tasks for project in SD Elements Raises an AlmException on encountering an error """ if not self.sde_plugin: raise AlmException('Requires initialization') try: if self.config['selected_tasks']: return self.sde_plugin.get_task_list(accepted=accepted) else: return self.sde_plugin.get_task_list(priority__gte=self.config['sde_min_priority'], accepted=accepted) except APIError as err: logger.error(err) raise AlmException('Unable to get tasks from SD Elements. Please ensure' ' the application and project are valid and that the user has' ' sufficient permission to access the project. Reason: %s' % (str(err))) def sde_get_task(self, task_id): """ Returns a single task from SD Elements w given task_id Raises an AlmException if task doesn't exist or any other error """ if not self.sde_plugin: raise AlmException('Requires initialization') try: return self.sde_plugin.get_task(task_id) except APIError as err: logger.error(err) raise AlmException('Unable to get task in SD Elements. Reason: %s' % (str(err))) def _add_note(self, task_id, note_msg): """ Convenience method to add note """ if not self.sde_plugin: raise AlmException('Requires initialization') try: self.sde_plugin.add_task_text_note(task_id, note_msg) logger.debug('Successfully set note for task %s' % task_id) except APIError as err: logger.error(err) raise AlmException('Unable to add note in SD Elements. Reason: %s' % (str(err))) def in_scope(self, task): """ Check to see if an SDE task is in scope For example, has one of the appropriate phases """ tid = self._extract_task_id(task['id']) if self.config['selected_tasks']: return tid in self.config['selected_tasks'] in_scope = (task['phase']['slug'] in self.config['alm_phases'] and task['priority'] >= self.config['sde_min_priority']) if in_scope and self.config['sde_tags_filter']: in_scope = in_scope and set(self.config['sde_tags_filter']).issubset(task['tags']) if in_scope and self.config['sde_verification_filter']: # Translate null to 'none' to match filter options if not task['verification_status']: task['verification_status'] = 'none' in_scope = in_scope and task['verification_status'] in self.config['sde_verification_filter'] return in_scope def sde_update_task_status(self, task, status): """ Updates the status of the given task in SD Elements Raises an AlmException on encountering an error Keyword arguments: task -- An SD Elements task representing the task to be updated status -- A string specifying the new status. Either STATUS.DONE, STATUS.TODO, or STATUS.NA """ if not self.sde_plugin: raise AlmException('Requires initialization') logger.debug('Attempting to update task %s to %s' % (task['id'], status)) try: self.sde_plugin.update_task_status(task['id'], status) except APIError as err: logger.error(err) raise AlmException('Unable to update the task status in SD ' 'Elements. Either the task no longer ' 'exists, there was a problem connecting ' 'to the server, or the status was invalid') readable_status = 'DONE' if status in STATUS.TODO_SET: readable_status = 'TODO' elif status in STATUS.NA_SET: readable_status = 'NA' logger.info('Status for task %s successfully set in SD Elements' % task['id']) note_msg = 'Task status changed to %s via %s' % (readable_status, self.alm_name) try: self._add_note(task['id'], note_msg) except APIError as err: logger.error('Unable to set a note to mark status ' 'for %s to %s. Reason: %s' % (task['id'], status, str(err))) def convert_markdown_to_alm(self, content, ref): return content def sde_get_task_content(self, task): """ Convenience method that returns the text that should go into content of an ALM ticket/defect/story for a given task. Raises an AlmException on encountering an error Keyword arguments: task -- An SD Elements task representing the task to enter in the ALM """ content = '%s\n\nImported from SD Elements: [%s](%s)' % (task['text'], task['url'], task['url']) if self.config['how_tos_in_scope'] and task['how_tos']: content += '\n\n# How Tos:\n\n' for implementation in task['how_tos']: content += '## %s\n\n' % (implementation['title']) content += implementation['text'] + '\n\n' content = RE_CODE_DOWNLOAD.sub(r'https://%s/\1' % self.config['sde_server'], content) return self.convert_markdown_to_alm(content, ref=task['id']) def output_progress(self, percent): if self.config['show_progress']: print str(percent) + "% complete" sys.stdout.flush() def status_match(self, alm_status, sde_status): if sde_status in STATUS.DONE_SET or sde_status in STATUS.NA_SET: return alm_status == STATUS.DONE else: return alm_status == STATUS.TODO def filter_tasks(self, tasks): return [task for task in tasks if self.in_scope(task)] def _determine_latest(self, alm_task, sde_task): """ Used in synchronize() for the timestamp conflict policy to determine which task status takes precedence. Extracted from synchronize() so it is easier to test. """ if 'status_updated' not in sde_task: raise AlmException('SDE does not support timestamp conflict policy') if sde_task['status_updated'] is None: return 'alm' sde_time = parse(sde_task['status_updated']) alm_time = alm_task.get_timestamp() logger.debug('Comparing timestamps for task %s - SDE: %s, ALM: %s' % (sde_task['id'], str(sde_time), str(alm_time))) return 'sde' if sde_time > alm_time else 'alm' def _check_supports_references(self, tasks): if not tasks: return for task in tasks: if 'references' in task: return self.supports_references = False def _check_has_migrated(self): alm_connection_id = self.config['alm_reference_context'] alm_connection = self.sde_plugin.get_alm_connection(alm_connection_id) # SDE 4.0 and below does not have get_alm_connection method or has_migrated flag if not alm_connection: return self.has_migrated = alm_connection.get('has_migrated', False) self.migration_required = 'has_migrated' in alm_connection and not self.has_migrated if self.migration_required: logger.debug("ALM Connection supports references - will perform a migration") if self.has_migrated: logger.debug("ALM Connection has already migrated to using references") def _update_alm_migration_status(self, has_migrated): alm_connection_id = self.config['alm_reference_context'] self.sde_plugin.update_alm_connection(alm_connection_id, has_migrated) def _make_reference_dict(self, task): return { 'alm_connection': self.config['alm_reference_context'], 'reference': str(task.reference), 'name': str(task.name), 'link': str(task.link) } def _create_task_reference(self, task_id, task): """ Create a Task Reference via SDE api. """ data = self._make_reference_dict(task) logger.debug('Creating reference for task %s' % task_id) self.sde_plugin.add_task_reference(task_id, data) return data def _update_task_reference(self, task_id, task, reference): """ Update a Task Reference via SDE api. """ data = self._make_reference_dict(task) data.pop('alm_connection') self.sde_plugin.update_task_reference(task_id, reference, **data) return data def _remove_task_reference(self, task_id, reference_id): """ Delete a Task Reference via SDE api. """ logger.debug('Removing a task reference for %s' % task_id) self.sde_plugin.remove_task_reference(task_id, reference_id) def _remove_alm_task(self, sde_task, task_reference): """ Given a task in SDE, remove it from an external system. """ alm_task = self.alm_get_task(sde_task) if alm_task: self.alm_remove_task(alm_task) if task_reference and self.supports_references: self.sde_plugin.remove_task_reference(sde_task, task_reference['id']) def _reference_out_of_date(self, old_reference, new_reference): """ Test whether an old Task Reference is stale, i.e. Jira task has moved projects, so its url reference is out of date. """ new_reference = self._make_reference_dict(new_reference) old_reference = old_reference.copy() old_reference.pop('id') changes = old_reference != new_reference return changes def _apply_conflict_policy(self, task_id, sde_task, alm_task): """ What takes precedence in case of a conflict of status? Starts with ALM """ precedence = 'alm' updated_system = 'SD Elements' status = alm_task.get_status() if (status in STATUS.DONE_SET or status in STATUS.NA_SET): updated_status = 'DONE' else: updated_status = 'TODO' if self.config['conflict_policy'] == 'sde': precedence = 'sde' elif self.config['conflict_policy'] == 'timestamp': precedence = self._determine_latest(alm_task, sde_task) if precedence == 'alm': self.sde_update_task_status(sde_task, alm_task.get_status()) else: self.alm_update_task_status(alm_task, sde_task['status']['id']) updated_system = self.alm_name updated_status = sde_task['status']['meaning'] self.emit.info((u'Updated status of task {0} in {1} to {2}' .format(task_id, updated_system, updated_status))) def _get_matching_reference(self, sde_task): """ SDE Tasks may be synced to multiple systems, so match the reference to the current alm. """ if not self.supports_references: return None task_references = sde_task.get('references', []) task_reference = None if task_references: task_reference = next((ref for ref in task_references if ref['alm_connection'] == self.config['alm_reference_context']), None) return task_reference def task_should_sync(self, sde_task): """ Test whether we should sync a task, i.e. it's status has changed in SDE or the ALM. """ if not sde_task['accepted']: return False if sde_task['id'] in self.ignored_tasks: return False if not self.config['selected_tasks'] and sde_task['status']['meaning'] not in self.config['sde_statuses_in_scope']: return False return True def synchronize(self): """ Synchronizes SDE project with ALM project. Reviews every task in the SDE project: - if the task exists in both SDE & ALM and the status is the same in both, nothing happens - if the task exists in both SDE & ALM and the status differs, then the conflict policy takes effect. Either the newest status based on timestamp is used, or the SDE status is used in every case, or the ALM tool status is used in every case. Default is ALM tool status - if the task only exists in SDE, the task is added to the ALM tool - NOTE: if a task that was previously imported from SDE into the ALM is later removed in the same SDE project, then the task is effectively orphaned. The task must be removed manually from the ALM tool Raises an AlmException on encountering an error """ class progress: count = 0 def update_progress(val, output=None): progress.count += val output = output or progress.count self.output_progress(output) try: if not self.sde_plugin: raise AlmException('Requires initialization') if self.config['test_alm']: self.alm_connect() return # Attempt to connect to SDE & ALM self.sde_connect() update_progress(2) self.alm_connect() update_progress(2) project = self.sde_get_project() # Attempt to get all tasks accepted_tasks = self.sde_get_tasks() unaccepted_tasks = self.sde_get_tasks(accepted=False) logger.info('Retrieved all tasks from SDE') # Filter tasks - progress must match reality tasks = self.filter_tasks(accepted_tasks + unaccepted_tasks) tasks = self.transform_tasks(tasks, project) self._check_supports_references(tasks) if self.supports_references: self._check_has_migrated() else: logger.debug(("ALM Connection doesn't support references - " "legacy lookup using titles will be used.")) logger.info('Filtered tasks') if self.config['start_fresh']: total_work = progress.count + len(tasks) * 2 else: total_work = progress.count + len(tasks) for sde_task in tasks: task_id = sde_task['task_id'] update_progress(1, 100 * progress.count / total_work) task_reference = self._get_matching_reference(sde_task) if self.config['start_fresh']: self._remove_alm_task(sde_task, task_reference) update_progress(1, 100 * progress.count / total_work) alm_task = self.alm_get_task(sde_task) if alm_task: if self.supports_references: if not task_reference: self._create_task_reference(sde_task['id'], alm_task) elif self._reference_out_of_date(task_reference, alm_task): reference_id = task_reference.pop('id') self._update_task_reference(sde_task['id'], alm_task, reference_id) if not self.config['alm_standard_workflow']: continue # Close tasks that are no longer part of the SDE project and remove the task reference if not sde_task['accepted']: if not self.status_match(alm_task.get_status(), STATUS.DONE): self.alm_update_task_status(alm_task, STATUS.DONE) if task_reference: self._remove_task_reference(sde_task['id'], task_reference['id']) continue # Has the status changed between SDE and ALM? if not self.status_match(alm_task.get_status(), sde_task['status']['id']): self._apply_conflict_policy(task_id, sde_task, alm_task) else: # Task only exists in SD Elements # Remove reference if it exists - ALM Task was probably deleted # Also pop any old references to force lookup using legacy if task_reference: self._remove_task_reference(sde_task['id'], task_reference['id']) sde_task.pop('references', []) if self.task_should_sync(sde_task): new_alm_task = self.alm_add_task(sde_task) if self.supports_references: self._create_task_reference(sde_task['id'], new_alm_task) ref = self.get_note_message(new_alm_task) self.emit.info(u'Added task %s to %s' % (task_id, self.alm_name)) note_msg = u'Task synchronized in %s. Reference: [%s]' % (self.alm_name, ref) self._add_note(sde_task['id'], note_msg) logger.debug(note_msg) logger.info('Synchronization complete') if self.supports_references and self.migration_required: self._update_alm_migration_status(has_migrated=True) self.alm_disconnect() except AlmException: self.alm_disconnect() raise def get_note_message(self, alm_task): pass def translate_priority(self, priority): """ Translates an SDE priority into an Alm label """ pmap = self.config[self.ALM_PRIORITY_MAP] if not pmap: return None try: priority = int(priority) except TypeError: logger.error('Could not coerce %s into an integer' % priority) raise AlmException("Error in translating SDE priority to %s: " "%s is not an integer priority" % (priority, self.alm_name)) for key in pmap: if '-' in key: lrange, hrange = key.split('-') lrange = int(lrange) hrange = int(hrange) if lrange <= priority <= hrange: return pmap[key] else: if int(key) == priority: return pmap[key] @property def new_issue_template(self): issue_template = {} issue_template.update(self.default_issue_template) if self.config['alm_custom_fields']: issue_template.update(self.config['alm_custom_fields']) return issue_template def get_new_issue(self, task, defaults=None): data = defaults.copy() if defaults else {} macros = task['__macros'] data.update( (field, self._transform_value(value, macros)) for field, value in self.new_issue_template.iteritems() ) if self.user_friendly_name_map: data = self.map_user_friendly_names(data) return data def map_user_friendly_names(self, task): def get_field_name(field_name): if not self.user_friendly_name_map: return field_name return self.user_friendly_name_map.get(field_name, field_name) return dict( (get_field_name(field), value) for field, value in task.iteritems() ) def transform_tasks(self, tasks, project=None): return [self.transform_task(task, project) for task in tasks] def transform_task(self, task, project=None): """ Take a task and add a new entry, __macros, containing a set of macro->value pairs so that this ALM's fields can be remapped to any of the values when a new issue is created """ task['alm_full_title'] = self.get_alm_task_title(self.config, task, fixed=False) task['alm_fixed_title'] = self.get_alm_task_title(self.config, task, fixed=True) task_id = self._extract_task_id(task['id']) # each macro is assumed to be a string or a list of strings macros = { 'business_unit': self.config['sde_businessunit'], 'application': self.config['sde_application'], 'project': self.config['sde_project'], 'context': self.config['alm_context'], 'alm_user': self.config['alm_user'], 'alm_project': self.config['alm_project'], 'id': task['id'], 'task_id': task_id, 'task_title': re.sub('^%s:\s+' % task_id, '', task['title']), 'task_alm_title': task['alm_full_title'], 'task_url': task['url'], 'task_content': task['text'], 'task_richcontent': self.sde_get_task_content(task), 'task_phase': task['phase']['slug'], 'task_status': task['status']['meaning'], 'task_priority': str(task['priority']), 'task_tags': task['tags'], 'problem_content': task['problem']['text'], 'problem_richcontent': self.convert_markdown_to_alm(task['problem']['text'], task['problem']['id']), 'problem_id': task['problem']['id'], 'problem_title': re.sub('^%s:\s+' % task['problem']['id'], '', task['problem']['title']), } if self.ALM_PRIORITY_MAP in self.config: macros['task_priority_translated'] = self.translate_priority(task['priority']) if self.sync_titles_only: macros['task_content'] = PUBLIC_TASK_CONTENT macros['task_richcontent'] = PUBLIC_TASK_CONTENT macros['problem_content'] = PUBLIC_TASK_CONTENT macros['problem_richcontent'] = PUBLIC_TASK_CONTENT if 'alm_issue_label' in self.config: macros['task_tags'].append(self.config['alm_issue_label']) if project and 'custom_attributes' in project: for attr, value in project['custom_attributes'].items(): macros['project_custom_attr_%s' % attr] = value task['__macros'] = macros return task
class Command(BaseCommand): help = 'Update task status and verification details given the contents of a CSV' sde_plugin = None def configure(self): self.sde_plugin = PlugInExperience(self.config) self.config.opts.add('csv_file', "CSV file") def sde_connect(self): if not self.sde_plugin: raise Error('Requires initialization') try: self.sde_plugin.connect() except APIError as err: raise Error('Unable to connect to SD Elements. Please review URL, id,' ' and password in configuration file. Reason: %s' % (str(err))) def handle(self): if not self.config['csv_file']: raise UsageError('Missing csv_file') self.sde_connect() with open(self.config['csv_file'], 'r') as csvfile: reader = csv.DictReader(csvfile) if not reader.fieldnames: raise Error('Empty CSV file') # Remove extra white space in headings for i in range(len(reader.fieldnames)): reader.fieldnames[i] = reader.fieldnames[i].strip() if len(reader.fieldnames) <= 1: raise Error('Please make sure CSV file has task id field and at least one more field.') try: for row in reader: if 'task id' not in reader.fieldnames: raise Error('Please make sure task id field exists.') task_id = row['task id'] if 'verification note' in reader.fieldnames and 'verification status' not in reader.fieldnames: raise Error('Please make sure a verification status field exists along with verification note.') if 'task status' in reader.fieldnames: status = row['task status'] self.sde_plugin.update_task_status(task_id, status) if 'task note' in reader.fieldnames: text_note = row['task note'] self.sde_plugin.add_task_text_note(task_id, text_note) if 'verification status' in reader.fieldnames: verification_status = row['verification status'] if 'verification note' in reader.fieldnames: finding_ref = row['verification note'] else: finding_ref = '' self.sde_plugin.add_manual_analysis_note(task_id, verification_status, finding_ref) except csv.Error, csv_err: raise Error('Unable to parse CSV file, line %d: %s' % (reader.line_num, csv_err)) return True
def configure(self): self.sde_plugin = PlugInExperience(self.config) self.config.opts.add('csv_file', "CSV file")