예제 #1
0
class AlmConnector(object):
    """
    Abstract base class for connectors to Application Lifecycle
    Management tools such as JIRA, Team Foundation Server, Rally, etc.
    """
    # This needs to be overwritten
    alm_name = 'ALM Module'
    ALM_PRIORITY_MAP = 'alm_priority_map'
    VERIFICATION_STATUSES = ['none', 'partial', 'pass', 'fail']
    TEST_OPTIONS = ['server', 'project', 'settings']
    STANDARD_STATUS_LIST = ['TODO', 'DONE', 'NA']
    FIELD_OPTIONS = ['task_id', 'title', 'context', 'application', 'project']
    DEFAULT_TITLE_FORMAT = '${task_id} ${title}'

    default_issue_template = {}
    default_priority_map = None
    feature_custom_lookup = False  # Assume the connector does not support custom lookups
    supports_references = False  # By default alm connections don't support task references
    sync_titles_only = False
    user_friendly_name_map = None

    # This is an abstract base class
    __metaclass__ = abc.ABCMeta

    def __init__(self, config, alm_plugin):
        """  Initialization of the Connector

        Keyword arguments:
        sde_plugin -- An SD Elements Plugin configuration object
        alm_plugin -- A plugin to connect to the ALM tool
        """
        self.config = config
        self.ignored_tasks = []
        self.sde_plugin = PlugInExperience(self.config)
        self.alm_plugin = alm_plugin
        self._add_alm_config_options()
        self.emit = self.config.emit

        self.has_migrated = False
        self.migration_required = False

    def _add_alm_config_options(self):
        """ Adds ALM config options to the config file"""
        self.config.opts.add('alm_phases', 'Phases to sync '
                             '(comma separated list, e.g. requirements,testing)',
                             default='requirements,architecture-design,development')
        self.config.opts.add('sde_statuses_in_scope', 'SDE statuses for adding to %s '
                             '(comma separated %s)' % (self.alm_name, ','.join(AlmConnector.STANDARD_STATUS_LIST)),
                             default='TODO')
        self.config.opts.add('sde_min_priority', 'Minimum SDE priority in scope',
                             default='7')
        self.config.opts.add('sde_tags_filter', 'Filter project tasks by tag (tag1, tag2)',
                             default='')
        self.config.opts.add('sde_verification_filter', 'Filter project tasks by verification statuses. Valid statuses '
                             'are: ' + ', '.join(AlmConnector.VERIFICATION_STATUSES),
                             default='')
        self.config.opts.add('how_tos_in_scope', 'Whether or not HowTos should be included',
                             default='False')
        self.config.opts.add('selected_tasks', 'Optionally limit the sync to certain tasks '
                             '(comma separated, e.g. T12,T13). Note: Overrides other selections.',
                             default='')
        self.config.opts.add('alm_project', 'Project in %s' % self.alm_name,
                             default='')
        self.config.opts.add('conflict_policy', 'Conflict policy to use',
                             default='alm')
        self.config.opts.add('alm_context', 'Add additional context to issues created in %s' % self.alm_name,
                             default='')
        self.config.opts.add('start_fresh', 'Delete any matching issues in %s' % self.alm_name,
                             default='False')
        self.config.opts.add('show_progress', 'Show progress',
                             default='False')
        self.config.opts.add('alm_title_format', 'Task title format in %s. May be composed of: %s' % (
                             self.alm_name, ','.join(AlmConnector.FIELD_OPTIONS)),
                             default=AlmConnector.DEFAULT_TITLE_FORMAT)
        self.config.opts.add('test_alm', 'Test Alm "server", "project" or "settings" configuration only',
                             default='')
        self.config.opts.add('alm_standard_workflow', 'Standard workflow in ALM?',
                             default='True')
        self.config.opts.add('alm_custom_fields',
                             'Customized fields to include when creating a task in %s '
                             '(JSON encoded dictionary of strings)' % self.alm_name,
                             default='')
        self.config.opts.add('alm_reference_context', 'Context to use to track external task references',
                             default='')

        if self.feature_custom_lookup:
            self.config.opts.add('alm_custom_lookup_fields',
                                 'Custom fields and values to use when finding a task in %s '
                                 '(JSON encoded dictionary of strings)' % self.alm_name,
                                 default='')

        if self.default_priority_map:
            self.config.opts.add(self.ALM_PRIORITY_MAP, 'Customized map from priority in SDE to %s '
                                 '(JSON encoded dictionary of strings)' % self.alm_name,
                                 default='')

    def initialize(self):
        """
        Verify that the configuration options are set properly
        """

        # Note: This will consider space as empty due to strip
        # We do this before checking if the config is non-empty later
        self.config.process_list_config('selected_tasks')
        for task in self.config['selected_tasks']:
            if not RE_TASK_IDS.match(task):
                raise UsageError('Invalid Task ID: %s' % task)

        self.config.process_list_config('sde_verification_filter')
        for verification_status in self.config['sde_verification_filter']:
            if verification_status not in AlmConnector.VERIFICATION_STATUSES:
                raise UsageError('Invalid status specified in sde_verification_filter: %s' % verification_status)

        if not self.config['selected_tasks']:
            self.config.process_list_config('alm_phases')
            if not self.config['alm_phases']:
                raise AlmException('Missing alm_phases in configuration')

            self.config.process_list_config('sde_statuses_in_scope')
            if not self.config['sde_statuses_in_scope']:
                raise AlmException('Missing the SD Elements statuses in scope')

            for status in self.config['sde_statuses_in_scope']:
                if status not in AlmConnector.STANDARD_STATUS_LIST:
                    raise AlmException('Invalid status specified in sde_statuses_in_scope')

            self.config.process_list_config('sde_tags_filter')

        if (not self.config['conflict_policy'] or
            not (self.config['conflict_policy'] == 'alm' or
                 self.config['conflict_policy'] == 'sde' or
                 self.config['conflict_policy'] == 'timestamp')):
            raise AlmException('Missing or incorrect conflict_policy '
                               'in configuration. Valid values are '
                               'alm, sde, or timestamp.')

        if self.config['conflict_policy'] == 'timestamp' and not self.alm_supports_timestamp_sync():
            raise AlmException('ALM does not support the timestamp conflict policy. Please use sde or alm')

        if self.config['sde_min_priority'] is not None:
            bad_priority_msg = 'Incorrect sde_min_priority specified in configuration. Valid values are > 0 '
            bad_priority_msg += 'and <= 10'

            try:
                self.config['sde_min_priority'] = int(self.config['sde_min_priority'])
            except:
                raise AlmException(bad_priority_msg)

            if self.config['sde_min_priority'] < 1 or self.config['sde_min_priority'] > 10:
                raise AlmException(bad_priority_msg)
        else:
            self.config['sde_min_priority'] = 1

        if self.config['test_alm'] and self.config['test_alm'] not in AlmConnector.TEST_OPTIONS:
            raise AlmException('Incorrect test_alm configuration setting. '
                               'Valid values are: %s' % ','.join(AlmConnector.TEST_OPTIONS))

        self.config.process_boolean_config('start_fresh')
        self.config.process_boolean_config('show_progress')
        self.config.process_boolean_config('how_tos_in_scope')
        self.config.process_boolean_config('alm_standard_workflow')
        self.config.process_json_dict('alm_custom_fields')

        if self.feature_custom_lookup:
            self.config.process_json_dict('alm_custom_lookup_fields')

        if self.config['start_fresh'] and not self.alm_supports_delete():
            raise AlmException('Incorrect start_fresh configuration: task deletion is not supported.')

        self.alm_plugin.post_conf_init()

        if self.ALM_PRIORITY_MAP in self.config:
            self.config.process_json_str_dict(self.ALM_PRIORITY_MAP)
            if not self.config[self.ALM_PRIORITY_MAP]:
                self.config[self.ALM_PRIORITY_MAP] = self.default_priority_map
            self._validate_alm_priority_map()

        matches = re.findall('\$\{?([a-zA-Z_]+)\}?', self.config['alm_title_format'])
        if not matches:
            raise AlmException('Incorrect alm_title_format configuration')
        if 'title' not in matches:
            raise AlmException('Incorrect alm_title_format configuration. Missing ${title}')
        if 'task_id' not in matches:
            raise AlmException('Incorrect alm_title_format configuration. Missing ${task_id}')
        if 'context' in matches and not self.config['alm_context']:
            raise AlmException('Missing alm_context in configuration')

        for match in matches:
            if match not in AlmConnector.FIELD_OPTIONS:
                raise AlmException('Incorrect alm_title_format configuration. Invalid field: ${%s}' % match)

        logger.info('*** AlmConnector initialized ***')

    def _transform_value(self, value, mapping):
        """
        Search for all macros in value and substitute them with their values
        """

        if isinstance(value, list):
            new_list = []
            for v in value:
                new_v = self._transform_value(v, mapping)
                if isinstance(new_v, list):
                    new_list.extend(new_v)
                else:
                    new_list.append(self._transform_value(v, mapping))
            return new_list
        elif isinstance(value, basestring):
            macros = re.findall('\$\{?([a-zA-Z_]+)\}?', value)
            for required_macro in macros:
                if required_macro not in mapping or not mapping[required_macro]:
                    raise AlmException("Missing value for configuration macro ${%s}" % required_macro)
                # the first non string macro that this finds will be returned
                # hence taking over the entire field
                if not isinstance(mapping[required_macro], basestring):
                    return mapping[required_macro]

            return Template(value).substitute(mapping).strip()
        else:
            raise TypeError('Unsupported type, cannot transform value: %s' % value)

    def transform_config_value(self, key, mapping):
        """
        Perform macro substitutions on configuration
         - key - Configuration key
         - mapping - dictionary of macro->substitution values
        """
        for field, value in self.config[key].iteritems():
            self.config[key][field] = self._transform_value(value, mapping)

    def alm_connect(self):
        self.alm_connect_server()

        if self.config['test_alm'] == 'server':
            return

        self.alm_connect_project()

        if self.config['test_alm'] == 'project':
            return

        self.alm_validate_configurations()

    @abstractmethod
    def alm_connect_server(self):
        """ Sets up a connection to the ALM tool.

        Raises an AlmException on encountering an error
        """
        pass

    def alm_validate_configurations(self):
        """ Validates alm-specific configurations

        Raises an AlmException on encountering an error
        """
        pass

    @abstractmethod
    def alm_connect_project(self):
        """ Sets up a connection to the ALM tool.

        Raises an AlmException on encountering an error
        """
        pass

    def alm_get_task_by_reference(self, task, reference):
        """ Returns an AlmTask that represents the value of this
        SD Elements task in the ALM, or None if the task doesn't exist

        Raises an AlmException on encountering an error

        Keyword arguments:
        task  -- An SDE task
        reference -- A task reference
        """
        return None

    def alm_get_task_legacy(self, task):
        """ Returns an AlmTask that represents the value of this
        SD Elements task in the ALM, or None if the task doesn't exist

        Raises an AlmException on encountering an error

        Keyword arguments:
        task  -- An SDE task
        """
        return None

    def alm_get_task(self, task):
        """ Returns an AlmTask that represents the value of this
        SD Elements task in the ALM, or None if the task doesn't exist

        Raises an AlmException on encountering an error

        Keyword arguments:
        task  -- An SDE task
        """
        reference = self._get_matching_reference(task)

        if self.supports_references:
            if reference:
                logger.debug('Looking up task %s via its reference' % task['task_id'])
                return self.alm_get_task_by_reference(task, reference)
            elif self.has_migrated:
                return None

        logger.debug('Looking up task %s via legacy lookup' % task['task_id'])
        return self.alm_get_task_legacy(task)

    @abstractmethod
    def alm_add_task(self, task):
        """ Adds SD Elements task to the ALM tool.

        Returns a string representing the task in the ALM tool,
        or None if that's not possible. This string will be
        added to a note for the task.

        Raises an AlmException on encountering an error.

        Keyword arguments:
        task  -- An SDE task
        """
        pass

    def alm_supports_timestamp_sync(self):
        """
        Returns True if we can retrieve the timestamp of the
        last status change on a Task.
        """
        return False

    def alm_supports_delete(self):
        """ Returns True if Task Delete is supported by the ALM
        """
        delete_method = getattr(self, "alm_remove_task", None)
        if delete_method:
            return callable(delete_method)
        return False

    @abstractmethod
    def alm_update_task_status(self, task, status):
        """ Updates the specified task in the ALM tool with a new status

        Raises an AlmException on encountering an error

        Keyword arguments:
        task  -- An AlmTask representing the task to be updated
        status -- A string specifying the new status. Either 'DONE', 'TODO',
                  or 'NA'
        """
        pass

    @abstractmethod
    def alm_disconnect(self):
        """ Attempt to disconnect from ALM, if necessary

        Raises an AlmException on encountering an error
        """
        pass

    def sde_connect(self):
        """ Connects to SD Elements server specified in plugin object

        Raises an AlmException on encountering an error
        """
        if not self.sde_plugin:
            raise AlmException('Requires initialization')
        try:
            self.sde_plugin.connect()
        except APIError as err:
            raise AlmException('Unable to connect to SD Elements. Please review URL, id,'
                               ' and password in configuration file. Reason: %s' % (str(err)))

        self.sde_validate_configuration()

        self.init_statuses()

    def init_statuses(self):
        # Set up the STATUS constant by making an api call
        statuses = self.sde_plugin.get_taskstatuses()
        done_list = []
        todo_list = []
        na_list = []

        for status in statuses:
            setattr(STATUS, status['slug'].encode('utf-8'), status['id'])
            if status['meaning'] == 'DONE':
                done_list.append(status['id'])
            elif status['meaning'] == 'NA':
                na_list.append(status['id'])
            else:
                todo_list.append(status['id'])

        setattr(STATUS, 'DONE_SET', set(done_list))
        setattr(STATUS, 'TODO_SET', set(todo_list))
        setattr(STATUS, 'NA_SET', set(na_list))

    def sde_validate_configuration(self):
        """ Validate selected phases, if applicable """
        if not self.config['selected_tasks']:
            result = self.sde_plugin.get_phases()
            if not result:
                raise AlmException('Unable to retrieve phases from SD Elements')

            all_phases_slugs = [phase['slug'] for phase in result]
            for selected_phase in self.config['alm_phases']:
                if selected_phase not in all_phases_slugs:
                    raise AlmException('Incorrect alm_phase configuration: %s is not a valid phase' % selected_phase)

    def is_sde_connected(self):
        """ Returns true if currently connected to SD Elements"""
        if not self.sde_plugin:
            return False
        return self.sde_plugin.connected

    def _validate_alm_priority_map(self):
        """
        Validate a priority mapping dictionary. The mapping specifies which value to use in another system
        based on the SD Elements task's priority numeric value. Priorities are numeric values from the range 1 to 10.

        This method ensures that:

         1. Keys represent a single priority {'10':'Critical'} or a range of priorities {'7-10':'High'}
         2. Priorities 1 to 10 are represented exactly once in the dictionary keys
         3. Mappings containing a range of priorities {'7-10':'High'} must have their values ordered from low to high.

        Valid example:
        {'1-3': 'Low', '4-6': 'Medium', '7-10': 'High'}

        All SD Elements tasks with priority 1 to 3 are to be mapped to the value "Low" in the other system.
        All SD Elements tasks with priority 4 to 6 are to be mapped to the value "Medium" in the other system.
        All SD Elements tasks with priority 7 to 10 are to be mapped to the value "High" in the other system.

        Invalid examples:
        {'1-3': 'Low', '4-6': 'Medium', '7-9': 'High'}
        {'1-3': 'Low', '4-6': 'Medium', '6-10': 'High'}
        {'3-1': 'Low', '6-4': 'Medium', '10-7': 'High'}
        """
        if self.ALM_PRIORITY_MAP not in self.config:
            return

        priority_set = set()
        for key, value in self.config[self.ALM_PRIORITY_MAP].iteritems():
            if not RE_MAP_RANGE_KEY.match(key):
                raise AlmException('Unable to process %s (not a JSON dictionary). '
                                   'Reason: Invalid range key %s' % (self.ALM_PRIORITY_MAP, key))

            if '-' in key:
                lrange, hrange = key.split('-')
                lrange = int(lrange)
                hrange = int(hrange)
                if lrange >= hrange:
                    raise AlmException('Invalid %s entry %s => %s: Priority %d should be less than %d' %
                                       (self.ALM_PRIORITY_MAP, key, value, lrange, hrange))
                for mapped_priority in range(lrange, hrange + 1):
                    if mapped_priority in priority_set:
                        raise AlmException('Invalid %s entry %s => %s: Priority %d is duplicated' %
                                           (self.ALM_PRIORITY_MAP, key, value, mapped_priority))
                    priority_set.add(mapped_priority)
            else:
                key_value = int(key)
                if key_value in priority_set:
                    raise AlmException('Invalid %s entry %s => %s: Priority %d is duplicated' %
                                       (self.ALM_PRIORITY_MAP, key, value, key_value))
                priority_set.add(key_value)

        for mapped_priority in xrange(1, 11):
            if mapped_priority not in priority_set:
                raise AlmException('Invalid %s: missing a value mapping for priority %d' %
                                   (self.ALM_PRIORITY_MAP, mapped_priority))

    @staticmethod
    def _extract_task_id(full_task_id):
        """
        Extract the task id e.g. "CT213" from the full project_id-task_id string e.g. "123-CT213"
        """
        task_id = None
        task_search = re.search('^(\d+)-([^\d]+\d+)$', full_task_id)
        if task_search:
            task_id = task_search.group(2)
        return task_id

    @staticmethod
    def get_alm_task_title(config, task, fixed=False):
        """
        Return the user-defined formatted fixed or full alm title for an SDE task. The fixed title can be used
        to uniquely identify an issue inside the ALM, allowing one ALM project to sync
        with more than one instance of the same SDE task
        """
        task_id = AlmConnector._extract_task_id(task['id'])

        # Remove the task id from the title to get the actual title
        title = re.sub('^%s:\s+' % task_id, '', task['title'])

        mapping = {
            'task_id': '%s:' % task_id,  # Force a suffix to avoid finding "T222" when searching with "T2"
            'context': config['alm_context'],
            'application': config['sde_application'],
            'project': config['sde_project'],
            'title': title,
        }
        if fixed:
            mapping['title'] = ''

        return Template(config['alm_title_format']).substitute(mapping).strip()

    def sde_get_project(self):
        """ Returns a single project from SD Elements with the given project_id

        Raises an AlmException on encountering an error
        """

        if not self.sde_plugin:
            raise AlmException('Requires initialization')

        try:
            return self.sde_plugin.get_project()
        except APIError as err:
            logger.error(err)
            raise AlmException('Unable to get project from SD Elements. Please ensure'
                               ' the project is valid and that the user has'
                               ' sufficient permission to access the project. Reason: %s' % (str(err)))

    def sde_get_tasks(self, accepted=True):
        """ Gets all tasks for project in SD Elements

        Raises an AlmException on encountering an error
        """

        if not self.sde_plugin:
            raise AlmException('Requires initialization')

        try:
            if self.config['selected_tasks']:
                return self.sde_plugin.get_task_list(accepted=accepted)
            else:
                return self.sde_plugin.get_task_list(priority__gte=self.config['sde_min_priority'], accepted=accepted)
        except APIError as err:
            logger.error(err)
            raise AlmException('Unable to get tasks from SD Elements. Please ensure'
                               ' the application and project are valid and that the user has'
                               ' sufficient permission to access the project. Reason: %s' % (str(err)))

    def sde_get_task(self, task_id):
        """ Returns a single task from SD Elements w given task_id

        Raises an AlmException if task doesn't exist or any other error
        """
        if not self.sde_plugin:
            raise AlmException('Requires initialization')

        try:
            return self.sde_plugin.get_task(task_id)
        except APIError as err:
            logger.error(err)
            raise AlmException('Unable to get task in SD Elements. Reason: %s' % (str(err)))

    def _add_note(self, task_id, note_msg):
        """ Convenience method to add note """
        if not self.sde_plugin:
            raise AlmException('Requires initialization')

        try:
            self.sde_plugin.add_task_text_note(task_id, note_msg)
            logger.debug('Successfully set note for task %s' % task_id)

        except APIError as err:
            logger.error(err)
            raise AlmException('Unable to add note in SD Elements. Reason: %s' % (str(err)))

    def in_scope(self, task):
        """ Check to see if an SDE task is in scope

        For example, has one of the appropriate phases
        """
        tid = self._extract_task_id(task['id'])
        if self.config['selected_tasks']:
            return tid in self.config['selected_tasks']

        in_scope = (task['phase']['slug'] in self.config['alm_phases'] and
                    task['priority'] >= self.config['sde_min_priority'])

        if in_scope and self.config['sde_tags_filter']:
            in_scope = in_scope and set(self.config['sde_tags_filter']).issubset(task['tags'])

        if in_scope and self.config['sde_verification_filter']:
            # Translate null to 'none' to match filter options
            if not task['verification_status']:
                task['verification_status'] = 'none'
            in_scope = in_scope and task['verification_status'] in self.config['sde_verification_filter']

        return in_scope

    def sde_update_task_status(self, task, status):
        """ Updates the status of the given task in SD Elements

        Raises an AlmException on encountering an error

        Keyword arguments:
        task  -- An SD Elements task representing the task to be updated
        status -- A string specifying the new status. Either STATUS.DONE, STATUS.TODO,
                  or STATUS.NA
        """
        if not self.sde_plugin:
            raise AlmException('Requires initialization')

        logger.debug('Attempting to update task %s to %s' % (task['id'], status))

        try:
            self.sde_plugin.update_task_status(task['id'], status)
        except APIError as err:
            logger.error(err)
            raise AlmException('Unable to update the task status in SD '
                               'Elements. Either the task no longer '
                               'exists, there was a problem connecting '
                               'to the server, or the status was invalid')
        readable_status = 'DONE'
        if status in STATUS.TODO_SET:
            readable_status = 'TODO'
        elif status in STATUS.NA_SET:
            readable_status = 'NA'
        logger.info('Status for task %s successfully set in SD Elements' % task['id'])

        note_msg = 'Task status changed to %s via %s' % (readable_status, self.alm_name)
        try:
            self._add_note(task['id'], note_msg)
        except APIError as err:
            logger.error('Unable to set a note to mark status '
                         'for %s to %s. Reason: %s' % (task['id'], status, str(err)))

    def convert_markdown_to_alm(self, content, ref):
        return content

    def sde_get_task_content(self, task):
        """ Convenience method that returns the text that should go into
        content of an ALM ticket/defect/story for a given task.

        Raises an AlmException on encountering an error

        Keyword arguments:
        task  -- An SD Elements task representing the task to enter in the
                 ALM
        """
        content = '%s\n\nImported from SD Elements: [%s](%s)' % (task['text'], task['url'], task['url'])
        if self.config['how_tos_in_scope'] and task['how_tos']:
            content += '\n\n# How Tos:\n\n'
            for implementation in task['how_tos']:
                content += '## %s\n\n' % (implementation['title'])
                content += implementation['text'] + '\n\n'

        content = RE_CODE_DOWNLOAD.sub(r'https://%s/\1' % self.config['sde_server'], content)

        return self.convert_markdown_to_alm(content, ref=task['id'])

    def output_progress(self, percent):
        if self.config['show_progress']:
            print str(percent) + "% complete"
            sys.stdout.flush()

    def status_match(self, alm_status, sde_status):
        if sde_status in STATUS.DONE_SET or sde_status in STATUS.NA_SET:
            return alm_status == STATUS.DONE
        else:
            return alm_status == STATUS.TODO

    def filter_tasks(self, tasks):
        return [task for task in tasks if self.in_scope(task)]

    def _determine_latest(self, alm_task, sde_task):
        """
        Used in synchronize() for the timestamp conflict policy to determine which
        task status takes precedence. Extracted from synchronize() so it is easier
        to test.
        """
        if 'status_updated' not in sde_task:
            raise AlmException('SDE does not support timestamp conflict policy')
        if sde_task['status_updated'] is None:
            return 'alm'

        sde_time = parse(sde_task['status_updated'])
        alm_time = alm_task.get_timestamp()
        logger.debug('Comparing timestamps for task %s - SDE: %s, ALM: %s' %
                     (sde_task['id'], str(sde_time), str(alm_time)))

        return 'sde' if sde_time > alm_time else 'alm'

    def _check_supports_references(self, tasks):
        if not tasks:
            return

        for task in tasks:
            if 'references' in task:
                return

        self.supports_references = False

    def _check_has_migrated(self):
        alm_connection_id = self.config['alm_reference_context']
        alm_connection = self.sde_plugin.get_alm_connection(alm_connection_id)

        # SDE 4.0 and below does not have get_alm_connection method or has_migrated flag
        if not alm_connection:
            return

        self.has_migrated = alm_connection.get('has_migrated', False)
        self.migration_required = 'has_migrated' in alm_connection and not self.has_migrated

        if self.migration_required:
            logger.debug("ALM Connection supports references - will perform a migration")

        if self.has_migrated:
            logger.debug("ALM Connection has already migrated to using references")

    def _update_alm_migration_status(self, has_migrated):
        alm_connection_id = self.config['alm_reference_context']
        self.sde_plugin.update_alm_connection(alm_connection_id, has_migrated)

    def _make_reference_dict(self, task):
        return {
            'alm_connection': self.config['alm_reference_context'],
            'reference': str(task.reference),
            'name': str(task.name),
            'link': str(task.link)
        }

    def _create_task_reference(self, task_id, task):
        """
        Create a Task Reference via SDE api.
        """
        data = self._make_reference_dict(task)
        logger.debug('Creating reference for task %s' % task_id)
        self.sde_plugin.add_task_reference(task_id, data)
        return data

    def _update_task_reference(self, task_id, task, reference):
        """
        Update a Task Reference via SDE api.
        """
        data = self._make_reference_dict(task)
        data.pop('alm_connection')
        self.sde_plugin.update_task_reference(task_id, reference, **data)
        return data

    def _remove_task_reference(self, task_id, reference_id):
        """
        Delete a Task Reference via SDE api.
        """
        logger.debug('Removing a task reference for %s' % task_id)
        self.sde_plugin.remove_task_reference(task_id, reference_id)

    def _remove_alm_task(self, sde_task, task_reference):
        """
        Given a task in SDE, remove it from an external system.
        """
        alm_task = self.alm_get_task(sde_task)
        if alm_task:
            self.alm_remove_task(alm_task)
        if task_reference and self.supports_references:
            self.sde_plugin.remove_task_reference(sde_task, task_reference['id'])

    def _reference_out_of_date(self, old_reference, new_reference):
        """
        Test whether an old Task Reference is stale, i.e. Jira task
        has moved projects, so its url reference is out of date.
        """
        new_reference = self._make_reference_dict(new_reference)
        old_reference = old_reference.copy()
        old_reference.pop('id')
        changes = old_reference != new_reference
        return changes

    def _apply_conflict_policy(self, task_id, sde_task, alm_task):
        """
        What takes precedence in case of a conflict of status? Starts with ALM
        """

        precedence = 'alm'
        updated_system = 'SD Elements'

        status = alm_task.get_status()
        if (status in STATUS.DONE_SET or status in STATUS.NA_SET):
            updated_status = 'DONE'
        else:
            updated_status = 'TODO'

        if self.config['conflict_policy'] == 'sde':
            precedence = 'sde'
        elif self.config['conflict_policy'] == 'timestamp':
            precedence = self._determine_latest(alm_task, sde_task)

        if precedence == 'alm':
            self.sde_update_task_status(sde_task, alm_task.get_status())
        else:
            self.alm_update_task_status(alm_task, sde_task['status']['id'])
            updated_system = self.alm_name
            updated_status = sde_task['status']['meaning']

        self.emit.info((u'Updated status of task {0} in {1} to {2}'
                        .format(task_id, updated_system, updated_status)))

    def _get_matching_reference(self, sde_task):
        """
        SDE Tasks may be synced to multiple systems, so match the reference
        to the current alm.
        """
        if not self.supports_references:
            return None

        task_references = sde_task.get('references', [])
        task_reference = None

        if task_references:
            task_reference = next((ref for ref in task_references
                                   if ref['alm_connection'] == self.config['alm_reference_context']), None)
        return task_reference

    def task_should_sync(self, sde_task):
        """
        Test whether we should sync a task, i.e. it's status has
        changed in SDE or the ALM.
        """
        if not sde_task['accepted']:
            return False

        if sde_task['id'] in self.ignored_tasks:
            return False

        if not self.config['selected_tasks'] and sde_task['status']['meaning'] not in self.config['sde_statuses_in_scope']:
            return False

        return True

    def synchronize(self):
        """ Synchronizes SDE project with ALM project.

        Reviews every task in the SDE project:
        - if the task exists in both SDE & ALM and the status is the same
          in both, nothing happens
        - if the task exists in both SDE & ALM and the status differs, then
          the conflict policy takes effect. Either the newest status based on
          timestamp is used, or the SDE status is used in every case, or
          the ALM tool status is used in every case. Default is ALM tool
          status
        - if the task only exists in SDE, the task is added to the ALM
          tool
        - NOTE: if a task that was previously imported from SDE into the
          ALM is later removed in the same SDE project, then the task is
          effectively orphaned. The task must be removed manually from the
          ALM tool

        Raises an AlmException on encountering an error
        """

        class progress:
            count = 0

        def update_progress(val, output=None):
            progress.count += val
            output = output or progress.count
            self.output_progress(output)

        try:
            if not self.sde_plugin:
                raise AlmException('Requires initialization')

            if self.config['test_alm']:
                self.alm_connect()
                return

            # Attempt to connect to SDE & ALM
            self.sde_connect()
            update_progress(2)

            self.alm_connect()
            update_progress(2)

            project = self.sde_get_project()

            # Attempt to get all tasks
            accepted_tasks = self.sde_get_tasks()
            unaccepted_tasks = self.sde_get_tasks(accepted=False)
            logger.info('Retrieved all tasks from SDE')

            # Filter tasks - progress must match reality
            tasks = self.filter_tasks(accepted_tasks + unaccepted_tasks)
            tasks = self.transform_tasks(tasks, project)

            self._check_supports_references(tasks)

            if self.supports_references:
                self._check_has_migrated()
            else:
                logger.debug(("ALM Connection doesn't support references - "
                              "legacy lookup using titles will be used."))

            logger.info('Filtered tasks')

            if self.config['start_fresh']:
                total_work = progress.count + len(tasks) * 2
            else:
                total_work = progress.count + len(tasks)

            for sde_task in tasks:
                task_id = sde_task['task_id']
                update_progress(1, 100 * progress.count / total_work)

                task_reference = self._get_matching_reference(sde_task)

                if self.config['start_fresh']:
                    self._remove_alm_task(sde_task, task_reference)
                    update_progress(1, 100 * progress.count / total_work)

                alm_task = self.alm_get_task(sde_task)

                if alm_task:
                    if self.supports_references:
                        if not task_reference:
                            self._create_task_reference(sde_task['id'], alm_task)
                        elif self._reference_out_of_date(task_reference, alm_task):
                            reference_id = task_reference.pop('id')
                            self._update_task_reference(sde_task['id'], alm_task, reference_id)

                    if not self.config['alm_standard_workflow']:
                        continue

                    # Close tasks that are no longer part of the SDE project and remove the task reference
                    if not sde_task['accepted']:
                        if not self.status_match(alm_task.get_status(), STATUS.DONE):
                            self.alm_update_task_status(alm_task, STATUS.DONE)
                        if task_reference:
                            self._remove_task_reference(sde_task['id'], task_reference['id'])
                        continue

                    # Has the status changed between SDE and ALM?
                    if not self.status_match(alm_task.get_status(), sde_task['status']['id']):
                        self._apply_conflict_policy(task_id, sde_task, alm_task)
                else:
                    # Task only exists in SD Elements
                    # Remove reference if it exists - ALM Task was probably deleted
                    # Also pop any old references to force lookup using legacy
                    if task_reference:
                        self._remove_task_reference(sde_task['id'], task_reference['id'])
                        sde_task.pop('references', [])

                    if self.task_should_sync(sde_task):
                        new_alm_task = self.alm_add_task(sde_task)

                        if self.supports_references:
                            self._create_task_reference(sde_task['id'], new_alm_task)

                        ref = self.get_note_message(new_alm_task)
                        self.emit.info(u'Added task %s to %s' % (task_id, self.alm_name))
                        note_msg = u'Task synchronized in %s. Reference: [%s]' % (self.alm_name, ref)
                        self._add_note(sde_task['id'], note_msg)
                        logger.debug(note_msg)

            logger.info('Synchronization complete')

            if self.supports_references and self.migration_required:
                self._update_alm_migration_status(has_migrated=True)

            self.alm_disconnect()

        except AlmException:
            self.alm_disconnect()
            raise

    def get_note_message(self, alm_task):
        pass

    def translate_priority(self, priority):
        """ Translates an SDE priority into an Alm label """
        pmap = self.config[self.ALM_PRIORITY_MAP]

        if not pmap:
            return None

        try:
            priority = int(priority)
        except TypeError:
            logger.error('Could not coerce %s into an integer' % priority)
            raise AlmException("Error in translating SDE priority to %s: "
                               "%s is not an integer priority" % (priority, self.alm_name))

        for key in pmap:
            if '-' in key:
                lrange, hrange = key.split('-')
                lrange = int(lrange)
                hrange = int(hrange)
                if lrange <= priority <= hrange:
                    return pmap[key]
            else:
                if int(key) == priority:
                    return pmap[key]

    @property
    def new_issue_template(self):
        issue_template = {}
        issue_template.update(self.default_issue_template)

        if self.config['alm_custom_fields']:
            issue_template.update(self.config['alm_custom_fields'])

        return issue_template

    def get_new_issue(self, task, defaults=None):
        data = defaults.copy() if defaults else {}

        macros = task['__macros']

        data.update(
            (field, self._transform_value(value, macros))
            for field, value in self.new_issue_template.iteritems()
        )

        if self.user_friendly_name_map:
            data = self.map_user_friendly_names(data)

        return data

    def map_user_friendly_names(self, task):
        def get_field_name(field_name):
            if not self.user_friendly_name_map:
                return field_name
            return self.user_friendly_name_map.get(field_name, field_name)

        return dict(
            (get_field_name(field), value)
            for field, value in task.iteritems()
        )

    def transform_tasks(self, tasks, project=None):
        return [self.transform_task(task, project) for task in tasks]

    def transform_task(self, task, project=None):
        """
        Take a task and add a new entry, __macros, containing a set of macro->value pairs
        so that this ALM's fields can be remapped to any of the values when a new issue is created
        """
        task['alm_full_title'] = self.get_alm_task_title(self.config, task, fixed=False)
        task['alm_fixed_title'] = self.get_alm_task_title(self.config, task, fixed=True)

        task_id = self._extract_task_id(task['id'])

        # each macro is assumed to be a string or a list of strings
        macros = {
            'business_unit': self.config['sde_businessunit'],
            'application': self.config['sde_application'],
            'project': self.config['sde_project'],
            'context': self.config['alm_context'],
            'alm_user': self.config['alm_user'],
            'alm_project': self.config['alm_project'],
            'id': task['id'],
            'task_id': task_id,
            'task_title': re.sub('^%s:\s+' % task_id, '', task['title']),
            'task_alm_title': task['alm_full_title'],
            'task_url': task['url'],
            'task_content': task['text'],
            'task_richcontent': self.sde_get_task_content(task),
            'task_phase': task['phase']['slug'],
            'task_status': task['status']['meaning'],
            'task_priority': str(task['priority']),
            'task_tags': task['tags'],
            'problem_content': task['problem']['text'],
            'problem_richcontent': self.convert_markdown_to_alm(task['problem']['text'], task['problem']['id']),
            'problem_id': task['problem']['id'],
            'problem_title': re.sub('^%s:\s+' % task['problem']['id'], '', task['problem']['title']),
        }

        if self.ALM_PRIORITY_MAP in self.config:
            macros['task_priority_translated'] = self.translate_priority(task['priority'])

        if self.sync_titles_only:
            macros['task_content'] = PUBLIC_TASK_CONTENT
            macros['task_richcontent'] = PUBLIC_TASK_CONTENT
            macros['problem_content'] = PUBLIC_TASK_CONTENT
            macros['problem_richcontent'] = PUBLIC_TASK_CONTENT

        if 'alm_issue_label' in self.config:
            macros['task_tags'].append(self.config['alm_issue_label'])

        if project and 'custom_attributes' in project:
            for attr, value in project['custom_attributes'].items():
                macros['project_custom_attr_%s' % attr] = value

        task['__macros'] = macros
        return task
예제 #2
0
class Command(BaseCommand):
    help = 'Update task status and verification details given the contents of a CSV'
    sde_plugin = None

    def configure(self):
        self.sde_plugin = PlugInExperience(self.config)
        self.config.opts.add('csv_file', "CSV file")

    def sde_connect(self):
        if not self.sde_plugin:
            raise Error('Requires initialization')
        try:
            self.sde_plugin.connect()
        except APIError as err:
            raise Error('Unable to connect to SD Elements. Please review URL, id,'
                        ' and password in configuration file. Reason: %s' % (str(err)))

    def handle(self):
        if not self.config['csv_file']:
            raise UsageError('Missing csv_file')

        self.sde_connect()

        with open(self.config['csv_file'], 'r') as csvfile:
            reader = csv.DictReader(csvfile)

            if not reader.fieldnames:
                raise Error('Empty CSV file')

            # Remove extra white space in headings
            for i in range(len(reader.fieldnames)):
                reader.fieldnames[i] = reader.fieldnames[i].strip()

            if len(reader.fieldnames) <= 1:
                raise Error('Please make sure CSV file has task id field and at least one more field.')

            try:
                for row in reader:
                    if 'task id' not in reader.fieldnames:
                        raise Error('Please make sure task id field exists.')
                    task_id = row['task id']
                    if 'verification note' in reader.fieldnames and 'verification status' not in reader.fieldnames:
                        raise Error('Please make sure a verification status field exists along with verification note.')
                    if 'task status' in reader.fieldnames:
                        status = row['task status']
                        self.sde_plugin.update_task_status(task_id, status)
                    if 'task note' in reader.fieldnames:
                        text_note = row['task note']
                        self.sde_plugin.add_task_text_note(task_id, text_note)
                    if 'verification status' in reader.fieldnames:
                        verification_status = row['verification status']
                        if 'verification note' in reader.fieldnames:
                            finding_ref = row['verification note']
                        else:
                            finding_ref = ''
                        self.sde_plugin.add_manual_analysis_note(task_id, verification_status, finding_ref)

            except csv.Error, csv_err:
                raise Error('Unable to parse CSV file, line %d: %s' % (reader.line_num, csv_err))

        return True