예제 #1
0
파일: gateway.py 프로젝트: zhan849/argo
class Gateway(object):
    """Repo Controller"""
    BASE_DIR = '/ax/data/repos'
    BRANCH_CACHE_TTL = 5 * 60  # 5 minutes TTL as we expect we won't finish upgrade within 5 minutes
    NAMESPACE = 'gateway'

    CLUSTER_NAME_ID = os.environ.get('AX_CLUSTER')
    CUSTOMER_ID = os.environ.get('AX_CUSTOMER_ID')
    S3_BUCKET_NAME = 'applatix-cluster-{account}-{seq}'.format(account=CUSTOMER_ID, seq=0)
    s3_bucket = boto3.resource('s3').Bucket(S3_BUCKET_NAME)

    def __init__(self):
        self.axdb_client = AxdbClient()
        self.axops_client = AxopsClient()
        self.axsys_client = AxsysClient()
        self.redis_client = RedisClient('redis', db=DB_REPORTING)
        self.event_notification_client = EventNotificationClient(FACILITY_GATEWAY)
        self.scm_clients = {
            ScmVendors.BITBUCKET: BitBucketClient(),
            ScmVendors.GITHUB: GitHubClient(),
            ScmVendors.GITLAB: GitLabClient()
        }
        self.repo_manager = RepoManager(DEFAULT_CONCURRENCY, DEFAULT_INTERVAL)
        self.event_trigger = EventTrigger()

    def get_repos(self, scm_type, url, username, password):
        """Get all repos owned by the user."""
        if scm_type in {ScmVendors.BITBUCKET, ScmVendors.GITHUB, ScmVendors.GITLAB}:
            try:
                repos = self.scm_clients[scm_type].get_repos(username, password)
            except Exception as e:
                logger.warning('Unable to connect to %s: %s', scm_type, e)
                detail = {
                    'type': scm_type,
                    'username': username,
                    'error': str(e.detail)
                }
                self.event_notification_client.send_message_to_notification_center(CODE_CONFIGURATION_SCM_CONNECTION_ERROR,
                                                                                   detail=detail)
                raise AXApiInvalidParam('Cannot connect to %s server' % scm_type)
            else:
                return repos
        elif scm_type == ScmVendors.GIT:
            _, vendor, repo_owner, repo_name = Gateway.parse_repo(url)
            path = '/tmp/{}/{}/{}'.format(vendor, repo_owner, repo_name)
            if os.path.isfile(path):
                os.remove(path)
            if os.path.isdir(path):
                shutil.rmtree(path)
            os.makedirs(path)
            client = GitClient(path=path, repo=url, username=username, password=password)
            try:
                client.list_remote()
            except Exception as e:
                logger.warning('Unable to connect to git server (%s): %s', url, e)
                detail = {
                    'type': scm_type,
                    'url': url,
                    'username': username,
                    'error': str(e)
                }
                self.event_notification_client.send_message_to_notification_center(CODE_CONFIGURATION_SCM_CONNECTION_ERROR,
                                                                                   detail=detail)
                raise AXApiInvalidParam('Cannot connect to git server')
            else:
                return {url: url}
        elif scm_type == ScmVendors.CODECOMMIT:
            repos = {}
            region = 'us-east-1'
            default_url_format = 'https://git-codecommit.{}.amazonaws.com/v1/repos/{}'
            client = boto3.client('codecommit', aws_access_key_id=username, aws_secret_access_key=password,
                                  region_name=region)
            try:
                response = client.list_repositories().get('repositories', [])
                for r in response:
                    repo_url = default_url_format.format(region, r['repositoryName'])
                    repos[repo_url] = repo_url
            except Exception as exc:
                detail = {
                    'type': scm_type,
                    'region': region,
                    'url': default_url_format.format(region, ''),
                    'username': username,
                    'error': 'Cannot connect to CodeCommit' + str(exc)
                }
                self.event_notification_client.send_message_to_notification_center(CODE_CONFIGURATION_SCM_CONNECTION_ERROR,
                                                                                   detail=detail)
                raise AXApiInvalidParam('Cannot connect to CodeCommit: %s' % exc)
            else:
                return repos
        else:
            return {}

    @staticmethod
    def parse_repo(repo):
        """Parse repo url into 4-tuple (protocol, vendor, repo_owner, repo_name).

        :param repo:
        :return:
        """
        parsed_url = urlparse(repo)
        protocol, vendor = parsed_url.scheme, parsed_url.hostname
        m = re.match(r'/([a-zA-Z0-9\-]+)/([a-zA-Z0-9_.\-/]+)', parsed_url.path)
        if not m:
            raise AXScmException('Illegal repo URL', detail='Illegal repo URL ({})'.format(repo))
        repo_owner, repo_name = m.groups()
        return protocol, vendor, repo_owner, repo_name

    def has_webhook(self, repo):
        """Test if there is any repo which uses webhook.

        :param repo:
        :return:
        """
        tools = self.axops_client.get_tools(category='scm')
        for i in range(len(tools)):
            use_webhook = tools[i].get('use_webhook', False)
            repos = set(tools[i].get('repos', []))
            repos -= {repo}
            if use_webhook and repos:
                return True
        return False

    def get_webhook(self, vendor, repo):
        """Get webhook

        :param vendor:
        :param repo:
        :returns:
        """
        logger.info('Retrieving webhook (repo: %s) ...', repo)
        return self.scm_clients[vendor].get_webhook(repo)

    def create_webhook(self, vendor, repo):
        """Create webhook

        :param vendor:
        :param repo:
        :returns:
        """

        @retry(wait_fixed=5000, stop_max_delay=20 * 60 * 1000)
        def _verify_elb(hostname):
            try:
                logger.info('Verifying ELB (%s) ...', hostname)
                ip = socket.gethostbyname(hostname)
                logger.info('Successfully resolved ELB (%s) to IP (%s)', hostname, ip)
            except Exception as e:
                logger.error('ELB not ready: %s', str(e))
                raise AXApiInternalError('ELB not ready', str(e))

        ip_range = self.scm_clients[vendor].get_webhook_whitelist()

        # Create ELB
        payload = {'ip_range': ip_range, 'external_port': 8443, 'internal_port': 8087}
        try:
            logger.info('Creating ELB for webhook ...')
            result = self.axsys_client.create_webhook(**payload)
        except Exception as e:
            logger.error('Failed to create ELB for webhook: %s', str(e))
            self.event_notification_client.send_message_to_notification_center(CODE_JOB_CI_ELB_CREATION_FAILURE,
                                                                               detail=payload)
            raise AXApiInternalError('Failed to create ELB for webhook', str(e))
        else:
            logger.info('Successfully created ELB for webhook')

        # Verify ELB
        hostname = result['hostname']
        try:
            _verify_elb(hostname)
        except Exception as e:
            logger.error('Timed out on waiting for ELB to be available: %s', str(e))
            self.event_notification_client.send_message_to_notification_center(CODE_JOB_CI_ELB_VERIFICATION_TIMEOUT,
                                                                               detail={'hostname': hostname})
            raise AXApiInternalError('Timed out on waiting for ELB to be available: %s' % str(e))

        # Create webhook
        try:
            logger.info('Creating webhook (repo: %s) ...', repo)
            self.scm_clients[vendor].create_webhook(repo)
        except AXApiAuthFailed as e:
            logger.error('Invalid credential supplied')
            detail = {
                'repo': repo,
                'error': 'Invalid credential supplied:' + str(e)
            }
            self.event_notification_client.send_message_to_notification_center(CODE_JOB_CI_WEBHOOK_CREATION_FAILURE,
                                                                               detail=detail)
            raise AXApiInvalidParam('User authentication failed', detail=str(e))
        except AXApiForbiddenReq as e:
            logger.error('Supplied credential is valid but having insufficient permission')
            detail = {
                'repo': repo,
                'error': 'Supplied credential is valid but having insufficient permission:' + str(e)
            }
            self.event_notification_client.send_message_to_notification_center(CODE_JOB_CI_WEBHOOK_CREATION_FAILURE,
                                                                               detail=detail)
            raise AXApiInvalidParam('User has insufficient permission', detail=str(e))
        except Exception as e:
            logger.error('Failed to configure webhook: %s', e)
            detail = {
                'repo': repo,
                'error': 'Failed to configure webhook:' + str(e)
            }
            self.event_notification_client.send_message_to_notification_center(CODE_JOB_CI_WEBHOOK_CREATION_FAILURE,
                                                                               detail=detail)
            raise AXApiInternalError('Failed to configure webhook', str(e))
        else:
            logger.info('Successfully created webhook (repo: %s)', repo)
            return {}

    def delete_webhook(self, vendor, repo):
        """Delete webhook

        :param vendor:
        :param repo:
        :returns:
        """
        # Delete webhook
        try:
            logger.info('Deleting webhook (repo: %s) ...', repo)
            self.scm_clients[vendor].delete_webhook(repo)
        except AXApiAuthFailed as e:
            logger.error('Invalid credential supplied')
            detail = {
                'repo': repo,
                'error': 'Invalid credential supplied:' + str(e)
            }
            self.event_notification_client.send_message_to_notification_center(CODE_JOB_CI_WEBHOOK_DELETION_FAILURE,
                                                                               detail=detail)
            raise AXApiInvalidParam('User authentication failed', detail=str(e))
        except AXApiForbiddenReq as e:
            logger.error('Supplied credential is valid but having insufficient permission')
            detail = {
                'repo': repo,
                'error': 'Supplied credential is valid but having insufficient permission:' + str(e)
            }
            self.event_notification_client.send_message_to_notification_center(CODE_JOB_CI_WEBHOOK_DELETION_FAILURE,
                                                                               detail=detail)
            raise AXApiInvalidParam('User has insufficient permission', detail=str(e))
        except Exception as e:
            logger.error('Failed to delete webhook: %s', e)
            detail = {
                'repo': repo,
                'error': 'Failed to delete webhook:' + str(e)
            }
            self.event_notification_client.send_message_to_notification_center(CODE_JOB_CI_WEBHOOK_DELETION_FAILURE,
                                                                               detail=detail)
            raise AXApiInternalError('Failed to delete webhook', str(e))
        else:
            logger.info('Successfully deleted webhook (repo: %s)', repo)

        # Delete ELB
        try:
            if not self.has_webhook(repo):
                logger.info('Deleting ELB for webhook ...')
                self.axsys_client.delete_webhook()
        except Exception as e:
            logger.error('Failed to delete ELB for webhook: %s', str(e))
            detail = {'repo': repo,
                      'error': 'Failed to delete ELB for webhook' + str(e)
                      }
            self.event_notification_client.send_message_to_notification_center(CODE_JOB_CI_ELB_DELETION_FAILURE,
                                                                               detail=detail)
            raise AXApiInternalError('Failed to delete ELB for webhook', str(e))
        else:
            logger.info('Successfully deleted ELB for webhook')
            return {}

    def purge_branches(self, repo, branch=None):
        """Purge branch heads.

        :param repo:
        :param branch:
        :return:
        """
        if not repo:
            raise AXApiInvalidParam('Missing required parameter', 'Missing required parameter (repo)')
        logger.info('Purging branch heads (repo: %s, branch: %s) ...', repo, branch)

        try:
            if not branch:
                self.axdb_client.purge_branch_heads(repo)
            else:
                self.axdb_client.purge_branch_head(repo, branch)
        except Exception as e:
            message = 'Unable to purge branch heads'
            detail = 'Unable to purge branch heads (repo: {}, branch: {}): {}'.format(repo, branch, str(e))
            logger.error(detail)
            raise AXApiInternalError(message, detail)
        else:
            logger.info('Successfully purged branch heads')

    def get_branches(self, repo=None, branch=None, order_by=None, limit=None):
        """Get branches.

        :param repo:
        :param branch:
        :param order_by:
        :param limit:
        :return:
        """

        def _get_branches(workspace):
            """Retrieve list of remote branches in the workspace.

            :param workspace:
            :return: a list of dictionaries.
            """
            try:
                key = '{}:{}'.format(Gateway.NAMESPACE, workspace)
                if self.redis_client.exists(key):
                    logger.info('Loading cache (workspace: %s) ...', workspace)
                    results = self.redis_client.get(key, decoder=json.loads)
                    return results
                else:
                    logger.info('Scanning workspace (%s) ...', workspace)
                    git_client = GitClient(path=workspace, read_only=True)
                    repo = git_client.get_remote()
                    branches = git_client.get_remote_heads()
                    results = []
                    for i in range(len(branches)):
                        results.append({
                            'repo': repo,
                            'name': branches[i]['reference'],
                            'revision': branches[i]['commit'],
                            'commit_date': branches[i]['commit_date']
                        })
                    logger.info('Saving cache (workspace: %s) ...', workspace)
                    self.redis_client.set(key, results, expire=Gateway.BRANCH_CACHE_TTL, encoder=json.dumps)
                    return results
            except Exception as e:
                logger.warning('Failed to scan workspace (%s): %s', workspace, e)
                return []

        logger.info('Retrieving branches (repo: %s, branch: %s) ...', repo, branch)
        if repo:
            repo = unquote(repo)
            _, vendor, repo_owner, repo_name = self.parse_repo(repo)
            workspaces = ['{}/{}/{}/{}'.format(Gateway.BASE_DIR, vendor, repo_owner, repo_name)]
        else:
            dirs = [dir_name[0] for dir_name in os.walk(Gateway.BASE_DIR) if dir_name[0].endswith('/.git')]
            workspaces = list(map(lambda v: v[:-5], dirs))

        branches = []
        with ThreadPoolExecutor(max_workers=20) as executor:
            futures = []
            for i in range(len(workspaces)):
                futures.append(executor.submit(_get_branches, workspaces[i]))
            for future in as_completed(futures):
                try:
                    data = future.result()
                except Exception as e:
                    logger.warning('Unexpected exception occurred during processing: %s', e)
                else:
                    for i in range(len(data)):
                        branches.append(data[i])
        if branch:
            pattern = '.*{}.*'.format(branch.replace('*', '.*'))
            branches = [branches[i] for i in range(len(branches)) if re.match(pattern, branches[i]['name'])]
        if order_by == 'commit_date':
            branches = sorted(branches, key=lambda v: v['commit_date'])
        elif order_by == '-commit_date':
            branches = sorted(branches, key=lambda v: v['commit_date'], reverse=True)
        elif order_by == '-native':
            branches = sorted(branches, key=lambda v: (v['repo'], v['name']), reverse=True)
        else:
            branches = sorted(branches, key=lambda v: (v['repo'], v['name']))
        if limit:
            branches = branches[:limit]
        logger.info('Successfully retrieved %s branches', len(branches))
        return branches

    @staticmethod
    def _get_commits(workspace, branch=None, since=None, until=None, commit=None, author=None, committer=None,
                     description=None, limit=None):
        """Search for commits in a workspace."""
        try:
            logger.info('Scanning workspace (%s) for commits ...', workspace)
            git_client = GitClient(path=workspace, read_only=True)
            if commit and commit.startswith('~'):
                commit = commit[1:]
            if author and author.startswith('~'):
                author = author[1:]
            if committer and committer.startswith('~'):
                committer = committer[1:]
            if description and description.startswith('~'):
                description = description[1:]
            return git_client.get_commits(branch=branch, commit=commit, since=since, until=until, author=author,
                                          committer=committer, description=description, limit=limit)
        except Exception as e:
            logger.warning('Failed to scan workspace (%s): %s', workspace, e)

    @staticmethod
    def _get_commit(workspace, commit):
        """Get a commit from a workspace."""
        try:
            logger.info('Scanning workspace (%s) for commit (%s) ...', workspace, commit)
            git_client = GitClient(path=workspace, read_only=True)
            return git_client.get_commit(commit)
        except Exception as e:
            logger.warning('Failed to scan workspace (%s): %s', workspace, e)

    @staticmethod
    def _parse_repo_branch(repo, branch, repo_branch):
        """Parse repo / branch / repo_branch."""
        if repo:
            try:
                repo = unquote(repo)
                _, vendor, repo_owner, repo_name = Gateway.parse_repo(repo)
            except Exception as e:
                msg = 'Unable to parse repo: %s', e
                logger.error(msg)
                raise AXApiInvalidParam('Unable to parse repo', msg)
            else:
                dir = '{}/{}/{}/{}'.format(Gateway.BASE_DIR, vendor, repo_owner, repo_name)
                workspaces = {dir: [branch] if branch else []}
        elif repo_branch:
            try:
                repo_branch = json.loads(repo_branch)
                workspaces = {}
                for repo in repo_branch.keys():
                    repo = unquote(repo)
                    _, vendor, repo_owner, repo_name = Gateway.parse_repo(repo)
                    dir = '{}/{}/{}/{}'.format(Gateway.BASE_DIR, vendor, repo_owner, repo_name)
                    if dir not in workspaces:
                        workspaces[dir] = set()
                    for branch in repo_branch[repo]:
                        workspaces[dir].add(branch)
            except Exception as e:
                msg = 'Unable to parse repo_branch: %s' % str(e)
                logger.error(msg)
                raise AXApiInvalidParam('Unable to parse repo_branch', msg)
        else:
            dirs = [dir[0] for dir in os.walk(Gateway.BASE_DIR) if dir[0].endswith('/.git')]
            workspaces = list(map(lambda v: v[:-5], dirs))
            workspaces = dict([(k, [branch] if branch else []) for k in workspaces])
        return workspaces

    @staticmethod
    def _put_file(repo, branch, path):
        """Put a file in s3.

        :param repo:
        :param branch:
        :param path:
        :return:
        """
        _, vendor, repo_owner, repo_name = Gateway.parse_repo(repo)
        workspace = '{}/{}/{}/{}'.format(Gateway.BASE_DIR, vendor, repo_owner, repo_name)
        if not os.path.isdir(workspace):
            raise AXApiInvalidParam('Invalid repository', 'Invalid repository ({})'.format(repo))
        try:
            logger.info('Extracting file content from repository (repo: %s, branch: %s, path: %s) ...',
                        repo, branch, path)
            git_client = GitClient(path=workspace, read_only=True)
            files = git_client.get_files(branch=branch, subdir=path, binary_mode=True)
        except Exception as e:
            message = 'Failed to extract file content'
            detail = '{}: {}'.format(message, str(e))
            logger.error(detail)
            raise AXApiInternalError(message, detail)
        else:
            if len(files) == 0:
                raise AXApiInvalidParam('Unable to locate file with given information')
            file_content = files[0]['content']
            logger.info('Successfully extracted file content')

        try:
            # Cluster name id always has the form <cluster_name>-<36_bytes_long_cluster_id>
            cluster_name, cluster_id = Gateway.CLUSTER_NAME_ID[:-37], Gateway.CLUSTER_NAME_ID[-36:]
            key = '{cluster_name}/{cluster_id}/{vendor}/{repo_owner}/{repo_name}/{branch}/{path}'.format(
                cluster_name=cluster_name, cluster_id=cluster_id, vendor=vendor,
                repo_owner=repo_owner, repo_name=repo_name, branch=branch, path=path)
            logger.info('Uploading file content to s3 (bucket: %s, key: %s) ...', Gateway.S3_BUCKET_NAME, key)
            response = Gateway.s3_bucket.Object(key).put(Body=file_content)
            etag = response.get('ETag')
            if etag:
                etag = json.loads(etag)
        except Exception as e:
            message = 'Failed to upload file content'
            detail = '{}: {}'.format(message, str(e))
            logger.error(detail)
            raise AXApiInternalError(message, detail)
        else:
            logger.info('Successfully uploaded file content')
            return {'bucket': Gateway.S3_BUCKET_NAME, 'key': key, 'etag': etag}

    @staticmethod
    def _delete_file(repo, branch, path):
        """Delete a file from s3.

        :param repo:
        :param branch:
        :param path:
        :return:
        """
        _, vendor, repo_owner, repo_name = Gateway.parse_repo(repo)
        try:
            cluster_name, cluster_id = Gateway.CLUSTER_NAME_ID[:-37], Gateway.CLUSTER_NAME_ID[-36:]
            key = '{cluster_name}/{cluster_id}/{vendor}/{repo_owner}/{repo_name}/{branch}/{path}'.format(
                cluster_name=cluster_name, cluster_id=cluster_id, vendor=vendor,
                repo_owner=repo_owner, repo_name=repo_name, branch=branch, path=path)
            logger.info('Deleting file from s3 (bucket: %s, key: %s) ...', Gateway.S3_BUCKET_NAME, key)
            Gateway.s3_bucket.Object(key).delete()
        except Exception as e:
            message = 'Failed to delete file'
            detail = '{}: {}'.format(message, str(e))
            logger.error(detail)
            raise AXApiInternalError(message, detail)
        else:
            logger.info('Successfully deleted file')
            return {'bucket': Gateway.S3_BUCKET_NAME, 'key': key}

    @staticmethod
    def init_jira_client(axops_client, url=None, username=None, password=None):
        """Initialize an Jira client"""

        def get_jira_configuration():
            js = axops_client.get_tools(category='issue_management', type='jira')
            if js:
                return {'url': js[0]['url'],
                        'username': js[0]['username'],
                        'password': js[0]['password']
                        }
            else:
                return dict()

        if url is None or username is None or password is None:
            conf = get_jira_configuration()
            if not conf:
                raise AXApiInvalidParam('No JIRA configured')
            else:
                url, username, password = conf['url'], conf['username'], conf['password']
        return JiraClient(url, username, password)

    # Verify whether this function is still needed
    def check_github_whitelist(self):
        if not self.is_github_webhook_enabled():
            logger.info('No GitHub webhook configured')
            return
        configured = self.get_from_cache()
        logger.info('The configured GitHub webhook whitelist is %s', configured)
        advertised = self.scm_clients[ScmVendors.GITHUB].get_webhook_whitelist()
        logger.info('The GitHub webhook whitelist is %s', advertised)
        if set(configured) == set(advertised):
            logger.info('No update needed')
        else:
            # Create ELB
            payload = {'ip_range': advertised, 'external_port': 8443, 'internal_port': 8087}
            try:
                logger.info('Creating ELB for webhook ...')
                self.axsys_client.create_webhook(**payload)
            except Exception as exc:
                logger.error('Failed to create ELB for webhook: %s', str(exc))
                self.event_notification_client.send_message_to_notification_center(CODE_JOB_CI_ELB_CREATION_FAILURE,
                                                                                   detail=payload)
            else:
                # Update cache
                self.write_to_cache(advertised)
                logger.info('Successfully updated ELB for webhook')

    def is_github_webhook_enabled(self):
        """ Check whether the webhook is configured or not"""
        github_data = self.axops_client.get_tools(type='github')
        use_webhook = [each for each in github_data if each['use_webhook']]
        return bool(use_webhook)

    @staticmethod
    def write_to_cache(ip_range):
        """ Store the webhook whitelist info"""
        cache_file = '/tmp/github_webhook_whitelist'
        with open(cache_file, 'w+') as f:
            f.write(json.dumps(ip_range))

    def get_from_cache(self):
        """ Get cached webhook whitelist info, otherwise get from axmon"""
        cache_file = '/tmp/github_webhook_whitelist'
        ip_range = list()
        if os.path.exists(cache_file):
            with open(cache_file, 'r+') as f:
                data = f.readlines()
                ip_range = json.loads(data[0])
        else:
            logger.debug('No cache file')
            try:
                data = self.axsys_client.get_webhook()
            except Exception as exc:
                logger.warning(exc)
            else:
                logger.info('Write whitelist info to cache file')
                ip_range = data['ip_ranges']
                self.write_to_cache(ip_range)
        return ip_range
예제 #2
0
class RepoManager(object):
    """Manage all repositories in track."""
    def __init__(self, concurrency, interval):
        self.axdb_client = AxdbClient()
        self.axops_client = AxopsClient()
        self.concurrency = concurrency
        self.interval = interval

    def run(self):
        """Create workspaces and perform initial/incremental fetch."""
        while True:
            logger.info('Start repository scan ...')
            try:
                self.connect()
                repos, has_change = self.synchronize()
                with ThreadPoolExecutor(
                        max_workers=self.concurrency) as executor:
                    futures = []
                    for i in range(len(repos)):
                        futures.append(
                            executor.submit(self.update_repo, **repos[i]))
                    for future in as_completed(futures):
                        try:
                            if not has_change and future.result():
                                has_change = True
                        except Exception as e:
                            logger.warning(
                                'Unexpected exception occurred during processing: %s',
                                e)
                # Notify UI backend about a change in repos
                if has_change:
                    key = '{}:repos_updated'.format(NAMESPACE)
                    redis_client.set(key, value=str(int(time.time())))
            except Exception as e:
                logger.warning('Repository scan failed: %s', str(e))
            else:
                logger.info('Repository scan completed\n')
            finally:
                time.sleep(self.interval)

    @retry(wait_fixed=5000)
    def connect(self):
        """Connect to axops."""
        connected = self.axops_client.ping()
        if not connected:
            msg = 'Unable to connect to axops'
            logger.warning(msg)
            raise ConnectionError(msg)

    def synchronize(self):
        """Synchronize all repos."""
        logger.info('Synchronizing repositories ...')

        # Get all repos
        repos = self.get_all_repos()
        logger.info('%s repositories currently in track', len(repos))

        # Get untracked repos currently on disk
        untracked_repos = self.get_untracked_repos(repos)
        logger.info('%s untracked repositories found on disk',
                    len(list(untracked_repos.keys())))

        for repo in untracked_repos:
            # Purge all branch heads
            logger.info('Purging branch heads (repo: %s) ...', repo)
            self.axdb_client.purge_branch_heads(repo)
            # Delete workspace
            logger.info('Deleting workspace (path: %s) ...',
                        untracked_repos[repo])
            shutil.rmtree(untracked_repos[repo])
            # Invalidate caches
            logger.info('Invalidating caches (workspace: %s) ...',
                        untracked_repos[repo])
            key_pattern = '^{}\:{}.*$'.format(NAMESPACE, untracked_repos[repo])
            keys = redis_client.keys(key_pattern)
            for k in keys:
                logger.debug('Invalidating cache (key: %s) ...', k)
                redis_client.delete(k)

        # Send event to trigger garbage collection from axops
        if untracked_repos:
            kafka_client = ProducerClient()
            ci_event = {
                'Op': "gc",
                'Payload': {
                    'details': "Repo or branch get deleted."
                }
            }
            kafka_client.send(AxSettings.TOPIC_GC_EVENT,
                              key=AxSettings.TOPIC_GC_EVENT,
                              value=ci_event,
                              timeout=120)

        return repos, len(untracked_repos) > 0

    def get_all_repos(self):
        """Retrieve all repos from axops."""
        tools = self.axops_client.get_tools(category='scm')
        repos = {}
        for i in range(len(tools)):
            _repos = tools[i].get('repos', [])
            for j in range(len(_repos)):
                parsed_url = urlparse(_repos[j])
                protocol, vendor = parsed_url.scheme, parsed_url.hostname
                m = re.match(r'/([a-zA-Z0-9-]+)/([a-zA-Z0-9_.-]+)',
                             parsed_url.path)
                if not m:
                    logger.warning('Illegal repo URL: %s, skip', parsed_url)
                    continue
                _, repo_owner, repo_name = parsed_url.path.split('/',
                                                                 maxsplit=2)
                key = (vendor, repo_owner, repo_name)
                if key in repos and repos[key]['protocol'] == 'https':
                    continue
                repos[key] = {
                    'repo_type': tools[i].get('type'),
                    'vendor': vendor,
                    'protocol': protocol,
                    'repo_owner': repo_owner,
                    'repo_name': repo_name,
                    'username': tools[i].get('username'),
                    'password': tools[i].get('password'),
                    'use_webhook': tools[i].get('use_webhook', False)
                }
        return list(repos.values())

    @staticmethod
    def get_untracked_repos(repos):
        """Get all untracked repos."""
        # Construct list of expected workspaces
        expected_workspaces = set()
        for i in range(len(repos)):
            expected_workspace = '{}/{}/{}/{}'.format(BASE_DIR,
                                                      repos[i]['vendor'],
                                                      repos[i]['repo_owner'],
                                                      repos[i]['repo_name'])
            expected_workspaces.add(expected_workspace)
        # Construct list of all workspaces currently on disk
        dirs = [
            dir[0] for dir in os.walk(BASE_DIR) if dir[0].endswith('/.git')
        ]
        workspaces = list(map(lambda v: v[:-5], dirs))
        # Construct list of untracked repos
        untracked_repos = {}
        for i in range(len(workspaces)):
            if workspaces[i] not in expected_workspaces:
                client = GitClient(path=workspaces[i])
                repo = client.get_remote()
                untracked_repos[repo] = workspaces[i]
        return untracked_repos

    @staticmethod
    def get_repo_workspace(repo_vendor, repo_owner, repo_name):
        return '{}/{}/{}/{}'.format(BASE_DIR, repo_vendor, repo_owner,
                                    repo_name)

    @staticmethod
    def get_repo_url(protocol, repo_vendor, repo_owner, repo_name):
        return '{}://{}/{}/{}'.format(protocol, repo_vendor, repo_owner,
                                      repo_name)

    @staticmethod
    def update_yaml(repo_client, kafka_client, repo, branch, head):
        """Using Kafka to send a event to axops to update the yamls in the axdb."""
        logger.info("Update yaml %s, %s, %s", repo, branch, head)
        try:
            yaml_contents = repo_client.get_files(commit=head,
                                                  subdir=TEMPLATE_DIR,
                                                  filter_yaml=True)
        except Exception as e:
            logger.error("Failed to obtain YAML files: %s", str(e))
            return -1

        if len(yaml_contents) >= 0:
            # This is a partition key defined as RepoName$$$$BranchName.
            # The key is used by Kafka partition, which means it allows concurrency
            #  if the events are for different repo/branch
            key = '{}$$$${}'.format(repo, branch)
            payload = {
                'Op': 'update',
                'Payload': {
                    'Revision':
                    head,
                    'Content':
                    [v['content']
                     for v in yaml_contents] if yaml_contents else []
                }
            }
            kafka_client.send('devops_template',
                              key=key,
                              value=payload,
                              timeout=120)
            logger.info("Updated YAML %s files (repo: %s, branch: %s)",
                        len(yaml_contents), repo, branch)
        return len(yaml_contents)

    def update_repo(self, repo_type, vendor, protocol, repo_owner, repo_name,
                    username, password, use_webhook):
        """Update a repo."""

        # Examples for the input variables
        # BASE_DIR:   /ax/data/repos
        # Repo_type:  github
        # Vendor:     github.com
        # Protocol:   https
        # Repo_owner: argo
        # Repo_name:  prod.git

        is_first_fetch = False
        do_send_gc_event = False
        workspace = '{}/{}/{}/{}'.format(BASE_DIR, vendor, repo_owner,
                                         repo_name)
        url = '{}://{}/{}/{}'.format(protocol, vendor, repo_owner, repo_name)
        kafka_client = ProducerClient()

        if not os.path.isdir(workspace):
            os.makedirs(workspace)
            # If we recreate the workspace, we need to purge all branch heads of this repo
            self.axdb_client.purge_branch_heads(url)

        logger.info("Start scanning repository (%s) ...", url)
        if repo_type == ScmVendors.CODECOMMIT:
            client = CodeCommitClient(path=workspace,
                                      repo=url,
                                      username=username,
                                      password=password)
        else:
            client = GitClient(path=workspace,
                               repo=url,
                               username=username,
                               password=password,
                               use_permanent_credentials=True)

        # Even if there is no change, performing a fetch is harmless but has a benefit
        # that, in case the workspace is destroyed without purging the history, we can
        # still update the workspace to the proper state
        logger.info("Start fetching ...")
        client.fetch()

        # Retrieve all previous branch heads and construct hash table
        prev_heads = self.axdb_client.get_branch_heads(url)
        logger.info("Have %s branch heads (repo: %s) from previous scan",
                    len(prev_heads), url)

        if len(prev_heads) == 0:
            is_first_fetch = True
            logger.debug(
                "This is an initial scan as no previous heads were found")

        prev_heads_map = dict()
        for prev_head in prev_heads:
            key = (prev_head['repo'], prev_head['branch'])
            prev_heads_map[key] = prev_head['head']

        # Retrieve all current branch heads
        current_heads = client.get_remote_heads()
        logger.info("Have %s branch heads (repo: %s) from current scan",
                    len(current_heads), url)
        current_heads = sorted(current_heads,
                               key=lambda v: v['commit_date'],
                               reverse=is_first_fetch)

        # Find out which branch heads need to be updated
        heads_to_update = list()
        heads_for_event = list()
        for current_head in current_heads:
            head, branch = current_head['commit'], current_head[
                'reference'].replace('refs/heads/', '')
            previous_head = prev_heads_map.pop((url, branch), None)
            if head != previous_head:
                event = {'repo': url, 'branch': branch, 'head': head}
                heads_to_update.append(event)

                if previous_head is None:
                    logger.info(
                        "New branch detected (branch: %s, current head: %s)",
                        branch, head)
                else:
                    logger.info(
                        "Existing ranch head updated (branch: %s, previous: %s, current: %s)",
                        branch, previous_head, head)
                    # Send CI event in case of policy
                    heads_for_event.append(event.copy())

        if prev_heads_map:
            logger.info("There are %s get deleted from repo: %s",
                        prev_heads_map.keys(), url)
            do_send_gc_event = True
            for key in prev_heads_map:
                self.axdb_client.purge_branch_head(repo=key[0], branch=key[1])

        # Invalidate cache if there is head update or branch deleted
        if heads_to_update or prev_heads_map:
            cache_key = '{}:{}'.format(NAMESPACE, workspace)
            logger.info('Invalidating cache (key: %s) ...', cache_key)
            if redis_client.exists(cache_key):
                redis_client.delete(cache_key)

        # Update YAML contents
        count = 0
        for event in heads_to_update:
            res_count = RepoManager.update_yaml(repo_client=client,
                                                kafka_client=kafka_client,
                                                repo=url,
                                                branch=event['branch'],
                                                head=event['head'])
            if res_count >= 0:
                self.axdb_client.set_branch_head(**event)
                count += res_count

        logger.info(
            "Updated %s YAML files (template/policy) for %s branches (repo: %s)",
            count, len(heads_to_update), url)
        logger.info("Updated %s branch heads (repo: %s)", len(heads_to_update),
                    url)

        # If garbarge collection needed due to branch or repo deletion
        if do_send_gc_event:
            logger.info(
                "Send gc event so that axops can garbage collect deleted branch / repo"
            )
            ci_event = {
                'Op': "gc",
                'Payload': {
                    'details': "Repo or branch get deleted."
                }
            }
            kafka_client.send(AxSettings.TOPIC_GC_EVENT,
                              key=AxSettings.TOPIC_GC_EVENT,
                              value=ci_event,
                              timeout=120)

        # If webhook is disabled, we need to send CI events
        if not use_webhook:
            for event in heads_for_event:
                commit = client.get_commit(event['head'])
                ci_event = {
                    'Op': "ci",
                    'Payload': {
                        'author':
                        commit['author'],
                        'branch':
                        event['branch'],
                        'commit':
                        commit['revision'],
                        'committer':
                        commit['committer'],
                        'date':
                        datetime.datetime.fromtimestamp(
                            commit['date']).strftime('%Y-%m-%dT%H:%M:%S'),
                        'description':
                        commit['description'],
                        'repo':
                        commit['repo'],
                        'type':
                        "push",
                        'vendor':
                        repo_type
                    }
                }
                kafka_client.send("devops_template",
                                  key="{}$$$${}".format(
                                      event['repo'], event['branch']),
                                  value=ci_event,
                                  timeout=120)
            logger.info('Webhook not enabled, send %s devops_ci_event events',
                        len(heads_for_event))

        kafka_client.close()
        logger.info('Successfully scanned repository (%s)', url)

        return len(heads_to_update) > 0 or len(prev_heads_map) > 0