예제 #1
0
 def __init__(self, dry_run=False):
     self.dry_run = dry_run
     self.gqlapi = gql.get_api()
     settings = queries.get_app_interface_settings()
     self.secret_reader = SecretReader(settings=settings)
     self.skopeo_cli = Skopeo(dry_run)
     self.push_creds = self._get_push_creds()
예제 #2
0
    def __init__(self, instance, dry_run):
        self.dry_run = dry_run
        self.instance = instance
        self.settings = queries.get_app_interface_settings()
        self.secret_reader = SecretReader(settings=self.settings)
        self.skopeo_cli = Skopeo(dry_run)
        self.error = False

        identifier = instance["identifier"]
        account = instance["account"]
        region = instance.get("region")

        self.aws_cli = AWSApi(
            thread_pool_size=1,
            accounts=[self._get_aws_account_info(account)],
            settings=self.settings,
            init_ecr_auth_tokens=True,
        )

        self.aws_cli.map_ecr_resources()

        self.ecr_uri = self._get_image_uri(
            account=account,
            repository=identifier,
        )
        if self.ecr_uri is None:
            self.error = True
            LOG.error(f"Could not find the ECR repository {identifier}")

        self.ecr_username, self.ecr_password = self._get_ecr_creds(
            account=account,
            region=region,
        )
        self.ecr_auth = f"{self.ecr_username}:{self.ecr_password}"

        self.image_username = None
        self.image_password = None
        self.image_auth = None
        pull_secret = self.instance["mirror"]["pullCredentials"]
        if pull_secret is not None:
            raw_data = self.secret_reader.read_all(pull_secret)
            self.image_username = raw_data["user"]
            self.image_password = raw_data["token"]
            self.image_auth = f"{self.image_username}:{self.image_password}"
예제 #3
0
class QuayMirror:

    QUAY_ORG_CATALOG_QUERY = """
    {
      quay_orgs: quay_orgs_v1 {
        name
        pushCredentials {
          path
          field
        }
      }
    }
    """

    QUAY_REPOS_QUERY = """
    {
      apps: apps_v1 {
        quayRepos {
          org {
            name
          }
          items {
            name
            mirror
          }
        }
      }
    }
    """

    def __init__(self, dry_run=False):
        self.dry_run = dry_run
        self.gqlapi = gql.get_api()
        self.settings = queries.get_app_interface_settings()
        self.skopeo_cli = Skopeo(dry_run)
        self.push_creds = self._get_push_creds()

    def run(self):
        sync_tasks = self.process_sync_tasks()
        for org, data in sync_tasks.items():
            for item in data:
                try:
                    self.skopeo_cli.copy(src_image=item['mirror_url'],
                                         dst_image=item['image_url'],
                                         dest_creds=self.push_creds[org])
                except SkopeoCmdError as details:
                    _LOG.error('[%s]', details)

    def process_repos_query(self):
        result = self.gqlapi.query(self.QUAY_REPOS_QUERY)

        summary = defaultdict(list)

        for app in result['apps']:
            quay_repos = app.get('quayRepos')

            if quay_repos is None:
                continue

            for quay_repo in quay_repos:
                org = quay_repo['org']['name']
                for item in quay_repo['items']:
                    if item['mirror'] is None:
                        continue

                    summary[org].append({
                        'name': item["name"],
                        'mirror': item['mirror']
                    })

        return summary

    def process_sync_tasks(self):
        eight_hours = 28800  # 60 * 60 * 8
        is_deep_sync = self._is_deep_sync(interval=eight_hours)

        summary = self.process_repos_query()

        sync_tasks = defaultdict(list)
        for org, data in summary.items():
            for item in data:
                image = Image(f'quay.io/{org}/{item["name"]}')
                image_mirror = Image(item['mirror'])

                for tag in image_mirror:
                    upstream = image_mirror[tag]
                    downstream = image[tag]
                    if tag not in image:
                        _LOG.debug('Image %s and mirror %s are out off sync',
                                   downstream, upstream)
                        sync_tasks[org].append({
                            'mirror_url': str(upstream),
                            'image_url': str(downstream)
                        })
                        continue

                    # Deep (slow) check only in non dry-run mode
                    if self.dry_run:
                        _LOG.debug('Image %s and mirror %s are in sync',
                                   downstream, upstream)
                        continue

                    # Deep (slow) check only from time to time
                    if not is_deep_sync:
                        _LOG.debug('Image %s and mirror %s are in sync',
                                   downstream, upstream)
                        continue

                    try:
                        if downstream == upstream:
                            _LOG.debug('Image %s and mirror %s are in sync',
                                       downstream, upstream)
                            continue
                    except ImageComparisonError as details:
                        _LOG.error('[%s]', details)
                        continue

                    _LOG.debug('Image %s and mirror %s are out of sync',
                               downstream, upstream)
                    sync_tasks[org].append({
                        'mirror_url': str(upstream),
                        'image_url': str(downstream)
                    })

        return sync_tasks

    def _is_deep_sync(self, interval):
        control_file_name = 'qontract-reconcile-quay-mirror.timestamp'
        control_file_path = os.path.join(tempfile.gettempdir(),
                                         control_file_name)
        try:
            with open(control_file_path, 'r') as file_obj:
                last_deep_sync = float(file_obj.read())
        except FileNotFoundError:
            self._record_timestamp(control_file_path)
            return True

        next_deep_sync = last_deep_sync + interval
        if time.time() >= next_deep_sync:
            self._record_timestamp(control_file_path)
            return True

        return False

    @staticmethod
    def _record_timestamp(path):
        with open(path, 'w') as file_object:
            file_object.write(str(time.time()))

    def _get_push_creds(self):
        result = self.gqlapi.query(self.QUAY_ORG_CATALOG_QUERY)

        creds = {}
        for org_data in result['quay_orgs']:
            push_secret = org_data['pushCredentials']
            if push_secret is None:
                continue

            raw_data = secret_reader.read_all(push_secret,
                                              settings=self.settings)
            org = org_data['name']
            creds[org] = f'{raw_data["user"]}:{raw_data["token"]}'

        return creds
예제 #4
0
 def __init__(self, dry_run=False):
     self.dry_run = dry_run
     self.skopeo_cli = Skopeo(dry_run)
     self.quay_api_store = get_quay_api_store()
예제 #5
0
class QuayMirrorOrg:
    def __init__(self, dry_run=False):
        self.dry_run = dry_run
        self.skopeo_cli = Skopeo(dry_run)
        self.quay_api_store = get_quay_api_store()

    def run(self):
        sync_tasks = self.process_sync_tasks()
        for org, data in sync_tasks.items():
            for item in data:
                try:
                    self.skopeo_cli.copy(src_image=item['mirror_url'],
                                         src_creds=item['mirror_creds'],
                                         dst_image=item['image_url'],
                                         dest_creds=self.get_push_creds(org))
                except SkopeoCmdError as details:
                    _LOG.error('[%s]', details)

    def process_org_mirrors(self, summary):
        """adds new keys to the summary dict with information about mirrored
        orgs

        It collects the list of repositories in the upstream org from an API
        call and not from App-Interface.

        :param summary: summary
        :type summary: dict
        :return: summary
        :rtype: dict
        """

        for org_key, org_info in self.quay_api_store.items():
            if not org_info.get('mirror'):
                continue

            quay_api = org_info['api']
            upstream_org_key = org_info['mirror']
            upstream_org = self.quay_api_store[upstream_org_key]
            upstream_quay_api = upstream_org['api']

            username = upstream_org['push_token']['user']
            token = upstream_org['push_token']['token']

            repos = [item['name'] for item in quay_api.list_images()]
            for repo in upstream_quay_api.list_images():
                if repo['name'] not in repos:
                    continue
                server_url = upstream_org['url']
                url = f"{server_url}/{org_key.org_name}/{repo['name']}"
                data = {
                    'name': repo['name'],
                    'mirror': {
                        'url': url,
                        'username': username,
                        'token': token,
                    }
                }
                summary[org_key].append(data)

        return summary

    def process_sync_tasks(self):
        eight_hours = 28800  # 60 * 60 * 8
        is_deep_sync = self._is_deep_sync(interval=eight_hours)

        summary = defaultdict(list)
        self.process_org_mirrors(summary)

        sync_tasks = defaultdict(list)
        for org_key, data in summary.items():
            org = self.quay_api_store[org_key]
            org_name = org_key.org_name

            server_url = org['url']
            username = org['push_token']['user']
            password = org['push_token']['token']

            for item in data:
                image = Image(f'{server_url}/{org_name}/{item["name"]}',
                              username=username, password=password)

                mirror_url = item['mirror']['url']

                mirror_username = None
                mirror_password = None
                mirror_creds = None

                if item['mirror'].get('username') and \
                        item['mirror'].get('token'):
                    mirror_username = item['mirror']['username']
                    mirror_password = item['mirror']['token']
                    mirror_creds = f'{mirror_username}:{mirror_password}'

                image_mirror = Image(mirror_url, username=mirror_username,
                                     password=mirror_password)

                for tag in image_mirror:
                    upstream = image_mirror[tag]
                    downstream = image[tag]
                    if tag not in image:
                        _LOG.debug('Image %s and mirror %s are out of sync',
                                   downstream, upstream)
                        task = {'mirror_url': str(upstream),
                                'mirror_creds': mirror_creds,
                                'image_url': str(downstream)}
                        sync_tasks[org_key].append(task)
                        continue

                    # Deep (slow) check only in non dry-run mode
                    if self.dry_run:
                        _LOG.debug('Image %s and mirror %s are in sync',
                                   downstream, upstream)
                        continue

                    # Deep (slow) check only from time to time
                    if not is_deep_sync:
                        _LOG.debug('Image %s and mirror %s are in sync',
                                   downstream, upstream)
                        continue

                    try:
                        if downstream == upstream:
                            _LOG.debug('Image %s and mirror %s are in sync',
                                       downstream, upstream)
                            continue
                    except ImageComparisonError as details:
                        _LOG.error('[%s]', details)
                        continue

                    _LOG.debug('Image %s and mirror %s are out of sync',
                               downstream, upstream)
                    sync_tasks[org_key].append({'mirror_url': str(upstream),
                                                'mirror_creds': mirror_creds,
                                                'image_url': str(downstream)})

        return sync_tasks

    def _is_deep_sync(self, interval):
        control_file_name = 'qontract-reconcile-quay-mirror-org.timestamp'
        control_file_path = os.path.join(tempfile.gettempdir(),
                                         control_file_name)
        try:
            with open(control_file_path, 'r') as file_obj:
                last_deep_sync = float(file_obj.read())
        except FileNotFoundError:
            self._record_timestamp(control_file_path)
            return True

        next_deep_sync = last_deep_sync + interval
        if time.time() >= next_deep_sync:
            self._record_timestamp(control_file_path)
            return True

        return False

    @staticmethod
    def _record_timestamp(path):
        with open(path, 'w') as file_object:
            file_object.write(str(time.time()))

    def get_push_creds(self, org_key):
        """returns username and password for the given org

        :param org_key: org_key
        :type org_key: tuple(instance, org_name)
        :return: tuple containing username and password
        :rtype: tuple(str, str)
        """

        push_token = self.quay_api_store[org_key]['push_token']
        username = push_token['user']
        password = push_token['token']
        return f"{username}:{password}"
예제 #6
0
class QuayMirror:

    GCR_PROJECT_CATALOG_QUERY = """
    {
      projects: gcp_projects_v1 {
        name
        pushCredentials {
          path
          field
        }
      }
    }
    """

    GCR_REPOS_QUERY = """
    {
      apps: apps_v1 {
        gcrRepos {
          project {
            name
          }
          items {
            name
            mirror {
              url
              pullCredentials {
                path
                field
              }
              tags
              tagsExclude
            }
          }
        }
      }
    }
    """

    def __init__(self, dry_run=False):
        self.dry_run = dry_run
        self.gqlapi = gql.get_api()
        self.settings = queries.get_app_interface_settings()
        self.skopeo_cli = Skopeo(dry_run)
        self.push_creds = self._get_push_creds()

    def run(self):
        sync_tasks = self.process_sync_tasks()
        for org, data in sync_tasks.items():
            for item in data:
                try:
                    self.skopeo_cli.copy(src_image=item['mirror_url'],
                                         src_creds=item['mirror_creds'],
                                         dst_image=item['image_url'],
                                         dest_creds=self.push_creds[org])
                except SkopeoCmdError as details:
                    _LOG.error('[%s]', details)

    def process_repos_query(self):
        result = self.gqlapi.query(self.GCR_REPOS_QUERY)

        summary = defaultdict(list)

        for app in result['apps']:
            gcr_repos = app.get('gcrRepos')

            if gcr_repos is None:
                continue

            for gcr_repo in gcr_repos:
                project = gcr_repo['project']['name']
                server_url = gcr_repo['project'].get('serverUrl') or 'gcr.io'
                for item in gcr_repo['items']:
                    if item['mirror'] is None:
                        continue

                    summary[project].append({
                        'name': item["name"],
                        'mirror': item['mirror'],
                        'server_url': server_url
                    })

        return summary

    def sync_tag(self, tags, tags_exclude, candidate):
        if tags is not None:
            for tag in tags:
                if re.match(tag, candidate):
                    return True
            # When tags is defined, we don't look at
            # tags_exclude
            return False

        if tags_exclude is not None:
            for tag_exclude in tags_exclude:
                if re.match(tag_exclude, candidate):
                    return False
            return True

        # Both tags and tags_exclude are None, so
        # tag must be synced
        return True

    def process_sync_tasks(self):
        eight_hours = 28800  # 60 * 60 * 8
        is_deep_sync = self._is_deep_sync(interval=eight_hours)

        summary = self.process_repos_query()

        sync_tasks = defaultdict(list)
        for org, data in summary.items():
            for item in data:
                image = Image(f'{item["server_url"]}/{org}/{item["name"]}')

                mirror_url = item['mirror']['url']

                username = None
                password = None
                mirror_creds = None
                if item['mirror']['pullCredentials'] is not None:
                    pull_credentials = item['mirror']['pullCredentials']
                    raw_data = secret_reader.read_all(pull_credentials,
                                                      settings=self.settings)
                    username = raw_data["user"]
                    password = raw_data["token"]
                    mirror_creds = f'{username}:{password}'

                image_mirror = Image(mirror_url,
                                     username=username,
                                     password=password)

                tags = item['mirror'].get('tags')
                tags_exclude = item['mirror'].get('tagsExclude')

                for tag in image_mirror:
                    if not self.sync_tag(tags=tags,
                                         tags_exclude=tags_exclude,
                                         candidate=tag):
                        continue

                    upstream = image_mirror[tag]
                    downstream = image[tag]
                    if tag not in image:
                        _LOG.debug('Image %s and mirror %s are out off sync',
                                   downstream, upstream)
                        sync_tasks[org].append({
                            'mirror_url': str(upstream),
                            'mirror_creds': mirror_creds,
                            'image_url': str(downstream)
                        })
                        continue

                    # Deep (slow) check only in non dry-run mode
                    if self.dry_run:
                        _LOG.debug('Image %s and mirror %s are in sync',
                                   downstream, upstream)
                        continue

                    # Deep (slow) check only from time to time
                    if not is_deep_sync:
                        _LOG.debug('Image %s and mirror %s are in sync',
                                   downstream, upstream)
                        continue

                    try:
                        if downstream == upstream:
                            _LOG.debug('Image %s and mirror %s are in sync',
                                       downstream, upstream)
                            continue
                    except ImageComparisonError as details:
                        _LOG.error('[%s]', details)
                        continue

                    _LOG.debug('Image %s and mirror %s are out of sync',
                               downstream, upstream)
                    sync_tasks[org].append({
                        'mirror_url': str(upstream),
                        'mirror_creds': mirror_creds,
                        'image_url': str(downstream)
                    })

        return sync_tasks

    def _is_deep_sync(self, interval):
        control_file_name = 'qontract-reconcile-gcr-mirror.timestamp'
        control_file_path = os.path.join(tempfile.gettempdir(),
                                         control_file_name)
        try:
            with open(control_file_path, 'r') as file_obj:
                last_deep_sync = float(file_obj.read())
        except FileNotFoundError:
            self._record_timestamp(control_file_path)
            return True

        next_deep_sync = last_deep_sync + interval
        if time.time() >= next_deep_sync:
            self._record_timestamp(control_file_path)
            return True

        return False

    @staticmethod
    def _record_timestamp(path):
        with open(path, 'w') as file_object:
            file_object.write(str(time.time()))

    def _get_push_creds(self):
        result = self.gqlapi.query(self.GCR_PROJECT_CATALOG_QUERY)

        creds = {}
        for project_data in result['projects']:
            push_secret = project_data['pushCredentials']
            if push_secret is None:
                continue

            raw_data = secret_reader.read_all(push_secret,
                                              settings=self.settings)
            project = project_data['name']
            token = base64.b64decode(raw_data["token"]).decode()
            creds[project] = f'{raw_data["user"]}:{token}'
        return creds
예제 #7
0
class QuayMirror:

    QUAY_ORG_CATALOG_QUERY = """
    {
      quay_orgs: quay_orgs_v1 {
        name
        pushCredentials {
          path
          field
        }
        instance {
            name
            url
        }
      }
    }
    """

    def __init__(self, dry_run=False):
        self.dry_run = dry_run
        self.gqlapi = gql.get_api()
        settings = queries.get_app_interface_settings()
        self.secret_reader = SecretReader(settings=settings)
        self.skopeo_cli = Skopeo(dry_run)
        self.push_creds = self._get_push_creds()

    def run(self):
        sync_tasks = self.process_sync_tasks()
        for org, data in sync_tasks.items():
            for item in data:
                try:
                    self.skopeo_cli.copy(src_image=item['mirror_url'],
                                         src_creds=item['mirror_creds'],
                                         dst_image=item['image_url'],
                                         dest_creds=self.push_creds[org])
                except SkopeoCmdError as details:
                    _LOG.error('[%s]', details)

    @staticmethod
    def process_repos_query():
        apps = queries.get_quay_repos()

        summary = defaultdict(list)

        for app in apps:
            quay_repos = app.get('quayRepos')

            if quay_repos is None:
                continue

            for quay_repo in quay_repos:
                org = quay_repo['org']['name']
                instance = quay_repo['org']['instance']['name']
                server_url = quay_repo['org']['instance']['url']

                for item in quay_repo['items']:
                    if item['mirror'] is None:
                        continue

                    mirror_image = Image(item['mirror']['url'])
                    if (mirror_image.registry == 'docker.io'
                            and mirror_image.repository == 'library'
                            and item['public']):
                        _LOG.error("Image %s can't be mirrored to a public "
                                   "quay repository.", mirror_image)
                        sys.exit(ExitCodes.ERROR)

                    org_key = OrgKey(instance, org)
                    summary[org_key].append({'name': item["name"],
                                             'mirror': item['mirror'],
                                             'server_url': server_url})

        return summary

    @staticmethod
    def sync_tag(tags, tags_exclude, candidate):
        if tags is not None:
            for tag in tags:
                if re.match(tag, candidate):
                    return True
            # When tags is defined, we don't look at
            # tags_exclude
            return False

        if tags_exclude is not None:
            for tag_exclude in tags_exclude:
                if re.match(tag_exclude, candidate):
                    return False
            return True

        # Both tags and tags_exclude are None, so
        # tag must be synced
        return True

    def process_sync_tasks(self):
        eight_hours = 28800  # 60 * 60 * 8
        is_deep_sync = self._is_deep_sync(interval=eight_hours)

        summary = self.process_repos_query()
        sync_tasks = defaultdict(list)
        for org_key, data in summary.items():
            org = org_key.org_name
            for item in data:
                push_creds = self.push_creds[org_key].split(':')
                image = Image(f'{item["server_url"]}/{org}/{item["name"]}',
                              username=push_creds[0], password=push_creds[1])

                mirror_url = item['mirror']['url']

                username = None
                password = None
                mirror_creds = None
                if item['mirror']['pullCredentials'] is not None:
                    pull_credentials = item['mirror']['pullCredentials']
                    raw_data = self.secret_reader.read_all(pull_credentials)
                    username = raw_data["user"]
                    password = raw_data["token"]
                    mirror_creds = f'{username}:{password}'

                image_mirror = Image(mirror_url, username=username,
                                     password=password)

                tags = item['mirror'].get('tags')
                tags_exclude = item['mirror'].get('tagsExclude')

                for tag in image_mirror:
                    if not self.sync_tag(tags=tags, tags_exclude=tags_exclude,
                                         candidate=tag):
                        continue

                    upstream = image_mirror[tag]
                    downstream = image[tag]
                    if tag not in image:
                        _LOG.debug('Image %s and mirror %s are out off sync',
                                   downstream, upstream)
                        task = {'mirror_url': str(upstream),
                                'mirror_creds': mirror_creds,
                                'image_url': str(downstream)}
                        sync_tasks[org_key].append(task)
                        continue

                    # Deep (slow) check only in non dry-run mode
                    if self.dry_run:
                        _LOG.debug('Image %s and mirror %s are in sync',
                                   downstream, upstream)
                        continue

                    # Deep (slow) check only from time to time
                    if not is_deep_sync:
                        _LOG.debug('Image %s and mirror %s are in sync',
                                   downstream, upstream)
                        continue

                    try:
                        if downstream == upstream:
                            _LOG.debug('Image %s and mirror %s are in sync',
                                       downstream, upstream)
                            continue
                    except ImageComparisonError as details:
                        _LOG.error('[%s]', details)
                        continue

                    _LOG.debug('Image %s and mirror %s are out of sync',
                               downstream, upstream)
                    sync_tasks[org_key].append({'mirror_url': str(upstream),
                                                'mirror_creds': mirror_creds,
                                                'image_url': str(downstream)})

        return sync_tasks

    def _is_deep_sync(self, interval):
        control_file_name = 'qontract-reconcile-quay-mirror.timestamp'
        control_file_path = os.path.join(tempfile.gettempdir(),
                                         control_file_name)
        try:
            with open(control_file_path, 'r') as file_obj:
                last_deep_sync = float(file_obj.read())
        except FileNotFoundError:
            self._record_timestamp(control_file_path)
            return True

        next_deep_sync = last_deep_sync + interval
        if time.time() >= next_deep_sync:
            self._record_timestamp(control_file_path)
            return True

        return False

    @staticmethod
    def _record_timestamp(path):
        with open(path, 'w') as file_object:
            file_object.write(str(time.time()))

    def _get_push_creds(self):
        result = self.gqlapi.query(self.QUAY_ORG_CATALOG_QUERY)

        creds = {}
        for org_data in result['quay_orgs']:
            push_secret = org_data['pushCredentials']
            if push_secret is None:
                continue

            raw_data = self.secret_reader.read_all(push_secret)
            org = org_data['name']
            instance = org_data['instance']['name']
            org_key = OrgKey(instance, org)
            creds[org_key] = f'{raw_data["user"]}:{raw_data["token"]}'

        return creds
예제 #8
0
class EcrMirror:
    def __init__(self, instance, dry_run):
        self.dry_run = dry_run
        self.instance = instance
        self.settings = queries.get_app_interface_settings()
        self.secret_reader = SecretReader(settings=self.settings)
        self.skopeo_cli = Skopeo(dry_run)
        self.error = False

        identifier = instance["identifier"]
        account = instance["account"]
        region = instance.get("region")

        self.aws_cli = AWSApi(
            thread_pool_size=1,
            accounts=[self._get_aws_account_info(account)],
            settings=self.settings,
            init_ecr_auth_tokens=True,
        )

        self.aws_cli.map_ecr_resources()

        self.ecr_uri = self._get_image_uri(
            account=account,
            repository=identifier,
        )
        if self.ecr_uri is None:
            self.error = True
            LOG.error(f"Could not find the ECR repository {identifier}")

        self.ecr_username, self.ecr_password = self._get_ecr_creds(
            account=account,
            region=region,
        )
        self.ecr_auth = f"{self.ecr_username}:{self.ecr_password}"

        self.image_username = None
        self.image_password = None
        self.image_auth = None
        pull_secret = self.instance["mirror"]["pullCredentials"]
        if pull_secret is not None:
            raw_data = self.secret_reader.read_all(pull_secret)
            self.image_username = raw_data["user"]
            self.image_password = raw_data["token"]
            self.image_auth = f"{self.image_username}:{self.image_password}"

    def run(self):
        if self.error:
            return

        ecr_mirror = Image(self.ecr_uri,
                           username=self.ecr_username,
                           password=self.ecr_password)

        image = Image(
            self.instance["mirror"]["url"],
            username=self.image_username,
            password=self.image_password,
        )

        LOG.debug("[checking %s -> %s]", image, ecr_mirror)
        for tag in image:
            if tag not in ecr_mirror:
                try:
                    self.skopeo_cli.copy(
                        src_image=image[tag],
                        src_creds=self.image_auth,
                        dst_image=ecr_mirror[tag],
                        dest_creds=self.ecr_auth,
                    )
                except SkopeoCmdError as details:
                    LOG.error("[%s]", details)

    def _get_ecr_creds(self, account, region):
        if region is None:
            region = self.aws_cli.accounts[account]["resourcesDefaultRegion"]
        auth_token = f"{account}/{region}"
        data = self.aws_cli.auth_tokens[auth_token]
        auth_data = data["authorizationData"][0]
        token = auth_data["authorizationToken"]
        password = base64.b64decode(token).decode("utf-8").split(":")[1]
        return "AWS", password

    def _get_image_uri(self, account, repository):
        for repo in self.aws_cli.resources[account]["ecr"]:
            if repo["repositoryName"] == repository:
                return repo["repositoryUri"]

    @staticmethod
    def _get_aws_account_info(account):
        for account_info in queries.get_aws_accounts():
            if "name" not in account_info:
                continue
            if account_info["name"] != account:
                continue
            return account_info
예제 #9
0
class EcrMirror:
    def __init__(self, instance, dry_run):
        self.dry_run = dry_run
        self.instance = instance
        self.settings = queries.get_app_interface_settings()
        self.secret_reader = SecretReader(settings=self.settings)
        self.skopeo_cli = Skopeo(dry_run)
        self.error = False

        identifier = instance['identifier']
        account = instance['account']
        region = instance.get('region')

        self.aws_cli = AWSApi(thread_pool_size=1,
                              accounts=[self._get_aws_account_info(account)],
                              settings=self.settings,
                              init_ecr_auth_tokens=True)

        self.aws_cli.map_ecr_resources()

        self.ecr_uri = self._get_image_uri(
            account=account,
            repository=identifier,
        )
        if self.ecr_uri is None:
            self.error = True
            LOG.error(f"Could not find the ECR repository {identifier}")

        self.ecr_username, self.ecr_password = self._get_ecr_creds(
            account=account,
            region=region,
        )
        self.ecr_auth = f'{self.ecr_username}:{self.ecr_password}'

        self.image_username = None
        self.image_password = None
        self.image_auth = None
        pull_secret = self.instance['mirror']['pullCredentials']
        if pull_secret is not None:
            raw_data = self.secret_reader.read_all(pull_secret)
            self.image_username = raw_data["user"]
            self.image_password = raw_data["token"]
            self.image_auth = f'{self.image_username}:{self.image_password}'

    def run(self):
        if self.error:
            return

        ecr_mirror = Image(self.ecr_uri,
                           username=self.ecr_username,
                           password=self.ecr_password)

        image = Image(self.instance['mirror']['url'],
                      username=self.image_username,
                      password=self.image_password)

        LOG.debug('[checking %s -> %s]', image, ecr_mirror)
        for tag in image:
            if tag not in ecr_mirror:
                try:
                    self.skopeo_cli.copy(src_image=image[tag],
                                         src_creds=self.image_auth,
                                         dst_image=ecr_mirror[tag],
                                         dest_creds=self.ecr_auth)
                except SkopeoCmdError as details:
                    LOG.error('[%s]', details)

    def _get_ecr_creds(self, account, region):
        if region is None:
            region = self.aws_cli.accounts[account]['resourcesDefaultRegion']
        auth_token = f'{account}/{region}'
        data = self.aws_cli.auth_tokens[auth_token]
        auth_data = data['authorizationData'][0]
        token = auth_data['authorizationToken']
        password = base64.b64decode(token).decode('utf-8').split(':')[1]
        return 'AWS', password

    def _get_image_uri(self, account, repository):
        for repo in self.aws_cli.resources[account]['ecr']:
            if repo['repositoryName'] == repository:
                return repo['repositoryUri']

    @staticmethod
    def _get_aws_account_info(account):
        for account_info in queries.get_aws_accounts():
            if 'name' not in account_info:
                continue
            if account_info['name'] != account:
                continue
            return account_info