class OcpReleaseMirror: def __init__(self, dry_run, instance): self.dry_run = dry_run self.settings = queries.get_app_interface_settings() cluster_info = instance['hiveCluster'] hive_cluster = instance['hiveCluster']['name'] # Getting the OCM Client for the hive cluster ocm_map = OCMMap(clusters=[cluster_info], integration=QONTRACT_INTEGRATION, settings=self.settings) self.ocm_cli = ocm_map.get(hive_cluster) if not self.ocm_cli: raise OcpReleaseMirrorError(f"Can't create ocm client for " f"cluster {hive_cluster}") # Getting the OC Client for the hive cluster oc_map = OC_Map(clusters=[cluster_info], integration=QONTRACT_INTEGRATION, settings=self.settings) self.oc_cli = oc_map.get(hive_cluster) if not self.oc_cli: raise OcpReleaseMirrorError(f"Can't create oc client for " f"cluster {hive_cluster}") namespace = instance['ecrResourcesNamespace'] ocp_release_identifier = instance['ocpReleaseEcrIdentifier'] ocp_art_dev_identifier = instance['ocpArtDevEcrIdentifier'] ocp_release_info = self._get_tf_resource_info(namespace, ocp_release_identifier) if ocp_release_info is None: raise OcpReleaseMirrorError(f"Could not find rds " f"identifier " f"{ocp_release_identifier} in " f"namespace {namespace['name']}") ocp_art_dev_info = self._get_tf_resource_info(namespace, ocp_art_dev_identifier) if ocp_art_dev_info is None: raise OcpReleaseMirrorError(f"Could not find rds identifier" f" {ocp_art_dev_identifier} in" f"namespace {namespace['name']}") # Getting the AWS Client for the accounts aws_accounts = [ self._get_aws_account_info(account=ocp_release_info['account']), self._get_aws_account_info(account=ocp_art_dev_info['account']) ] self.aws_cli = AWSApi(thread_pool_size=1, accounts=aws_accounts, settings=self.settings, init_ecr_auth_tokens=True) self.aws_cli.map_ecr_resources() self.ocp_release_ecr_uri = self._get_image_uri( account=ocp_release_info['account'], repository=ocp_release_identifier) if self.ocp_release_ecr_uri is None: raise OcpReleaseMirrorError(f"Could not find the " f"ECR repository " f"{ocp_release_identifier}") self.ocp_art_dev_ecr_uri = self._get_image_uri( account=ocp_art_dev_info['account'], repository=ocp_art_dev_identifier) if self.ocp_art_dev_ecr_uri is None: raise OcpReleaseMirrorError(f"Could not find the " f"ECR repository " f"{ocp_art_dev_identifier}") # Process all the quayOrgTargets quay_api_store = get_quay_api_store() self.quay_target_orgs = [] for quayTargetOrg in instance['quayTargetOrgs']: org_name = quayTargetOrg['name'] instance_name = quayTargetOrg['instance']['name'] org_key = OrgKey(instance_name, org_name) org_info = quay_api_store[org_key] if not org_info['push_token']: raise OcpReleaseMirrorError( f'{org_key} has no push_token defined.') url = org_info['url'] user = org_info['push_token']['user'] token = org_info['push_token']['token'] self.quay_target_orgs.append({ 'url': url, 'dest_ocp_release': f"{url}/{org_name}/ocp-release", 'dest_ocp_art_dev': f"{url}/{org_name}/ocp-v4.0-art-dev", 'auths': self._build_quay_auths(url, user, token) }) # Getting all the credentials quay_creds = self._get_quay_creds() ocp_release_creds = self._get_ecr_creds( account=ocp_release_info['account'], region=ocp_release_info['region']) ocp_art_dev_creds = self._get_ecr_creds( account=ocp_art_dev_info['account'], region=ocp_art_dev_info['region']) # Creating a single dictionary with all credentials to be used by the # "oc adm release mirror" command self.registry_creds = { 'auths': { **quay_creds['auths'], **ocp_release_creds['auths'], **ocp_art_dev_creds['auths'], } } # Append quay_target_orgs auths to registry_creds for quay_target_org in self.quay_target_orgs: url = quay_target_org['url'] if url in self.registry_creds['auths'].keys(): OcpReleaseMirrorError('Cannot mirror to the same Quay ' f'instance multiple times: {url}') self.registry_creds['auths'].update(quay_target_org['auths']) # Initiate channel groups self.channel_groups = instance['mirrorChannels'] def run(self): ocp_releases = self._get_ocp_releases() if not ocp_releases: raise RuntimeError('No OCP Releases found') for ocp_release_info in ocp_releases: ocp_release = ocp_release_info.ocp_release tag = ocp_release_info.tag # mirror to ecr dest_ocp_release = f'{self.ocp_release_ecr_uri}:{tag}' self._run_mirror(ocp_release=ocp_release, dest_ocp_release=dest_ocp_release, dest_ocp_art_dev=self.ocp_art_dev_ecr_uri) # mirror to all quay target orgs for quay_target_org in self.quay_target_orgs: dest_ocp_release = (f'{quay_target_org["dest_ocp_release"]}:' f'{tag}') dest_ocp_art_dev = quay_target_org["dest_ocp_art_dev"] self._run_mirror(ocp_release=ocp_release, dest_ocp_release=dest_ocp_release, dest_ocp_art_dev=dest_ocp_art_dev) def _run_mirror(self, ocp_release, dest_ocp_release, dest_ocp_art_dev): # Checking if the image is already there if self._is_image_there(dest_ocp_release): LOG.debug(f'Image {ocp_release} already in ' f'the mirror. Skipping.') return LOG.info(f'Mirroring {ocp_release} to {dest_ocp_art_dev} ' f'to_release {dest_ocp_release}') if self.dry_run: return # Creating a new, bare, OC client since we don't # want to run this against any cluster or via # a jump host oc_cli = OC('cluster', None, None, local=True) oc_cli.release_mirror(from_release=ocp_release, to=dest_ocp_art_dev, to_release=dest_ocp_release, dockerconfig=self.registry_creds) def _is_image_there(self, image): image_obj = Image(image) for registry, creds in self.registry_creds['auths'].items(): # Getting the credentials for the image_obj if '//' not in registry: registry = '//' + registry registry_obj = urlparse(registry) if registry_obj.netloc != image_obj.registry: continue image_obj.auth = (creds['username'], creds['password']) # Checking if the image is already # in the registry if image_obj: return True return False @staticmethod def _get_aws_account_info(account): for account_info in queries.get_aws_accounts(): if 'name' not in account_info: continue if account_info['name'] != account: continue return account_info def _get_ocp_releases(self): ocp_releases = [] clusterimagesets = self.oc_cli.get_all('ClusterImageSet') for clusterimageset in clusterimagesets['items']: release_image = clusterimageset['spec']['releaseImage'] # There are images in some ClusterImagesSets not coming # from quay.io, e.g.: # registry.svc.ci.openshift.org/ocp/release:4.2.0-0.nightly-2020-11-04-053758 # Let's filter out everything not from quay.io if not release_image.startswith('quay.io'): continue labels = clusterimageset['metadata']['labels'] name = clusterimageset['metadata']['name'] # ClusterImagesSets may be enabled or disabled. # Let's only mirror enabled ones enabled = labels['api.openshift.com/enabled'] if enabled == 'false': continue # ClusterImageSets may be in different channels. channel_group = labels['api.openshift.com/channel-group'] if channel_group not in self.channel_groups: continue ocp_releases.append(OcpReleaseInfo(release_image, name)) return ocp_releases def _get_quay_creds(self): return self.ocm_cli.get_pull_secrets() def _get_ecr_creds(self, account, region): if region is None: region = self.aws_cli.accounts[account]['resourcesDefaultRegion'] auth_token = f'{account}/{region}' data = self.aws_cli.auth_tokens[auth_token] auth_data = data['authorizationData'][0] server = auth_data['proxyEndpoint'] token = auth_data['authorizationToken'] password = base64.b64decode(token).decode('utf-8').split(':')[1] return { 'auths': { server: { 'username': '******', 'password': password, 'email': '*****@*****.**', 'auth': token } } } @staticmethod def _get_tf_resource_info(namespace, identifier): tf_resources = namespace['terraformResources'] for tf_resource in tf_resources: if 'identifier' not in tf_resource: continue if tf_resource['identifier'] != identifier: continue if tf_resource['provider'] != 'ecr': continue return { 'account': tf_resource['account'], 'region': tf_resource.get('region'), } def _get_image_uri(self, account, repository): for repo in self.aws_cli.resources[account]['ecr']: if repo['repositoryName'] == repository: return repo['repositoryUri'] @staticmethod def _build_quay_auths(url, user, token): auth_bytes = bytes(f"{user}:{token}", 'utf-8') auth = base64.b64encode(auth_bytes).decode('utf-8') return { url: { 'username': user, 'password': token, 'email': '', 'auth': auth } }
class EcrMirror: def __init__(self, instance, dry_run): self.dry_run = dry_run self.instance = instance self.settings = queries.get_app_interface_settings() self.secret_reader = SecretReader(settings=self.settings) self.skopeo_cli = Skopeo(dry_run) self.error = False identifier = instance["identifier"] account = instance["account"] region = instance.get("region") self.aws_cli = AWSApi( thread_pool_size=1, accounts=[self._get_aws_account_info(account)], settings=self.settings, init_ecr_auth_tokens=True, ) self.aws_cli.map_ecr_resources() self.ecr_uri = self._get_image_uri( account=account, repository=identifier, ) if self.ecr_uri is None: self.error = True LOG.error(f"Could not find the ECR repository {identifier}") self.ecr_username, self.ecr_password = self._get_ecr_creds( account=account, region=region, ) self.ecr_auth = f"{self.ecr_username}:{self.ecr_password}" self.image_username = None self.image_password = None self.image_auth = None pull_secret = self.instance["mirror"]["pullCredentials"] if pull_secret is not None: raw_data = self.secret_reader.read_all(pull_secret) self.image_username = raw_data["user"] self.image_password = raw_data["token"] self.image_auth = f"{self.image_username}:{self.image_password}" def run(self): if self.error: return ecr_mirror = Image(self.ecr_uri, username=self.ecr_username, password=self.ecr_password) image = Image( self.instance["mirror"]["url"], username=self.image_username, password=self.image_password, ) LOG.debug("[checking %s -> %s]", image, ecr_mirror) for tag in image: if tag not in ecr_mirror: try: self.skopeo_cli.copy( src_image=image[tag], src_creds=self.image_auth, dst_image=ecr_mirror[tag], dest_creds=self.ecr_auth, ) except SkopeoCmdError as details: LOG.error("[%s]", details) def _get_ecr_creds(self, account, region): if region is None: region = self.aws_cli.accounts[account]["resourcesDefaultRegion"] auth_token = f"{account}/{region}" data = self.aws_cli.auth_tokens[auth_token] auth_data = data["authorizationData"][0] token = auth_data["authorizationToken"] password = base64.b64decode(token).decode("utf-8").split(":")[1] return "AWS", password def _get_image_uri(self, account, repository): for repo in self.aws_cli.resources[account]["ecr"]: if repo["repositoryName"] == repository: return repo["repositoryUri"] @staticmethod def _get_aws_account_info(account): for account_info in queries.get_aws_accounts(): if "name" not in account_info: continue if account_info["name"] != account: continue return account_info
def run(dry_run, gitlab_project_id=None): settings = queries.get_app_interface_settings() namespaces = queries.get_namespaces() # This is a list of app-interface ECR resources and their # mirrors osd_mirrors = [] for namespace in namespaces: # We are only interested on the ECR resources from # this specific namespace if namespace['name'] != 'osd-operators-ecr-mirrors': continue if namespace['terraformResources'] is None: continue for tfr in namespace['terraformResources']: if tfr['provider'] != 'ecr': continue if tfr['mirror'] is None: continue osd_mirrors.append(tfr) # Now the tricky part. The "OCP Release ECR Mirror" is a stand-alone # object in app-interface. We have to process it so we get the # upstream and the mirror repositories instances = queries.get_ocp_release_ecr_mirror() for instance in instances: namespace = instance['ecrResourcesNamespace'] ocp_release_identifier = instance['ocpReleaseEcrIdentifier'] ocp_art_dev_identifier = instance['ocpArtDevEcrIdentifier'] ocp_release_tf_info = get_ecr_tf_resource_info(namespace, ocp_release_identifier) # We get an ECR resource from app-interface, but it has # no mirror property as the mirroring is done differently # there (see qontract-reconcile-ocp-release-ecr-mirror). # The quay repositories are not managed in app-interface, but # we know where they are by looking at the ClusterImageSets # in Hive. # Let's just manually inject the mirror information so we # process all the ECR resources the same way ocp_release_tf_info['mirror'] = { 'url': 'quay.io/openshift-release-dev/ocp-release', 'pullCredentials': None, 'tags': None, 'tagsExclude': None } osd_mirrors.append(ocp_release_tf_info) ocp_art_dev_tf_info = get_ecr_tf_resource_info(namespace, ocp_art_dev_identifier) ocp_art_dev_tf_info['mirror'] = { 'url': 'quay.io/openshift-release-dev/ocp-v4.0-art-dev', 'pullCredentials': None, 'tags': None, 'tagsExclude': None } osd_mirrors.append(ocp_art_dev_tf_info) # Initializing the AWS Client for all the accounts # with ECR resources of interest accounts = [] for tfr in osd_mirrors: account = get_aws_account_info(tfr['account']) if account not in accounts: accounts.append(account) aws_cli = AWSApi(thread_pool_size=1, accounts=accounts, settings=settings, init_ecr_auth_tokens=True) aws_cli.map_ecr_resources() # Building up the mirrors information in the # install-config.yaml compatible format mirrors_info = [] for tfr in osd_mirrors: image_url = get_image_uri(aws_cli=aws_cli, account=tfr['account'], repository=tfr['identifier']) mirrors_info.append({ 'source': tfr['mirror']['url'], 'mirrors': [ image_url, ] }) if not dry_run: # Creating the MR to app-interface mr_cli = mr_client_gateway.init(gitlab_project_id=gitlab_project_id) mr = CSInstallConfig(mirrors_info=mirrors_info) mr.submit(cli=mr_cli)
class OcpReleaseMirror: def __init__(self, dry_run, instance): self.dry_run = dry_run self.settings = queries.get_app_interface_settings() cluster_info = instance["hiveCluster"] hive_cluster = instance["hiveCluster"]["name"] # Getting the OCM Client for the hive cluster ocm_map = OCMMap( clusters=[cluster_info], integration=QONTRACT_INTEGRATION, settings=self.settings, ) self.ocm_cli = ocm_map.get(hive_cluster) if not self.ocm_cli: raise OcpReleaseMirrorError( f"Can't create ocm client for " f"cluster {hive_cluster}" ) # Getting the OC Client for the hive cluster oc_map = OC_Map( clusters=[cluster_info], integration=QONTRACT_INTEGRATION, settings=self.settings, ) self.oc_cli = oc_map.get(hive_cluster) if not self.oc_cli: raise OcpReleaseMirrorError( f"Can't create oc client for " f"cluster {hive_cluster}" ) namespace = instance["ecrResourcesNamespace"] ocp_release_identifier = instance["ocpReleaseEcrIdentifier"] ocp_art_dev_identifier = instance["ocpArtDevEcrIdentifier"] ocp_release_info = self._get_tf_resource_info(namespace, ocp_release_identifier) if ocp_release_info is None: raise OcpReleaseMirrorError( f"Could not find rds " f"identifier " f"{ocp_release_identifier} in " f"namespace {namespace['name']}" ) ocp_art_dev_info = self._get_tf_resource_info(namespace, ocp_art_dev_identifier) if ocp_art_dev_info is None: raise OcpReleaseMirrorError( f"Could not find rds identifier" f" {ocp_art_dev_identifier} in" f"namespace {namespace['name']}" ) # Getting the AWS Client for the accounts aws_accounts = [ self._get_aws_account_info(account=ocp_release_info["account"]), self._get_aws_account_info(account=ocp_art_dev_info["account"]), ] self.aws_cli = AWSApi( thread_pool_size=1, accounts=aws_accounts, settings=self.settings, init_ecr_auth_tokens=True, ) self.aws_cli.map_ecr_resources() self.ocp_release_ecr_uri = self._get_image_uri( account=ocp_release_info["account"], repository=ocp_release_identifier ) if self.ocp_release_ecr_uri is None: raise OcpReleaseMirrorError( f"Could not find the " f"ECR repository " f"{ocp_release_identifier}" ) self.ocp_art_dev_ecr_uri = self._get_image_uri( account=ocp_art_dev_info["account"], repository=ocp_art_dev_identifier ) if self.ocp_art_dev_ecr_uri is None: raise OcpReleaseMirrorError( f"Could not find the " f"ECR repository " f"{ocp_art_dev_identifier}" ) # Process all the quayOrgTargets quay_api_store = get_quay_api_store() self.quay_target_orgs = [] for quayTargetOrg in instance["quayTargetOrgs"]: org_name = quayTargetOrg["name"] instance_name = quayTargetOrg["instance"]["name"] org_key = OrgKey(instance_name, org_name) org_info = quay_api_store[org_key] if not org_info["push_token"]: raise OcpReleaseMirrorError(f"{org_key} has no push_token defined.") url = org_info["url"] user = org_info["push_token"]["user"] token = org_info["push_token"]["token"] self.quay_target_orgs.append( { "url": url, "dest_ocp_release": f"{url}/{org_name}/ocp-release", "dest_ocp_art_dev": f"{url}/{org_name}/ocp-v4.0-art-dev", "auths": self._build_quay_auths(url, user, token), } ) # Getting all the credentials quay_creds = self._get_quay_creds() ocp_release_creds = self._get_ecr_creds( account=ocp_release_info["account"], region=ocp_release_info["region"] ) ocp_art_dev_creds = self._get_ecr_creds( account=ocp_art_dev_info["account"], region=ocp_art_dev_info["region"] ) # Creating a single dictionary with all credentials to be used by the # "oc adm release mirror" command self.registry_creds = { "auths": { **quay_creds["auths"], **ocp_release_creds["auths"], **ocp_art_dev_creds["auths"], } } # Append quay_target_orgs auths to registry_creds for quay_target_org in self.quay_target_orgs: url = quay_target_org["url"] if url in self.registry_creds["auths"].keys(): OcpReleaseMirrorError( "Cannot mirror to the same Quay " f"instance multiple times: {url}" ) self.registry_creds["auths"].update(quay_target_org["auths"]) # Initiate channel groups self.channel_groups = instance["mirrorChannels"] def run(self): ocp_releases = self._get_ocp_releases() if not ocp_releases: raise RuntimeError("No OCP Releases found") for ocp_release_info in ocp_releases: ocp_release = ocp_release_info.ocp_release tag = ocp_release_info.tag # mirror to ecr dest_ocp_release = f"{self.ocp_release_ecr_uri}:{tag}" self._run_mirror( ocp_release=ocp_release, dest_ocp_release=dest_ocp_release, dest_ocp_art_dev=self.ocp_art_dev_ecr_uri, ) # mirror to all quay target orgs for quay_target_org in self.quay_target_orgs: dest_ocp_release = f'{quay_target_org["dest_ocp_release"]}:' f"{tag}" dest_ocp_art_dev = quay_target_org["dest_ocp_art_dev"] self._run_mirror( ocp_release=ocp_release, dest_ocp_release=dest_ocp_release, dest_ocp_art_dev=dest_ocp_art_dev, ) def _run_mirror(self, ocp_release, dest_ocp_release, dest_ocp_art_dev): # Checking if the image is already there if self._is_image_there(dest_ocp_release): LOG.debug(f"Image {ocp_release} already in " f"the mirror. Skipping.") return LOG.info( f"Mirroring {ocp_release} to {dest_ocp_art_dev} " f"to_release {dest_ocp_release}" ) if self.dry_run: return # Creating a new, bare, OC client since we don't # want to run this against any cluster or via # a jump host oc_cli = OC("cluster", None, None, local=True) oc_cli.release_mirror( from_release=ocp_release, to=dest_ocp_art_dev, to_release=dest_ocp_release, dockerconfig=self.registry_creds, ) def _is_image_there(self, image): image_obj = Image(image) for registry, creds in self.registry_creds["auths"].items(): # Getting the credentials for the image_obj if "//" not in registry: registry = "//" + registry registry_obj = urlparse(registry) if registry_obj.netloc != image_obj.registry: continue image_obj.auth = (creds["username"], creds["password"]) # Checking if the image is already # in the registry if image_obj: return True return False @staticmethod def _get_aws_account_info(account): for account_info in queries.get_aws_accounts(): if "name" not in account_info: continue if account_info["name"] != account: continue return account_info def _get_ocp_releases(self): ocp_releases = [] clusterimagesets = self.oc_cli.get_all("ClusterImageSet") for clusterimageset in clusterimagesets["items"]: release_image = clusterimageset["spec"]["releaseImage"] # There are images in some ClusterImagesSets not coming # from quay.io, e.g.: # registry.svc.ci.openshift.org/ocp/release:4.2.0-0.nightly-2020-11-04-053758 # Let's filter out everything not from quay.io if not release_image.startswith("quay.io"): continue labels = clusterimageset["metadata"]["labels"] name = clusterimageset["metadata"]["name"] # ClusterImagesSets may be enabled or disabled. # Let's only mirror enabled ones enabled = labels["api.openshift.com/enabled"] if enabled == "false": continue # ClusterImageSets may be in different channels. channel_group = labels["api.openshift.com/channel-group"] if channel_group not in self.channel_groups: continue ocp_releases.append(OcpReleaseInfo(release_image, name)) return ocp_releases def _get_quay_creds(self): return self.ocm_cli.get_pull_secrets() def _get_ecr_creds(self, account, region): if region is None: region = self.aws_cli.accounts[account]["resourcesDefaultRegion"] auth_token = f"{account}/{region}" data = self.aws_cli.auth_tokens[auth_token] auth_data = data["authorizationData"][0] server = auth_data["proxyEndpoint"] token = auth_data["authorizationToken"] password = base64.b64decode(token).decode("utf-8").split(":")[1] return { "auths": { server: { "username": "******", "password": password, "email": "*****@*****.**", "auth": token, } } } @staticmethod def _get_tf_resource_info(namespace, identifier): tf_resources = namespace["terraformResources"] for tf_resource in tf_resources: if "identifier" not in tf_resource: continue if tf_resource["identifier"] != identifier: continue if tf_resource["provider"] != "ecr": continue return { "account": tf_resource["account"], "region": tf_resource.get("region"), } def _get_image_uri(self, account, repository): for repo in self.aws_cli.resources[account]["ecr"]: if repo["repositoryName"] == repository: return repo["repositoryUri"] @staticmethod def _build_quay_auths(url, user, token): auth_bytes = bytes(f"{user}:{token}", "utf-8") auth = base64.b64encode(auth_bytes).decode("utf-8") return {url: {"username": user, "password": token, "email": "", "auth": auth}}
class EcrMirror: def __init__(self, instance, dry_run): self.dry_run = dry_run self.instance = instance self.settings = queries.get_app_interface_settings() self.secret_reader = SecretReader(settings=self.settings) self.skopeo_cli = Skopeo(dry_run) self.error = False identifier = instance['identifier'] account = instance['account'] region = instance.get('region') self.aws_cli = AWSApi(thread_pool_size=1, accounts=[self._get_aws_account_info(account)], settings=self.settings, init_ecr_auth_tokens=True) self.aws_cli.map_ecr_resources() self.ecr_uri = self._get_image_uri( account=account, repository=identifier, ) if self.ecr_uri is None: self.error = True LOG.error(f"Could not find the ECR repository {identifier}") self.ecr_username, self.ecr_password = self._get_ecr_creds( account=account, region=region, ) self.ecr_auth = f'{self.ecr_username}:{self.ecr_password}' self.image_username = None self.image_password = None self.image_auth = None pull_secret = self.instance['mirror']['pullCredentials'] if pull_secret is not None: raw_data = self.secret_reader.read_all(pull_secret) self.image_username = raw_data["user"] self.image_password = raw_data["token"] self.image_auth = f'{self.image_username}:{self.image_password}' def run(self): if self.error: return ecr_mirror = Image(self.ecr_uri, username=self.ecr_username, password=self.ecr_password) image = Image(self.instance['mirror']['url'], username=self.image_username, password=self.image_password) LOG.debug('[checking %s -> %s]', image, ecr_mirror) for tag in image: if tag not in ecr_mirror: try: self.skopeo_cli.copy(src_image=image[tag], src_creds=self.image_auth, dst_image=ecr_mirror[tag], dest_creds=self.ecr_auth) except SkopeoCmdError as details: LOG.error('[%s]', details) def _get_ecr_creds(self, account, region): if region is None: region = self.aws_cli.accounts[account]['resourcesDefaultRegion'] auth_token = f'{account}/{region}' data = self.aws_cli.auth_tokens[auth_token] auth_data = data['authorizationData'][0] token = auth_data['authorizationToken'] password = base64.b64decode(token).decode('utf-8').split(':')[1] return 'AWS', password def _get_image_uri(self, account, repository): for repo in self.aws_cli.resources[account]['ecr']: if repo['repositoryName'] == repository: return repo['repositoryUri'] @staticmethod def _get_aws_account_info(account): for account_info in queries.get_aws_accounts(): if 'name' not in account_info: continue if account_info['name'] != account: continue return account_info
class OcpReleaseEcrMirror: def __init__(self, dry_run, instance): self.dry_run = dry_run self.settings = queries.get_app_interface_settings() cluster_info = instance['hiveCluster'] hive_cluster = instance['hiveCluster']['name'] # Getting the OCM Client for the hive cluster ocm_map = OCMMap(clusters=[cluster_info], integration=QONTRACT_INTEGRATION, settings=self.settings) self.ocm_cli = ocm_map.get(hive_cluster) if not self.ocm_cli: raise OcpReleaseEcrMirrorError(f"Can't create ocm client for " f"cluster {hive_cluster}") # Getting the OC Client for the hive cluster oc_map = OC_Map(clusters=[cluster_info], integration=QONTRACT_INTEGRATION, settings=self.settings) self.oc_cli = oc_map.get(hive_cluster) if not self.oc_cli: raise OcpReleaseEcrMirrorError(f"Can't create oc client for " f"cluster {hive_cluster}") namespace = instance['ecrResourcesNamespace'] ocp_release_identifier = instance['ocpReleaseEcrIdentifier'] ocp_art_dev_identifier = instance['ocpArtDevEcrIdentifier'] ocp_release_info = self._get_tf_resource_info(namespace, ocp_release_identifier) if ocp_release_info is None: raise OcpReleaseEcrMirrorError(f"Could not find rds " f"identifier " f"{ocp_release_identifier} in " f"namespace {namespace['name']}") ocp_art_dev_info = self._get_tf_resource_info(namespace, ocp_art_dev_identifier) if ocp_art_dev_info is None: raise OcpReleaseEcrMirrorError(f"Could not find rds identifier" f" {ocp_art_dev_identifier} in" f"namespace {namespace['name']}") # Getting the AWS Client for the accounts aws_accounts = [ self._get_aws_account_info(account=ocp_release_info['account']), self._get_aws_account_info(account=ocp_art_dev_info['account']) ] self.aws_cli = AWSApi(thread_pool_size=1, accounts=aws_accounts, settings=self.settings, init_ecr_auth_tokens=True) self.aws_cli.map_ecr_resources() self.ocp_release_ecr_uri = self._get_image_uri( account=ocp_release_info['account'], repository=ocp_release_identifier) if self.ocp_release_ecr_uri is None: raise OcpReleaseEcrMirrorError(f"Could not find the " f"ECR repository " f"{ocp_release_identifier}") self.ocp_art_dev_ecr_uri = self._get_image_uri( account=ocp_art_dev_info['account'], repository=ocp_art_dev_identifier) if self.ocp_art_dev_ecr_uri is None: raise OcpReleaseEcrMirrorError(f"Could not find the " f"ECR repository " f"{ocp_art_dev_identifier}") # Getting all the credentials quay_creds = self._get_quay_creds() ocp_release_creds = self._get_ecr_creds( account=ocp_release_info['account'], region=ocp_release_info['region']) ocp_art_dev_creds = self._get_ecr_creds( account=ocp_art_dev_info['account'], region=ocp_art_dev_info['region']) # Creating a single dictionary with all credentials to be used by the # "oc adm release mirror" command self.registry_creds = { 'auths': { **quay_creds['auths'], **ocp_release_creds['auths'], **ocp_art_dev_creds['auths'], } } def run(self): ocp_releases = self._get_ocp_releases() if not ocp_releases: raise RuntimeError('No OCP Releases found') for ocp_release in ocp_releases: tag = ocp_release.split(':')[-1] dest_ocp_release = f'{self.ocp_release_ecr_uri}:{tag}' self._run_mirror(ocp_release=ocp_release, dest_ocp_release=dest_ocp_release, dest_ocp_art_dev=self.ocp_art_dev_ecr_uri) def _run_mirror(self, ocp_release, dest_ocp_release, dest_ocp_art_dev): # Checking if the image is already there if self._is_image_there(dest_ocp_release): LOG.info(f'Image {ocp_release} already in ' f'the mirror. Skipping.') return LOG.info(f'Mirroring {ocp_release} to {dest_ocp_art_dev} ' f'to_release {dest_ocp_release}') if self.dry_run: return # Creating a new, bare, OC client since we don't # want to run this against any cluster or via # a jump host oc_cli = OC(server='', token='', jh=None, settings=None, init_projects=False, init_api_resources=False) oc_cli.release_mirror(from_release=ocp_release, to=dest_ocp_art_dev, to_release=dest_ocp_release, dockerconfig=self.registry_creds) def _is_image_there(self, image): image_obj = Image(image) for registry, creds in self.registry_creds['auths'].items(): # Getting the credentials for the image_obj registry_obj = urlparse(registry) if registry_obj.netloc != image_obj.registry: continue image_obj.auth = (creds['username'], creds['password']) # Checking if the image is already # in the registry if image_obj: return True return False @staticmethod def _get_aws_account_info(account): for account_info in queries.get_aws_accounts(): if 'name' not in account_info: continue if account_info['name'] != account: continue return account_info def _get_ocp_releases(self): ocp_releases = list() clusterimagesets = self.oc_cli.get_all('clusterimageset') for clusterimageset in clusterimagesets['items']: release_image = clusterimageset['spec']['releaseImage'] # There are images in some ClusterImagesSets not coming # from quay.io, e.g.: # registry.svc.ci.openshift.org/ocp/release:4.2.0-0.nightly-2020-11-04-053758 # Let's filter out everything not from quay.io if not release_image.startswith('quay.io'): continue labels = clusterimageset['metadata']['labels'] # ClusterImagesSets may be enabled or disabled. # Let's only mirror enabled ones enabled = labels['api.openshift.com/enabled'] if enabled == 'false': continue # ClusterImageSets may be in different channels. # Let's only mirror stable channel_group = labels['api.openshift.com/channel-group'] if channel_group != 'stable': continue ocp_releases.append(release_image) return ocp_releases def _get_quay_creds(self): return self.ocm_cli.get_pull_secrets() def _get_ecr_creds(self, account, region): if region is None: region = self.aws_cli.accounts[account]['resourcesDefaultRegion'] auth_token = f'{account}/{region}' data = self.aws_cli.auth_tokens[auth_token] auth_data = data['authorizationData'][0] server = auth_data['proxyEndpoint'] token = auth_data['authorizationToken'] password = base64.b64decode(token).decode('utf-8').split(':')[1] return { 'auths': { server: { 'username': '******', 'password': password, 'email': '*****@*****.**', 'auth': token } } } @staticmethod def _get_tf_resource_info(namespace, identifier): tf_resources = namespace['terraformResources'] for tf_resource in tf_resources: if 'identifier' not in tf_resource: continue if tf_resource['identifier'] != identifier: continue if tf_resource['provider'] != 'ecr': continue return { 'account': tf_resource['account'], 'region': tf_resource.get('region'), } def _get_image_uri(self, account, repository): for repo in self.aws_cli.resources[account]['ecr']: if repo['repositoryName'] == repository: return repo['repositoryUri']