class OcpReleaseMirror:
def __init__(self, dry_run, instance):
self.dry_run = dry_run
self.settings = queries.get_app_interface_settings()
cluster_info = instance['hiveCluster']
hive_cluster = instance['hiveCluster']['name']
# Getting the OCM Client for the hive cluster
ocm_map = OCMMap(clusters=[cluster_info],
integration=QONTRACT_INTEGRATION,
settings=self.settings)
self.ocm_cli = ocm_map.get(hive_cluster)
if not self.ocm_cli:
raise OcpReleaseMirrorError(f"Can't create ocm client for "
f"cluster {hive_cluster}")
# Getting the OC Client for the hive cluster
oc_map = OC_Map(clusters=[cluster_info],
integration=QONTRACT_INTEGRATION,
settings=self.settings)
self.oc_cli = oc_map.get(hive_cluster)
if not self.oc_cli:
raise OcpReleaseMirrorError(f"Can't create oc client for "
f"cluster {hive_cluster}")
namespace = instance['ecrResourcesNamespace']
ocp_release_identifier = instance['ocpReleaseEcrIdentifier']
ocp_art_dev_identifier = instance['ocpArtDevEcrIdentifier']
ocp_release_info = self._get_tf_resource_info(namespace,
ocp_release_identifier)
if ocp_release_info is None:
raise OcpReleaseMirrorError(f"Could not find rds "
f"identifier "
f"{ocp_release_identifier} in "
f"namespace {namespace['name']}")
ocp_art_dev_info = self._get_tf_resource_info(namespace,
ocp_art_dev_identifier)
if ocp_art_dev_info is None:
raise OcpReleaseMirrorError(f"Could not find rds identifier"
f" {ocp_art_dev_identifier} in"
f"namespace {namespace['name']}")
# Getting the AWS Client for the accounts
aws_accounts = [
self._get_aws_account_info(account=ocp_release_info['account']),
self._get_aws_account_info(account=ocp_art_dev_info['account'])
]
self.aws_cli = AWSApi(thread_pool_size=1,
accounts=aws_accounts,
settings=self.settings,
init_ecr_auth_tokens=True)
self.aws_cli.map_ecr_resources()
self.ocp_release_ecr_uri = self._get_image_uri(
account=ocp_release_info['account'],
repository=ocp_release_identifier)
if self.ocp_release_ecr_uri is None:
raise OcpReleaseMirrorError(f"Could not find the "
f"ECR repository "
f"{ocp_release_identifier}")
self.ocp_art_dev_ecr_uri = self._get_image_uri(
account=ocp_art_dev_info['account'],
repository=ocp_art_dev_identifier)
if self.ocp_art_dev_ecr_uri is None:
raise OcpReleaseMirrorError(f"Could not find the "
f"ECR repository "
f"{ocp_art_dev_identifier}")
# Process all the quayOrgTargets
quay_api_store = get_quay_api_store()
self.quay_target_orgs = []
for quayTargetOrg in instance['quayTargetOrgs']:
org_name = quayTargetOrg['name']
instance_name = quayTargetOrg['instance']['name']
org_key = OrgKey(instance_name, org_name)
org_info = quay_api_store[org_key]
if not org_info['push_token']:
raise OcpReleaseMirrorError(
f'{org_key} has no push_token defined.')
url = org_info['url']
user = org_info['push_token']['user']
token = org_info['push_token']['token']
self.quay_target_orgs.append({
'url':
url,
'dest_ocp_release':
f"{url}/{org_name}/ocp-release",
'dest_ocp_art_dev':
f"{url}/{org_name}/ocp-v4.0-art-dev",
'auths':
self._build_quay_auths(url, user, token)
})
# Getting all the credentials
quay_creds = self._get_quay_creds()
ocp_release_creds = self._get_ecr_creds(
account=ocp_release_info['account'],
region=ocp_release_info['region'])
ocp_art_dev_creds = self._get_ecr_creds(
account=ocp_art_dev_info['account'],
region=ocp_art_dev_info['region'])
# Creating a single dictionary with all credentials to be used by the
# "oc adm release mirror" command
self.registry_creds = {
'auths': {
**quay_creds['auths'],
**ocp_release_creds['auths'],
**ocp_art_dev_creds['auths'],
}
}
# Append quay_target_orgs auths to registry_creds
for quay_target_org in self.quay_target_orgs:
url = quay_target_org['url']
if url in self.registry_creds['auths'].keys():
OcpReleaseMirrorError('Cannot mirror to the same Quay '
f'instance multiple times: {url}')
self.registry_creds['auths'].update(quay_target_org['auths'])
# Initiate channel groups
self.channel_groups = instance['mirrorChannels']
def run(self):
ocp_releases = self._get_ocp_releases()
if not ocp_releases:
raise RuntimeError('No OCP Releases found')
for ocp_release_info in ocp_releases:
ocp_release = ocp_release_info.ocp_release
tag = ocp_release_info.tag
# mirror to ecr
dest_ocp_release = f'{self.ocp_release_ecr_uri}:{tag}'
self._run_mirror(ocp_release=ocp_release,
dest_ocp_release=dest_ocp_release,
dest_ocp_art_dev=self.ocp_art_dev_ecr_uri)
# mirror to all quay target orgs
for quay_target_org in self.quay_target_orgs:
dest_ocp_release = (f'{quay_target_org["dest_ocp_release"]}:'
f'{tag}')
dest_ocp_art_dev = quay_target_org["dest_ocp_art_dev"]
self._run_mirror(ocp_release=ocp_release,
dest_ocp_release=dest_ocp_release,
dest_ocp_art_dev=dest_ocp_art_dev)
def _run_mirror(self, ocp_release, dest_ocp_release, dest_ocp_art_dev):
# Checking if the image is already there
if self._is_image_there(dest_ocp_release):
LOG.debug(f'Image {ocp_release} already in '
f'the mirror. Skipping.')
return
LOG.info(f'Mirroring {ocp_release} to {dest_ocp_art_dev} '
f'to_release {dest_ocp_release}')
if self.dry_run:
return
# Creating a new, bare, OC client since we don't
# want to run this against any cluster or via
# a jump host
oc_cli = OC('cluster', None, None, local=True)
oc_cli.release_mirror(from_release=ocp_release,
to=dest_ocp_art_dev,
to_release=dest_ocp_release,
dockerconfig=self.registry_creds)
def _is_image_there(self, image):
image_obj = Image(image)
for registry, creds in self.registry_creds['auths'].items():
# Getting the credentials for the image_obj
if '//' not in registry:
registry = '//' + registry
registry_obj = urlparse(registry)
if registry_obj.netloc != image_obj.registry:
continue
image_obj.auth = (creds['username'], creds['password'])
# Checking if the image is already
# in the registry
if image_obj:
return True
return False
@staticmethod
def _get_aws_account_info(account):
for account_info in queries.get_aws_accounts():
if 'name' not in account_info:
continue
if account_info['name'] != account:
continue
return account_info
def _get_ocp_releases(self):
ocp_releases = []
clusterimagesets = self.oc_cli.get_all('ClusterImageSet')
for clusterimageset in clusterimagesets['items']:
release_image = clusterimageset['spec']['releaseImage']
# There are images in some ClusterImagesSets not coming
# from quay.io, e.g.:
# registry.svc.ci.openshift.org/ocp/release:4.2.0-0.nightly-2020-11-04-053758
# Let's filter out everything not from quay.io
if not release_image.startswith('quay.io'):
continue
labels = clusterimageset['metadata']['labels']
name = clusterimageset['metadata']['name']
# ClusterImagesSets may be enabled or disabled.
# Let's only mirror enabled ones
enabled = labels['api.openshift.com/enabled']
if enabled == 'false':
continue
# ClusterImageSets may be in different channels.
channel_group = labels['api.openshift.com/channel-group']
if channel_group not in self.channel_groups:
continue
ocp_releases.append(OcpReleaseInfo(release_image, name))
return ocp_releases
def _get_quay_creds(self):
return self.ocm_cli.get_pull_secrets()
def _get_ecr_creds(self, account, region):
if region is None:
region = self.aws_cli.accounts[account]['resourcesDefaultRegion']
auth_token = f'{account}/{region}'
data = self.aws_cli.auth_tokens[auth_token]
auth_data = data['authorizationData'][0]
server = auth_data['proxyEndpoint']
token = auth_data['authorizationToken']
password = base64.b64decode(token).decode('utf-8').split(':')[1]
return {
'auths': {
server: {
'username': '******',
'password': password,
'email': '*****@*****.**',
'auth': token
}
}
}
@staticmethod
def _get_tf_resource_info(namespace, identifier):
tf_resources = namespace['terraformResources']
for tf_resource in tf_resources:
if 'identifier' not in tf_resource:
continue
if tf_resource['identifier'] != identifier:
continue
if tf_resource['provider'] != 'ecr':
continue
return {
'account': tf_resource['account'],
'region': tf_resource.get('region'),
}
def _get_image_uri(self, account, repository):
for repo in self.aws_cli.resources[account]['ecr']:
if repo['repositoryName'] == repository:
return repo['repositoryUri']
@staticmethod
def _build_quay_auths(url, user, token):
auth_bytes = bytes(f"{user}:{token}", 'utf-8')
auth = base64.b64encode(auth_bytes).decode('utf-8')
return {
url: {
'username': user,
'password': token,
'email': '',
'auth': auth
}
}
class EcrMirror:
def __init__(self, instance, dry_run):
self.dry_run = dry_run
self.instance = instance
self.settings = queries.get_app_interface_settings()
self.secret_reader = SecretReader(settings=self.settings)
self.skopeo_cli = Skopeo(dry_run)
self.error = False
identifier = instance["identifier"]
account = instance["account"]
region = instance.get("region")
self.aws_cli = AWSApi(
thread_pool_size=1,
accounts=[self._get_aws_account_info(account)],
settings=self.settings,
init_ecr_auth_tokens=True,
)
self.aws_cli.map_ecr_resources()
self.ecr_uri = self._get_image_uri(
account=account,
repository=identifier,
)
if self.ecr_uri is None:
self.error = True
LOG.error(f"Could not find the ECR repository {identifier}")
self.ecr_username, self.ecr_password = self._get_ecr_creds(
account=account,
region=region,
)
self.ecr_auth = f"{self.ecr_username}:{self.ecr_password}"
self.image_username = None
self.image_password = None
self.image_auth = None
pull_secret = self.instance["mirror"]["pullCredentials"]
if pull_secret is not None:
raw_data = self.secret_reader.read_all(pull_secret)
self.image_username = raw_data["user"]
self.image_password = raw_data["token"]
self.image_auth = f"{self.image_username}:{self.image_password}"
def run(self):
if self.error:
return
ecr_mirror = Image(self.ecr_uri,
username=self.ecr_username,
password=self.ecr_password)
image = Image(
self.instance["mirror"]["url"],
username=self.image_username,
password=self.image_password,
)
LOG.debug("[checking %s -> %s]", image, ecr_mirror)
for tag in image:
if tag not in ecr_mirror:
try:
self.skopeo_cli.copy(
src_image=image[tag],
src_creds=self.image_auth,
dst_image=ecr_mirror[tag],
dest_creds=self.ecr_auth,
)
except SkopeoCmdError as details:
LOG.error("[%s]", details)
def _get_ecr_creds(self, account, region):
if region is None:
region = self.aws_cli.accounts[account]["resourcesDefaultRegion"]
auth_token = f"{account}/{region}"
data = self.aws_cli.auth_tokens[auth_token]
auth_data = data["authorizationData"][0]
token = auth_data["authorizationToken"]
password = base64.b64decode(token).decode("utf-8").split(":")[1]
return "AWS", password
def _get_image_uri(self, account, repository):
for repo in self.aws_cli.resources[account]["ecr"]:
if repo["repositoryName"] == repository:
return repo["repositoryUri"]
@staticmethod
def _get_aws_account_info(account):
for account_info in queries.get_aws_accounts():
if "name" not in account_info:
continue
if account_info["name"] != account:
continue
return account_info
def run(dry_run, gitlab_project_id=None):
settings = queries.get_app_interface_settings()
namespaces = queries.get_namespaces()
# This is a list of app-interface ECR resources and their
# mirrors
osd_mirrors = []
for namespace in namespaces:
# We are only interested on the ECR resources from
# this specific namespace
if namespace['name'] != 'osd-operators-ecr-mirrors':
continue
if namespace['terraformResources'] is None:
continue
for tfr in namespace['terraformResources']:
if tfr['provider'] != 'ecr':
continue
if tfr['mirror'] is None:
continue
osd_mirrors.append(tfr)
# Now the tricky part. The "OCP Release ECR Mirror" is a stand-alone
# object in app-interface. We have to process it so we get the
# upstream and the mirror repositories
instances = queries.get_ocp_release_ecr_mirror()
for instance in instances:
namespace = instance['ecrResourcesNamespace']
ocp_release_identifier = instance['ocpReleaseEcrIdentifier']
ocp_art_dev_identifier = instance['ocpArtDevEcrIdentifier']
ocp_release_tf_info = get_ecr_tf_resource_info(namespace,
ocp_release_identifier)
# We get an ECR resource from app-interface, but it has
# no mirror property as the mirroring is done differently
# there (see qontract-reconcile-ocp-release-ecr-mirror).
# The quay repositories are not managed in app-interface, but
# we know where they are by looking at the ClusterImageSets
# in Hive.
# Let's just manually inject the mirror information so we
# process all the ECR resources the same way
ocp_release_tf_info['mirror'] = {
'url': 'quay.io/openshift-release-dev/ocp-release',
'pullCredentials': None,
'tags': None,
'tagsExclude': None
}
osd_mirrors.append(ocp_release_tf_info)
ocp_art_dev_tf_info = get_ecr_tf_resource_info(namespace,
ocp_art_dev_identifier)
ocp_art_dev_tf_info['mirror'] = {
'url': 'quay.io/openshift-release-dev/ocp-v4.0-art-dev',
'pullCredentials': None,
'tags': None,
'tagsExclude': None
}
osd_mirrors.append(ocp_art_dev_tf_info)
# Initializing the AWS Client for all the accounts
# with ECR resources of interest
accounts = []
for tfr in osd_mirrors:
account = get_aws_account_info(tfr['account'])
if account not in accounts:
accounts.append(account)
aws_cli = AWSApi(thread_pool_size=1,
accounts=accounts,
settings=settings,
init_ecr_auth_tokens=True)
aws_cli.map_ecr_resources()
# Building up the mirrors information in the
# install-config.yaml compatible format
mirrors_info = []
for tfr in osd_mirrors:
image_url = get_image_uri(aws_cli=aws_cli,
account=tfr['account'],
repository=tfr['identifier'])
mirrors_info.append({
'source': tfr['mirror']['url'],
'mirrors': [
image_url,
]
})
if not dry_run:
# Creating the MR to app-interface
mr_cli = mr_client_gateway.init(gitlab_project_id=gitlab_project_id)
mr = CSInstallConfig(mirrors_info=mirrors_info)
mr.submit(cli=mr_cli)
class OcpReleaseMirror:
def __init__(self, dry_run, instance):
self.dry_run = dry_run
self.settings = queries.get_app_interface_settings()
cluster_info = instance["hiveCluster"]
hive_cluster = instance["hiveCluster"]["name"]
# Getting the OCM Client for the hive cluster
ocm_map = OCMMap(
clusters=[cluster_info],
integration=QONTRACT_INTEGRATION,
settings=self.settings,
)
self.ocm_cli = ocm_map.get(hive_cluster)
if not self.ocm_cli:
raise OcpReleaseMirrorError(
f"Can't create ocm client for " f"cluster {hive_cluster}"
)
# Getting the OC Client for the hive cluster
oc_map = OC_Map(
clusters=[cluster_info],
integration=QONTRACT_INTEGRATION,
settings=self.settings,
)
self.oc_cli = oc_map.get(hive_cluster)
if not self.oc_cli:
raise OcpReleaseMirrorError(
f"Can't create oc client for " f"cluster {hive_cluster}"
)
namespace = instance["ecrResourcesNamespace"]
ocp_release_identifier = instance["ocpReleaseEcrIdentifier"]
ocp_art_dev_identifier = instance["ocpArtDevEcrIdentifier"]
ocp_release_info = self._get_tf_resource_info(namespace, ocp_release_identifier)
if ocp_release_info is None:
raise OcpReleaseMirrorError(
f"Could not find rds "
f"identifier "
f"{ocp_release_identifier} in "
f"namespace {namespace['name']}"
)
ocp_art_dev_info = self._get_tf_resource_info(namespace, ocp_art_dev_identifier)
if ocp_art_dev_info is None:
raise OcpReleaseMirrorError(
f"Could not find rds identifier"
f" {ocp_art_dev_identifier} in"
f"namespace {namespace['name']}"
)
# Getting the AWS Client for the accounts
aws_accounts = [
self._get_aws_account_info(account=ocp_release_info["account"]),
self._get_aws_account_info(account=ocp_art_dev_info["account"]),
]
self.aws_cli = AWSApi(
thread_pool_size=1,
accounts=aws_accounts,
settings=self.settings,
init_ecr_auth_tokens=True,
)
self.aws_cli.map_ecr_resources()
self.ocp_release_ecr_uri = self._get_image_uri(
account=ocp_release_info["account"], repository=ocp_release_identifier
)
if self.ocp_release_ecr_uri is None:
raise OcpReleaseMirrorError(
f"Could not find the " f"ECR repository " f"{ocp_release_identifier}"
)
self.ocp_art_dev_ecr_uri = self._get_image_uri(
account=ocp_art_dev_info["account"], repository=ocp_art_dev_identifier
)
if self.ocp_art_dev_ecr_uri is None:
raise OcpReleaseMirrorError(
f"Could not find the " f"ECR repository " f"{ocp_art_dev_identifier}"
)
# Process all the quayOrgTargets
quay_api_store = get_quay_api_store()
self.quay_target_orgs = []
for quayTargetOrg in instance["quayTargetOrgs"]:
org_name = quayTargetOrg["name"]
instance_name = quayTargetOrg["instance"]["name"]
org_key = OrgKey(instance_name, org_name)
org_info = quay_api_store[org_key]
if not org_info["push_token"]:
raise OcpReleaseMirrorError(f"{org_key} has no push_token defined.")
url = org_info["url"]
user = org_info["push_token"]["user"]
token = org_info["push_token"]["token"]
self.quay_target_orgs.append(
{
"url": url,
"dest_ocp_release": f"{url}/{org_name}/ocp-release",
"dest_ocp_art_dev": f"{url}/{org_name}/ocp-v4.0-art-dev",
"auths": self._build_quay_auths(url, user, token),
}
)
# Getting all the credentials
quay_creds = self._get_quay_creds()
ocp_release_creds = self._get_ecr_creds(
account=ocp_release_info["account"], region=ocp_release_info["region"]
)
ocp_art_dev_creds = self._get_ecr_creds(
account=ocp_art_dev_info["account"], region=ocp_art_dev_info["region"]
)
# Creating a single dictionary with all credentials to be used by the
# "oc adm release mirror" command
self.registry_creds = {
"auths": {
**quay_creds["auths"],
**ocp_release_creds["auths"],
**ocp_art_dev_creds["auths"],
}
}
# Append quay_target_orgs auths to registry_creds
for quay_target_org in self.quay_target_orgs:
url = quay_target_org["url"]
if url in self.registry_creds["auths"].keys():
OcpReleaseMirrorError(
"Cannot mirror to the same Quay " f"instance multiple times: {url}"
)
self.registry_creds["auths"].update(quay_target_org["auths"])
# Initiate channel groups
self.channel_groups = instance["mirrorChannels"]
def run(self):
ocp_releases = self._get_ocp_releases()
if not ocp_releases:
raise RuntimeError("No OCP Releases found")
for ocp_release_info in ocp_releases:
ocp_release = ocp_release_info.ocp_release
tag = ocp_release_info.tag
# mirror to ecr
dest_ocp_release = f"{self.ocp_release_ecr_uri}:{tag}"
self._run_mirror(
ocp_release=ocp_release,
dest_ocp_release=dest_ocp_release,
dest_ocp_art_dev=self.ocp_art_dev_ecr_uri,
)
# mirror to all quay target orgs
for quay_target_org in self.quay_target_orgs:
dest_ocp_release = f'{quay_target_org["dest_ocp_release"]}:' f"{tag}"
dest_ocp_art_dev = quay_target_org["dest_ocp_art_dev"]
self._run_mirror(
ocp_release=ocp_release,
dest_ocp_release=dest_ocp_release,
dest_ocp_art_dev=dest_ocp_art_dev,
)
def _run_mirror(self, ocp_release, dest_ocp_release, dest_ocp_art_dev):
# Checking if the image is already there
if self._is_image_there(dest_ocp_release):
LOG.debug(f"Image {ocp_release} already in " f"the mirror. Skipping.")
return
LOG.info(
f"Mirroring {ocp_release} to {dest_ocp_art_dev} "
f"to_release {dest_ocp_release}"
)
if self.dry_run:
return
# Creating a new, bare, OC client since we don't
# want to run this against any cluster or via
# a jump host
oc_cli = OC("cluster", None, None, local=True)
oc_cli.release_mirror(
from_release=ocp_release,
to=dest_ocp_art_dev,
to_release=dest_ocp_release,
dockerconfig=self.registry_creds,
)
def _is_image_there(self, image):
image_obj = Image(image)
for registry, creds in self.registry_creds["auths"].items():
# Getting the credentials for the image_obj
if "//" not in registry:
registry = "//" + registry
registry_obj = urlparse(registry)
if registry_obj.netloc != image_obj.registry:
continue
image_obj.auth = (creds["username"], creds["password"])
# Checking if the image is already
# in the registry
if image_obj:
return True
return False
@staticmethod
def _get_aws_account_info(account):
for account_info in queries.get_aws_accounts():
if "name" not in account_info:
continue
if account_info["name"] != account:
continue
return account_info
def _get_ocp_releases(self):
ocp_releases = []
clusterimagesets = self.oc_cli.get_all("ClusterImageSet")
for clusterimageset in clusterimagesets["items"]:
release_image = clusterimageset["spec"]["releaseImage"]
# There are images in some ClusterImagesSets not coming
# from quay.io, e.g.:
# registry.svc.ci.openshift.org/ocp/release:4.2.0-0.nightly-2020-11-04-053758
# Let's filter out everything not from quay.io
if not release_image.startswith("quay.io"):
continue
labels = clusterimageset["metadata"]["labels"]
name = clusterimageset["metadata"]["name"]
# ClusterImagesSets may be enabled or disabled.
# Let's only mirror enabled ones
enabled = labels["api.openshift.com/enabled"]
if enabled == "false":
continue
# ClusterImageSets may be in different channels.
channel_group = labels["api.openshift.com/channel-group"]
if channel_group not in self.channel_groups:
continue
ocp_releases.append(OcpReleaseInfo(release_image, name))
return ocp_releases
def _get_quay_creds(self):
return self.ocm_cli.get_pull_secrets()
def _get_ecr_creds(self, account, region):
if region is None:
region = self.aws_cli.accounts[account]["resourcesDefaultRegion"]
auth_token = f"{account}/{region}"
data = self.aws_cli.auth_tokens[auth_token]
auth_data = data["authorizationData"][0]
server = auth_data["proxyEndpoint"]
token = auth_data["authorizationToken"]
password = base64.b64decode(token).decode("utf-8").split(":")[1]
return {
"auths": {
server: {
"username": "******",
"password": password,
"email": "*****@*****.**",
"auth": token,
}
}
}
@staticmethod
def _get_tf_resource_info(namespace, identifier):
tf_resources = namespace["terraformResources"]
for tf_resource in tf_resources:
if "identifier" not in tf_resource:
continue
if tf_resource["identifier"] != identifier:
continue
if tf_resource["provider"] != "ecr":
continue
return {
"account": tf_resource["account"],
"region": tf_resource.get("region"),
}
def _get_image_uri(self, account, repository):
for repo in self.aws_cli.resources[account]["ecr"]:
if repo["repositoryName"] == repository:
return repo["repositoryUri"]
@staticmethod
def _build_quay_auths(url, user, token):
auth_bytes = bytes(f"{user}:{token}", "utf-8")
auth = base64.b64encode(auth_bytes).decode("utf-8")
return {url: {"username": user, "password": token, "email": "", "auth": auth}}
class EcrMirror:
def __init__(self, instance, dry_run):
self.dry_run = dry_run
self.instance = instance
self.settings = queries.get_app_interface_settings()
self.secret_reader = SecretReader(settings=self.settings)
self.skopeo_cli = Skopeo(dry_run)
self.error = False
identifier = instance['identifier']
account = instance['account']
region = instance.get('region')
self.aws_cli = AWSApi(thread_pool_size=1,
accounts=[self._get_aws_account_info(account)],
settings=self.settings,
init_ecr_auth_tokens=True)
self.aws_cli.map_ecr_resources()
self.ecr_uri = self._get_image_uri(
account=account,
repository=identifier,
)
if self.ecr_uri is None:
self.error = True
LOG.error(f"Could not find the ECR repository {identifier}")
self.ecr_username, self.ecr_password = self._get_ecr_creds(
account=account,
region=region,
)
self.ecr_auth = f'{self.ecr_username}:{self.ecr_password}'
self.image_username = None
self.image_password = None
self.image_auth = None
pull_secret = self.instance['mirror']['pullCredentials']
if pull_secret is not None:
raw_data = self.secret_reader.read_all(pull_secret)
self.image_username = raw_data["user"]
self.image_password = raw_data["token"]
self.image_auth = f'{self.image_username}:{self.image_password}'
def run(self):
if self.error:
return
ecr_mirror = Image(self.ecr_uri,
username=self.ecr_username,
password=self.ecr_password)
image = Image(self.instance['mirror']['url'],
username=self.image_username,
password=self.image_password)
LOG.debug('[checking %s -> %s]', image, ecr_mirror)
for tag in image:
if tag not in ecr_mirror:
try:
self.skopeo_cli.copy(src_image=image[tag],
src_creds=self.image_auth,
dst_image=ecr_mirror[tag],
dest_creds=self.ecr_auth)
except SkopeoCmdError as details:
LOG.error('[%s]', details)
def _get_ecr_creds(self, account, region):
if region is None:
region = self.aws_cli.accounts[account]['resourcesDefaultRegion']
auth_token = f'{account}/{region}'
data = self.aws_cli.auth_tokens[auth_token]
auth_data = data['authorizationData'][0]
token = auth_data['authorizationToken']
password = base64.b64decode(token).decode('utf-8').split(':')[1]
return 'AWS', password
def _get_image_uri(self, account, repository):
for repo in self.aws_cli.resources[account]['ecr']:
if repo['repositoryName'] == repository:
return repo['repositoryUri']
@staticmethod
def _get_aws_account_info(account):
for account_info in queries.get_aws_accounts():
if 'name' not in account_info:
continue
if account_info['name'] != account:
continue
return account_info
class OcpReleaseEcrMirror:
def __init__(self, dry_run, instance):
self.dry_run = dry_run
self.settings = queries.get_app_interface_settings()
cluster_info = instance['hiveCluster']
hive_cluster = instance['hiveCluster']['name']
# Getting the OCM Client for the hive cluster
ocm_map = OCMMap(clusters=[cluster_info],
integration=QONTRACT_INTEGRATION,
settings=self.settings)
self.ocm_cli = ocm_map.get(hive_cluster)
if not self.ocm_cli:
raise OcpReleaseEcrMirrorError(f"Can't create ocm client for "
f"cluster {hive_cluster}")
# Getting the OC Client for the hive cluster
oc_map = OC_Map(clusters=[cluster_info],
integration=QONTRACT_INTEGRATION,
settings=self.settings)
self.oc_cli = oc_map.get(hive_cluster)
if not self.oc_cli:
raise OcpReleaseEcrMirrorError(f"Can't create oc client for "
f"cluster {hive_cluster}")
namespace = instance['ecrResourcesNamespace']
ocp_release_identifier = instance['ocpReleaseEcrIdentifier']
ocp_art_dev_identifier = instance['ocpArtDevEcrIdentifier']
ocp_release_info = self._get_tf_resource_info(namespace,
ocp_release_identifier)
if ocp_release_info is None:
raise OcpReleaseEcrMirrorError(f"Could not find rds "
f"identifier "
f"{ocp_release_identifier} in "
f"namespace {namespace['name']}")
ocp_art_dev_info = self._get_tf_resource_info(namespace,
ocp_art_dev_identifier)
if ocp_art_dev_info is None:
raise OcpReleaseEcrMirrorError(f"Could not find rds identifier"
f" {ocp_art_dev_identifier} in"
f"namespace {namespace['name']}")
# Getting the AWS Client for the accounts
aws_accounts = [
self._get_aws_account_info(account=ocp_release_info['account']),
self._get_aws_account_info(account=ocp_art_dev_info['account'])
]
self.aws_cli = AWSApi(thread_pool_size=1,
accounts=aws_accounts,
settings=self.settings,
init_ecr_auth_tokens=True)
self.aws_cli.map_ecr_resources()
self.ocp_release_ecr_uri = self._get_image_uri(
account=ocp_release_info['account'],
repository=ocp_release_identifier)
if self.ocp_release_ecr_uri is None:
raise OcpReleaseEcrMirrorError(f"Could not find the "
f"ECR repository "
f"{ocp_release_identifier}")
self.ocp_art_dev_ecr_uri = self._get_image_uri(
account=ocp_art_dev_info['account'],
repository=ocp_art_dev_identifier)
if self.ocp_art_dev_ecr_uri is None:
raise OcpReleaseEcrMirrorError(f"Could not find the "
f"ECR repository "
f"{ocp_art_dev_identifier}")
# Getting all the credentials
quay_creds = self._get_quay_creds()
ocp_release_creds = self._get_ecr_creds(
account=ocp_release_info['account'],
region=ocp_release_info['region'])
ocp_art_dev_creds = self._get_ecr_creds(
account=ocp_art_dev_info['account'],
region=ocp_art_dev_info['region'])
# Creating a single dictionary with all credentials to be used by the
# "oc adm release mirror" command
self.registry_creds = {
'auths': {
**quay_creds['auths'],
**ocp_release_creds['auths'],
**ocp_art_dev_creds['auths'],
}
}
def run(self):
ocp_releases = self._get_ocp_releases()
if not ocp_releases:
raise RuntimeError('No OCP Releases found')
for ocp_release in ocp_releases:
tag = ocp_release.split(':')[-1]
dest_ocp_release = f'{self.ocp_release_ecr_uri}:{tag}'
self._run_mirror(ocp_release=ocp_release,
dest_ocp_release=dest_ocp_release,
dest_ocp_art_dev=self.ocp_art_dev_ecr_uri)
def _run_mirror(self, ocp_release, dest_ocp_release, dest_ocp_art_dev):
# Checking if the image is already there
if self._is_image_there(dest_ocp_release):
LOG.info(f'Image {ocp_release} already in '
f'the mirror. Skipping.')
return
LOG.info(f'Mirroring {ocp_release} to {dest_ocp_art_dev} '
f'to_release {dest_ocp_release}')
if self.dry_run:
return
# Creating a new, bare, OC client since we don't
# want to run this against any cluster or via
# a jump host
oc_cli = OC(server='',
token='',
jh=None,
settings=None,
init_projects=False,
init_api_resources=False)
oc_cli.release_mirror(from_release=ocp_release,
to=dest_ocp_art_dev,
to_release=dest_ocp_release,
dockerconfig=self.registry_creds)
def _is_image_there(self, image):
image_obj = Image(image)
for registry, creds in self.registry_creds['auths'].items():
# Getting the credentials for the image_obj
registry_obj = urlparse(registry)
if registry_obj.netloc != image_obj.registry:
continue
image_obj.auth = (creds['username'], creds['password'])
# Checking if the image is already
# in the registry
if image_obj:
return True
return False
@staticmethod
def _get_aws_account_info(account):
for account_info in queries.get_aws_accounts():
if 'name' not in account_info:
continue
if account_info['name'] != account:
continue
return account_info
def _get_ocp_releases(self):
ocp_releases = list()
clusterimagesets = self.oc_cli.get_all('clusterimageset')
for clusterimageset in clusterimagesets['items']:
release_image = clusterimageset['spec']['releaseImage']
# There are images in some ClusterImagesSets not coming
# from quay.io, e.g.:
# registry.svc.ci.openshift.org/ocp/release:4.2.0-0.nightly-2020-11-04-053758
# Let's filter out everything not from quay.io
if not release_image.startswith('quay.io'):
continue
labels = clusterimageset['metadata']['labels']
# ClusterImagesSets may be enabled or disabled.
# Let's only mirror enabled ones
enabled = labels['api.openshift.com/enabled']
if enabled == 'false':
continue
# ClusterImageSets may be in different channels.
# Let's only mirror stable
channel_group = labels['api.openshift.com/channel-group']
if channel_group != 'stable':
continue
ocp_releases.append(release_image)
return ocp_releases
def _get_quay_creds(self):
return self.ocm_cli.get_pull_secrets()
def _get_ecr_creds(self, account, region):
if region is None:
region = self.aws_cli.accounts[account]['resourcesDefaultRegion']
auth_token = f'{account}/{region}'
data = self.aws_cli.auth_tokens[auth_token]
auth_data = data['authorizationData'][0]
server = auth_data['proxyEndpoint']
token = auth_data['authorizationToken']
password = base64.b64decode(token).decode('utf-8').split(':')[1]
return {
'auths': {
server: {
'username': '******',
'password': password,
'email': '*****@*****.**',
'auth': token
}
}
}
@staticmethod
def _get_tf_resource_info(namespace, identifier):
tf_resources = namespace['terraformResources']
for tf_resource in tf_resources:
if 'identifier' not in tf_resource:
continue
if tf_resource['identifier'] != identifier:
continue
if tf_resource['provider'] != 'ecr':
continue
return {
'account': tf_resource['account'],
'region': tf_resource.get('region'),
}
def _get_image_uri(self, account, repository):
for repo in self.aws_cli.resources[account]['ecr']:
if repo['repositoryName'] == repository:
return repo['repositoryUri']