def run(dry_run, gitlab_project_id=None, thread_pool_size=10, enable_deletion=False): accounts = queries.get_aws_accounts() settings = queries.get_app_interface_settings() aws = AWSApi(thread_pool_size, accounts, settings=settings) deleted_keys = get_deleted_keys(accounts) existing_keys = aws.get_users_keys() aws_support_cases = aws.get_support_cases() keys_to_delete_from_cases = get_keys_to_delete(aws_support_cases) keys_to_delete = [] for ktd in keys_to_delete_from_cases: ktd_account = ktd['account'] ktd_key = ktd['key'] account_deleted_keys = deleted_keys.get(ktd_account) if account_deleted_keys and ktd_key in account_deleted_keys: continue account_existing_keys = existing_keys.get(ktd_account) if account_existing_keys: keys_only = \ itertools.chain.from_iterable(account_existing_keys.values()) if ktd_key not in keys_only: continue keys_to_delete.append(ktd) act(dry_run, gitlab_project_id, accounts, keys_to_delete)
def clusters_egress_ips(ctx): settings = queries.get_app_interface_settings() clusters = queries.get_clusters() clusters = [ c for c in clusters if c.get('ocm') is not None and c.get('awsInfrastructureAccess') is not None ] ocm_map = OCMMap(clusters=clusters, settings=settings) results = [] for cluster in clusters: cluster_name = cluster['name'] account = tfvpc.aws_account_from_infrastructure_access( cluster, 'network-mgmt', ocm_map) aws_api = AWSApi(1, [account], settings=settings) egress_ips = \ aws_api.get_cluster_nat_gateways_egress_ips(account) item = { 'cluster': cluster_name, 'egress_ips': ', '.join(sorted(egress_ips)) } results.append(item) columns = ['cluster', 'egress_ips'] print_output(ctx.obj['options'], results, columns)
def clusters_egress_ips(ctx): settings = queries.get_app_interface_settings() clusters = queries.get_clusters() clusters = [c for c in clusters if c.get('ocm') is not None and c.get('awsInfrastructureManagementAccounts') is not None] ocm_map = OCMMap(clusters=clusters, settings=settings) results = [] for cluster in clusters: cluster_name = cluster['name'] management_account = tfvpc._get_default_management_account(cluster) account = tfvpc._build_infrastructure_assume_role( management_account, cluster, ocm_map.get(cluster_name) ) account['resourcesDefaultRegion'] = \ management_account['resourcesDefaultRegion'] aws_api = AWSApi(1, [account], settings=settings) egress_ips = \ aws_api.get_cluster_nat_gateways_egress_ips(account) item = { 'cluster': cluster_name, 'egress_ips': ', '.join(sorted(egress_ips)) } results.append(item) columns = ['cluster', 'egress_ips'] print_output(ctx.obj['options'], results, columns)
def run(dry_run, gitlab_project_id=None, thread_pool_size=10): accounts = queries.get_aws_accounts() settings = queries.get_app_interface_settings() aws = AWSApi(thread_pool_size, accounts, settings=settings) existing_keys = aws.get_users_keys() existing_keys_list = [ key for user_key in existing_keys.values() for keys in user_key.values() for key in keys ] logging.info("found {} existing keys".format(len(existing_keys_list))) app_int_github_repos = queries.get_repos(server="https://github.com") all_repos = get_all_repos_to_scan(app_int_github_repos) logging.info("about to scan {} repos".format(len(all_repos))) results = threaded.run( git_secrets.scan_history, all_repos, thread_pool_size, existing_keys=existing_keys_list, ) all_leaked_keys = [key for keys in results for key in keys] deleted_keys = aws_sos.get_deleted_keys(accounts) keys_to_delete = [ { "account": account, "key": key } for key in all_leaked_keys for account, user_keys in existing_keys.items() if key in [uk for uks in user_keys.values() for uk in uks] and key not in deleted_keys[account] ] aws_sos.act(dry_run, gitlab_project_id, accounts, keys_to_delete)
def run(dry_run): accounts = queries.get_aws_accounts(reset_passwords=True) settings = queries.get_app_interface_settings() state = State(integration=QONTRACT_INTEGRATION, accounts=accounts, settings=settings) for a in accounts: aws_api = None account_name = a['name'] reset_passwords = a.get('resetPasswords') if not reset_passwords: continue for r in reset_passwords: user_name = r['user']['org_username'] request_id = r['requestId'] state_key = f"{account_name}/{user_name}/{request_id}" if state.exists(state_key): continue logging.info(['reset_password', account_name, user_name]) if dry_run: continue if aws_api is None: aws_api = AWSApi(1, [a], settings=settings) aws_api.reset_password(account_name, user_name) state.add(state_key)
def __init__(self, accounts, settings=None): queue_url = os.environ['gitlab_pr_submitter_queue_url'] account = self.get_queue_account(accounts, queue_url) aws_api = AWSApi(1, accounts, settings=settings) session = aws_api.get_session(account) self.sqs = session.client('sqs') self.queue_url = queue_url
def __init__(self, accounts, settings=None): queue_url = os.environ["gitlab_pr_submitter_queue_url"] account = self.get_queue_account(accounts, queue_url) accounts = [a for a in accounts if a["name"] == account] aws_api = AWSApi(1, accounts, settings=settings, init_users=False) session = aws_api.get_session(account) self.sqs = session.client("sqs") self.queue_url = queue_url
def __init__(self, integration, accounts, settings=None): """Initiates S3 client from AWSApi.""" self.state_path = f"state/{integration}" self.bucket = os.environ['APP_INTERFACE_STATE_BUCKET'] account = os.environ['APP_INTERFACE_STATE_BUCKET_ACCOUNT'] accounts = [a for a in accounts if a['name'] == account] aws_api = AWSApi(1, accounts, settings=settings) session = aws_api.get_session(account) self.client = session.client('s3')
def run(dry_run, thread_pool_size=10, disable_service_account_keys=False, defer=None): accounts = queries.get_aws_accounts() settings = queries.get_app_interface_settings() aws = AWSApi(thread_pool_size, accounts, settings=settings) keys_to_delete = get_keys_to_delete(accounts) working_dirs = init_tf_working_dirs(accounts, thread_pool_size, settings) defer(lambda: cleanup(working_dirs)) error = aws.delete_keys(dry_run, keys_to_delete, working_dirs, disable_service_account_keys) if error: sys.exit(1)
def run(dry_run): accounts = queries.get_aws_accounts(sharing=True) sharing_accounts = filter_accounts(accounts) settings = queries.get_app_interface_settings() aws_api = AWSApi(1, sharing_accounts, settings=settings, init_users=False) for src_account in sharing_accounts: sharing = src_account.get("sharing") if not sharing: continue for share in sharing: if share["provider"] != "ami": continue dst_account = share["account"] regex = share["regex"] region = get_region(share, src_account, dst_account) src_amis = aws_api.get_amis_details(src_account, src_account, regex, region) dst_amis = aws_api.get_amis_details(dst_account, src_account, regex, region) for src_ami in src_amis: src_ami_id = src_ami["image_id"] found_dst_amis = [d for d in dst_amis if d["image_id"] == src_ami_id] if not found_dst_amis: logging.info( [ "share_ami", src_account["name"], dst_account["name"], src_ami_id, ] ) if not dry_run: aws_api.share_ami( src_account, dst_account["uid"], src_ami_id, region ) # we assume an unshared ami does not have tags found_dst_amis = [{"image_id": src_ami_id, "tags": []}] dst_ami = found_dst_amis[0] dst_ami_id = dst_ami["image_id"] dst_ami_tags = dst_ami["tags"] if MANAGED_TAG not in dst_ami_tags: logging.info( ["tag_shared_ami", dst_account["name"], dst_ami_id, MANAGED_TAG] ) if not dry_run: aws_api.create_tag(dst_account, dst_ami_id, MANAGED_TAG) src_ami_tags = src_ami["tags"] for src_tag in src_ami_tags: if src_tag not in dst_ami_tags: logging.info( ["tag_shared_ami", dst_account["name"], dst_ami_id, src_tag] ) if not dry_run: aws_api.create_tag(dst_account, dst_ami_id, src_tag)
def run(dry_run, thread_pool_size=10, io_dir="throughput/"): accounts = [a for a in queries.get_aws_accounts() if a.get("garbageCollection")] settings = queries.get_app_interface_settings() aws = AWSApi(thread_pool_size, accounts, settings=settings) if dry_run: aws.simulate_deleted_users(io_dir) aws.map_resources() aws.delete_resources_without_owner(dry_run)
def build_desired_state_vpc_single_cluster(cluster_info, ocm: OCM, awsapi: AWSApi): desired_state = [] peering_info = cluster_info['peering'] peer_connections = peering_info['connections'] cluster = cluster_info['name'] for peer_connection in peer_connections: # We only care about account-vpc peering providers peer_connection_provider = peer_connection['provider'] if not peer_connection_provider == 'account-vpc': continue # requester is the cluster's AWS account requester = { 'cidr_block': cluster_info['network']['vpc'], 'region': cluster_info['spec']['region'] } connection_name = peer_connection['name'] peer_vpc = peer_connection['vpc'] # accepter is the peered AWS account accepter = { 'vpc_id': peer_vpc['vpc_id'], 'cidr_block': peer_vpc['cidr_block'], 'region': peer_vpc['region'] } account = peer_vpc['account'] # assume_role is the role to assume to provision the peering # connection request, through the accepter AWS account. account['assume_role'] = \ ocm.get_aws_infrastructure_access_terraform_assume_role( cluster, peer_vpc['account']['uid'], peer_vpc['account']['terraformUsername'] ) account['assume_region'] = requester['region'] account['assume_cidr'] = requester['cidr_block'] requester_vpc_id, requester_route_table_ids, _ = \ awsapi.get_cluster_vpc_details( account, route_tables=peer_connection.get('manageRoutes') ) if requester_vpc_id is None: raise BadTerraformPeeringState( f'[{cluster} could not find VPC ID for cluster') requester['vpc_id'] = requester_vpc_id requester['route_table_ids'] = requester_route_table_ids requester['account'] = account accepter['account'] = account item = { 'connection_provider': peer_connection_provider, 'connection_name': connection_name, 'requester': requester, 'accepter': accepter, 'deleted': peer_connection.get('delete', False) } desired_state.append(item) return desired_state
def aws_api(accounts, mocker): mock_secret_reader = mocker.patch("reconcile.utils.aws_api.SecretReader", autospec=True) mock_secret_reader.return_value.read_all.return_value = { "aws_access_key_id": "key_id", "aws_secret_access_key": "access_key", "region": "tf_state_bucket_region", } return AWSApi(1, accounts, init_users=False)
def run(dry_run, gitlab_project_id=None, thread_pool_size=10, enable_deletion=False): accounts = queries.get_aws_accounts() settings = queries.get_app_interface_settings() aws = AWSApi(thread_pool_size, accounts, settings=settings) deleted_keys = get_deleted_keys(accounts) existing_keys = aws.get_users_keys() aws_support_cases = aws.get_support_cases() keys_to_delete_from_cases = get_keys_to_delete(aws_support_cases) keys_to_delete = [ ktd for ktd in keys_to_delete_from_cases if deleted_keys.get(ktd['account']) is not None and existing_keys.get( ktd['account']) is not None and ktd['key'] not in deleted_keys[ ktd['account']] and ktd['key'] in existing_keys[ktd['account']] ] act(dry_run, gitlab_project_id, accounts, keys_to_delete)
def __init__(self, instance, dry_run): self.dry_run = dry_run self.instance = instance self.settings = queries.get_app_interface_settings() self.secret_reader = SecretReader(settings=self.settings) self.skopeo_cli = Skopeo(dry_run) self.error = False identifier = instance["identifier"] account = instance["account"] region = instance.get("region") self.aws_cli = AWSApi( thread_pool_size=1, accounts=[self._get_aws_account_info(account)], settings=self.settings, init_ecr_auth_tokens=True, ) self.aws_cli.map_ecr_resources() self.ecr_uri = self._get_image_uri( account=account, repository=identifier, ) if self.ecr_uri is None: self.error = True LOG.error(f"Could not find the ECR repository {identifier}") self.ecr_username, self.ecr_password = self._get_ecr_creds( account=account, region=region, ) self.ecr_auth = f"{self.ecr_username}:{self.ecr_password}" self.image_username = None self.image_password = None self.image_auth = None pull_secret = self.instance["mirror"]["pullCredentials"] if pull_secret is not None: raw_data = self.secret_reader.read_all(pull_secret) self.image_username = raw_data["user"] self.image_password = raw_data["token"] self.image_auth = f"{self.image_username}:{self.image_password}"
def run(dry_run, vault_output_path=""): accounts = [a for a in queries.get_aws_accounts() if a.get("ecrs")] settings = queries.get_app_interface_settings() aws = AWSApi(1, accounts, settings=settings, init_ecr_auth_tokens=True) for account, data in aws.auth_tokens.items(): dockercfg_secret_data = construct_dockercfg_secret_data(data) basic_auth_secret_data = construct_basic_auth_secret_data(data) write_output_to_vault(dry_run, vault_output_path, account, dockercfg_secret_data, "dockercfg") write_output_to_vault(dry_run, vault_output_path, account, basic_auth_secret_data, "basic-auth")
def __init__(self, integration: str, accounts: Iterable[Mapping[str, Any]], settings: Optional[Mapping[str, Any]] = None) -> None: """Initiates S3 client from AWSApi.""" self.state_path = f'state/{integration}' if integration else 'state' self.bucket = os.environ['APP_INTERFACE_STATE_BUCKET'] account = os.environ['APP_INTERFACE_STATE_BUCKET_ACCOUNT'] accounts = [a for a in accounts if a['name'] == account] aws_api = AWSApi(1, accounts, settings=settings, init_users=False) session = aws_api.get_session(account) self.client = session.client('s3') # check if the bucket exists try: self.client.head_bucket(Bucket=self.bucket) except ClientError as details: raise StateInaccessibleException( f"Bucket {self.bucket} is not accessible - {str(details)}")
def run( dry_run, thread_pool_size=10, disable_service_account_keys=False, account_name=None, defer=None, ): accounts = filter_accounts(queries.get_aws_accounts(), account_name) if not accounts: logging.debug("nothing to do here") # using return because terraform-resources # may be the calling entity, and has more to do return settings = queries.get_app_interface_settings() aws = AWSApi(thread_pool_size, accounts, settings=settings) keys_to_delete = get_keys_to_delete(accounts) working_dirs = init_tf_working_dirs(accounts, thread_pool_size, settings) defer(lambda: cleanup(working_dirs)) error = aws.delete_keys(dry_run, keys_to_delete, working_dirs, disable_service_account_keys) if error: sys.exit(1)
def setup(print_to_file, thread_pool_size: int) \ -> tuple[list[dict[str, Any]], dict[str, str], bool, AWSApi]: gqlapi = gql.get_api() accounts = queries.get_aws_accounts() settings = queries.get_app_interface_settings() roles = expiration.filter(gqlapi.query(TF_QUERY)['roles']) tf_roles = [ r for r in roles if r['aws_groups'] is not None or r['user_policies'] is not None ] ts = Terrascript(QONTRACT_INTEGRATION, QONTRACT_TF_PREFIX, thread_pool_size, accounts, settings=settings) err = ts.populate_users(tf_roles) working_dirs = ts.dump(print_to_file) aws_api = AWSApi(1, accounts, settings=settings, init_users=False) return accounts, working_dirs, err, aws_api
def setup(dry_run, print_to_file, thread_pool_size, internal, use_jump_host, account_name, extra_labels): gqlapi = gql.get_api() accounts = queries.get_aws_accounts() if account_name: accounts = [n for n in accounts if n['name'] == account_name] if not accounts: raise ValueError(f"aws account {account_name} is not found") extra_labels['shard_key'] = account_name settings = queries.get_app_interface_settings() namespaces = gqlapi.query(TF_NAMESPACES_QUERY)['namespaces'] tf_namespaces = filter_tf_namespaces(namespaces, account_name) ri, oc_map = fetch_current_state(dry_run, tf_namespaces, thread_pool_size, internal, use_jump_host, account_name) ts, working_dirs = init_working_dirs(accounts, thread_pool_size, settings=settings) aws_api = AWSApi(1, accounts, settings=settings, init_users=False) tf = Terraform(QONTRACT_INTEGRATION, QONTRACT_INTEGRATION_VERSION, QONTRACT_TF_PREFIX, accounts, working_dirs, thread_pool_size, aws_api) existing_secrets = tf.get_terraform_output_secrets() clusters = [c for c in queries.get_clusters() if c.get('ocm') is not None] if clusters: ocm_map = OCMMap(clusters=clusters, integration=QONTRACT_INTEGRATION, settings=settings) else: ocm_map = None ts.populate_resources(tf_namespaces, existing_secrets, account_name, ocm_map=ocm_map) ts.dump(print_to_file, existing_dirs=working_dirs) return ri, oc_map, tf, tf_namespaces
def build_desired_state(zones: Iterable[Mapping], all_accounts: Iterable[Mapping], settings: Mapping) -> list[dict]: """ Build the desired state from the app-interface resources :param zones: List of zone resources to build state for :type zones: list of dict :return: State :rtype: list of dict """ desired_state = [] for zone in zones: account = zone["account"] account_name = account["name"] zone_name = zone["name"] zone_values = { "name": zone_name, "account_name": account_name, "records": [] } # a vpc will be referenced for a zone to be considered private vpc = zone.get("vpc") if vpc: zone_values["vpc"] = { "vpc_id": vpc["vpc_id"], "vpc_region": vpc["region"] } # Check if we have unmanaged_record_names (urn) and compile them # all as regular expressions urn_compiled = [] for urn in zone.get("unmanaged_record_names", []): urn_compiled.append(re.compile(urn)) for record in zone["records"]: record_name = record["name"] record_type = record["type"] # Check if this record should be ignored # as per 'unmanaged_record_names' ignored = False for regex in urn_compiled: if regex.fullmatch(record["name"]): logging.debug(f"{zone_name}: excluding unmanaged " f"record {record_name} because it matched " f"unmanaged_record_names pattern " f"'{regex.pattern}'") ignored = True if ignored: continue # We use the record object as-is from the list as the terraform # data to apply. This makes things simpler and map 1-to-1 with # Terraform's capabilities. As such we need to remove (pop) some of # the keys we use for our own features # Process '_target_cluster' target_cluster = record.pop("_target_cluster", None) if target_cluster: target_cluster_elb = target_cluster["elbFQDN"] if target_cluster_elb is None or target_cluster_elb == "": msg = (f"{zone_name}: field `_target_cluster` for record " f"{record_name} of type {record_type} points to a " f"cluster that has an empty elbFQDN field.") logging.error(msg) sys.exit(ExitCodes.ERROR) record_values = [] if record_type == "A": record_values = dnsutils.get_a_records(target_cluster_elb) elif record_type == "CNAME": record_values = [target_cluster_elb] else: msg = (f"{zone_name}: field `_target_cluster` found " f"for record {record_name} of type {record_type}. " f"The use of _target_cluster on this record type " f"is not supported by the integration.") logging.error(msg) sys.exit(ExitCodes.ERROR) if not record_values: msg = (f"{zone_name}: field `_target_cluster` found " f"for record {record_name} of type {record_type} " f"has no values! (invalid elb FQDN?)") logging.error(msg) sys.exit(ExitCodes.ERROR) msg = (f"{zone_name}: field `_target_cluster` found " f"for record {record_name} of type {record_type}. " f"Value will be set to {record_values}") logging.debug(msg) record["records"] = record_values # Process '_target_namespace_zone' target_namespace_zone = record.pop("_target_namespace_zone", None) if target_namespace_zone: tf_resources = target_namespace_zone["namespace"][ "terraformResources"] tf_zone_name = target_namespace_zone["name"] tf_zone_resources = [ tfr for tfr in tf_resources if tfr["provider"] == "route53-zone" and tfr["name"] == tf_zone_name ] if not tf_zone_resources: logging.error( f"{zone_name}: field `_target_namespace_zone` found " f"for record {record_name}, but target zone not found: " f"{tf_zone_name}") sys.exit(ExitCodes.ERROR) tf_zone_resource = tf_zone_resources[0] tf_zone_account_name = tf_zone_resource["account"] zone_account = [ a for a in all_accounts if a["name"] == tf_zone_account_name ][0] awsapi = AWSApi(1, [zone_account], settings=settings, init_users=False) tf_zone_region = (tf_zone_resource.get("region") or zone_account["resourcesDefaultRegion"]) tf_zone_ns_records = awsapi.get_route53_zone_ns_records( tf_zone_account_name, tf_zone_name, tf_zone_region) if not tf_zone_ns_records: logging.warning( f"{zone_name}: field `_target_namespace_zone` found " f"for record {record_name}, but target zone not found (yet): " f"{tf_zone_name}") continue logging.debug( f"{zone_name}: field `_target_namespace_zone` found " f"for record {record_name}, Values are: " f"{tf_zone_ns_records}") record["records"] = tf_zone_ns_records # Process '_healthcheck' healthcheck = record.pop("_healthcheck", None) if healthcheck: logging.debug(f"{zone_name}: field `_healthcheck` found " f"for record {record_name}. Values are: " f"{healthcheck}") record["healthcheck"] = healthcheck zone_values["records"].append(record) desired_state.append(zone_values) return desired_state
def build_desired_state_cluster(clusters, ocm_map, settings): """ Fetch state for VPC peerings between two OCM clusters """ desired_state = [] error = False for cluster_info in clusters: cluster_name = cluster_info['name'] # Find an aws account with the "network-mgmt" access level on the # requester cluster and use that as the account for the requester req_aws = aws_account_from_infrastructure_access( cluster_info, 'network-mgmt', ocm_map) if not req_aws: msg = f"could not find an AWS account with the " \ f"'network-mgmt' access level on the cluster {cluster_name}" logging.error(msg) error = True continue req_aws['assume_region'] = cluster_info['spec']['region'] req_aws['assume_cidr'] = cluster_info['network']['vpc'] peering_info = cluster_info['peering'] peer_connections = peering_info['connections'] for peer_connection in peer_connections: # We only care about cluster-vpc-requester peering providers peer_connection_provider = peer_connection['provider'] if not peer_connection_provider == 'cluster-vpc-requester': continue peer_connection_name = peer_connection['name'] peer_cluster = peer_connection['cluster'] peer_cluster_name = peer_cluster['name'] requester_manage_routes = peer_connection.get('manageRoutes') # Ensure we have a matching peering connection peer_info = find_matching_peering(cluster_info, peer_connection, peer_cluster, 'cluster-vpc-accepter') if not peer_info: msg = f"could not find a matching peering connection for " \ f"cluster {cluster_name}, " \ f"connection {peer_connection_name}" logging.error(msg) error = True continue accepter_manage_routes = peer_info.get('manageRoutes') aws_api = AWSApi(1, [req_aws], settings=settings) requester_vpc_id, requester_route_table_ids, _ = \ aws_api.get_cluster_vpc_details( req_aws, route_tables=requester_manage_routes ) if requester_vpc_id is None: msg = f'[{cluster_name} could not find VPC ID for cluster' logging.error(msg) error = True continue requester = { 'cidr_block': cluster_info['network']['vpc'], 'region': cluster_info['spec']['region'], 'vpc_id': requester_vpc_id, 'route_table_ids': requester_route_table_ids, 'account': req_aws } # Find an aws account with the "network-mgmt" access level on the # peer cluster and use that as the account for the accepter acc_aws = aws_account_from_infrastructure_access( peer_cluster, 'network-mgmt', ocm_map) if not acc_aws: msg = "could not find an AWS account with the " \ "'network-mgmt' access level on the cluster" logging.error(msg) error = True continue acc_aws['assume_region'] = peer_cluster['spec']['region'] acc_aws['assume_cidr'] = peer_cluster['network']['vpc'] aws_api = AWSApi(1, [acc_aws], settings=settings) accepter_vpc_id, accepter_route_table_ids, _ = \ aws_api.get_cluster_vpc_details( acc_aws, route_tables=accepter_manage_routes ) if accepter_vpc_id is None: msg = f'[{peer_cluster_name} could not find VPC ID for cluster' logging.error(msg) error = True continue requester['peer_owner_id'] = acc_aws['assume_role'].split(':')[4] accepter = { 'cidr_block': peer_cluster['network']['vpc'], 'region': peer_cluster['spec']['region'], 'vpc_id': accepter_vpc_id, 'route_table_ids': accepter_route_table_ids, 'account': acc_aws } item = { 'connection_provider': peer_connection_provider, 'connection_name': peer_connection_name, 'requester': requester, 'accepter': accepter, 'deleted': peer_connection.get('delete', False) } desired_state.append(item) return desired_state, error
def build_desired_state_tgw_attachments(clusters, ocm_map, settings): """ Fetch state for TGW attachments between a cluster and all TGWs in an account in the same region as the cluster """ desired_state = [] error = False for cluster_info in clusters: cluster = cluster_info['name'] ocm = ocm_map.get(cluster) peering_info = cluster_info['peering'] peer_connections = peering_info['connections'] for peer_connection in peer_connections: # We only care about account-tgw peering providers peer_connection_provider = peer_connection['provider'] if not peer_connection_provider == 'account-tgw': continue # accepter is the cluster's AWS account cluster_region = cluster_info['spec']['region'] cluster_cidr_block = cluster_info['network']['vpc'] accepter = { 'cidr_block': cluster_cidr_block, 'region': cluster_region } account = peer_connection['account'] # assume_role is the role to assume to provision the # peering connection request, through the accepter AWS account. account['assume_role'] = \ ocm.get_aws_infrastructure_access_terraform_assume_role( cluster, account['uid'], account['terraformUsername'] ) account['assume_region'] = accepter['region'] account['assume_cidr'] = accepter['cidr_block'] aws_api = AWSApi(1, [account], settings=settings) accepter_vpc_id, accepter_route_table_ids, \ accepter_subnets_id_az = \ aws_api.get_cluster_vpc_details( account, route_tables=peer_connection.get('manageRoutes'), subnets=True, ) if accepter_vpc_id is None: logging.error(f'[{cluster} could not find VPC ID for cluster') error = True continue accepter['vpc_id'] = accepter_vpc_id accepter['route_table_ids'] = accepter_route_table_ids accepter['subnets_id_az'] = accepter_subnets_id_az accepter['account'] = account account_tgws = \ aws_api.get_tgws_details( account, cluster_region, cluster_cidr_block, tags=json.loads(peer_connection.get('tags') or {}), route_tables=peer_connection.get('manageRoutes'), security_groups=peer_connection.get( 'manageSecurityGroups'), ) for tgw in account_tgws: tgw_id = tgw['tgw_id'] connection_name = \ f"{peer_connection['name']}_" + \ f"{account['name']}-{tgw_id}" requester = { 'tgw_id': tgw_id, 'tgw_arn': tgw['tgw_arn'], 'region': tgw['region'], 'routes': tgw.get('routes'), 'rules': tgw.get('rules'), 'cidr_block': peer_connection.get('cidrBlock'), 'account': account, } item = { 'connection_provider': peer_connection_provider, 'connection_name': connection_name, 'requester': requester, 'accepter': accepter, 'deleted': peer_connection.get('delete', False) } desired_state.append(item) return desired_state, error
def build_desired_state_vpc(clusters, ocm_map, settings): """ Fetch state for VPC peerings between a cluster and a VPC (account) """ desired_state = [] error = False for cluster_info in clusters: cluster = cluster_info['name'] ocm = ocm_map.get(cluster) peering_info = cluster_info['peering'] peer_connections = peering_info['connections'] for peer_connection in peer_connections: # We only care about account-vpc peering providers peer_connection_provider = peer_connection['provider'] if not peer_connection_provider == 'account-vpc': continue # requester is the cluster's AWS account requester = { 'cidr_block': cluster_info['network']['vpc'], 'region': cluster_info['spec']['region'] } connection_name = peer_connection['name'] peer_vpc = peer_connection['vpc'] # accepter is the peered AWS account accepter = { 'vpc_id': peer_vpc['vpc_id'], 'cidr_block': peer_vpc['cidr_block'], 'region': peer_vpc['region'] } account = peer_vpc['account'] # assume_role is the role to assume to provision the # peering connection request, through the accepter AWS account. account['assume_role'] = \ ocm.get_aws_infrastructure_access_terraform_assume_role( cluster, peer_vpc['account']['uid'], peer_vpc['account']['terraformUsername'] ) account['assume_region'] = requester['region'] account['assume_cidr'] = requester['cidr_block'] aws_api = AWSApi(1, [account], settings=settings) requester_vpc_id, requester_route_table_ids, _ = \ aws_api.get_cluster_vpc_details( account, route_tables=peer_connection.get('manageRoutes') ) if requester_vpc_id is None: logging.error(f'[{cluster} could not find VPC ID for cluster') error = True continue requester['vpc_id'] = requester_vpc_id requester['route_table_ids'] = requester_route_table_ids requester['account'] = account accepter['account'] = account item = { 'connection_provider': peer_connection_provider, 'connection_name': connection_name, 'requester': requester, 'accepter': accepter, 'deleted': peer_connection.get('delete', False) } desired_state.append(item) return desired_state, error
def run(dry_run, print_to_file=None, enable_deletion=False, thread_pool_size=10, defer=None): settings = queries.get_app_interface_settings() clusters = [ c for c in queries.get_clusters() if c.get("peering") is not None ] with_ocm = any(c.get("ocm") for c in clusters) if with_ocm: ocm_map = OCMMap(clusters=clusters, integration=QONTRACT_INTEGRATION, settings=settings) else: # this is a case for an OCP cluster which is not provisioned # through OCM. it is expected that an 'assume_role' is provided # on the tgw defition in the cluster file. ocm_map = {} accounts = queries.get_aws_accounts() awsapi = AWSApi(1, accounts, settings=settings, init_users=False) # Fetch desired state for cluster-to-vpc(account) VPCs desired_state, err = build_desired_state_tgw_attachments( clusters, ocm_map, awsapi) if err: sys.exit(1) # check there are no repeated vpc connection names connection_names = [c["connection_name"] for c in desired_state] if len(set(connection_names)) != len(connection_names): logging.error("duplicate vpc connection names found") sys.exit(1) participating_accounts = [ item["requester"]["account"] for item in desired_state ] participating_accounts += [ item["accepter"]["account"] for item in desired_state ] participating_account_names = [a["name"] for a in participating_accounts] accounts = [ a for a in queries.get_aws_accounts() if a["name"] in participating_account_names ] ts = Terrascript(QONTRACT_INTEGRATION, "", thread_pool_size, accounts, settings=settings) ts.populate_additional_providers(participating_accounts) ts.populate_tgw_attachments(desired_state) working_dirs = ts.dump(print_to_file=print_to_file) aws_api = AWSApi(1, accounts, settings=settings, init_users=False) if print_to_file: sys.exit() tf = Terraform( QONTRACT_INTEGRATION, QONTRACT_INTEGRATION_VERSION, "", accounts, working_dirs, thread_pool_size, aws_api, ) if tf is None: sys.exit(1) defer(tf.cleanup) disabled_deletions_detected, err = tf.plan(enable_deletion) if err: sys.exit(1) if disabled_deletions_detected: sys.exit(1) if dry_run: return err = tf.apply() if err: sys.exit(1)
def build_desired_state_vpc_single_cluster(cluster_info, ocm: Optional[OCM], awsapi: AWSApi): desired_state = [] peering_info = cluster_info['peering'] peer_connections = peering_info['connections'] cluster = cluster_info['name'] for peer_connection in peer_connections: # We only care about account-vpc peering providers peer_connection_provider = peer_connection['provider'] if not peer_connection_provider == 'account-vpc': continue # requester is the cluster's AWS account requester = { 'cidr_block': cluster_info['network']['vpc'], 'region': cluster_info['spec']['region'] } connection_name = peer_connection['name'] peer_vpc = peer_connection['vpc'] # accepter is the peered AWS account accepter = { 'vpc_id': peer_vpc['vpc_id'], 'cidr_block': peer_vpc['cidr_block'], 'region': peer_vpc['region'] } account = peer_vpc['account'] # assume_role is the role to assume to provision the peering # connection request, through the accepter AWS account. provided_assume_role = peer_connection.get('assumeRole') # if an assume_role is provided, it means we don't need # to get the information from OCM. it likely means that # there is no OCM at all. if provided_assume_role: account['assume_role'] = provided_assume_role elif ocm is not None: account['assume_role'] = \ ocm.get_aws_infrastructure_access_terraform_assume_role( cluster, peer_vpc['account']['uid'], peer_vpc['account']['terraformUsername'] ) else: raise KeyError( f'[{cluster}] peering connection ' f'{connection_name} must either specify assumeRole ' 'or ocm should be defined to obtain role to assume') account['assume_region'] = requester['region'] account['assume_cidr'] = requester['cidr_block'] requester_vpc_id, requester_route_table_ids, _ = \ awsapi.get_cluster_vpc_details( account, route_tables=peer_connection.get('manageRoutes') ) if requester_vpc_id is None: raise BadTerraformPeeringState( f'[{cluster}] could not find VPC ID for cluster' ) requester['vpc_id'] = requester_vpc_id requester['route_table_ids'] = requester_route_table_ids requester['account'] = account accepter['account'] = account item = { 'connection_provider': peer_connection_provider, 'connection_name': connection_name, 'requester': requester, 'accepter': accepter, 'deleted': peer_connection.get('delete', False) } desired_state.append(item) return desired_state
def build_desired_state_single_cluster(cluster_info, ocm: OCM, awsapi: AWSApi): cluster_name = cluster_info['name'] peerings = [] peering_info = cluster_info['peering'] peer_connections = peering_info['connections'] for peer_connection in peer_connections: # We only care about cluster-vpc-requester peering providers peer_connection_provider = peer_connection['provider'] if peer_connection_provider != 'cluster-vpc-requester': continue peer_connection_name = peer_connection['name'] peer_cluster = peer_connection['cluster'] peer_cluster_name = peer_cluster['name'] requester_manage_routes = peer_connection.get('manageRoutes') # Ensure we have a matching peering connection peer_info = find_matching_peering(cluster_info, peer_cluster, 'cluster-vpc-accepter') if not peer_info: raise BadTerraformPeeringState( "[no_matching_peering] could not find a matching peering " f"connection for cluster {cluster_name}, connection " f"{peer_connection_name}" ) accepter_manage_routes = peer_info.get('manageRoutes') req_aws, acc_aws = aws_assume_roles_for_cluster_vpc_peering( cluster_info, peer_info, peer_cluster, ocm ) requester_vpc_id, requester_route_table_ids, _ = \ awsapi.get_cluster_vpc_details( req_aws, route_tables=requester_manage_routes ) if requester_vpc_id is None: raise BadTerraformPeeringState( f'[{cluster_name}] could not find VPC ID for cluster' ) requester = { 'cidr_block': cluster_info['network']['vpc'], 'region': cluster_info['spec']['region'], 'vpc_id': requester_vpc_id, 'route_table_ids': requester_route_table_ids, 'account': req_aws } accepter_vpc_id, accepter_route_table_ids, _ = \ awsapi.get_cluster_vpc_details( acc_aws, route_tables=accepter_manage_routes ) if accepter_vpc_id is None: raise BadTerraformPeeringState( f'[{peer_cluster_name}] could not find VPC ID for cluster' ) requester['peer_owner_id'] = acc_aws['assume_role'].split(':')[4] accepter = { 'cidr_block': peer_cluster['network']['vpc'], 'region': peer_cluster['spec']['region'], 'vpc_id': accepter_vpc_id, 'route_table_ids': accepter_route_table_ids, 'account': acc_aws } item = { 'connection_provider': peer_connection_provider, 'connection_name': peer_connection_name, 'requester': requester, 'accepter': accepter, 'deleted': peer_connection.get('delete', False) } peerings.append(item) return peerings
def __init__(self, dry_run, instance): self.dry_run = dry_run self.settings = queries.get_app_interface_settings() cluster_info = instance['hiveCluster'] hive_cluster = instance['hiveCluster']['name'] # Getting the OCM Client for the hive cluster ocm_map = OCMMap(clusters=[cluster_info], integration=QONTRACT_INTEGRATION, settings=self.settings) self.ocm_cli = ocm_map.get(hive_cluster) if not self.ocm_cli: raise OcpReleaseMirrorError(f"Can't create ocm client for " f"cluster {hive_cluster}") # Getting the OC Client for the hive cluster oc_map = OC_Map(clusters=[cluster_info], integration=QONTRACT_INTEGRATION, settings=self.settings) self.oc_cli = oc_map.get(hive_cluster) if not self.oc_cli: raise OcpReleaseMirrorError(f"Can't create oc client for " f"cluster {hive_cluster}") namespace = instance['ecrResourcesNamespace'] ocp_release_identifier = instance['ocpReleaseEcrIdentifier'] ocp_art_dev_identifier = instance['ocpArtDevEcrIdentifier'] ocp_release_info = self._get_tf_resource_info(namespace, ocp_release_identifier) if ocp_release_info is None: raise OcpReleaseMirrorError(f"Could not find rds " f"identifier " f"{ocp_release_identifier} in " f"namespace {namespace['name']}") ocp_art_dev_info = self._get_tf_resource_info(namespace, ocp_art_dev_identifier) if ocp_art_dev_info is None: raise OcpReleaseMirrorError(f"Could not find rds identifier" f" {ocp_art_dev_identifier} in" f"namespace {namespace['name']}") # Getting the AWS Client for the accounts aws_accounts = [ self._get_aws_account_info(account=ocp_release_info['account']), self._get_aws_account_info(account=ocp_art_dev_info['account']) ] self.aws_cli = AWSApi(thread_pool_size=1, accounts=aws_accounts, settings=self.settings, init_ecr_auth_tokens=True) self.aws_cli.map_ecr_resources() self.ocp_release_ecr_uri = self._get_image_uri( account=ocp_release_info['account'], repository=ocp_release_identifier) if self.ocp_release_ecr_uri is None: raise OcpReleaseMirrorError(f"Could not find the " f"ECR repository " f"{ocp_release_identifier}") self.ocp_art_dev_ecr_uri = self._get_image_uri( account=ocp_art_dev_info['account'], repository=ocp_art_dev_identifier) if self.ocp_art_dev_ecr_uri is None: raise OcpReleaseMirrorError(f"Could not find the " f"ECR repository " f"{ocp_art_dev_identifier}") # Process all the quayOrgTargets quay_api_store = get_quay_api_store() self.quay_target_orgs = [] for quayTargetOrg in instance['quayTargetOrgs']: org_name = quayTargetOrg['name'] instance_name = quayTargetOrg['instance']['name'] org_key = OrgKey(instance_name, org_name) org_info = quay_api_store[org_key] if not org_info['push_token']: raise OcpReleaseMirrorError( f'{org_key} has no push_token defined.') url = org_info['url'] user = org_info['push_token']['user'] token = org_info['push_token']['token'] self.quay_target_orgs.append({ 'url': url, 'dest_ocp_release': f"{url}/{org_name}/ocp-release", 'dest_ocp_art_dev': f"{url}/{org_name}/ocp-v4.0-art-dev", 'auths': self._build_quay_auths(url, user, token) }) # Getting all the credentials quay_creds = self._get_quay_creds() ocp_release_creds = self._get_ecr_creds( account=ocp_release_info['account'], region=ocp_release_info['region']) ocp_art_dev_creds = self._get_ecr_creds( account=ocp_art_dev_info['account'], region=ocp_art_dev_info['region']) # Creating a single dictionary with all credentials to be used by the # "oc adm release mirror" command self.registry_creds = { 'auths': { **quay_creds['auths'], **ocp_release_creds['auths'], **ocp_art_dev_creds['auths'], } } # Append quay_target_orgs auths to registry_creds for quay_target_org in self.quay_target_orgs: url = quay_target_org['url'] if url in self.registry_creds['auths'].keys(): OcpReleaseMirrorError('Cannot mirror to the same Quay ' f'instance multiple times: {url}') self.registry_creds['auths'].update(quay_target_org['auths']) # Initiate channel groups self.channel_groups = instance['mirrorChannels']
def build_desired_state_tgw_attachments(clusters, ocm_map: OCMMap, awsapi: AWSApi): """ Fetch state for TGW attachments between a cluster and all TGWs in an account in the same region as the cluster """ desired_state = [] error = False for cluster_info in clusters: cluster = cluster_info["name"] ocm = ocm_map.get(cluster) peering_info = cluster_info["peering"] peer_connections = peering_info["connections"] for peer_connection in peer_connections: # We only care about account-tgw peering providers peer_connection_provider = peer_connection["provider"] if not peer_connection_provider == "account-tgw": continue # accepter is the cluster's AWS account cluster_region = cluster_info["spec"]["region"] cluster_cidr_block = cluster_info["network"]["vpc"] accepter = { "cidr_block": cluster_cidr_block, "region": cluster_region } account = peer_connection["account"] # assume_role is the role to assume to provision the # peering connection request, through the accepter AWS account. provided_assume_role = peer_connection.get("assumeRole") # if an assume_role is provided, it means we don't need # to get the information from OCM. it likely means that # there is no OCM at all. if provided_assume_role: account["assume_role"] = provided_assume_role else: account[ "assume_role"] = ocm.get_aws_infrastructure_access_terraform_assume_role( cluster, account["uid"], account["terraformUsername"]) account["assume_region"] = accepter["region"] account["assume_cidr"] = accepter["cidr_block"] ( accepter_vpc_id, accepter_route_table_ids, accepter_subnets_id_az, ) = awsapi.get_cluster_vpc_details( account, route_tables=peer_connection.get("manageRoutes"), subnets=True, ) if accepter_vpc_id is None: logging.error(f"[{cluster} could not find VPC ID for cluster") error = True continue accepter["vpc_id"] = accepter_vpc_id accepter["route_table_ids"] = accepter_route_table_ids accepter["subnets_id_az"] = accepter_subnets_id_az accepter["account"] = account account_tgws = awsapi.get_tgws_details( account, cluster_region, cluster_cidr_block, tags=json.loads(peer_connection.get("tags") or "{}"), route_tables=peer_connection.get("manageRoutes"), security_groups=peer_connection.get("manageSecurityGroups"), ) for tgw in account_tgws: tgw_id = tgw["tgw_id"] connection_name = (f"{peer_connection['name']}_" + f"{account['name']}-{tgw_id}") requester = { "tgw_id": tgw_id, "tgw_arn": tgw["tgw_arn"], "region": tgw["region"], "routes": tgw.get("routes"), "rules": tgw.get("rules"), "cidr_block": peer_connection.get("cidrBlock"), "account": account, } item = { "connection_provider": peer_connection_provider, "connection_name": connection_name, "requester": requester, "accepter": accepter, "deleted": peer_connection.get("delete", False), } desired_state.append(item) return desired_state, error
class OcpReleaseMirror: def __init__(self, dry_run, instance): self.dry_run = dry_run self.settings = queries.get_app_interface_settings() cluster_info = instance['hiveCluster'] hive_cluster = instance['hiveCluster']['name'] # Getting the OCM Client for the hive cluster ocm_map = OCMMap(clusters=[cluster_info], integration=QONTRACT_INTEGRATION, settings=self.settings) self.ocm_cli = ocm_map.get(hive_cluster) if not self.ocm_cli: raise OcpReleaseMirrorError(f"Can't create ocm client for " f"cluster {hive_cluster}") # Getting the OC Client for the hive cluster oc_map = OC_Map(clusters=[cluster_info], integration=QONTRACT_INTEGRATION, settings=self.settings) self.oc_cli = oc_map.get(hive_cluster) if not self.oc_cli: raise OcpReleaseMirrorError(f"Can't create oc client for " f"cluster {hive_cluster}") namespace = instance['ecrResourcesNamespace'] ocp_release_identifier = instance['ocpReleaseEcrIdentifier'] ocp_art_dev_identifier = instance['ocpArtDevEcrIdentifier'] ocp_release_info = self._get_tf_resource_info(namespace, ocp_release_identifier) if ocp_release_info is None: raise OcpReleaseMirrorError(f"Could not find rds " f"identifier " f"{ocp_release_identifier} in " f"namespace {namespace['name']}") ocp_art_dev_info = self._get_tf_resource_info(namespace, ocp_art_dev_identifier) if ocp_art_dev_info is None: raise OcpReleaseMirrorError(f"Could not find rds identifier" f" {ocp_art_dev_identifier} in" f"namespace {namespace['name']}") # Getting the AWS Client for the accounts aws_accounts = [ self._get_aws_account_info(account=ocp_release_info['account']), self._get_aws_account_info(account=ocp_art_dev_info['account']) ] self.aws_cli = AWSApi(thread_pool_size=1, accounts=aws_accounts, settings=self.settings, init_ecr_auth_tokens=True) self.aws_cli.map_ecr_resources() self.ocp_release_ecr_uri = self._get_image_uri( account=ocp_release_info['account'], repository=ocp_release_identifier) if self.ocp_release_ecr_uri is None: raise OcpReleaseMirrorError(f"Could not find the " f"ECR repository " f"{ocp_release_identifier}") self.ocp_art_dev_ecr_uri = self._get_image_uri( account=ocp_art_dev_info['account'], repository=ocp_art_dev_identifier) if self.ocp_art_dev_ecr_uri is None: raise OcpReleaseMirrorError(f"Could not find the " f"ECR repository " f"{ocp_art_dev_identifier}") # Process all the quayOrgTargets quay_api_store = get_quay_api_store() self.quay_target_orgs = [] for quayTargetOrg in instance['quayTargetOrgs']: org_name = quayTargetOrg['name'] instance_name = quayTargetOrg['instance']['name'] org_key = OrgKey(instance_name, org_name) org_info = quay_api_store[org_key] if not org_info['push_token']: raise OcpReleaseMirrorError( f'{org_key} has no push_token defined.') url = org_info['url'] user = org_info['push_token']['user'] token = org_info['push_token']['token'] self.quay_target_orgs.append({ 'url': url, 'dest_ocp_release': f"{url}/{org_name}/ocp-release", 'dest_ocp_art_dev': f"{url}/{org_name}/ocp-v4.0-art-dev", 'auths': self._build_quay_auths(url, user, token) }) # Getting all the credentials quay_creds = self._get_quay_creds() ocp_release_creds = self._get_ecr_creds( account=ocp_release_info['account'], region=ocp_release_info['region']) ocp_art_dev_creds = self._get_ecr_creds( account=ocp_art_dev_info['account'], region=ocp_art_dev_info['region']) # Creating a single dictionary with all credentials to be used by the # "oc adm release mirror" command self.registry_creds = { 'auths': { **quay_creds['auths'], **ocp_release_creds['auths'], **ocp_art_dev_creds['auths'], } } # Append quay_target_orgs auths to registry_creds for quay_target_org in self.quay_target_orgs: url = quay_target_org['url'] if url in self.registry_creds['auths'].keys(): OcpReleaseMirrorError('Cannot mirror to the same Quay ' f'instance multiple times: {url}') self.registry_creds['auths'].update(quay_target_org['auths']) # Initiate channel groups self.channel_groups = instance['mirrorChannels'] def run(self): ocp_releases = self._get_ocp_releases() if not ocp_releases: raise RuntimeError('No OCP Releases found') for ocp_release_info in ocp_releases: ocp_release = ocp_release_info.ocp_release tag = ocp_release_info.tag # mirror to ecr dest_ocp_release = f'{self.ocp_release_ecr_uri}:{tag}' self._run_mirror(ocp_release=ocp_release, dest_ocp_release=dest_ocp_release, dest_ocp_art_dev=self.ocp_art_dev_ecr_uri) # mirror to all quay target orgs for quay_target_org in self.quay_target_orgs: dest_ocp_release = (f'{quay_target_org["dest_ocp_release"]}:' f'{tag}') dest_ocp_art_dev = quay_target_org["dest_ocp_art_dev"] self._run_mirror(ocp_release=ocp_release, dest_ocp_release=dest_ocp_release, dest_ocp_art_dev=dest_ocp_art_dev) def _run_mirror(self, ocp_release, dest_ocp_release, dest_ocp_art_dev): # Checking if the image is already there if self._is_image_there(dest_ocp_release): LOG.debug(f'Image {ocp_release} already in ' f'the mirror. Skipping.') return LOG.info(f'Mirroring {ocp_release} to {dest_ocp_art_dev} ' f'to_release {dest_ocp_release}') if self.dry_run: return # Creating a new, bare, OC client since we don't # want to run this against any cluster or via # a jump host oc_cli = OC('cluster', None, None, local=True) oc_cli.release_mirror(from_release=ocp_release, to=dest_ocp_art_dev, to_release=dest_ocp_release, dockerconfig=self.registry_creds) def _is_image_there(self, image): image_obj = Image(image) for registry, creds in self.registry_creds['auths'].items(): # Getting the credentials for the image_obj if '//' not in registry: registry = '//' + registry registry_obj = urlparse(registry) if registry_obj.netloc != image_obj.registry: continue image_obj.auth = (creds['username'], creds['password']) # Checking if the image is already # in the registry if image_obj: return True return False @staticmethod def _get_aws_account_info(account): for account_info in queries.get_aws_accounts(): if 'name' not in account_info: continue if account_info['name'] != account: continue return account_info def _get_ocp_releases(self): ocp_releases = [] clusterimagesets = self.oc_cli.get_all('ClusterImageSet') for clusterimageset in clusterimagesets['items']: release_image = clusterimageset['spec']['releaseImage'] # There are images in some ClusterImagesSets not coming # from quay.io, e.g.: # registry.svc.ci.openshift.org/ocp/release:4.2.0-0.nightly-2020-11-04-053758 # Let's filter out everything not from quay.io if not release_image.startswith('quay.io'): continue labels = clusterimageset['metadata']['labels'] name = clusterimageset['metadata']['name'] # ClusterImagesSets may be enabled or disabled. # Let's only mirror enabled ones enabled = labels['api.openshift.com/enabled'] if enabled == 'false': continue # ClusterImageSets may be in different channels. channel_group = labels['api.openshift.com/channel-group'] if channel_group not in self.channel_groups: continue ocp_releases.append(OcpReleaseInfo(release_image, name)) return ocp_releases def _get_quay_creds(self): return self.ocm_cli.get_pull_secrets() def _get_ecr_creds(self, account, region): if region is None: region = self.aws_cli.accounts[account]['resourcesDefaultRegion'] auth_token = f'{account}/{region}' data = self.aws_cli.auth_tokens[auth_token] auth_data = data['authorizationData'][0] server = auth_data['proxyEndpoint'] token = auth_data['authorizationToken'] password = base64.b64decode(token).decode('utf-8').split(':')[1] return { 'auths': { server: { 'username': '******', 'password': password, 'email': '*****@*****.**', 'auth': token } } } @staticmethod def _get_tf_resource_info(namespace, identifier): tf_resources = namespace['terraformResources'] for tf_resource in tf_resources: if 'identifier' not in tf_resource: continue if tf_resource['identifier'] != identifier: continue if tf_resource['provider'] != 'ecr': continue return { 'account': tf_resource['account'], 'region': tf_resource.get('region'), } def _get_image_uri(self, account, repository): for repo in self.aws_cli.resources[account]['ecr']: if repo['repositoryName'] == repository: return repo['repositoryUri'] @staticmethod def _build_quay_auths(url, user, token): auth_bytes = bytes(f"{user}:{token}", 'utf-8') auth = base64.b64encode(auth_bytes).decode('utf-8') return { url: { 'username': user, 'password': token, 'email': '', 'auth': auth } }