def clusters_egress_ips(ctx): settings = queries.get_app_interface_settings() clusters = queries.get_clusters() clusters = [c for c in clusters if c.get('ocm') is not None and c.get('awsInfrastructureManagementAccounts') is not None] ocm_map = OCMMap(clusters=clusters, settings=settings) results = [] for cluster in clusters: cluster_name = cluster['name'] management_account = tfvpc._get_default_management_account(cluster) account = tfvpc._build_infrastructure_assume_role( management_account, cluster, ocm_map.get(cluster_name) ) account['resourcesDefaultRegion'] = \ management_account['resourcesDefaultRegion'] aws_api = AWSApi(1, [account], settings=settings) egress_ips = \ aws_api.get_cluster_nat_gateways_egress_ips(account) item = { 'cluster': cluster_name, 'egress_ips': ', '.join(sorted(egress_ips)) } results.append(item) columns = ['cluster', 'egress_ips'] print_output(ctx.obj['options'], results, columns)
def aws_account_from_infrastructure_access(cluster, access_level: str, ocm_map: OCMMap): """ Generate an AWS account object from a cluster's awsInfrastructureAccess groups and access levels """ ocm = ocm_map.get(cluster['name']) account = None for awsAccess in cluster['awsInfrastructureAccess']: if awsAccess.get('accessLevel', "") == access_level: account = { 'name': awsAccess['awsGroup']['account']['name'], 'uid': awsAccess['awsGroup']['account']['uid'], 'terraformUsername': awsAccess['awsGroup']['account']['terraformUsername'], 'automationToken': awsAccess['awsGroup']['account']['automationToken'], 'assume_role': ocm.get_aws_infrastructure_access_terraform_assume_role( cluster['name'], awsAccess['awsGroup']['account']['uid'], awsAccess['awsGroup']['account']['terraformUsername'], ), 'assume_region': cluster['spec']['region'], 'assume_cidr': cluster['network']['vpc'] } return account
def fetch_current_state(): current_state = [] current_failed = [] current_deleting = [] settings = queries.get_app_interface_settings() clusters = [c for c in queries.get_clusters() if c.get('ocm') is not None] ocm_map = OCMMap(clusters=clusters, integration=QONTRACT_INTEGRATION, settings=settings) for cluster_info in clusters: cluster = cluster_info['name'] ocm = ocm_map.get(cluster) role_grants = ocm.get_aws_infrastructure_access_role_grants(cluster) for user_arn, access_level, state, _ in role_grants: item = { 'cluster': cluster, 'user_arn': user_arn, 'access_level': access_level } if state == STATUS_FAILED: current_failed.append(item) elif state == STATUS_DELETING: current_deleting.append(item) else: current_state.append(item) return ocm_map, current_state, current_failed, current_deleting
def fetch_current_state(clusters, settings): current_state = [] ocm_map = OCMMap( clusters=clusters, integration=QONTRACT_INTEGRATION, settings=settings ) for cluster_info in clusters: cluster = cluster_info["name"] ocm = ocm_map.get(cluster) idps = ocm.get_github_idp_teams(cluster) current_state.extend(idps) return ocm_map, current_state
def cluster_upgrades(ctx, name): settings = queries.get_app_interface_settings() clusters = queries.get_clusters() clusters_ocm = [ c for c in clusters if c.get('ocm') is not None and c.get('auth') is not None ] ocm_map = OCMMap(clusters=clusters_ocm, settings=settings) clusters_data = [] for c in clusters: if name and c['name'] != name: continue if not c.get('spec'): continue data = { 'name': c['name'], 'upgrade': c['spec']['upgrade'], 'id': c['spec']['id'], 'external_id': c['spec'].get('external_id'), } upgrade_policy = c['upgradePolicy'] if upgrade_policy: data['upgradePolicy'] = upgrade_policy.get('schedule_type') if data.get('upgradePolicy') == 'automatic': data['schedule'] = c['upgradePolicy']['schedule'] ocm = ocm_map.get(c['name']) if ocm: upgrade_policy = ocm.get_upgrade_policies(c['name']) next_run = upgrade_policy[0].get('next_run') if next_run: data['next_run'] = next_run else: data['upgradePolicy'] = 'manual' clusters_data.append(data) clusters_data = sorted(clusters_data, key=lambda k: k['upgrade']) columns = ['name', 'upgrade', 'upgradePolicy', 'schedule', 'next_run'] print_output(ctx.obj['output'], clusters_data, columns)
def fetch_current_state(clusters): settings = queries.get_app_interface_settings() ocm_map = OCMMap(clusters=clusters, integration=QONTRACT_INTEGRATION, settings=settings) current_state = [] for cluster in clusters: cluster_name = cluster['name'] ocm = ocm_map.get(cluster_name) machine_pools = ocm.get_machine_pools(cluster_name) for machine_pool in machine_pools: machine_pool['cluster'] = cluster_name current_state.append(machine_pool) return ocm_map, current_state
def fetch_current_state(clusters): settings = queries.get_app_interface_settings() ocm_map = OCMMap(clusters=clusters, integration=QONTRACT_INTEGRATION, settings=settings) current_state = [] for cluster in clusters: cluster_name = cluster['name'] ocm = ocm_map.get(cluster_name) upgrade_policies = \ ocm.get_upgrade_policies(cluster_name) for upgrade_policy in upgrade_policies: upgrade_policy['cluster'] = cluster_name current_state.append(upgrade_policy) return ocm_map, current_state
def fetch_current_state(clusters): settings = queries.get_app_interface_settings() ocm_map = OCMMap(clusters=clusters, integration=QONTRACT_INTEGRATION, settings=settings) current_state = [] for cluster in clusters: cluster_name = cluster["name"] ocm = ocm_map.get(cluster_name) routers = ocm.get_additional_routers(cluster_name) for router in routers: router["cluster"] = cluster_name current_state.append(router) return ocm_map, current_state
def fetch_current_state(clusters): settings = queries.get_app_interface_settings() ocm_map = OCMMap(clusters=clusters, integration=QONTRACT_INTEGRATION, settings=settings) current_state = [] for cluster in clusters: cluster_name = cluster['name'] ocm = ocm_map.get(cluster_name) addons = ocm.get_cluster_addons(cluster_name) if addons: for addon in addons: addon['cluster'] = cluster_name current_state.append(addon) return ocm_map, current_state
def clusters_aws_account_ids(ctx): settings = queries.get_app_interface_settings() clusters = [c for c in queries.get_clusters() if c.get('ocm') is not None] ocm_map = OCMMap(clusters=clusters, settings=settings) results = [] for cluster in clusters: cluster_name = cluster['name'] ocm = ocm_map.get(cluster_name) aws_account_id = ocm.get_cluster_aws_account_id(cluster_name) item = { 'cluster': cluster_name, 'aws_account_id': aws_account_id, } results.append(item) columns = ['cluster', 'aws_account_id'] print_output(ctx.obj['options'], results, columns)
def build_desired_state_vpc(clusters, ocm_map: OCMMap, awsapi: AWSApi): """ Fetch state for VPC peerings between a cluster and a VPC (account) """ desired_state = [] error = False for cluster_info in clusters: try: cluster = cluster_info['name'] ocm = ocm_map.get(cluster) items = build_desired_state_vpc_single_cluster( cluster_info, ocm, awsapi) desired_state.extend(items) except (KeyError, BadTerraformPeeringState, aws_api.MissingARNError): logging.exception(f"Unable to process {cluster_info['name']}") error = True return desired_state, error
def fetch_current_state(clusters): settings = queries.get_app_interface_settings() ocm_map = OCMMap( clusters=clusters, integration=QONTRACT_INTEGRATION, settings=settings, init_addons=True, ) current_state = [] for cluster in clusters: cluster_name = cluster["name"] ocm = ocm_map.get(cluster_name) addons = ocm.get_cluster_addons(cluster_name) if addons: for addon in addons: addon["cluster"] = cluster_name sort_parameters(addon) current_state.append(addon) return ocm_map, current_state
def fetch_current_state(clusters): settings = queries.get_app_interface_settings() ocm_map = OCMMap(clusters=clusters, integration=QONTRACT_INTEGRATION, settings=settings) current_state = [] for cluster in clusters: cluster_name = cluster['name'] ocm = ocm_map.get(cluster_name) labels = ocm.get_external_configuration_labels(cluster_name) for key, value in labels.items(): item = { 'label': { 'key': key, 'value': value }, 'cluster': cluster_name } current_state.append(item) return ocm_map, current_state
def build_desired_state_all_clusters(clusters, ocm_map: OCMMap, awsapi: AWSApi): """ Fetch state for VPC peerings between two OCM clusters """ desired_state = [] error = False for cluster_info in clusters: try: cluster = cluster_info['name'] ocm = ocm_map.get(cluster) items = build_desired_state_single_cluster( cluster_info, ocm, awsapi ) desired_state.extend(items) except (KeyError, BadTerraformPeeringState, aws_api.MissingARNError): logging.exception( f"Failed to get desired state for {cluster}" ) error = True return desired_state, error
def ocm_aws_infrastructure_access_switch_role_links(ctx): settings = queries.get_app_interface_settings() clusters = queries.get_clusters() clusters = [c for c in clusters if c.get('ocm') is not None] ocm_map = OCMMap(clusters=clusters, settings=settings) results = [] for cluster in clusters: cluster_name = cluster['name'] ocm = ocm_map.get(cluster_name) role_grants = \ ocm.get_aws_infrastructure_access_role_grants(cluster_name) for user_arn, access_level, _, switch_role_link in role_grants: item = { 'cluster': cluster_name, 'user_arn': user_arn, 'access_level': access_level, 'switch_role_link': switch_role_link, } results.append(item) columns = ['cluster', 'user_arn', 'access_level', 'switch_role_link'] print_output(ctx.obj['output'], results, columns)
def run(dry_run, thread_pool_size=10, internal=None, use_jump_host=True, vault_throughput_path=None, defer=None): if not vault_throughput_path: logging.error('must supply vault throughput path') sys.exit(ExitCodes.ERROR) kafka_clusters = queries.get_kafka_clusters() if not kafka_clusters: logging.debug("No Kafka clusters found in app-interface") sys.exit(ExitCodes.SUCCESS) settings = queries.get_app_interface_settings() ocm_map = OCMMap(clusters=kafka_clusters, integration=QONTRACT_INTEGRATION, settings=settings) namespaces = [] for kafka_cluster in kafka_clusters: namespaces.extend(kafka_cluster['namespaces']) ri, oc_map = ob.fetch_current_state( namespaces=namespaces, thread_pool_size=thread_pool_size, integration=QONTRACT_INTEGRATION, integration_version=QONTRACT_INTEGRATION_VERSION, override_managed_types=['Secret'], internal=internal, use_jump_host=use_jump_host) defer(oc_map.cleanup) current_state = ocm_map.kafka_cluster_specs() desired_state = fetch_desired_state(kafka_clusters) kafka_service_accounts = ocm_map.kafka_service_account_specs() for kafka_cluster in kafka_clusters: kafka_cluster_name = kafka_cluster['name'] desired_cluster = [ c for c in desired_state if kafka_cluster_name == c['name'] ][0] current_cluster = [ c for c in current_state if kafka_cluster_name == c['name'] ] # check if cluster exists. if not - create it if not current_cluster: logging.info(['create_cluster', kafka_cluster_name]) if not dry_run: ocm = ocm_map.get(kafka_cluster_name) ocm.create_kafka_cluster(desired_cluster) continue # there should only be one cluster current_cluster = current_cluster[0] # check if desired cluster matches current cluster. if not - error if not all(k in current_cluster.keys() for k in desired_cluster.keys()): logging.error( '[%s] desired spec %s is different ' + 'from current spec %s', kafka_cluster_name, desired_cluster, current_cluster) ri.register_error() continue # check if cluster is ready. if not - wait status = current_cluster['status'] if status != STATUS_READY: # check if cluster is failed if status == STATUS_FAILED: failed_reason = current_cluster['failed_reason'] logging.error( f'[{kafka_cluster_name}] cluster status is {status}. ' f'reason: {failed_reason}') ri.register_error() else: logging.warning( f'[{kafka_cluster_name}] cluster status is {status}') continue # we have a ready cluster! # get a service account for the cluster kafka_service_account = get_kafa_service_account( kafka_service_accounts, kafka_cluster_name, vault_throughput_path, dry_run, ocm_map, ) # let's create a Secret in all referencing namespaces kafka_namespaces = kafka_cluster['namespaces'] secret_fields = ['bootstrap_server_host'] data = {k: v for k, v in current_cluster.items() if k in secret_fields} data.update(kafka_service_account) resource = construct_oc_resource(data) for namespace_info in kafka_namespaces: ri.add_desired(namespace_info['cluster']['name'], namespace_info['name'], resource.kind, resource.name, resource) if not dry_run: write_output_to_vault(vault_throughput_path, kafka_cluster_name, resource.body['data']) ob.realize_data(dry_run, oc_map, ri, thread_pool_size) if ri.has_error_registered(): sys.exit(ExitCodes.ERROR)
def run(dry_run, thread_pool_size=10, internal=None, use_jump_host=True, defer=None): kafka_clusters = queries.get_kafka_clusters() if not kafka_clusters: logging.debug("No Kafka clusters found in app-interface") sys.exit(ExitCodes.SUCCESS) settings = queries.get_app_interface_settings() ocm_map = OCMMap(clusters=kafka_clusters, integration=QONTRACT_INTEGRATION, settings=settings) namespaces = [] for kafka_cluster in kafka_clusters: namespaces.extend(kafka_cluster['namespaces']) ri, oc_map = ob.fetch_current_state( namespaces=namespaces, thread_pool_size=thread_pool_size, integration=QONTRACT_INTEGRATION, integration_version=QONTRACT_INTEGRATION_VERSION, override_managed_types=['Secret'], internal=internal, use_jump_host=use_jump_host) defer(lambda: oc_map.cleanup()) current_state = ocm_map.kafka_cluster_specs() desired_state = fetch_desired_state(kafka_clusters) error = False for kafka_cluster in kafka_clusters: kafka_cluster_name = kafka_cluster['name'] desired_cluster = [ c for c in desired_state if kafka_cluster_name == c['name'] ][0] current_cluster = [ c for c in current_state if kafka_cluster_name == c['name'] ] # check if cluster exists. if not - create it if not current_cluster: logging.info(['create_cluster', kafka_cluster_name]) if not dry_run: ocm = ocm_map.get(kafka_cluster_name) ocm.create_kafka_cluster(desired_cluster) continue # there should only be one cluster current_cluster = current_cluster[0] # check if desired cluster matches current cluster. if not - error if not all(k in current_cluster.keys() for k in desired_cluster.keys()): logging.error( '[%s] desired spec %s is different ' + 'from current spec %s', kafka_cluster_name, desired_cluster, current_cluster) error = True continue # check if cluster is ready. if not - wait if current_cluster['status'] != 'complete': continue # we have a ready cluster! # let's create a Secret in all referencing namespaces kafka_namespaces = kafka_cluster['namespaces'] secret_fields = ['bootstrapServerHost'] data = {k: v for k, v in current_cluster.items() if k in secret_fields} resource = construct_oc_resource(data) for namespace_info in kafka_namespaces: ri.add_desired(namespace_info['cluster']['name'], namespace_info['name'], resource.kind, resource.name, resource) ob.realize_data(dry_run, oc_map, ri) if error: sys.exit(ExitCodes.ERROR)
def build_desired_state_tgw_attachments(clusters, ocm_map: OCMMap, awsapi: AWSApi): """ Fetch state for TGW attachments between a cluster and all TGWs in an account in the same region as the cluster """ desired_state = [] error = False for cluster_info in clusters: cluster = cluster_info['name'] ocm = ocm_map.get(cluster) peering_info = cluster_info['peering'] peer_connections = peering_info['connections'] for peer_connection in peer_connections: # We only care about account-tgw peering providers peer_connection_provider = peer_connection['provider'] if not peer_connection_provider == 'account-tgw': continue # accepter is the cluster's AWS account cluster_region = cluster_info['spec']['region'] cluster_cidr_block = cluster_info['network']['vpc'] accepter = { 'cidr_block': cluster_cidr_block, 'region': cluster_region } account = peer_connection['account'] # assume_role is the role to assume to provision the # peering connection request, through the accepter AWS account. provided_assume_role = peer_connection.get('assumeRole') # if an assume_role is provided, it means we don't need # to get the information from OCM. it likely means that # there is no OCM at all. if provided_assume_role: account['assume_role'] = provided_assume_role else: account['assume_role'] = \ ocm.get_aws_infrastructure_access_terraform_assume_role( cluster, account['uid'], account['terraformUsername'] ) account['assume_region'] = accepter['region'] account['assume_cidr'] = accepter['cidr_block'] accepter_vpc_id, accepter_route_table_ids, \ accepter_subnets_id_az = \ awsapi.get_cluster_vpc_details( account, route_tables=peer_connection.get('manageRoutes'), subnets=True, ) if accepter_vpc_id is None: logging.error(f'[{cluster} could not find VPC ID for cluster') error = True continue accepter['vpc_id'] = accepter_vpc_id accepter['route_table_ids'] = accepter_route_table_ids accepter['subnets_id_az'] = accepter_subnets_id_az accepter['account'] = account account_tgws = \ awsapi.get_tgws_details( account, cluster_region, cluster_cidr_block, tags=json.loads(peer_connection.get('tags') or '{}'), route_tables=peer_connection.get('manageRoutes'), security_groups=peer_connection.get( 'manageSecurityGroups'), ) for tgw in account_tgws: tgw_id = tgw['tgw_id'] connection_name = \ f"{peer_connection['name']}_" + \ f"{account['name']}-{tgw_id}" requester = { 'tgw_id': tgw_id, 'tgw_arn': tgw['tgw_arn'], 'region': tgw['region'], 'routes': tgw.get('routes'), 'rules': tgw.get('rules'), 'cidr_block': peer_connection.get('cidrBlock'), 'account': account, } item = { 'connection_provider': peer_connection_provider, 'connection_name': connection_name, 'requester': requester, 'accepter': accepter, 'deleted': peer_connection.get('delete', False) } desired_state.append(item) return desired_state, error
def __init__(self, dry_run, instance): self.dry_run = dry_run self.settings = queries.get_app_interface_settings() cluster_info = instance['hiveCluster'] hive_cluster = instance['hiveCluster']['name'] # Getting the OCM Client for the hive cluster ocm_map = OCMMap(clusters=[cluster_info], integration=QONTRACT_INTEGRATION, settings=self.settings) self.ocm_cli = ocm_map.get(hive_cluster) if not self.ocm_cli: raise OcpReleaseMirrorError(f"Can't create ocm client for " f"cluster {hive_cluster}") # Getting the OC Client for the hive cluster oc_map = OC_Map(clusters=[cluster_info], integration=QONTRACT_INTEGRATION, settings=self.settings) self.oc_cli = oc_map.get(hive_cluster) if not self.oc_cli: raise OcpReleaseMirrorError(f"Can't create oc client for " f"cluster {hive_cluster}") namespace = instance['ecrResourcesNamespace'] ocp_release_identifier = instance['ocpReleaseEcrIdentifier'] ocp_art_dev_identifier = instance['ocpArtDevEcrIdentifier'] ocp_release_info = self._get_tf_resource_info(namespace, ocp_release_identifier) if ocp_release_info is None: raise OcpReleaseMirrorError(f"Could not find rds " f"identifier " f"{ocp_release_identifier} in " f"namespace {namespace['name']}") ocp_art_dev_info = self._get_tf_resource_info(namespace, ocp_art_dev_identifier) if ocp_art_dev_info is None: raise OcpReleaseMirrorError(f"Could not find rds identifier" f" {ocp_art_dev_identifier} in" f"namespace {namespace['name']}") # Getting the AWS Client for the accounts aws_accounts = [ self._get_aws_account_info(account=ocp_release_info['account']), self._get_aws_account_info(account=ocp_art_dev_info['account']) ] self.aws_cli = AWSApi(thread_pool_size=1, accounts=aws_accounts, settings=self.settings, init_ecr_auth_tokens=True) self.aws_cli.map_ecr_resources() self.ocp_release_ecr_uri = self._get_image_uri( account=ocp_release_info['account'], repository=ocp_release_identifier) if self.ocp_release_ecr_uri is None: raise OcpReleaseMirrorError(f"Could not find the " f"ECR repository " f"{ocp_release_identifier}") self.ocp_art_dev_ecr_uri = self._get_image_uri( account=ocp_art_dev_info['account'], repository=ocp_art_dev_identifier) if self.ocp_art_dev_ecr_uri is None: raise OcpReleaseMirrorError(f"Could not find the " f"ECR repository " f"{ocp_art_dev_identifier}") # Process all the quayOrgTargets quay_api_store = get_quay_api_store() self.quay_target_orgs = [] for quayTargetOrg in instance['quayTargetOrgs']: org_name = quayTargetOrg['name'] instance_name = quayTargetOrg['instance']['name'] org_key = OrgKey(instance_name, org_name) org_info = quay_api_store[org_key] if not org_info['push_token']: raise OcpReleaseMirrorError( f'{org_key} has no push_token defined.') url = org_info['url'] user = org_info['push_token']['user'] token = org_info['push_token']['token'] self.quay_target_orgs.append({ 'url': url, 'dest_ocp_release': f"{url}/{org_name}/ocp-release", 'dest_ocp_art_dev': f"{url}/{org_name}/ocp-v4.0-art-dev", 'auths': self._build_quay_auths(url, user, token) }) # Getting all the credentials quay_creds = self._get_quay_creds() ocp_release_creds = self._get_ecr_creds( account=ocp_release_info['account'], region=ocp_release_info['region']) ocp_art_dev_creds = self._get_ecr_creds( account=ocp_art_dev_info['account'], region=ocp_art_dev_info['region']) # Creating a single dictionary with all credentials to be used by the # "oc adm release mirror" command self.registry_creds = { 'auths': { **quay_creds['auths'], **ocp_release_creds['auths'], **ocp_art_dev_creds['auths'], } } # Append quay_target_orgs auths to registry_creds for quay_target_org in self.quay_target_orgs: url = quay_target_org['url'] if url in self.registry_creds['auths'].keys(): OcpReleaseMirrorError('Cannot mirror to the same Quay ' f'instance multiple times: {url}') self.registry_creds['auths'].update(quay_target_org['auths']) # Initiate channel groups self.channel_groups = instance['mirrorChannels']
def run(dry_run, gitlab_project_id=None, thread_pool_size=10): settings = queries.get_app_interface_settings() clusters = queries.get_clusters() clusters = [c for c in clusters if c.get('ocm') is not None] ocm_map = OCMMap(clusters=clusters, integration=QONTRACT_INTEGRATION, settings=settings, skip_provision_shards=False) current_state, pending_state = ocm_map.cluster_specs() desired_state = fetch_desired_state(clusters) if not dry_run: mr_cli = mr_client_gateway.init(gitlab_project_id=gitlab_project_id) error = False clusters_updates = {} for cluster_name, desired_spec in desired_state.items(): current_spec = current_state.get(cluster_name) if current_spec: clusters_updates[cluster_name] = {} cluster_path = 'data' + \ [c['path'] for c in clusters if c['name'] == cluster_name][0] # validate version desired_spec['spec'].pop('initial_version') desired_version = desired_spec['spec'].pop('version') current_version = current_spec['spec'].pop('version') compare_result = 1 # default value in case version is empty if desired_version: compare_result = \ semver.compare(current_version, desired_version) if compare_result > 0: # current version is larger due to an upgrade. # submit MR to update cluster version logging.info( '[%s] desired version %s is different ' + 'from current version %s. ' + 'version will be updated automatically in app-interface.', cluster_name, desired_version, current_version) clusters_updates[cluster_name]['version'] = current_version elif compare_result < 0: logging.error( '[%s] desired version %s is different ' + 'from current version %s', cluster_name, desired_version, current_version) error = True if not desired_spec['spec'].get('id'): clusters_updates[cluster_name]['id'] = \ current_spec['spec']['id'] if not desired_spec['spec'].get('external_id'): clusters_updates[cluster_name]['external_id'] = \ current_spec['spec']['external_id'] desired_provision_shard_id = \ desired_spec['spec'].get('provision_shard_id') current_provision_shard_id = \ current_spec['spec']['provision_shard_id'] if desired_provision_shard_id != current_provision_shard_id: clusters_updates[cluster_name]['provision_shard_id'] = \ current_provision_shard_id if clusters_updates[cluster_name]: clusters_updates[cluster_name]['path'] = cluster_path # exclude params we don't want to check in the specs for k in ['id', 'external_id', 'provision_shard_id']: current_spec['spec'].pop(k, None) desired_spec['spec'].pop(k, None) if current_spec != desired_spec: # check if cluster update is valid update_spec, err = get_cluster_update_spec( cluster_name, current_spec, desired_spec, ) if err: error = True continue # update cluster logging.debug( '[%s] desired spec %s is different ' + 'from current spec %s', cluster_name, desired_spec, current_spec) logging.info(['update_cluster', cluster_name]) # TODO(mafriedm): check dry_run in OCM API patch if not dry_run: ocm = ocm_map.get(cluster_name) ocm.update_cluster(cluster_name, update_spec, dry_run) else: # create cluster if cluster_name in pending_state: continue logging.info(['create_cluster', cluster_name]) ocm = ocm_map.get(cluster_name) ocm.create_cluster(cluster_name, desired_spec, dry_run) create_update_mr = False for cluster_name, cluster_updates in clusters_updates.items(): for k, v in cluster_updates.items(): if k == 'path': continue logging.info(f"[{cluster_name}] desired key " + f"{k} will be updated automatically " + f"with value {v}.") create_update_mr = True if create_update_mr and not dry_run: mr = CreateClustersUpdates(clusters_updates) mr.submit(cli=mr_cli) if error: sys.exit(1)
def cluster_upgrade_policies(ctx, cluster=None, workload=None, show_only_soaking_upgrades=False, by_workload=False): settings = queries.get_app_interface_settings() clusters = queries.get_clusters() clusters = [c for c in clusters if c.get('upgradePolicy') is not None] if cluster: clusters = [c for c in clusters if cluster == c['name']] if workload: clusters = [ c for c in clusters if workload in c['upgradePolicy'].get('workloads', []) ] desired_state = ous.fetch_desired_state(clusters) ocm_map = OCMMap(clusters=clusters, settings=settings) history = ous.get_version_history(dry_run=True, upgrade_policies=[], ocm_map=ocm_map) results = [] upgrades_cache = {} def soaking_str(soaking, soakdays): sorted_soaking = sorted(soaking.items(), key=lambda x: x[1]) if ctx.obj['options']['output'] == 'md': for i, data in enumerate(sorted_soaking): v, s = data if s > soakdays: sorted_soaking[i] = (v, f'{s} :tada:') return ', '.join([f'{v} ({s})' for v, s in sorted_soaking]) for c in desired_state: cluster_name, version = c['cluster'], c['current_version'] channel, schedule = c['channel'], c.get('schedule') soakdays = c.get('conditions', {}).get('soakDays') item = { 'cluster': cluster_name, 'version': parse_semver(version), 'channel': channel, 'schedule': schedule, 'soak_days': soakdays, } ocm = ocm_map.get(cluster_name) if 'workloads' not in c: results.append(item) continue upgrades = upgrades_cache.get((version, channel)) if not upgrades: upgrades = ocm.get_available_upgrades(version, channel) upgrades_cache[(version, channel)] = upgrades workload_soaking_upgrades = {} for w in c.get('workloads', []): if not workload or workload == w: s = soaking_days(history, upgrades, w, show_only_soaking_upgrades) workload_soaking_upgrades[w] = s if by_workload: for w, soaking in workload_soaking_upgrades.items(): i = item.copy() i.update({ 'workload': w, 'soaking_upgrades': soaking_str(soaking, soakdays) }) results.append(i) else: workloads = sorted(c.get('workloads', [])) w = ', '.join(workloads) soaking = {} for v in upgrades: soaks = [ s.get(v, 0) for s in workload_soaking_upgrades.values() ] min_soaks = min(soaks) if not show_only_soaking_upgrades or min_soaks > 0: soaking[v] = min_soaks item.update({ 'workload': w, 'soaking_upgrades': soaking_str(soaking, soakdays) }) results.append(item) if ctx.obj['options']['output'] == 'md': print(""" The table below regroups upgrade information for each clusters: * `version` is the current openshift version on the cluster * `channel` is the OCM upgrade channel being tracked by the cluster * `schedule` is the cron-formatted schedule for cluster upgrades * `soak_days` is the minimum number of days a given version must have been running on other clusters with the same workload to be considered for an upgrade. * `workload` is a list of workload names that are running on the cluster * `soaking_upgrades` lists all available upgrades available on the OCM channel for that cluster. The number in parenthesis shows the number of days this version has been running on other clusters with the same workloads. By comparing with the `soak_days` columns, you can see when a version is close to be upgraded to. A :tada: sign is displayed for versions which have soaked enough and are ready to be upgraded to. """) columns = [ 'cluster', 'version', 'channel', 'schedule', 'soak_days', 'workload', 'soaking_upgrades' ] ctx.obj['options']['to_string'] = True print_output(ctx.obj['options'], results, columns)
def build_desired_state_tgw_attachments(clusters, ocm_map: OCMMap, awsapi: AWSApi): """ Fetch state for TGW attachments between a cluster and all TGWs in an account in the same region as the cluster """ desired_state = [] error = False for cluster_info in clusters: cluster = cluster_info["name"] ocm = ocm_map.get(cluster) peering_info = cluster_info["peering"] peer_connections = peering_info["connections"] for peer_connection in peer_connections: # We only care about account-tgw peering providers peer_connection_provider = peer_connection["provider"] if not peer_connection_provider == "account-tgw": continue # accepter is the cluster's AWS account cluster_region = cluster_info["spec"]["region"] cluster_cidr_block = cluster_info["network"]["vpc"] accepter = { "cidr_block": cluster_cidr_block, "region": cluster_region } account = peer_connection["account"] # assume_role is the role to assume to provision the # peering connection request, through the accepter AWS account. provided_assume_role = peer_connection.get("assumeRole") # if an assume_role is provided, it means we don't need # to get the information from OCM. it likely means that # there is no OCM at all. if provided_assume_role: account["assume_role"] = provided_assume_role else: account[ "assume_role"] = ocm.get_aws_infrastructure_access_terraform_assume_role( cluster, account["uid"], account["terraformUsername"]) account["assume_region"] = accepter["region"] account["assume_cidr"] = accepter["cidr_block"] ( accepter_vpc_id, accepter_route_table_ids, accepter_subnets_id_az, ) = awsapi.get_cluster_vpc_details( account, route_tables=peer_connection.get("manageRoutes"), subnets=True, ) if accepter_vpc_id is None: logging.error(f"[{cluster} could not find VPC ID for cluster") error = True continue accepter["vpc_id"] = accepter_vpc_id accepter["route_table_ids"] = accepter_route_table_ids accepter["subnets_id_az"] = accepter_subnets_id_az accepter["account"] = account account_tgws = awsapi.get_tgws_details( account, cluster_region, cluster_cidr_block, tags=json.loads(peer_connection.get("tags") or "{}"), route_tables=peer_connection.get("manageRoutes"), security_groups=peer_connection.get("manageSecurityGroups"), ) for tgw in account_tgws: tgw_id = tgw["tgw_id"] connection_name = (f"{peer_connection['name']}_" + f"{account['name']}-{tgw_id}") requester = { "tgw_id": tgw_id, "tgw_arn": tgw["tgw_arn"], "region": tgw["region"], "routes": tgw.get("routes"), "rules": tgw.get("rules"), "cidr_block": peer_connection.get("cidrBlock"), "account": account, } item = { "connection_provider": peer_connection_provider, "connection_name": connection_name, "requester": requester, "accepter": accepter, "deleted": peer_connection.get("delete", False), } desired_state.append(item) return desired_state, error
def __init__(self, dry_run, instance): self.dry_run = dry_run self.settings = queries.get_app_interface_settings() cluster_info = instance['hiveCluster'] hive_cluster = instance['hiveCluster']['name'] # Getting the OCM Client for the hive cluster ocm_map = OCMMap(clusters=[cluster_info], integration=QONTRACT_INTEGRATION, settings=self.settings) self.ocm_cli = ocm_map.get(hive_cluster) if not self.ocm_cli: raise OcpReleaseEcrMirrorError(f"Can't create ocm client for " f"cluster {hive_cluster}") # Getting the OC Client for the hive cluster oc_map = OC_Map(clusters=[cluster_info], integration=QONTRACT_INTEGRATION, settings=self.settings) self.oc_cli = oc_map.get(hive_cluster) if not self.oc_cli: raise OcpReleaseEcrMirrorError(f"Can't create oc client for " f"cluster {hive_cluster}") namespace = instance['ecrResourcesNamespace'] ocp_release_identifier = instance['ocpReleaseEcrIdentifier'] ocp_art_dev_identifier = instance['ocpArtDevEcrIdentifier'] ocp_release_info = self._get_tf_resource_info(namespace, ocp_release_identifier) if ocp_release_info is None: raise OcpReleaseEcrMirrorError(f"Could not find rds " f"identifier " f"{ocp_release_identifier} in " f"namespace {namespace['name']}") ocp_art_dev_info = self._get_tf_resource_info(namespace, ocp_art_dev_identifier) if ocp_art_dev_info is None: raise OcpReleaseEcrMirrorError(f"Could not find rds identifier" f" {ocp_art_dev_identifier} in" f"namespace {namespace['name']}") # Getting the AWS Client for the accounts aws_accounts = [ self._get_aws_account_info(account=ocp_release_info['account']), self._get_aws_account_info(account=ocp_art_dev_info['account']) ] self.aws_cli = AWSApi(thread_pool_size=1, accounts=aws_accounts, settings=self.settings, init_ecr_auth_tokens=True) self.aws_cli.map_ecr_resources() self.ocp_release_ecr_uri = self._get_image_uri( account=ocp_release_info['account'], repository=ocp_release_identifier) if self.ocp_release_ecr_uri is None: raise OcpReleaseEcrMirrorError(f"Could not find the " f"ECR repository " f"{ocp_release_identifier}") self.ocp_art_dev_ecr_uri = self._get_image_uri( account=ocp_art_dev_info['account'], repository=ocp_art_dev_identifier) if self.ocp_art_dev_ecr_uri is None: raise OcpReleaseEcrMirrorError(f"Could not find the " f"ECR repository " f"{ocp_art_dev_identifier}") # Getting all the credentials quay_creds = self._get_quay_creds() ocp_release_creds = self._get_ecr_creds( account=ocp_release_info['account'], region=ocp_release_info['region']) ocp_art_dev_creds = self._get_ecr_creds( account=ocp_art_dev_info['account'], region=ocp_art_dev_info['region']) # Creating a single dictionary with all credentials to be used by the # "oc adm release mirror" command self.registry_creds = { 'auths': { **quay_creds['auths'], **ocp_release_creds['auths'], **ocp_art_dev_creds['auths'], } }