def test_missing_cluster_automation_token(self, mock_secret_reader, mock_oc): cluster_1 = { "name": "cl1", "serverUrl": "http://localhost", "automationToken": { "path": "some-path", "field": "some-field" }, } namespace_1 = { "name": "ns1", "clusterAdmin": True, "cluster": cluster_1 } oc_map = OC_Map(namespaces=[namespace_1]) # check that non-priv OC got instantiated but priv one not self.assertEqual(oc_map.clusters(), ["cl1"]) self.assertEqual(oc_map.clusters(privileged=True), []) self.assertEqual(oc_map.clusters(include_errors=True, privileged=True), [cluster_1["name"]]) self.assertIsInstance(oc_map.get(cluster_1["name"]), OC) self.assertFalse(oc_map.get(cluster_1["name"], privileged=True))
def test_missing_cluster_automation_token(self, mock_secret_reader, mock_oc): cluster_1 = { 'name': 'cl1', 'serverUrl': 'http://localhost', 'automationToken': { 'path': 'some-path', 'field': 'some-field' } } namespace_1 = { 'name': 'ns1', 'clusterAdmin': True, 'cluster': cluster_1 } oc_map = OC_Map(namespaces=[namespace_1]) # check that non-priv OC got instantiated but priv one not self.assertEqual(oc_map.clusters(), ["cl1"]) self.assertEqual(oc_map.clusters(privileged=True), []) self.assertEqual(oc_map.clusters(include_errors=True, privileged=True), [cluster_1['name']]) self.assertIsInstance(oc_map.get(cluster_1['name']), OC) self.assertFalse(oc_map.get(cluster_1['name'], privileged=True))
def root_owner(ctx, cluster, namespace, kind, name): settings = queries.get_app_interface_settings() clusters = [ c for c in queries.get_clusters(minimal=True) if c['name'] == cluster ] oc_map = OC_Map(clusters=clusters, integration='qontract-cli', thread_pool_size=1, settings=settings, init_api_resources=True) oc = oc_map.get(cluster) obj = oc.get(namespace, kind, name) root_owner = oc.get_obj_root_owner(namespace, obj, allow_not_found=True, allow_not_controller=True) # TODO(mafriedm): fix this # do not sort ctx.obj['options']['sort'] = False # a bit hacky, but ¯\_(ツ)_/¯ if ctx.obj['options']['output'] != 'json': ctx.obj['options']['output'] = 'yaml' print_output(ctx.obj['options'], root_owner)
def manage_namespaces(spec: Mapping[str, str], oc_map: OC_Map, dry_run: bool) -> None: cluster = spec['cluster'] namespace = spec['namespace'] desired_state = spec["desired_state"] oc = oc_map.get(cluster) if not oc: logging.log(level=oc.log_level, msg=oc.message) return None act = { NS_ACTION_CREATE: oc.new_project, NS_ACTION_DELETE: oc.delete_project } exists = oc.project_exists(namespace) action = None if not exists and desired_state == NS_STATE_PRESENT: action = NS_ACTION_CREATE elif exists and desired_state == NS_STATE_ABSENT: action = NS_ACTION_DELETE if action: logging.info([action, cluster, namespace]) if not dry_run: act[action](namespace)
def run(dry_run, thread_pool_size=10, internal=None, use_jump_host=True, defer=None): settings = queries.get_app_interface_settings() accounts = queries.get_state_aws_accounts() clusters = [c for c in queries.get_clusters(minimal=True) if c.get("ocm")] oc_map = OC_Map( clusters=clusters, integration=QONTRACT_INTEGRATION, settings=settings, internal=internal, use_jump_host=use_jump_host, thread_pool_size=thread_pool_size, ) defer(oc_map.cleanup) state = State(integration=QONTRACT_INTEGRATION, accounts=accounts, settings=settings) if not dry_run: slack = slackapi_from_queries(QONTRACT_INTEGRATION) now = datetime.utcnow() for cluster in oc_map.clusters(include_errors=True): oc = oc_map.get(cluster) if not oc: logging.log(level=oc.log_level, msg=oc.message) continue upgrade_config = oc.get( namespace="openshift-managed-upgrade-operator", kind="UpgradeConfig", allow_not_found=True, )["items"] if not upgrade_config: logging.debug(f"[{cluster}] UpgradeConfig not found.") continue [upgrade_config] = upgrade_config upgrade_spec = upgrade_config["spec"] upgrade_at = upgrade_spec["upgradeAt"] version = upgrade_spec["desired"]["version"] upgrade_at_obj = datetime.strptime(upgrade_at, "%Y-%m-%dT%H:%M:%SZ") state_key = f"{cluster}-{upgrade_at}" # if this is the first iteration in which 'now' had passed # the upgrade at date time, we send a notification if upgrade_at_obj < now: if state.exists(state_key): # already notified continue logging.info(["cluster_upgrade", cluster]) if not dry_run: state.add(state_key) usergroup = f"{cluster}-cluster" usergroup_id = slack.get_usergroup_id(usergroup) slack.chat_post_message( f"Heads up <!subteam^{usergroup_id}>! " + f"cluster `{cluster}` is currently " + f"being upgraded to version `{version}`")
def test_namespace_with_cluster_admin(self, mock_secret_reader, mock_oc): cluster_1 = { "name": "cl1", "serverUrl": "http://localhost", "clusterAdminAutomationToken": { "path": "some-path", "field": "some-field" }, "automationToken": { "path": "some-path", "field": "some-field" }, } cluster_2 = { "name": "cl2", "serverUrl": "http://localhost", "clusterAdminAutomationToken": { "path": "some-path", "field": "some-field" }, "automationToken": { "path": "some-path", "field": "some-field" }, } namespace_1 = { "name": "ns1", "clusterAdmin": True, "cluster": cluster_1 } namespace_2 = {"name": "ns2", "cluster": cluster_2} oc_map = OC_Map(namespaces=[namespace_1, namespace_2]) self.assertEqual(oc_map.clusters(), ["cl1", "cl2"]) self.assertEqual(oc_map.clusters(privileged=True), ["cl1"]) # both clusters are present as non privileged clusters in the map self.assertIsInstance(oc_map.get(cluster_1["name"]), OC) self.assertIsInstance(oc_map.get(cluster_2["name"]), OC) # only cluster_1 is present as privileged cluster in the map self.assertIsInstance(oc_map.get(cluster_1["name"], privileged=True), OC) self.assertIsInstance(oc_map.get(cluster_2["name"], privileged=True), OCLogMsg)
def test_namespace_with_cluster_admin(self, mock_secret_reader, mock_oc): cluster_1 = { 'name': 'cl1', 'serverUrl': 'http://localhost', 'clusterAdminAutomationToken': { 'path': 'some-path', 'field': 'some-field' }, 'automationToken': { 'path': 'some-path', 'field': 'some-field' } } cluster_2 = { 'name': 'cl2', 'serverUrl': 'http://localhost', 'clusterAdminAutomationToken': { 'path': 'some-path', 'field': 'some-field' }, 'automationToken': { 'path': 'some-path', 'field': 'some-field' } } namespace_1 = { 'name': 'ns1', 'clusterAdmin': True, 'cluster': cluster_1 } namespace_2 = {'name': 'ns2', 'cluster': cluster_2} oc_map = OC_Map(namespaces=[namespace_1, namespace_2]) self.assertEqual(oc_map.clusters(), ["cl1", "cl2"]) self.assertEqual(oc_map.clusters(privileged=True), ["cl1"]) # both clusters are present as non privileged clusters in the map self.assertIsInstance(oc_map.get(cluster_1['name']), OC) self.assertIsInstance(oc_map.get(cluster_2['name']), OC) # only cluster_1 is present as privileged cluster in the map self.assertIsInstance(oc_map.get(cluster_1['name'], privileged=True), OC) self.assertIsInstance(oc_map.get(cluster_2['name'], privileged=True), OCLogMsg)
def test_missing_automationtoken(self): """ When a cluster with a missing automationToken is passed into OC_Map, it should be skipped. """ cluster = { 'name': 'test-1', 'serverUrl': 'http://localhost', 'automationToken': None } oc_map = OC_Map(clusters=[cluster]) self.assertIsInstance(oc_map.get(cluster['name']), OCLogMsg) self.assertEqual( oc_map.get(cluster['name']).message, f'[{cluster["name"]}] has no automation token') self.assertEqual(len(oc_map.clusters()), 0)
def test_missing_automationtoken(self): """ When a cluster with a missing automationToken is passed into OC_Map, it should be skipped. """ cluster = { "name": "test-1", "serverUrl": "http://localhost", "automationToken": None, } oc_map = OC_Map(clusters=[cluster]) self.assertIsInstance(oc_map.get(cluster["name"]), OCLogMsg) self.assertEqual( oc_map.get(cluster["name"]).message, f'[{cluster["name"]}] has no automation token', ) self.assertEqual(len(oc_map.clusters()), 0)
def run(dry_run, thread_pool_size=10, internal=None, use_jump_host=True, defer=None): settings = queries.get_app_interface_settings() accounts = queries.get_aws_accounts() clusters = [c for c in queries.get_clusters(minimal=True) if c.get('ocm')] oc_map = OC_Map(clusters=clusters, integration=QONTRACT_INTEGRATION, settings=settings, internal=internal, use_jump_host=use_jump_host, thread_pool_size=thread_pool_size) defer(oc_map.cleanup) state = State(integration=QONTRACT_INTEGRATION, accounts=accounts, settings=settings) if not dry_run: slack = init_slack_workspace(QONTRACT_INTEGRATION) now = datetime.utcnow() for cluster in oc_map.clusters(include_errors=True): oc = oc_map.get(cluster) if not oc: logging.log(level=oc.log_level, msg=oc.message) continue upgrade_config = oc.get(namespace='openshift-managed-upgrade-operator', kind='UpgradeConfig', allow_not_found=True)['items'] if not upgrade_config: logging.debug(f'[{cluster}] UpgradeConfig not found.') continue [upgrade_config] = upgrade_config upgrade_spec = upgrade_config['spec'] upgrade_at = upgrade_spec['upgradeAt'] version = upgrade_spec['desired']['version'] upgrade_at_obj = datetime.strptime(upgrade_at, '%Y-%m-%dT%H:%M:%SZ') state_key = f'{cluster}-{upgrade_at}' # if this is the first iteration in which 'now' had passed # the upgrade at date time, we send a notification if upgrade_at_obj < now: if state.exists(state_key): # already notified continue logging.info(['cluster_upgrade', cluster]) if not dry_run: state.add(state_key) usergroup = f'{cluster}-cluster' usergroup_id = slack.get_usergroup_id(usergroup) slack.chat_post_message( f'Heads up <!subteam^{usergroup_id}>! ' + f'cluster `{cluster}` is currently ' + f'being upgraded to version `{version}`')
def test_internal_clusters(self, selfmock_secret_reader, mock_oc): cluster = { 'name': 'cl1', 'serverUrl': 'http://localhost', 'internal': True, 'automationToken': { 'path': 'some-path', 'field': 'some-field' } } namespace = {'name': 'ns1', 'cluster': cluster} # internal cluster must be in oc_map when internal is enabled internal_oc_map = OC_Map(internal=True, namespaces=[namespace]) self.assertIsInstance(internal_oc_map.get(cluster['name']), OC) # internal cluster must not be in oc_map when internal is disabled oc_map = OC_Map(internal=False, namespaces=[namespace]) self.assertFalse(oc_map.get(cluster['name']))
def test_internal_clusters(self, selfmock_secret_reader, mock_oc): cluster = { "name": "cl1", "serverUrl": "http://localhost", "internal": True, "automationToken": { "path": "some-path", "field": "some-field" }, } namespace = {"name": "ns1", "cluster": cluster} # internal cluster must be in oc_map when internal is enabled internal_oc_map = OC_Map(internal=True, namespaces=[namespace]) self.assertIsInstance(internal_oc_map.get(cluster["name"]), OC) # internal cluster must not be in oc_map when internal is disabled oc_map = OC_Map(internal=False, namespaces=[namespace]) self.assertFalse(oc_map.get(cluster["name"]))
def test_automationtoken_not_found(self, mock_secret_reader): mock_secret_reader.side_effect = SecretNotFound cluster = { 'name': 'test-1', 'serverUrl': 'http://localhost', 'automationToken': { 'path': 'some-path', 'field': 'some-field' } } oc_map = OC_Map(clusters=[cluster]) self.assertIsInstance(oc_map.get(cluster['name']), OCLogMsg) self.assertEqual( oc_map.get(cluster['name']).message, f'[{cluster["name"]}] secret not found') self.assertEqual(len(oc_map.clusters()), 0)
def test_missing_serverurl(self): """ When a cluster with a missing serverUrl is passed into OC_Map, it should be skipped. """ cluster = { "name": "test-1", "serverUrl": "", "automationToken": { "path": "some-path", "field": "some-field" }, } oc_map = OC_Map(clusters=[cluster]) self.assertIsInstance(oc_map.get(cluster["name"]), OCLogMsg) self.assertEqual( oc_map.get(cluster["name"]).message, f'[{cluster["name"]}] has no serverUrl') self.assertEqual(len(oc_map.clusters()), 0)
def test_automationtoken_not_found(self, mock_secret_reader): mock_secret_reader.side_effect = SecretNotFound cluster = { "name": "test-1", "serverUrl": "http://localhost", "automationToken": { "path": "some-path", "field": "some-field" }, } oc_map = OC_Map(clusters=[cluster]) self.assertIsInstance(oc_map.get(cluster["name"]), OCLogMsg) self.assertEqual( oc_map.get(cluster["name"]).message, f'[{cluster["name"]}] secret not found') self.assertEqual(len(oc_map.clusters()), 0)
def test_missing_serverurl(self): """ When a cluster with a missing serverUrl is passed into OC_Map, it should be skipped. """ cluster = { 'name': 'test-1', 'serverUrl': '', 'automationToken': { 'path': 'some-path', 'field': 'some-field' } } oc_map = OC_Map(clusters=[cluster]) self.assertIsInstance(oc_map.get(cluster['name']), OCLogMsg) self.assertEqual( oc_map.get(cluster['name']).message, f'[{cluster["name"]}] has no serverUrl') self.assertEqual(len(oc_map.clusters()), 0)
def label(inv_item: Tuple[str, str, Types], oc_map: OC_Map, dry_run: bool, inventory: LabelInventory): cluster, namespace, types = inv_item if inventory.errors(cluster, namespace): return changed = types.get(CHANGED, {}) if changed: prefix = "[dry-run] " if dry_run else "" _LOG.info(prefix + f'Updating labels on {cluster}/{namespace}: {changed}') if not dry_run: oc: OCNative = oc_map.get(cluster) oc.label(None, 'Namespace', namespace, changed, overwrite=True)
def run(dry_run, thread_pool_size=10, internal=None, use_jump_host=True, defer=None): settings = queries.get_app_interface_settings() pipelines_providers = queries.get_pipelines_providers() tkn_namespaces = [ pp['namespace'] for pp in pipelines_providers if pp['provider'] == Providers.TEKTON ] oc_map = OC_Map(namespaces=tkn_namespaces, integration=QONTRACT_INTEGRATION, settings=settings, internal=internal, use_jump_host=use_jump_host, thread_pool_size=thread_pool_size) defer(lambda: oc_map.cleanup()) for pp in pipelines_providers: retention = pp.get('retention') if not retention: continue if pp['provider'] == Providers.TEKTON: ns_info = pp['namespace'] namespace = ns_info['name'] cluster = ns_info['cluster']['name'] oc = oc_map.get(cluster) pipeline_runs = sorted( oc.get(namespace, 'PipelineRun')['items'], key=lambda k: k['metadata']['creationTimestamp']) retention_min = retention.get('minimum') if retention_min: pipeline_runs = pipeline_runs[retention_min:] retention_days = retention.get('days') for pr in pipeline_runs: name = pr['metadata']['name'] if retention_days and \ within_retention_days(pr, retention_days): continue logging.info([ 'delete_trigger', cluster, namespace, 'PipelineRun', name ]) if not dry_run: oc.delete(namespace, 'PipelineRun', name)
def test_disabled_integration(self, selfmock_secret_reader, mock_oc): calling_int = 'calling_integration' cluster = { 'name': 'cl1', 'serverUrl': 'http://localhost', 'disable': { 'integrations': [calling_int.replace('_', '-')] }, 'automationToken': { 'path': 'some-path', 'field': 'some-field' } } namespace = {'name': 'ns1', 'cluster': cluster} oc_map = OC_Map(integration=calling_int, namespaces=[namespace]) self.assertFalse(oc_map.get(cluster['name']))
def test_disabled_integration(self, selfmock_secret_reader, mock_oc): calling_int = "calling_integration" cluster = { "name": "cl1", "serverUrl": "http://localhost", "disable": { "integrations": [calling_int.replace("_", "-")] }, "automationToken": { "path": "some-path", "field": "some-field" }, } namespace = {"name": "ns1", "cluster": cluster} oc_map = OC_Map(integration=calling_int, namespaces=[namespace]) self.assertFalse(oc_map.get(cluster["name"]))
def run(dry_run, vault_output_path): """Get Hive ClusterDeployments from clusters and save mapping to Vault""" if not vault_output_path: logging.error('must supply vault output path') sys.exit(ExitCodes.ERROR) clusters = queries.get_clusters() settings = queries.get_app_interface_settings() oc_map = OC_Map(clusters=clusters, integration=QONTRACT_INTEGRATION, thread_pool_size=1, settings=settings, init_api_resources=True) results = [] for c in clusters: name = c['name'] oc = oc_map.get(name) if not oc: continue if 'ClusterDeployment' not in oc.api_resources: continue logging.info(f'[{name}] getting ClusterDeployments') cds = oc.get_all('ClusterDeployment', all_namespaces=True)['items'] for cd in cds: try: item = { 'id': cd['spec']['clusterMetadata']['clusterID'], 'cluster': name, } results.append(item) except KeyError: pass if not dry_run: logging.info('writing ClusterDeployments to vault') vault_client = VaultClient() secret = { 'path': f"{vault_output_path}/{QONTRACT_INTEGRATION}", 'data': { 'map': '\n'.join(f"{item['id']}: {item['cluster']}" for item in results) } } vault_client.write(secret, decode_base64=False)
def run(dry_run, vault_output_path): """Get Hive ClusterDeployments from clusters and save mapping to Vault""" if not vault_output_path: logging.error("must supply vault output path") sys.exit(ExitCodes.ERROR) clusters = queries.get_clusters() settings = queries.get_app_interface_settings() oc_map = OC_Map( clusters=clusters, integration=QONTRACT_INTEGRATION, thread_pool_size=1, settings=settings, init_api_resources=True, ) results = [] for c in clusters: name = c["name"] oc = oc_map.get(name) if not oc: continue if "ClusterDeployment" not in oc.api_resources: continue logging.info(f"[{name}] getting ClusterDeployments") cds = oc.get_all("ClusterDeployment", all_namespaces=True)["items"] for cd in cds: try: item = { "id": cd["spec"]["clusterMetadata"]["clusterID"], "cluster": name, } results.append(item) except KeyError: pass if not dry_run: logging.info("writing ClusterDeployments to vault") vault_client = VaultClient() secret = { "path": f"{vault_output_path}/{QONTRACT_INTEGRATION}", "data": { "map": "\n".join(f"{item['id']}: {item['cluster']}" for item in results) }, } vault_client.write(secret, decode_base64=False)
def delete(dry_run: bool, oc_map: OC_Map, cluster: str, namespace: str, resource_type: str, name: str, enable_deletion: bool, privileged: bool = False) -> None: logging.info(['delete', cluster, namespace, resource_type, name]) if not enable_deletion: logging.error('\'delete\' action is disabled due to previous errors.') return oc = oc_map.get(cluster, privileged) if not oc: logging.log(level=oc.log_level, msg=oc.message) return None if not dry_run: oc.delete(namespace, resource_type, name)
def _fetch_oc_secret(self) -> str: parts = self._command_data.openshift_path.split("/") if len(parts) != 3: raise ArgumentException( f"Wrong format! --openshift-path must be of format {{cluster}}/{{namespace}}/{{secret}}. Got {self._command_data.openshift_path}" ) cluster_name, namespace, secret = parts clusters = queries.get_clusters_by( filter=queries.ClusterFilter( name=cluster_name, ) ) if not clusters: raise ArgumentException(f"No cluster found with name '{cluster_name}'") settings = queries.get_app_interface_settings() data = {} try: oc_map = OC_Map( clusters=clusters, integration="qontract-cli", settings=settings, use_jump_host=True, thread_pool_size=1, init_projects=False, ) oc = oc_map.get(cluster_name) data = oc.get(namespace, "Secret", name=secret, allow_not_found=False)[ "data" ] except Exception as e: raise OpenshiftException( f"Could not fetch secret from Openshift cluster {cluster_name}" ) from e return GPGEncryptCommand._format(data)
def lookup_namespaces(cluster: str, oc_map: OC_Map): """ Retrieve all namespaces from the given cluster """ try: oc = oc_map.get(cluster) if not oc: # cluster is not reachable (may be used --internal / --external ?) _LOG.debug(f"Skipping not-handled cluster: {cluster}") return cluster, None _LOG.debug(f"Looking up namespaces on {cluster}") namespaces = oc.get_all("Namespace") if namespaces: return cluster, namespaces["items"] except StatusCodeError as e: msg = "cluster: {}, exception: {}" msg = msg.format(cluster, str(e)) _LOG.error(msg) except ApiException as e: _LOG.error(f"Cluster {cluster} skipped: " f"APIException [{e.status}:{e.reason}] {e.body}") return cluster, None
def apply(dry_run: bool, oc_map: OC_Map, cluster: str, namespace: str, resource_type: OR, resource, wait_for_namespace: bool, recycle_pods: bool = True, privileged: bool = False) -> None: logging.info(['apply', cluster, namespace, resource_type, resource.name]) oc = oc_map.get(cluster, privileged) if not oc: logging.log(level=oc.log_level, msg=oc.message) return None if not dry_run: annotated = resource.annotate() # skip if namespace does not exist (as it will soon) # do not skip if this is a cluster scoped integration if namespace != 'cluster' and not oc.project_exists(namespace): msg = f"[{cluster}/{namespace}] namespace does not exist (yet)." if wait_for_namespace: logging.info(msg + ' waiting...') wait_for_namespace_exists(oc, namespace) else: logging.warning(msg) return try: oc.apply(namespace, annotated) except InvalidValueApplyError: oc.remove_last_applied_configuration(namespace, resource_type, resource.name) oc.apply(namespace, annotated) except (MetaDataAnnotationsTooLongApplyError, UnsupportedMediaTypeError): if not oc.get( namespace, resource_type, resource.name, allow_not_found=True): oc.create(namespace, annotated) oc.replace(namespace, annotated) except FieldIsImmutableError: # Add more resources types to the list when you're # sure they're safe. if resource_type not in ['Route', 'Service', 'Secret']: raise oc.delete(namespace=namespace, kind=resource_type, name=resource.name) oc.apply(namespace=namespace, resource=annotated) except (MayNotChangeOnceSetError, PrimaryClusterIPCanNotBeUnsetError): if resource_type not in ['Service']: raise oc.delete(namespace=namespace, kind=resource_type, name=resource.name) oc.apply(namespace=namespace, resource=annotated) except StatefulSetUpdateForbidden: if resource_type != 'StatefulSet': raise logging.info([ 'delete_sts_and_apply', cluster, namespace, resource_type, resource.name ]) current_resource = oc.get(namespace, resource_type, resource.name) current_storage = oc.get_storage(current_resource) desired_storage = oc.get_storage(resource.body) resize_required = current_storage != desired_storage if resize_required: owned_pods = oc.get_owned_pods(namespace, resource) owned_pvc_names = oc.get_pod_owned_pvc_names(owned_pods) oc.delete(namespace=namespace, kind=resource_type, name=resource.name, cascade=False) oc.apply(namespace=namespace, resource=annotated) # the resource was applied without cascading. # if the change was in the storage, we need to # take care of the resize ourselves. # ref: https://github.com/kubernetes/enhancements/pull/2842 if resize_required: logging.info( ['resizing_pvcs', cluster, namespace, owned_pvc_names]) oc.resize_pvcs(namespace, owned_pvc_names, desired_storage) if recycle_pods: oc.recycle_pods(dry_run, namespace, resource_type, resource)
def __init__(self, dry_run, instance): self.dry_run = dry_run self.settings = queries.get_app_interface_settings() cluster_info = instance['hiveCluster'] hive_cluster = instance['hiveCluster']['name'] # Getting the OCM Client for the hive cluster ocm_map = OCMMap(clusters=[cluster_info], integration=QONTRACT_INTEGRATION, settings=self.settings) self.ocm_cli = ocm_map.get(hive_cluster) if not self.ocm_cli: raise OcpReleaseMirrorError(f"Can't create ocm client for " f"cluster {hive_cluster}") # Getting the OC Client for the hive cluster oc_map = OC_Map(clusters=[cluster_info], integration=QONTRACT_INTEGRATION, settings=self.settings) self.oc_cli = oc_map.get(hive_cluster) if not self.oc_cli: raise OcpReleaseMirrorError(f"Can't create oc client for " f"cluster {hive_cluster}") namespace = instance['ecrResourcesNamespace'] ocp_release_identifier = instance['ocpReleaseEcrIdentifier'] ocp_art_dev_identifier = instance['ocpArtDevEcrIdentifier'] ocp_release_info = self._get_tf_resource_info(namespace, ocp_release_identifier) if ocp_release_info is None: raise OcpReleaseMirrorError(f"Could not find rds " f"identifier " f"{ocp_release_identifier} in " f"namespace {namespace['name']}") ocp_art_dev_info = self._get_tf_resource_info(namespace, ocp_art_dev_identifier) if ocp_art_dev_info is None: raise OcpReleaseMirrorError(f"Could not find rds identifier" f" {ocp_art_dev_identifier} in" f"namespace {namespace['name']}") # Getting the AWS Client for the accounts aws_accounts = [ self._get_aws_account_info(account=ocp_release_info['account']), self._get_aws_account_info(account=ocp_art_dev_info['account']) ] self.aws_cli = AWSApi(thread_pool_size=1, accounts=aws_accounts, settings=self.settings, init_ecr_auth_tokens=True) self.aws_cli.map_ecr_resources() self.ocp_release_ecr_uri = self._get_image_uri( account=ocp_release_info['account'], repository=ocp_release_identifier) if self.ocp_release_ecr_uri is None: raise OcpReleaseMirrorError(f"Could not find the " f"ECR repository " f"{ocp_release_identifier}") self.ocp_art_dev_ecr_uri = self._get_image_uri( account=ocp_art_dev_info['account'], repository=ocp_art_dev_identifier) if self.ocp_art_dev_ecr_uri is None: raise OcpReleaseMirrorError(f"Could not find the " f"ECR repository " f"{ocp_art_dev_identifier}") # Process all the quayOrgTargets quay_api_store = get_quay_api_store() self.quay_target_orgs = [] for quayTargetOrg in instance['quayTargetOrgs']: org_name = quayTargetOrg['name'] instance_name = quayTargetOrg['instance']['name'] org_key = OrgKey(instance_name, org_name) org_info = quay_api_store[org_key] if not org_info['push_token']: raise OcpReleaseMirrorError( f'{org_key} has no push_token defined.') url = org_info['url'] user = org_info['push_token']['user'] token = org_info['push_token']['token'] self.quay_target_orgs.append({ 'url': url, 'dest_ocp_release': f"{url}/{org_name}/ocp-release", 'dest_ocp_art_dev': f"{url}/{org_name}/ocp-v4.0-art-dev", 'auths': self._build_quay_auths(url, user, token) }) # Getting all the credentials quay_creds = self._get_quay_creds() ocp_release_creds = self._get_ecr_creds( account=ocp_release_info['account'], region=ocp_release_info['region']) ocp_art_dev_creds = self._get_ecr_creds( account=ocp_art_dev_info['account'], region=ocp_art_dev_info['region']) # Creating a single dictionary with all credentials to be used by the # "oc adm release mirror" command self.registry_creds = { 'auths': { **quay_creds['auths'], **ocp_release_creds['auths'], **ocp_art_dev_creds['auths'], } } # Append quay_target_orgs auths to registry_creds for quay_target_org in self.quay_target_orgs: url = quay_target_org['url'] if url in self.registry_creds['auths'].keys(): OcpReleaseMirrorError('Cannot mirror to the same Quay ' f'instance multiple times: {url}') self.registry_creds['auths'].update(quay_target_org['auths']) # Initiate channel groups self.channel_groups = instance['mirrorChannels']
def init_specs_to_fetch( ri: ResourceInventory, oc_map: OC_Map, namespaces: Optional[Iterable[Mapping]] = None, clusters: Optional[Iterable[Mapping]] = None, override_managed_types: Optional[Iterable[str]] = None, managed_types_key: str = 'managedResourceTypes') -> list[StateSpec]: state_specs = [] if clusters and namespaces: raise KeyError('expected only one of clusters or namespaces.') elif namespaces: for namespace_info in namespaces: if override_managed_types is None: managed_types = set( namespace_info.get(managed_types_key) or []) else: managed_types = set(override_managed_types) if not managed_types: continue cluster = namespace_info['cluster']['name'] privileged = namespace_info.get("clusterAdmin", False) is True oc = oc_map.get(cluster, privileged) if not oc: if oc.log_level >= logging.ERROR: ri.register_error() logging.log(level=oc.log_level, msg=oc.message) continue namespace = namespace_info['name'] # These may exit but have a value of None managed_resource_names = \ namespace_info.get('managedResourceNames') or [] managed_resource_type_overrides = \ namespace_info.get('managedResourceTypeOverrides') or [] # Initialize current state specs for resource_type in managed_types: ri.initialize_resource_type(cluster, namespace, resource_type) resource_names = {} resource_type_overrides = {} for mrn in managed_resource_names: # Current implementation guarantees only one # managed_resource_name of each managed type if mrn['resource'] in managed_types: resource_names[mrn['resource']] = mrn['resourceNames'] elif override_managed_types: logging.debug( f"Skipping resource {mrn['resource']} in {cluster}/" f"{namespace} because the integration explicitly " "dismisses it") else: raise KeyError( f"Non-managed resource name {mrn} listed on " f"{cluster}/{namespace} (valid kinds: {managed_types})" ) for o in managed_resource_type_overrides: # Current implementation guarantees only one # override of each managed type if o['resource'] in managed_types: resource_type_overrides[o['resource']] = o['override'] elif override_managed_types: logging.debug( f"Skipping resource type override {o} listed on" f"{cluster}/{namespace} because the integration " "dismisses it explicitly") else: raise KeyError( f"Non-managed override {o} listed on " f"{cluster}/{namespace} (valid kinds: {managed_types})" ) for kind, names in resource_names.items(): c_spec = StateSpec( "current", oc, cluster, namespace, kind, resource_type_override=resource_type_overrides.get(kind), resource_names=names) state_specs.append(c_spec) managed_types.remove(kind) # Produce "empty" StateSpec's for any resource type that # doesn't have an explicit managedResourceName listed in # the namespace state_specs.extend( StateSpec("current", oc, cluster, namespace, t, resource_type_override=resource_type_overrides.get( t), resource_names=None) for t in managed_types) # Initialize desired state specs openshift_resources = namespace_info.get('openshiftResources') for openshift_resource in openshift_resources or []: d_spec = StateSpec("desired", oc, cluster, namespace, openshift_resource, namespace_info, privileged=privileged) state_specs.append(d_spec) elif clusters: # set namespace to something indicative namespace = 'cluster' for cluster_info in clusters: cluster = cluster_info['name'] oc = oc_map.get(cluster) if not oc: if oc.log_level >= logging.ERROR: ri.register_error() logging.log(level=oc.log_level, msg=oc.message) continue # we currently only use override_managed_types, # and not allow a `managedResourcesTypes` field in a cluster file for resource_type in override_managed_types or []: ri.initialize_resource_type(cluster, namespace, resource_type) # Initialize current state specs c_spec = StateSpec("current", oc, cluster, namespace, resource_type) state_specs.append(c_spec) # Initialize desired state specs d_spec = StateSpec("desired", oc, cluster, namespace, resource_type) state_specs.append(d_spec) else: raise KeyError('expected one of clusters or namespaces.') return state_specs
def __init__(self, dry_run, instance): self.dry_run = dry_run self.settings = queries.get_app_interface_settings() cluster_info = instance['hiveCluster'] hive_cluster = instance['hiveCluster']['name'] # Getting the OCM Client for the hive cluster ocm_map = OCMMap(clusters=[cluster_info], integration=QONTRACT_INTEGRATION, settings=self.settings) self.ocm_cli = ocm_map.get(hive_cluster) if not self.ocm_cli: raise OcpReleaseEcrMirrorError(f"Can't create ocm client for " f"cluster {hive_cluster}") # Getting the OC Client for the hive cluster oc_map = OC_Map(clusters=[cluster_info], integration=QONTRACT_INTEGRATION, settings=self.settings) self.oc_cli = oc_map.get(hive_cluster) if not self.oc_cli: raise OcpReleaseEcrMirrorError(f"Can't create oc client for " f"cluster {hive_cluster}") namespace = instance['ecrResourcesNamespace'] ocp_release_identifier = instance['ocpReleaseEcrIdentifier'] ocp_art_dev_identifier = instance['ocpArtDevEcrIdentifier'] ocp_release_info = self._get_tf_resource_info(namespace, ocp_release_identifier) if ocp_release_info is None: raise OcpReleaseEcrMirrorError(f"Could not find rds " f"identifier " f"{ocp_release_identifier} in " f"namespace {namespace['name']}") ocp_art_dev_info = self._get_tf_resource_info(namespace, ocp_art_dev_identifier) if ocp_art_dev_info is None: raise OcpReleaseEcrMirrorError(f"Could not find rds identifier" f" {ocp_art_dev_identifier} in" f"namespace {namespace['name']}") # Getting the AWS Client for the accounts aws_accounts = [ self._get_aws_account_info(account=ocp_release_info['account']), self._get_aws_account_info(account=ocp_art_dev_info['account']) ] self.aws_cli = AWSApi(thread_pool_size=1, accounts=aws_accounts, settings=self.settings, init_ecr_auth_tokens=True) self.aws_cli.map_ecr_resources() self.ocp_release_ecr_uri = self._get_image_uri( account=ocp_release_info['account'], repository=ocp_release_identifier) if self.ocp_release_ecr_uri is None: raise OcpReleaseEcrMirrorError(f"Could not find the " f"ECR repository " f"{ocp_release_identifier}") self.ocp_art_dev_ecr_uri = self._get_image_uri( account=ocp_art_dev_info['account'], repository=ocp_art_dev_identifier) if self.ocp_art_dev_ecr_uri is None: raise OcpReleaseEcrMirrorError(f"Could not find the " f"ECR repository " f"{ocp_art_dev_identifier}") # Getting all the credentials quay_creds = self._get_quay_creds() ocp_release_creds = self._get_ecr_creds( account=ocp_release_info['account'], region=ocp_release_info['region']) ocp_art_dev_creds = self._get_ecr_creds( account=ocp_art_dev_info['account'], region=ocp_art_dev_info['region']) # Creating a single dictionary with all credentials to be used by the # "oc adm release mirror" command self.registry_creds = { 'auths': { **quay_creds['auths'], **ocp_release_creds['auths'], **ocp_art_dev_creds['auths'], } }