def create_connectedk8s(cmd, client, resource_group_name, cluster_name, https_proxy="", http_proxy="", no_proxy="", location=None, kube_config=None, kube_context=None, no_wait=False, tags=None): logger.warning( "Ensure that you have the latest helm version installed before proceeding." ) logger.warning("This operation might take a while...\n") # Setting subscription id subscription_id = get_subscription_id(cmd.cli_ctx) # Send cloud information to telemetry send_cloud_telemetry(cmd) # Fetching Tenant Id graph_client = _graph_client_factory(cmd.cli_ctx) onboarding_tenant_id = graph_client.config.tenant_id # Setting kubeconfig kube_config = set_kube_config(kube_config) # Escaping comma, forward slash present in https proxy urls, needed for helm params. https_proxy = escape_proxy_settings(https_proxy) # Escaping comma, forward slash present in http proxy urls, needed for helm params. http_proxy = escape_proxy_settings(http_proxy) # Escaping comma, forward slash present in no proxy urls, needed for helm params. no_proxy = escape_proxy_settings(no_proxy) # Checking whether optional extra values file has been provided. values_file_provided = False values_file = os.getenv('HELMVALUESPATH') if (values_file is not None) and (os.path.isfile(values_file)): values_file_provided = True logger.warning( "Values files detected. Reading additional helm parameters from same." ) # trimming required for windows os if (values_file.startswith("'") or values_file.startswith('"')): values_file = values_file[1:] if (values_file.endswith("'") or values_file.endswith('"')): values_file = values_file[:-1] # Validate the helm environment file for Dogfood. dp_endpoint_dogfood = None release_train_dogfood = None if cmd.cli_ctx.cloud.endpoints.resource_manager == consts.Dogfood_RMEndpoint: dp_endpoint_dogfood, release_train_dogfood = validate_env_file_dogfood( values_file, values_file_provided) # Loading the kubeconfig file in kubernetes client configuration try: config.load_kube_config(config_file=kube_config, context=kube_context) except Exception as e: telemetry.set_user_fault() telemetry.set_exception(exception=e, fault_type=consts.Load_Kubeconfig_Fault_Type, summary='Problem loading the kubeconfig file') raise CLIError("Problem loading the kubeconfig file." + str(e)) configuration = kube_client.Configuration() # Checking the connection to kubernetes cluster. # This check was added to avoid large timeouts when connecting to AAD Enabled AKS clusters # if the user had not logged in. check_kube_connection(configuration) # Get kubernetes cluster info for telemetry kubernetes_version = get_server_version(configuration) kubernetes_distro = get_kubernetes_distro(configuration) kubernetes_properties = { 'Context.Default.AzureCLI.KubernetesVersion': kubernetes_version, 'Context.Default.AzureCLI.KubernetesDistro': kubernetes_distro } telemetry.add_extension_event('connectedk8s', kubernetes_properties) # Checking helm installation check_helm_install(kube_config, kube_context) # Check helm version helm_version = check_helm_version(kube_config, kube_context) telemetry.add_extension_event( 'connectedk8s', {'Context.Default.AzureCLI.HelmVersion': helm_version}) # Validate location utils.validate_location(cmd, location) resourceClient = _resource_client_factory(cmd.cli_ctx, subscription_id=subscription_id) # Check Release Existance release_namespace = get_release_namespace(kube_config, kube_context) if release_namespace: # Loading config map api_instance = kube_client.CoreV1Api( kube_client.ApiClient(configuration)) try: configmap = api_instance.read_namespaced_config_map( 'azure-clusterconfig', 'azure-arc') except Exception as e: # pylint: disable=broad-except utils.kubernetes_exception_handler( e, consts.Read_ConfigMap_Fault_Type, 'Unable to read ConfigMap', error_message= "Unable to read ConfigMap 'azure-clusterconfig' in 'azure-arc' namespace: ", message_for_not_found= "The helm release 'azure-arc' is present but the azure-arc namespace/configmap is missing. Please run 'helm delete azure-arc --no-hooks' to cleanup the release before onboarding the cluster again." ) configmap_rg_name = configmap.data["AZURE_RESOURCE_GROUP"] configmap_cluster_name = configmap.data["AZURE_RESOURCE_NAME"] if connected_cluster_exists(client, configmap_rg_name, configmap_cluster_name): if (configmap_rg_name.lower() == resource_group_name.lower() and configmap_cluster_name.lower() == cluster_name.lower()): # Re-put connected cluster try: public_key = client.get( configmap_rg_name, configmap_cluster_name).agent_public_key_certificate except Exception as e: # pylint: disable=broad-except utils.arm_exception_handler( e, consts.Get_ConnectedCluster_Fault_Type, 'Failed to check if connected cluster resource already exists.' ) cc = generate_request_payload(configuration, location, public_key, tags) create_cc_resource(client, resource_group_name, cluster_name, cc, no_wait) else: telemetry.set_user_fault() telemetry.set_exception( exception='The kubernetes cluster is already onboarded', fault_type=consts.Cluster_Already_Onboarded_Fault_Type, summary='Kubernetes cluster already onboarded') raise CLIError( "The kubernetes cluster you are trying to onboard " + "is already onboarded to the resource group" + " '{}' with resource name '{}'.".format( configmap_rg_name, configmap_cluster_name)) else: # Cleanup agents and continue with put delete_arc_agents(release_namespace, kube_config, kube_context, configuration) else: if connected_cluster_exists(client, resource_group_name, cluster_name): telemetry.set_user_fault() telemetry.set_exception( exception='The connected cluster resource already exists', fault_type=consts.Resource_Already_Exists_Fault_Type, summary='Connected cluster resource already exists') raise CLIError( "The connected cluster resource {} already exists ".format( cluster_name) + "in the resource group {} ".format(resource_group_name) + "and corresponds to a different Kubernetes cluster. To onboard this Kubernetes cluster" + "to Azure, specify different resource name or resource group name." ) # Resource group Creation if resource_group_exists(cmd.cli_ctx, resource_group_name, subscription_id) is False: resource_group_params = {'location': location} try: resourceClient.resource_groups.create_or_update( resource_group_name, resource_group_params) except Exception as e: # pylint: disable=broad-except utils.arm_exception_handler(e, consts.Create_ResourceGroup_Fault_Type, 'Failed to create the resource group') # Adding helm repo if os.getenv('HELMREPONAME') and os.getenv('HELMREPOURL'): utils.add_helm_repo(kube_config, kube_context) # Retrieving Helm chart OCI Artifact location registry_path = os.getenv('HELMREGISTRY') if os.getenv( 'HELMREGISTRY') else utils.get_helm_registry( cmd, location, dp_endpoint_dogfood, release_train_dogfood) # Get azure-arc agent version for telemetry azure_arc_agent_version = registry_path.split(':')[1] telemetry.add_extension_event( 'connectedk8s', {'Context.Default.AzureCLI.AgentVersion': azure_arc_agent_version}) # Get helm chart path chart_path = utils.get_chart_path(registry_path, kube_config, kube_context) # Generate public-private key pair try: key_pair = RSA.generate(4096) except Exception as e: telemetry.set_exception( exception=e, fault_type=consts.KeyPair_Generate_Fault_Type, summary='Failed to generate public-private key pair') raise CLIError("Failed to generate public-private key pair. " + str(e)) try: public_key = get_public_key(key_pair) except Exception as e: telemetry.set_exception(exception=e, fault_type=consts.PublicKey_Export_Fault_Type, summary='Failed to export public key') raise CLIError("Failed to export public key." + str(e)) try: private_key_pem = get_private_key(key_pair) except Exception as e: telemetry.set_exception(exception=e, fault_type=consts.PrivateKey_Export_Fault_Type, summary='Failed to export private key') raise CLIError("Failed to export private key." + str(e)) # Generate request payload cc = generate_request_payload(configuration, location, public_key, tags) # Create connected cluster resource put_cc_response = create_cc_resource(client, resource_group_name, cluster_name, cc, no_wait) # Install azure-arc agents helm_install_release(chart_path, subscription_id, kubernetes_distro, resource_group_name, cluster_name, location, onboarding_tenant_id, http_proxy, https_proxy, no_proxy, private_key_pem, kube_config, kube_context, no_wait, values_file_provided, values_file) return put_cc_response
def create_connectedk8s(cmd, client, resource_group_name, cluster_name, location=None, kube_config=None, kube_context=None, no_wait=False, tags=None): logger.warning("Ensure that you have the latest helm version installed before proceeding.") logger.warning("This operation might take a while...\n") # Setting subscription id subscription_id = get_subscription_id(cmd.cli_ctx) # Setting user profile profile = Profile(cli_ctx=cmd.cli_ctx) # Fetching Tenant Id graph_client = _graph_client_factory(cmd.cli_ctx) onboarding_tenant_id = graph_client.config.tenant_id # Setting kubeconfig kube_config = set_kube_config(kube_config) # Removing quotes from kubeconfig path. This is necessary for windows OS. trim_kube_config(kube_config) # Loading the kubeconfig file in kubernetes client configuration try: config.load_kube_config(config_file=kube_config, context=kube_context) except Exception as e: telemetry.set_user_fault() telemetry.set_exception(exception=e, fault_type=Load_Kubeconfig_Fault_Type, summary='Problem loading the kubeconfig file') raise CLIError("Problem loading the kubeconfig file." + str(e)) configuration = kube_client.Configuration() # Checking the connection to kubernetes cluster. # This check was added to avoid large timeouts when connecting to AAD Enabled AKS clusters # if the user had not logged in. check_kube_connection(configuration) # Get kubernetes cluster info for telemetry kubernetes_version = get_server_version(configuration) kubernetes_distro = get_kubernetes_distro(configuration) kubernetes_properties = { 'Context.Default.AzureCLI.KubernetesVersion': kubernetes_version, 'Context.Default.AzureCLI.KubernetesDistro': kubernetes_distro } telemetry.add_extension_event('connectedk8s', kubernetes_properties) # Checking helm installation check_helm_install(kube_config, kube_context) # Check helm version helm_version = check_helm_version(kube_config, kube_context) telemetry.add_extension_event('connectedk8s', {'Context.Default.AzureCLI.HelmVersion': helm_version}) # Validate location rp_locations = [] resourceClient = _resource_client_factory(cmd.cli_ctx, subscription_id=subscription_id) providerDetails = resourceClient.providers.get('Microsoft.Kubernetes') for resourceTypes in providerDetails.resource_types: if resourceTypes.resource_type == 'connectedClusters': rp_locations = [location.replace(" ", "").lower() for location in resourceTypes.locations] if location.lower() not in rp_locations: telemetry.set_user_fault() telemetry.set_exception(exception='Location not supported', fault_type=Invalid_Location_Fault_Type, summary='Provided location is not supported for creating connected clusters') raise CLIError("Connected cluster resource creation is supported only in the following locations: " + ', '.join(map(str, rp_locations)) + ". Use the --location flag to specify one of these locations.") break # Check Release Existance release_namespace = get_release_namespace(kube_config, kube_context) if release_namespace is not None: # Loading config map api_instance = kube_client.CoreV1Api(kube_client.ApiClient(configuration)) try: configmap = api_instance.read_namespaced_config_map('azure-clusterconfig', 'azure-arc') except Exception as e: # pylint: disable=broad-except telemetry.set_exception(exception=e, fault_type=Read_ConfigMap_Fault_Type, summary='Unable to read ConfigMap') raise CLIError("Unable to read ConfigMap 'azure-clusterconfig' in 'azure-arc' namespace: %s\n" % e) configmap_rg_name = configmap.data["AZURE_RESOURCE_GROUP"] configmap_cluster_name = configmap.data["AZURE_RESOURCE_NAME"] if connected_cluster_exists(client, configmap_rg_name, configmap_cluster_name): if (configmap_rg_name.lower() == resource_group_name.lower() and configmap_cluster_name.lower() == cluster_name.lower()): # Re-put connected cluster public_key = client.get(configmap_rg_name, configmap_cluster_name).agent_public_key_certificate cc = generate_request_payload(configuration, location, public_key, tags) try: return sdk_no_wait(no_wait, client.create, resource_group_name=resource_group_name, cluster_name=cluster_name, connected_cluster=cc) except CloudError as ex: telemetry.set_exception(exception=ex, fault_type=Create_ConnectedCluster_Fault_Type, summary='Unable to create connected cluster resource') raise CLIError(ex) else: telemetry.set_user_fault() telemetry.set_exception(exception='The kubernetes cluster is already onboarded', fault_type=Cluster_Already_Onboarded_Fault_Type, summary='Kubernetes cluster already onboarded') raise CLIError("The kubernetes cluster you are trying to onboard " + "is already onboarded to the resource group" + " '{}' with resource name '{}'.".format(configmap_rg_name, configmap_cluster_name)) else: # Cleanup agents and continue with put delete_arc_agents(release_namespace, kube_config, kube_context, configuration) else: if connected_cluster_exists(client, resource_group_name, cluster_name): telemetry.set_user_fault() telemetry.set_exception(exception='The connected cluster resource already exists', fault_type=Resource_Already_Exists_Fault_Type, summary='Connected cluster resource already exists') raise CLIError("The connected cluster resource {} already exists ".format(cluster_name) + "in the resource group {} ".format(resource_group_name) + "and corresponds to a different Kubernetes cluster. To onboard this Kubernetes cluster" + "to Azure, specify different resource name or resource group name.") # Resource group Creation if resource_group_exists(cmd.cli_ctx, resource_group_name, subscription_id) is False: resource_group_params = {'location': location} try: resourceClient.resource_groups.create_or_update(resource_group_name, resource_group_params) except Exception as e: telemetry.set_exception(exception=e, fault_type=Create_ResourceGroup_Fault_Type, summary='Failed to create the resource group') raise CLIError("Failed to create the resource group {} :".format(resource_group_name) + str(e)) # Adding helm repo if os.getenv('HELMREPONAME') and os.getenv('HELMREPOURL'): repo_name = os.getenv('HELMREPONAME') repo_url = os.getenv('HELMREPOURL') cmd_helm_repo = ["helm", "repo", "add", repo_name, repo_url, "--kubeconfig", kube_config] if kube_context: cmd_helm_repo.extend(["--kube-context", kube_context]) response_helm_repo = Popen(cmd_helm_repo, stdout=PIPE, stderr=PIPE) _, error_helm_repo = response_helm_repo.communicate() if response_helm_repo.returncode != 0: telemetry.set_exception(exception=error_helm_repo.decode("ascii"), fault_type=Add_HelmRepo_Fault_Type, summary='Failed to add helm repository') raise CLIError("Unable to add repository {} to helm: ".format(repo_url) + error_helm_repo.decode("ascii")) # Retrieving Helm chart OCI Artifact location registry_path = os.getenv('HELMREGISTRY') if os.getenv('HELMREGISTRY') else get_helm_registry(profile, location) # Get azure-arc agent version for telemetry azure_arc_agent_version = registry_path.split(':')[1] telemetry.add_extension_event('connectedk8s', {'Context.Default.AzureCLI.AgentVersion': azure_arc_agent_version}) # Pulling helm chart from registry os.environ['HELM_EXPERIMENTAL_OCI'] = '1' pull_helm_chart(registry_path, kube_config, kube_context) # Exporting helm chart chart_export_path = os.path.join(os.path.expanduser('~'), '.azure', 'AzureArcCharts') export_helm_chart(registry_path, chart_export_path, kube_config, kube_context) # Generate public-private key pair try: key_pair = RSA.generate(4096) except Exception as e: telemetry.set_exception(exception=e, fault_type=KeyPair_Generate_Fault_Type, summary='Failed to generate public-private key pair') raise CLIError("Failed to generate public-private key pair. " + str(e)) try: public_key = get_public_key(key_pair) except Exception as e: telemetry.set_exception(exception=e, fault_type=PublicKey_Export_Fault_Type, summary='Failed to export public key') raise CLIError("Failed to export public key." + str(e)) try: private_key_pem = get_private_key(key_pair) except Exception as e: telemetry.set_exception(exception=e, fault_type=PrivateKey_Export_Fault_Type, summary='Failed to export private key') raise CLIError("Failed to export private key." + str(e)) # Helm Install helm_chart_path = os.path.join(chart_export_path, 'azure-arc-k8sagents') chart_path = os.getenv('HELMCHART') if os.getenv('HELMCHART') else helm_chart_path cmd_helm_install = ["helm", "upgrade", "--install", "azure-arc", chart_path, "--set", "global.subscriptionId={}".format(subscription_id), "--set", "global.kubernetesDistro={}".format(kubernetes_distro), "--set", "global.resourceGroupName={}".format(resource_group_name), "--set", "global.resourceName={}".format(cluster_name), "--set", "global.location={}".format(location), "--set", "global.tenantId={}".format(onboarding_tenant_id), "--set", "global.onboardingPrivateKey={}".format(private_key_pem), "--set", "systemDefaultValues.spnOnboarding=false", "--kubeconfig", kube_config, "--output", "json"] if kube_context: cmd_helm_install.extend(["--kube-context", kube_context]) response_helm_install = Popen(cmd_helm_install, stdout=PIPE, stderr=PIPE) _, error_helm_install = response_helm_install.communicate() if response_helm_install.returncode != 0: telemetry.set_exception(exception=error_helm_install.decode("ascii"), fault_type=Install_HelmRelease_Fault_Type, summary='Unable to install helm release') raise CLIError("Unable to install helm release: " + error_helm_install.decode("ascii")) # Create connected cluster resource cc = generate_request_payload(configuration, location, public_key, tags) try: put_cc_response = sdk_no_wait(no_wait, client.create, resource_group_name=resource_group_name, cluster_name=cluster_name, connected_cluster=cc) if no_wait: return put_cc_response except CloudError as ex: telemetry.set_exception(exception=ex, fault_type=Create_ConnectedCluster_Fault_Type, summary='Unable to create connected cluster resource') raise CLIError(ex) # Getting total number of pods scheduled to run in azure-arc namespace api_instance = kube_client.CoreV1Api(kube_client.ApiClient(configuration)) pod_dict = get_pod_dict(api_instance) # Checking azure-arc pod statuses try: check_pod_status(pod_dict) except Exception as e: # pylint: disable=broad-except telemetry.set_exception(exception=e, fault_type=Check_PodStatus_Fault_Type, summary='Failed to check arc agent pods statuses') logger.warning("Failed to check arc agent pods statuses: %s", e) return put_cc_response