def authenticate_cli(self) -> Credentials: """ Implements authentication for the Azure provider """ try: # Set logging level to error for libraries as otherwise generates a lot of warnings logging.getLogger('adal-python').setLevel(logging.ERROR) logging.getLogger('msrest').setLevel(logging.ERROR) logging.getLogger('msrestazure.azure_active_directory').setLevel(logging.ERROR) logging.getLogger('urllib3').setLevel(logging.ERROR) arm_credentials, subscription_id, tenant_id = get_azure_cli_credentials(with_tenant=True) aad_graph_credentials, placeholder_1, placeholder_2 = get_azure_cli_credentials( with_tenant=True, resource='https://graph.windows.net', ) profile = get_cli_profile() return Credentials( arm_credentials, aad_graph_credentials, tenant_id=tenant_id, current_user=profile.get_current_account_user(), subscription_id=subscription_id, ) except HttpResponseError as e: if ', AdalError: Unsupported wstrust endpoint version. ' \ 'Current supported version is wstrust2005 or wstrust13.' in e.args: logger.error( f'You are likely authenticating with a Microsoft Account. \ This authentication mode only supports Azure Active Directory principal authentication.\ {e}', ) raise e
def skip_without_credentials(func): try: get_azure_cli_credentials() except FileNotFoundError: return pytest.mark.skip( reason=""" You must configure your Azure credentials to run this test. $ az login """ )(func) rg = dask.config.get("cloudprovider.azure.azurevm.resource_group", None) vnet = dask.config.get("cloudprovider.azure.azurevm.vnet", None) security_group = dask.config.get("cloudprovider.azure.azurevm.security_group", None) location = dask.config.get("cloudprovider.azure.location", None) if rg is None or vnet is None or security_group is None or location is None: return pytest.mark.skip( reason=""" You must configure your Azure resource group and vnet to run this test. $ export DASK_CLOUDPROVIDER__AZURE__LOCATION="<LOCATION>" $ export DASK_CLOUDPROVIDER__AZURE__AZUREVM__RESOURCE_GROUP="<RESOURCE GROUP>" $ export DASK_CLOUDPROVIDER__AZURE__AZUREVM__VNET="<VNET>" $ export DASK_CLOUDPROVIDER__AZURE__AZUREVM__SECURITY_GROUP="<SECUROTY GROUP>" """ )(func) return func
def main(n: int = None) -> None: """ Show all Azure Subscriptions in current profile using the `az` command-line utility. Ask user input for switching to another subscription. """ try: subscriptions = json.loads( subprocess.getoutput('az account list --output json')) current_nr = _print_options(subscriptions) if not n: n = click.prompt('Switch', type=int, default=current_nr) if n not in range(1, len(subscriptions) + 1): raise ValueError("Value not in range! Not changing subscription.") _select_subscription(n, subscriptions) _, subscription_id = get_azure_cli_credentials() active = next( filter(lambda x: x['id'] == subscription_id, subscriptions)) click.echo("Active: " + click.style( active['id'] + ": " + active['name'], fg='green', bold=True)) except subprocess.CalledProcessError: # Issue is already printed to stderr. pass except click.exceptions.Abort: # No need to raise exception when CTRL-C out of the cli pass except ValueError as e: print(e)
def auth_azuread(self, workspace_resource_id=None, token_callback=None, subscription_id=None, resource_group=None, workspace_name=None): if token_callback is None: from azure.common.credentials import get_azure_cli_credentials credentials, subscription_id = get_azure_cli_credentials() def token_callback(resource): return credentials.get_token(resource).token if workspace_resource_id is None: if resource_group is None or workspace_name is None: raise ValueError( "Either workspace_resource_id or both of " "resource_group and workspace_name must be provided") if subscription_id is None: raise ValueError("subscription_id must be provided") workspace_resource_id = ( '/subscriptions/%s/resourceGroups/%s/providers/' 'Microsoft.Databricks/workspaces/%s' % (subscription_id, resource_group, workspace_name)) adb_token = token_callback('2ff814a6-3304-4ab8-85cb-cd0e6f879c1d') arm_token = token_callback('https://management.core.windows.net/') self.dbricks_auth = { 'Authorization': 'Bearer %s' % adb_token, 'X-Databricks-Azure-SP-Management-Token': arm_token, 'X-Databricks-Azure-Workspace-Resource-Id': workspace_resource_id, }
def get_azure_client(): client_id = environ.get("CLIENT_ID", "") client_secret = environ.get("CLIENT_SECRET", "") tenant_id = environ.get("TENANT_ID", "") if client_id and client_secret and tenant_id: logger.info( "Found credentials in environment variables, using service principle for auth" ) credentials = ServicePrincipalCredentials( tenant=tenant_id, client_id=client_id, secret=client_secret, resource="https://graph.windows.net", ) return GraphRbacManagementClient(credentials, tenant_id) try: credentials, _, tenant_id = get_azure_cli_credentials( resource="https://graph.windows.net", with_tenant=True) except FileNotFoundError: logger.fatal( "Failed to find Azure credentials. Please configure environment variables or an az login JSON." ) raise return GraphRbacManagementClient(credentials, tenant_id)
def _configure_resource_group(config): # TODO: look at availability sets # https://docs.microsoft.com/en-us/azure/virtual-machines/windows/tutorial-availability-sets resource_client = _get_client(ResourceManagementClient, config) _, cli_subscription_id = get_azure_cli_credentials( resource=ResourceManagementClient) subscription_id = config["provider"].get("subscription_id", cli_subscription_id) logger.info("Using subscription id: %s", subscription_id) config["provider"]["subscription_id"] = subscription_id assert "resource_group" in config["provider"], ( "Provider config must include resource_group field") resource_group = config["provider"]["resource_group"] assert "location" in config["provider"], ( "Provider config must include location field") params = {"location": config["provider"]["location"]} if "tags" in config["provider"]: params["tags"] = config["provider"]["tags"] logger.info("Creating/Updating Resource Group: %s", resource_group) resource_client.resource_groups.create_or_update( resource_group_name=resource_group, parameters=params) # load the template file current_path = Path(__file__).parent template_path = current_path.joinpath("azure-config-template.json") with open(template_path, "r") as template_fp: template = json.load(template_fp) # choose a random subnet, skipping most common value of 0 random.seed(resource_group) subnet_mask = "10.{}.0.0/16".format(random.randint(1, 254)) parameters = { "properties": { "mode": DeploymentMode.incremental, "template": template, "parameters": { "subnet": { "value": subnet_mask } } } } if hasattr(resource_client.deployments, "create_or_update"): create_or_update = resource_client.deployments.create_or_update else: create_or_update = resource_client.deployments.begin_create_or_update create_or_update(resource_group_name=resource_group, deployment_name="ray-config", parameters=parameters).wait() return config
def _get_azure_cli_credentials(self): credentials, subscription_id = get_azure_cli_credentials() cloud_environment = get_cli_active_cloud() cli_credentials = { 'credentials': credentials, 'subscription_id': subscription_id, 'cloud_environment': cloud_environment } return cli_credentials
def _get_azure_cli_credentials(self): credentials, subscription_id = get_azure_cli_credentials() cloud_environment = get_cli_active_cloud() cli_credentials = { 'credentials': credentials, 'subscription_id': subscription_id, 'cloud_environment': cloud_environment } return cli_credentials
def get_subscription(self): try: return get_azure_cli_credentials()[1] except Exception as error: logging.warning("Couldn't get subscription id from Azure CLI.") try: client, secret, tenant, sub = self.get_config(self.config_path) return sub except Exception as error: logging.error("Failed to get a subscription id.")
def _get_azure_cli_profile(self): if not HAS_AZURE_CLI_CORE: self.fail("Do you have azure-cli-core installed? Try `pip install 'azure-cli-core' --upgrade`") try: credentials, subscription_id = get_azure_cli_credentials() self._cloud_environment = get_cli_active_cloud() return { 'credentials': credentials, 'subscription_id': subscription_id } except CLIError as err: self.fail("AzureCLI profile cannot be loaded - {0}".format(err))
def _get_azure_cli_profile(self): if not HAS_AZURE_CLI_CORE: self.fail("Do you have azure-cli-core installed? Try `pip install 'azure-cli-core' --upgrade`") try: credentials, subscription_id = get_azure_cli_credentials() self._cloud_environment = get_cli_active_cloud() return { 'credentials': credentials, 'subscription_id': subscription_id } except CLIError as err: self.fail("AzureCLI profile cannot be loaded - {0}".format(err))
def _get_azure_cli_profile(self): if not HAS_AZURE_CLI_CORE: self.fail("Do you have azure-cli-core installed? Try `pip install 'azure-cli-core' --upgrade`") try: credentials, subscription_id = get_azure_cli_credentials() base_url = get_cli_active_cloud().endpoints.resource_manager return { 'credentials': credentials, 'subscription_id': subscription_id, 'base_url': base_url } except CLIError as err: self.fail("AzureCLI profile cannot be loaded - {0}".format(err))
def main(req: func.HttpRequest) -> func.HttpResponse: if "MSI_ENDPOINT" in os.environ: credentials = MSIAuthentication() else: credentials, *_ = get_azure_cli_credentials() subscription_client = SubscriptionClient(credentials) subscription = next(subscription_client.subscriptions.list()) subscription_id = subscription.subscription_id client = ResourceManagementClient(credentials, subscription_id) resource_groups = [g.name for g in client.resource_groups.list()] return func.HttpResponse(json.dumps(resource_groups), mimetype="application/json")
async def main(req: func.HttpRequest) -> func.HttpResponse: """ The main entry point to the function. """ if "MSI_ENDPOINT" in os.environ: credentials = MSIAuthentication() else: credentials, *_ = get_azure_cli_credentials() subscription_id = os.environ.get('AZURE_SUBSCRIPTION_ID', '11111111-1111-1111-1111-111111111111') list_of_rgs = await list_rgs(credentials, subscription_id) return func.HttpResponse(list_of_rgs, mimetype="application/json")
def main(n: int = None, v: bool = False) -> None: """ Show all Azure Subscriptions in current profile using the `az` command-line utility. Ask user input for switching to another subscription. """ try: # Using --query to map subset of fields and sort by name (ascending) list_cmd = 'az account list --all --output json ' \ '--query "sort_by([].{name:name, isDefault:isDefault, id:id, state:state}, &name)"' if v: click.echo(f'Issuing AZ CLI command: {list_cmd}') subscriptions = json.loads(subprocess.getoutput(list_cmd)) current_nr = _print_options(subscriptions) if not n: n = click.prompt('Switch', type=int, default=current_nr) if n not in range(1, len(subscriptions) + 1): raise ValueError("Value not in range! Not changing subscription.") if n == current_nr: click.echo( "Selection is same as current. Not changing subscription.") else: _select_subscription(n, v, subscriptions) _, subscription_id = get_azure_cli_credentials() active = next( filter(lambda x: x['id'] == subscription_id, subscriptions)) click.echo("Active: " + click.style( active['id'] + ": " + active['name'], fg='green', bold=True)) if active['state'].lower() == 'disabled': click.echo( click.style( "Subscription state is Disabled, requires: az login!", fg='yellow')) except subprocess.CalledProcessError: # Issue is already printed to stderr. pass except click.exceptions.Abort: # No need to raise exception when CTRL-C out of the cli pass except ValueError as e: print(e)
def resource_apply_tags(id, subscription, provider, tags): # NOTE: I've observed that the app serice needs to be restarted once # you add the web app managed identity to the subscription. Otherwise it_Owner ends up blank. if "LOCAL_DEBUG" in os.environ and os.environ["LOCAL_DEBUG"] == '1': # When running locally for debug/development sys.stderr.write("****** USING CLI AUTHENTICATION ***********") creds, _ = credentials.get_azure_cli_credentials(resource=None, with_tenant=False) else: # When running on Azure, and managed identity is used to grant tag priviledge. sys.stderr.write("****** USING MSI AUTHENTICATION ***********") creds = MSIAuthentication() resource_client = ResourceManagementClient(creds, subscription) try: r = resource_client.resources.get_by_id(id, api_version_lookup[provider]) except: print("Lookup Failed: Skipped...", id) return False # The tag operation is not additive. Preserve tags already there and add new ones. try: current_tags = r.tags if 'it_Owner' in current_tags and len(current_tags['it_Owner']) > 0: sys.stderr.write( "**************it_Owner exists. Do Nothing. ***************") return False except: assert False, "Invalid tags provided." merged_tags = {} if current_tags: merged_tags.update(current_tags) merged_tags.update(tags) r.tags = merged_tags resource_client.resources.create_or_update_by_id( id, api_version_lookup[provider], r) return True
def resizeDisksToStandardSSD(diskIDs): #credentials = get_credentials() cred = get_azure_cli_credentials() newSKU = DiskSku(name="StandardSSD_LRS") for sub in subscriptionIds: compute_client = ComputeManagementClient(cred[0], sub) logging.warning("Changing level in subscription {}".format(sub)) logging.warning("Resource group \t diskname") for disk in diskIDs: if (disk['id'].split('/')[2] == sub): rgName = disk['id'].split('/')[4] diskName = disk['id'].split('/')[8] logging.warning("{}\t{}".format(rgName, diskName)) disk = compute_client.disks.get(rgName, diskName) disk.sku = newSKU async_disk_update = compute_client.disks.begin_create_or_update( rgName, diskName, disk) async_disk_update.wait() return True
def authenticate_from_cli(self, resource): """ Create the azure resource manager client from the user logged into azure cli If a user is not logged in the run the az login command so they can authenticate """ print(f"Attempting to get cli credentials for resource {resource}") from azure.common.credentials import get_azure_cli_credentials try: credential, subscription_id, self.tenant_id = get_azure_cli_credentials( resource=resource, with_tenant=True) except: print("We were not able to find a cli profile attempting to login") from subprocess import run import platform shell = platform.system() == "Windows" run(["az", "login", "--use-device-code"], shell=shell) return self.authenticate_from_cli(resource) return credential
def main(mytimer: func.TimerRequest, template: func.InputStream) -> None: """main is run from the timer function Args: mytimer (func.TimerRequest): [description] """ logging.info('RandomDeployer started at %s', formatted_time()) subscription_id = None if "MSI_ENDPOINT" in os.environ: credentials = MSIAuthentication() else: credentials, subscription_id = get_azure_cli_credentials() subscription_id = os.environ.get('RANDOM_DEPLOY_SUBSCRIPTION_ID', subscription_id) logging.info('Using subscription %s', subscription_id) deployer = RandomDeployer(subscription_id, credentials, json.load(template)) deployer.deploy(int(os.environ.get('RANDOM_DEPLOY_LIFETIME', 60*60*24))) logging.info('RandomDeployer done at %s', formatted_time())
try: tenant = os.environ["AZ_AD_TENANT"] client_id = os.environ["AZ_AD_ID"] secret = os.environ["AZ_AD_PASS"] except KeyError as ke: print("set the following env variables:") print("AZ_AD_ID") print("AZ_AD_TENANT") print("AZ_AD_PASS") sys.exit(1) credentials = ServicePrincipalCredentials(client_id=client_id, secret=secret, tenant=tenant) subscription = get_azure_cli_credentials()[1] new_resource_groups = ["wewewe", "ererere", "fgfgfgfg"] res_client = azure.mgmt.resource.ResourceManagementClient( credentials, subscription) def print_resource_group_info(res_client): for res_group in res_client.resource_groups.list(): for thing in ["name", "id", "location", "tags"]: print("{:.<10}{}{}{}".format(thing, "{0.", thing, "}").format(res_group)) print("." * 20) print("===============Before")
def main(): """ Builds the Azure ML pipeline for data engineering and model training. """ databricks_workspace_name = os.environ['DATABRICKS_WORKSPACE_NAME'] training_data_account_name = os.environ['TRAINING_DATA_ACCOUNT_NAME'] build_id = os.getenv('BUILD_BUILDID', 0) # Get Azure machine learning workspace aml_workspace = Workspace.get( name=os.environ['AML_WORKSPACE_NAME'], subscription_id=os.environ['SUBSCRIPTION_ID'], resource_group=os.environ['RESOURCE_GROUP'], ) print(aml_workspace) # Generate Databricks credentials, see https://aka.ms/databricks-aad dbricks_region = aml_workspace.location dbricks_api = f"https://{dbricks_region}.azuredatabricks.net/api/2.0" dbricks_client = databricks_client.create(dbricks_api) dbricks_client.auth_azuread(resource_group=aml_workspace.resource_group, workspace_name=databricks_workspace_name) dbricks_client.ensure_available() # Attach Databricks as Azure ML training compute dbricks_compute_name = "databricks" dbricks_compute = get_databricks_compute( aml_workspace, dbricks_compute_name, ) if dbricks_compute is None: pat_token = dbricks_client.post( 'token/create', json={"comment": "Azure ML Token generated by Build " + build_id })['token_value'] dbricks_compute = create_databricks_compute( aml_workspace, databricks_workspace_name, dbricks_compute_name, pat_token, ) print("dbricks_compute:") print(dbricks_compute) # Create Databricks instance pool pool_name = "azureml_training" instance_pool_id = get_instance_pool(dbricks_client, pool_name) if not instance_pool_id: dbricks_client.post('instance-pools/create', json={ "instance_pool_name": pool_name, "node_type_id": "Standard_D3_v2", "idle_instance_autotermination_minutes": 10, "preloaded_spark_versions": [DATABRICKS_RUNTIME_VERSION], }) instance_pool_id = get_instance_pool(dbricks_client, pool_name) notebook_folder = f"/Shared/AzureMLDeployed" workspace_datastore = Datastore(aml_workspace, "workspaceblobstore") # Create a datastore for the training data container credentials, subscription = get_azure_cli_credentials() storage_client = StorageManagementClient(credentials, subscription) training_storage_keys = storage_client.storage_accounts.list_keys( aml_workspace.resource_group, training_data_account_name) training_datastore = Datastore.register_azure_blob_container( workspace=aml_workspace, datastore_name="trainingdata", container_name="trainingdata", account_name=training_data_account_name, account_key=training_storage_keys.keys[0].value, ) # FEATURE ENGINEERING STEP (DATABRICKS) # Create feature engineering pipeline step training_data_input = DataReference(datastore=training_datastore, path_on_datastore="/", data_reference_name="training") feature_eng_output = PipelineData("feature_engineered", datastore=workspace_datastore) notebook_path = upload_notebook(dbricks_client, notebook_folder, "code/prepare", "feature_engineering") training_dataprep_step = DatabricksStep( name="FeatureEngineering", inputs=[training_data_input], outputs=[feature_eng_output], spark_version=DATABRICKS_RUNTIME_VERSION, instance_pool_id=instance_pool_id, num_workers=3, notebook_path=notebook_path, run_name="FeatureEngineering", compute_target=dbricks_compute, allow_reuse=True, ) # You can add Azure ML model training tasks using # feature_eng_output as input. # ... # Create Azure ML Pipeline steps = [training_dataprep_step] ml_pipeline = Pipeline(workspace=aml_workspace, steps=steps) ml_pipeline.validate() published_pipeline = ml_pipeline.publish( name="Feature Engineering", description="Feature engineering pipeline", version=build_id, ) print(f"Published pipeline: {published_pipeline.name}") print(f"for build {published_pipeline.version}") # When running in Azure DevOps, set AMLPIPELINE_ID variable # for AML Pipeline task in next job print("Setting Azure DevOps variable") print(f"##vso[task.setvariable variable=AMLPIPELINE_ID;isOutput=true]" f"{published_pipeline.id}")
def main(): print('Azure Import Tags Util v1.4') # Check that we have args if (len(sys.argv) < 10): print('YOU MUST SPECIFY THE CORRECT COMMAND LINE PARAMETERS:') print('Usage:') print( 'azureimporttags.py [vm_input_file] [input_file] [migrated_from] [migrate_project] [azure_region] [subscriptiom] [client_id] [secret] [tenant]' ) print( 'NOTE: the migrated_from field should contain one of these values:' ) print(' On Premise') print(' AWS') print(' Azure') print(' Equinix') print(' CGI') print('') sys.exit(1) # Get command line args if (sys.argv[0] == 'python'): vm_input_file = sys.argv[2] inputfile = sys.argv[3] migrated_from = sys.argv[4] migrate_project = sys.argv[5] region = sys.argv[6] sub_id = sys.argv[7] client_id = sys.argv[8] secret = sys.argv[9] tenant = sys.argv[10] else: vm_input_file = sys.argv[1] inputfile = sys.argv[2] migrated_from = sys.argv[3] migrate_project = sys.argv[4] region = sys.argv[5] sub_id = sys.argv[6] client_id = sys.argv[7] secret = sys.argv[8] tenant = sys.argv[9] print('VM List file: ', vm_input_file) print('Inputfile: ', inputfile) print('Region: ', region) print('MigrateProject: ', migrate_project) print('sub_id: ', sub_id) print('client_id: ', client_id) print('secret: ', secret) print('tenant: ', tenant) # Start logging updatelog('------ Starting run ------') updatelog('Region: ', region) # If not a valid migrated_from field, abort with error migrated_from_cases = { 'On Premise': 'On Premise', 'AWS': 'AWS', 'Azure': 'Azure', 'Equinix': 'Equinix', 'CGI': 'CGI', 'Native': 'Native' } migrated_from = migrated_from_cases.get(migrated_from, 'INVALID') if (migrated_from == 'INVALID'): print('') print('ERROR: invalid migrated_from arg was passed') updatelog('ERROR: invalid migrated_from arg was passed: ', migrated_from) print('') exit(2) print('migrated_from: ', migrated_from) updatelog('migrated_from: ', migrated_from) # Get creds updatelog('Calling ServicePrincipalCredentials()') subscription_id = sub_id #credentials = ServicePrincipalCredentials(client_id = client_id, secret = secret, tenant = tenant) # Testing this code credentials = creds.get_azure_cli_credentials(resource=None, with_tenant=False)[0] sub_client = SubscriptionClient(credentials) # Create object for compute-related interactions updatelog('Calling ComputeManagementClient()') compute_client = ComputeManagementClient(credentials, subscription_id) # Read all tag data into taglist var taglist = [] updatelog('Calling loadrecordsfromfile()') loadrecordsfromfile(inputfile, taglist) # Get VM input list file target_vm_list = get_target_vm_list(vm_input_file) # Get list of all Azure VMs and add to list #az_vm_list = [] updatelog('Calling getallazvms()') #getallazvms(compute_client, az_vm_list, target_vm_list) az_vm_list = getallazvms(compute_client, target_vm_list) print('VMs: ', az_vm_list) # Tag'em updatelog('Calling tageachvm()') tageachvmfromlist(credentials, subscription_id, compute_client, migrated_from, migrate_project, region, taglist, az_vm_list) updatelog('------ Exiting ------')
def __init__( self, location: str = None, resource_group: str = None, vnet: str = None, security_group: str = None, public_ingress: bool = None, vm_size: str = None, scheduler_vm_size: str = None, vm_image: dict = {}, disk_size: int = None, bootstrap: bool = None, auto_shutdown: bool = None, docker_image=None, debug: bool = False, marketplace_plan: dict = {}, **kwargs, ): self.config = dask.config.get("cloudprovider.azure.azurevm", {}) self.scheduler_class = AzureVMScheduler self.worker_class = AzureVMWorker self.location = (location if location is not None else dask.config.get("cloudprovider.azure.location")) if self.location is None: raise ConfigError("You must configure a location") self.resource_group = (resource_group if resource_group is not None else self.config.get("resource_group")) if self.resource_group is None: raise ConfigError("You must configure a resource_group") self.public_ingress = (public_ingress if public_ingress is not None else self.config.get("public_ingress")) self.credentials, self.subscription_id = get_azure_cli_credentials() self.compute_client = ComputeManagementClient(self.credentials, self.subscription_id) self.network_client = NetworkManagementClient(self.credentials, self.subscription_id) self.vnet = vnet if vnet is not None else self.config.get("vnet") if self.vnet is None: raise ConfigError("You must configure a vnet") self.security_group = (security_group if security_group is not None else self.config.get("security_group")) if self.security_group is None: raise ConfigError( "You must configure a security group which allows traffic on 8786 and 8787" ) self.vm_size = vm_size if vm_size is not None else self.config.get( "vm_size") self.disk_size = (disk_size if disk_size is not None else self.config.get("disk_size")) if self.disk_size > 1023: raise ValueError( "VM OS disk canot be larger than 1023. Please change the ``disk_size`` config option." ) self.scheduler_vm_size = (scheduler_vm_size if scheduler_vm_size is not None else self.config.get("scheduler_vm_size")) if self.scheduler_vm_size is None: self.scheduler_vm_size = self.vm_size self.gpu_instance = ("_NC" in self.vm_size.upper() or "_ND" in self.vm_size.upper()) self.vm_image = self.config.get("vm_image") for key in vm_image: self.vm_image[key] = vm_image[key] self.bootstrap = (bootstrap if bootstrap is not None else self.config.get("bootstrap")) self.auto_shutdown = (auto_shutdown if auto_shutdown is not None else self.config.get("auto_shutdown")) self.docker_image = docker_image or self.config.get("docker_image") self.debug = debug self.marketplace_plan = marketplace_plan or self.config.get( "marketplace_plan") if self.marketplace_plan: # Check that self.marketplace_plan contains the right options with values if not all( self.marketplace_plan.get(item, "") != "" for item in ["name", "publisher", "product"]): raise ConfigError( """To create a virtual machine from Marketplace image or a custom image sourced from a Marketplace image with a plan, all 3 fields 'name', 'publisher' and 'product' must be passed.""" ) self.options = { "cluster": self, "config": self.config, "security_group": self.security_group, "location": self.location, "vm_image": self.vm_image, "disk_size": self.disk_size, "gpu_instance": self.gpu_instance, "bootstrap": self.bootstrap, "auto_shutdown": self.auto_shutdown, "docker_image": self.docker_image, "marketplace_plan": self.marketplace_plan, } self.scheduler_options = { "vm_size": self.scheduler_vm_size, "public_ingress": self.public_ingress, **self.options, } self.worker_options = {"vm_size": self.vm_size, **self.options} super().__init__(debug=debug, **kwargs)
#------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for # license information. #-------------------------------------------------------------------------- import os from azure.mgmt.peering import PeeringManagementClient from azure.mgmt.resource import ResourceManagementClient # from azure.common.credentials import ServicePrincipalCredentials from azure.common.credentials import get_azure_cli_credentials cred, sub, tenant = get_azure_cli_credentials(with_tenant=True) AZURE_LOCATION = 'eastus' RESOURCE_GROUP = "myResourceGroup" # credentials from environment #SUBSCRIPTION_ID = os.environ['AZURE_SUBSCRIPTION_ID'] SUBSCRIPTION_ID = 'xxx' #TENANT_ID = os.environ['AZURE_TENANT'] #CLIENT_ID = os.environ['AZURE_CLIENT_ID'] #CLIENT_SECRET = os.environ['AZURE_SECRET'] PEER_ASN_NAME = "myPeerAsn" PEERING_NAME = "myPeering" + "B" REGISTERED_ASN_NAME = "myRegisteredAsn" REGISTERED_PREFIX_NAME = "myRegisteredPrefix" PEERING_SERVICE_NAME = "myPeeringService" PREFIX_NAME = "myPrefix" # management client #credentials = ServicePrincipalCredentials( # client_id=CLIENT_ID,
def delete_spn(dbr_tmp_pat, spn_id): # Delete SPN only works with PAT token response = requests.delete( f"{dbricks_api}/preview/scim/v2/ServicePrincipals/{spn_id}", headers={ "Accept": "application/scim+json", "Authorization": "Bearer " + dbr_tmp_pat, }) print(response.status_code) if __name__ == "__main__": # 1. get admin credentials, subscription and tenant credentials, subscription_id, tenant_id = get_azure_cli_credentials( with_tenant=True) # 2.1 Deploy new Databricks workspace create_databricks_workspace() # 2.2 Wait 5 minutes, since it takes some time before Workspace is initialized time.sleep(300) # 3. Get token to authenticate to DataBricks, user needs to be admin in Databricks admin_adb_token = get_admin_token(credentials, "2ff814a6-3304-4ab8-85cb-cd0e6f879c1d") admin_az_token = get_admin_token(credentials, "https://management.core.windows.net/") dbricks_admin_auth = get_dbr_auth(admin_adb_token, admin_az_token) # 4. Create tmp dbr pat token to authenticate create/delete SPN in SCIM interface (works only with PAT) dbr_tmp_pat = create_tmp_dbrpat(dbricks_admin_auth) # 3. Add spn to Databricks and provide rights to SPN to run manage clusters spn_dbr_id = check_spn_exists(dbricks_admin_auth) if spn_dbr_id == "":
def __init__( self, location: str = None, resource_group: str = None, vnet: str = None, security_group: str = None, public_ingress: bool = None, vm_size: str = None, scheduler_vm_size: str = None, vm_image: dict = {}, disk_size: int = None, bootstrap: bool = None, auto_shutdown: bool = None, docker_image=None, debug: bool = False, **kwargs, ): self.config = dask.config.get("cloudprovider.azure.azurevm", {}) self.scheduler_class = AzureVMScheduler self.worker_class = AzureVMWorker self.location = (location if location is not None else dask.config.get("cloudprovider.azure.location")) if self.location is None: raise ConfigError("You must configure a location") self.resource_group = (resource_group if resource_group is not None else self.config.get("resource_group")) if self.resource_group is None: raise ConfigError("You must configure a resource_group") self.public_ingress = (public_ingress if public_ingress is not None else self.config.get("public_ingress")) self.credentials, self.subscription_id = get_azure_cli_credentials() self.compute_client = ComputeManagementClient(self.credentials, self.subscription_id) self.network_client = NetworkManagementClient(self.credentials, self.subscription_id) self.vnet = vnet if vnet is not None else self.config.get("vnet") if self.vnet is None: raise ConfigError("You must configure a vnet") self.security_group = (security_group if security_group is not None else self.config.get("security_group")) if self.security_group is None: raise ConfigError( "You must configure a security group which allows traffic on 8786 and 8787" ) self.vm_size = vm_size if vm_size is not None else self.config.get( "vm_size") self.disk_size = (disk_size if disk_size is not None else self.config.get("disk_size")) if self.disk_size > 1023: raise ValueError( "VM OS disk canot be larger than 1023. Please change the ``disk_size`` config option." ) self.scheduler_vm_size = (scheduler_vm_size if scheduler_vm_size is not None else self.config.get("scheduler_vm_size")) if self.scheduler_vm_size is None: self.scheduler_vm_size = self.vm_size self.gpu_instance = ("_NC" in self.vm_size.upper() or "_ND" in self.vm_size.upper()) self.vm_image = self.config.get("vm_image") for key in vm_image: self.vm_image[key] = vm_image[key] self.bootstrap = (bootstrap if bootstrap is not None else self.config.get("bootstrap")) self.auto_shutdown = (auto_shutdown if auto_shutdown is not None else self.config.get("auto_shutdown")) self.docker_image = docker_image or self.config.get("docker_image") self.debug = debug self.options = { "cluster": self, "config": self.config, "security_group": self.security_group, "location": self.location, "vm_image": self.vm_image, "disk_size": self.disk_size, "gpu_instance": self.gpu_instance, "bootstrap": self.bootstrap, "auto_shutdown": self.auto_shutdown, "docker_image": self.docker_image, } self.scheduler_options = { "vm_size": self.scheduler_vm_size, "public_ingress": self.public_ingress, **self.options, } self.worker_options = {"vm_size": self.vm_size, **self.options} super().__init__(debug=debug, **kwargs)
import databricks_client import os import pytest import requests import requests_mock import adal from azure.common.credentials import get_azure_cli_credentials credentials, subscription_id = get_azure_cli_credentials() dbricks_api = os.environ["DATABRICKS_HOST"] resource_group = os.environ["DATABRICKS_RG"] workspace_name = os.environ["DATABRICKS_WORKSPACE"] resource_id = ('/subscriptions/%s/resourceGroups/%s/providers/' 'Microsoft.Databricks/workspaces/%s' % (subscription_id, resource_group, workspace_name)) non_provisioned_response = { "error_code": "INVALID_PARAMETER_VALUE", "message": "Unknown worker environment WorkerEnvId(workerenv-63435k2940201085)" } client_id = os.environ["CLIENT_ID"] client_secret = os.environ["CLIENT_SECRET"] tenant_id = os.environ["TENANT_ID"] authority_host_uri = 'https://login.microsoftonline.com' authority_uri = authority_host_uri + '/' + tenant_id context = adal.AuthenticationContext(authority_uri) def get_clusters_list(client):
def authenticate(self, cli=None, user_account=None, user_account_browser=None, service_principal=None, file_auth=None, msi=None, tenant_id=None, subscription_id=None, client_id=None, client_secret=None, username=None, password=None, programmatic_execution=False, **kargs): """ Implements authentication for the Azure provider """ try: # Set logging level to error for libraries as otherwise generates a lot of warnings logging.getLogger('adal-python').setLevel(logging.ERROR) logging.getLogger('msrest').setLevel(logging.ERROR) logging.getLogger('urllib3').setLevel(logging.ERROR) if cli: arm_credentials, subscription_id, tenant_id = get_azure_cli_credentials(with_tenant=True) aad_graph_credentials, placeholder_1, placeholder_2 = \ get_azure_cli_credentials(with_tenant=True, resource='https://graph.windows.net') elif user_account: if not (username and password): if not programmatic_execution: username = username if username else input("Username: "******"Password: "******"Tenant ID: ") else: raise AuthenticationException('No Tenant ID set') if not client_id: if not programmatic_execution: client_id = input("Client ID: ") else: raise AuthenticationException('No Client ID set') if not client_secret: if not programmatic_execution: client_secret = getpass("Client secret: ") else: raise AuthenticationException('No Client Secret set') arm_credentials = ServicePrincipalCredentials( client_id=client_id, secret=client_secret, tenant=tenant_id ) aad_graph_credentials = ServicePrincipalCredentials( client_id=client_id, secret=client_secret, tenant=tenant_id, resource='https://graph.windows.net' ) elif file_auth: data = json.loads(file_auth.read()) tenant_id = data.get('tenantId') client_id = data.get('clientId') client_secret = data.get('clientSecret') arm_credentials = ServicePrincipalCredentials( client_id=client_id, secret=client_secret, tenant=tenant_id ) aad_graph_credentials = ServicePrincipalCredentials( client_id=client_id, secret=client_secret, tenant=tenant_id, resource='https://graph.windows.net' ) elif msi: arm_credentials = MSIAuthentication() aad_graph_credentials = MSIAuthentication(resource='https://graph.windows.net') else: raise AuthenticationException('Unknown authentication method') return AzureCredentials(arm_credentials, aad_graph_credentials, tenant_id, subscription_id) except Exception as e: raise AuthenticationException(e)
def get_az_credentials(): credentials, _, tenant = get_azure_cli_credentials( resource='https://graph.windows.net', with_tenant=True) return credentials, tenant
import os, uuid, sys from azure.mgmt.iotcentral import IotCentralClient from azure.mgmt.iotcentral.models import App, AppSkuInfo, AppPatch from msrestazure.azure_active_directory import MSIAuthentication from azure.common.credentials import UserPassCredentials, get_azure_cli_credentials # login with az login creds = get_azure_cli_credentials() subId = "FILL IN SUB ID" appName = "iot-central-app-tocreate" resourceGroup = "myResourceGroup" print(creds[0]) print(creds[1]) client = IotCentralClient(creds[0], subId) result = client.apps.check_name_availability(appName) print(result) app = App(location="unitedstates", sku=AppSkuInfo(name="ST2")) app.subdomain = appName app.display_name = appName createResult = client.apps.create_or_update(resourceGroup, appName, app) print(createResult) getResult = client.apps.get(resourceGroup, appName) print(getResult)
def main(): parser = argparse.ArgumentParser(description="Library path in ADF") parser.add_argument("-r", "--resource_group", help="Resource group", required=True) parser.add_argument("-a", "--adf_name", help="ADF NAME", required=True) parser.add_argument("-p", "--adf_pipeline_name", help="ADF pipeline name", required=True) parser.add_argument("-o", "--output_file_path", help="Output file path", required=True) parser.add_argument("-pa", "--parameters", help="Parameters", required=False) args = parser.parse_args() resource_group = args.resource_group adf_name = args.adf_name adf_pipeline_name = args.adf_pipeline_name output_file_path = args.output_file_path parameters = args.parameters print(f"-resource_group is {resource_group}") print(f"-adf_name is {adf_name}") print(f"-adf_pipeline_name is {adf_pipeline_name}") print(f"-output_file_path is {output_file_path}") print(f"-parameters is {parameters}") credentials, subscription_id = get_azure_cli_credentials() # The data factory name. It must be globally unique. get_azure_cli_credentials() adf_client = DataFactoryManagementClient(credentials, subscription_id) # Create a pipeline run run_response = adf_client.pipelines.create_run(resource_group, adf_name, adf_pipeline_name, parameters=parameters) # Monitor the pipeline run time.sleep(5) pipeline_run = adf_client.pipeline_runs.get(resource_group, adf_name, run_response.run_id) print("\n\tPipeline run status: {}".format(pipeline_run.status)) filter_params = RunFilterParameters( last_updated_after=datetime.utcnow() - timedelta(1), last_updated_before=datetime.utcnow() + timedelta(1)) query_response = adf_client.activity_runs.query_by_pipeline_run( resource_group, adf_name, pipeline_run.run_id, filter_params) while query_response.value[0].status in ['InProgress']: print_activity_run_details(query_response.value[0]) time.sleep(3) query_response = adf_client.activity_runs.query_by_pipeline_run( resource_group, adf_name, pipeline_run.run_id, filter_params) print_activity_run_details(query_response.value[0])
def authenticate(self, key_file=None, user_account=None, service_account=None, azure_cli=None, azure_msi=None, azure_service_principal=None, azure_file_auth=None, azure_user_credentials=None, **kargs): """ Implements authentication for the Azure provider using azure-cli. Refer to https://docs.microsoft.com/en-us/python/azure/python-sdk-azure-authenticate?view=azure-python. :return: """ try: if azure_cli: cli_credentials, self.aws_account_id = get_azure_cli_credentials() # TODO: Remove aws_account_id self.credentials = AzureCredentials(cli_credentials, self.aws_account_id) return True elif azure_msi: credentials = MSIAuthentication() # Get the subscription ID subscription_client = SubscriptionClient(credentials) try: # Tries to read the subscription list subscription = next(subscription_client.subscriptions.list()) self.aws_account_id = subscription.subscription_id except StopIteration: # If the VM cannot read subscription list, ask Subscription ID: self.aws_account_id = input('Subscription ID: ') self.credentials = AzureCredentials(credentials, self.aws_account_id) return True elif azure_file_auth: with open(azure_file_auth) as f: data = json.loads(f.read()) subscription_id = data.get('subscriptionId') tenant_id = data.get('tenantId') client_id = data.get('clientId') client_secret = data.get('clientSecret') self.aws_account_id = tenant_id # TODO this is for AWS credentials = ServicePrincipalCredentials( client_id=client_id, secret=client_secret, tenant=tenant_id ) self.credentials = AzureCredentials(credentials, subscription_id) return True elif azure_service_principal: subscription_id = input("Subscription ID: ") tenant_id = input("Tenant ID: ") client_id = input("Client ID: ") client_secret = getpass("Client secret: ") self.aws_account_id = tenant_id # TODO this is for AWS credentials = ServicePrincipalCredentials( client_id=client_id, secret=client_secret, tenant=tenant_id ) self.credentials = AzureCredentials(credentials, subscription_id) return True elif azure_user_credentials: username = input("Username: "******"Password: "******"" # TODO this is for AWS self.credentials = AzureCredentials(credentials, self.aws_account_id) return True except Exception as e: printError('Failed to authenticate to Azure') printException(e) return False