def get_auth(): """Authentication to access workspace""" try: auth = AzureCliAuthentication() auth.get_authentication_header() except AuthenticationException: logger.info("Authentication Error Occured") return auth
def get_auth(): logger = logging.getLogger(__name__) logger.debug("Trying to create Workspace with CLI Authentication") try: auth = AzureCliAuthentication() auth.get_authentication_header() except AuthenticationException: logger.debug("Trying to create Workspace with Interactive login") auth = InteractiveLoginAuthentication() return auth
def get_auth(): ''' Authentication to access workspace ''' try: auth = AzureCliAuthentication() auth.get_authentication_header() except AuthenticationException: print("Authentication Error Occured") return auth
def get_auth(): """ Method to get the correct Azure ML Authentication type Always start with CLI Authentication and if it fails, fall back to interactive login """ try: auth_type = AzureCliAuthentication() auth_type.get_authentication_header() except AuthenticationException: auth_type = InteractiveLoginAuthentication() return auth_type
def main(name: str, subscription_id: str, resource_group: str, location: str): """Create Workspace on Azure ML Service - https://docs.microsoft.com/pt-br/azure/machine-learning /how-to-manage-workspace?tab=python&tabs=python#create-multi-tenant """ print(location) click.secho("[ML SERVICE] - Creating Workspace...", fg="green") cli_auth = AzureCliAuthentication() ws = Workspace.create( name=name, subscription_id=subscription_id, resource_group=resource_group, create_resource_group=True, location=location, auth=cli_auth, ) ws.write_config(".azureml") print( "\tWorkspace name: " + ws.name, "Azure region: " + ws.location, "Subscription id: " + ws.subscription_id, "Resource group: " + ws.resource_group, sep="\n\t", )
def get_auth(env_path: str) -> AbstractAuthentication: """ :param env_path: :return: """ logger = logging.getLogger(__name__) if get_key(env_path, "password") != "YOUR_SERVICE_PRINCIPAL_PASSWORD": logger.debug("Trying to create Workspace with Service Principal") aml_sp_password = get_key(env_path, "password") aml_sp_tennant_id = get_key(env_path, "tenant_id") aml_sp_username = get_key(env_path, "username") auth = ServicePrincipalAuthentication( tenant_id=aml_sp_tennant_id, service_principal_id=aml_sp_username, service_principal_password=aml_sp_password, ) else: logger.debug("Trying to create Workspace with CLI Authentication") try: auth = AzureCliAuthentication() auth.get_authentication_header() except AuthenticationException: logger.debug("Trying to create Workspace with Interactive login") auth = InteractiveLoginAuthentication() return auth
def trigger_data_prep(): # get the parameter values workspace = sys.argv[1] subscription_id = sys.argv[2] resource_grp = sys.argv[3] domain = sys.argv[4] DBR_PAT_TOKEN = bytes(sys.argv[5], encoding='utf-8') # adding b' notebook = "lgb_eq_sec.py" experiment_name = "experiment_model_release" # Print AML Version print("Azure ML SDK Version: ", azureml.core.VERSION) # Point file to conf directory containing details for the aml service cli_auth = AzureCliAuthentication() ws = Workspace(workspace_name = workspace, subscription_id = subscription_id, resource_group = resource_grp, auth=cli_auth) # Create a new experiment print("Starting to create new experiment") Experiment(workspace=ws, name=experiment_name) # Upload notebook to Databricks print("Upload notebook to databricks") upload_notebook(domain, DBR_PAT_TOKEN, notebook)
def trigger_env_prep(): # Define Vars < Change the vars>. # In a production situation, don't put secrets in source code, but as secret variables, # see https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch#secret-variables workspace = "<Name of your workspace>" subscription_id = "<Subscription id>" resource_grp = "<Name of your resource group where aml service is created>" domain = "westeurope.azuredatabricks.net" # change location in case databricks instance is not in westeurope databricks_name = "<<Your Databricks Name>>" dbr_pat_token_raw = "<<your Databricks Personal Access Token>>" DBR_PAT_TOKEN = bytes(dbr_pat_token_raw, encoding='utf-8') # adding b' databricks_grp = resource_grp dataset = "AdultCensusIncome.csv" notebook = "3_IncomeNotebookDevops.py" experiment_name = "experiment_model_release" db_compute_name = "dbr-amls-comp" # Print AML Version print("Azure ML SDK Version: ", azureml.core.VERSION) # Point file to conf directory containing details for the aml service cli_auth = AzureCliAuthentication() ws = Workspace(workspace_name=workspace, subscription_id=subscription_id, resource_group=resource_grp, auth=cli_auth) print(ws.name, ws._workspace_name, ws.resource_group, ws.location, sep='\t') # Create a new experiment print("Starting to create new experiment") Experiment(workspace=ws, name=experiment_name) # Upload notebook to Databricks print("Upload notebook to databricks") upload_notebook(domain, DBR_PAT_TOKEN, notebook) print("Add databricks env to Azure ML Service Compute") # Create databricks workspace in AML SDK try: databricks_compute = DatabricksCompute(workspace=ws, name=db_compute_name) print('Compute target {} already exists'.format(db_compute_name)) except ComputeTargetException: print('Compute not found, will use below parameters to attach new one') config = DatabricksCompute.attach_configuration( resource_group=databricks_grp, workspace_name=databricks_name, access_token=dbr_pat_token_raw) databricks_compute = ComputeTarget.attach(ws, db_compute_name, config) databricks_compute.wait_for_completion(True)
def get_workspace(): env = EnvironmentVariables() cli_auth = AzureCliAuthentication() workspace = Workspace(workspace_name=env.workspace_name, subscription_id=env.subscription_id, resource_group=env.resource_group, auth=cli_auth) return workspace
def get_auth(): ''' Retreive the user authentication. If they aren't logged in this will prompt the standard interactive login method. PARAMS: None RETURNS: Authentication object ''' auth = None print("Get auth...") try: auth = AzureCliAuthentication() auth.get_authentication_header() except AuthenticationException: auth = InteractiveLoginAuthentication() return auth
def register_model(model_name, run_id): """Register the model to the AML Workspace""" cli_auth = AzureCliAuthentication() experiment = Experiment.from_directory(".", auth=cli_auth) run = Run(experiment, run_id) run.register_model(model_name, model_path='outputs/final_model.hdf5', model_framework='TfKeras', model_framework_version='1.13')
def setUpClass(cls): subscription_id = os.getenv("SUBSCRIPTION_ID") resource_group = os.getenv("RESOURCE_GROUP") workspace_name = os.getenv("WORKSPACE_NAME") auth = AzureCliAuthentication() setAutomatedMLWorkspace(auth=auth, subscription_id=subscription_id, resource_group=resource_group, workspace_name=workspace_name)
def main(epochs, iterations, compute_target, concurrent_runs): cli_auth = AzureCliAuthentication() experiment = Experiment.from_directory(".", auth=cli_auth) ws = experiment.workspace cluster = ws.compute_targets[compute_target] food_data = ws.datastores['food_images'] script_arguments = {"--data-dir": food_data.as_mount(), "--epochs": epochs} tf_est = TensorFlow(source_directory=".", entry_script='code/train/train.py', script_params=script_arguments, compute_target=cluster, conda_packages=['pillow', 'pandas'], pip_packages=['click', 'seaborn'], use_docker=True, use_gpu=True, framework_version='1.13') # Run on subset of food categories tf_est.run_config.arguments.extend( ['apple_pie', 'baby_back_ribs', 'baklava', 'beef_carpaccio']) param_sampler = RandomParameterSampling({ '--minibatch-size': choice(16, 32, 64), '--learning-rate': loguniform(-9, -6), '--optimizer': choice('rmsprop', 'adagrad', 'adam') }) # Create Early Termination Policy etpolicy = BanditPolicy(evaluation_interval=2, slack_factor=0.1) # Create HyperDrive Run Configuration hyper_drive_config = HyperDriveConfig( estimator=tf_est, hyperparameter_sampling=param_sampler, policy=etpolicy, primary_metric_name='acc', primary_metric_goal=PrimaryMetricGoal.MAXIMIZE, max_total_runs=iterations, max_concurrent_runs=concurrent_runs) # Submit the Hyperdrive Run print("Submitting Hyperdrive Run") hd_run = experiment.submit(hyper_drive_config) hd_run.wait_for_completion(raise_on_error=True, show_output=True) print("Finishing Run") best_run = hd_run.get_best_run_by_primary_metric() print(f'##vso[task.setvariable variable=run_id]{best_run.id}')
def __init__(self, ws_config_file): self.ws = Workspace.from_config( auth=AzureCliAuthentication(), path=os.path.join( os.path.dirname(os.path.realpath(__file__)), ws_config_file ) ) print( f"Found workspace {self.ws.name} \n\tat location {self.ws.location}\n\t with the id:{self.ws._workspace_id}") self.experiments = {}
def __init__(self, ws_config_file: str, subscription_id: str, aml_resource_group: str, aml_workspace_name: str): """Initializes an AML workspace object""" auth = AzureCliAuthentication() if ws_config_file != None: self.ws = Workspace.from_config( auth=auth, path=os.path.join(os.path.dirname(os.path.realpath(__file__)), ws_config_file)) else: self.ws = Workspace.get(aml_workspace_name, auth, subscription_id, aml_resource_group) _logger.info(f"Found workspace {self.ws.name}")
def get_cli_specific_auth(): """ Returns the cli specific auth. For azure.cli, returns an object of azureml.core.authentication.AzureCliAuthentication For azureml._cli, returns an object of azureml.core.authentication.ArmTokenAuthentication :return: :rtype: azureml.core.authentication.AbstractAuthentication """ if AZUREML_CLI_IN_USE: if AZUREML_ARM_ACCESS_TOKEN in os.environ: return ArmTokenAuthentication(os.environ[AZUREML_ARM_ACCESS_TOKEN]) else: return InteractiveLoginAuthentication() # az cli case return AzureCliAuthentication()
def trigger_data_prep(): # Define Vars < Change the vars>. # In a production situation, don't put secrets in source code, but as secret variables, # see https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch#secret-variables # workspace="<Name of your workspace>" # subscription_id="<Subscription id>" # resource_grp="<Name of your resource group where aml service is created>" # domain = "westeurope.azuredatabricks.net" # change location in case databricks instance is not in westeurope # DBR_PAT_TOKEN = bytes("<<your Databricks Personal Access Token>>", encoding='utf-8') # adding b' # inserted new code for my run workspace="aiml-ws1" subscription_id="05c034fe-a6e2-42b7-bdfe-519a3b3a40cf" resource_grp="vikram-aiml" domain = "eastus.azuredatabricks.net" # change location in case databricks instance is not in westeurope DBR_PAT_TOKEN = bytes("dapi2dd4008fef79f1f64392ca27b7a2888e", encoding='utf-8') # adding b' dataset = "AdultCensusIncome.csv" notebook = "3_IncomeNotebookDevops.py" experiment_name = "experiment_model_release" # Print AML Version print("Azure ML SDK Version: ", azureml.core.VERSION) # Point file to conf directory containing details for the aml service cli_auth = AzureCliAuthentication() ws = Workspace(workspace_name = workspace, subscription_id = subscription_id, resource_group = resource_grp, auth=cli_auth) print(ws.name, ws._workspace_name, ws.resource_group, ws.location, sep = '\t') # Create a new experiment print("Starting to create new experiment") Experiment(workspace=ws, name=experiment_name) # Upload notebook to Databricks print("Upload notebook to databricks") upload_notebook(domain, DBR_PAT_TOKEN, notebook)
def test_response(service_name, data_path, auth_enabled): cli_auth = AzureCliAuthentication() ws = Workspace.from_config(auth=cli_auth) with open(data_path, "r") as file_obj: data = json.load(file_obj) service = ws.webservices[service_name] headers = {} if auth_enabled: auth_key = service.get_keys()[0] headers = {"Authorization": "Bearer {0}".format(auth_key)} response = requests.post(service.scoring_uri, json=data, headers=headers) response_data = json.loads(response.content) assert response.status_code == 200
def workspace(*args, **kwargs): ws = None try: click.secho("[ML SERVICE] - Connection with workspace...", fg="green") cli_auth = AzureCliAuthentication() ws = Workspace.from_config(path=os.path.join( ".azureml", "config.json"), auth=cli_auth) print( "\tWorkspace name: " + ws.name, "Azure region: " + ws.location, "Subscription id: " + ws.subscription_id, "Resource group: " + ws.resource_group, sep="\n\t", ) except WorkspaceException: raise WorkspaceException("Failed connection with workspace") return f(*args, **kwargs, ws=ws)
def get_auth(env_path): logger = logging.getLogger(__name__) if os.environ.get("AML_SP_PASSWORD", None): logger.debug("Trying to create Workspace with Service Principal") aml_sp_password = os.environ.get("AML_SP_PASSWORD") aml_sp_tennant_id = os.environ.get("AML_SP_TENNANT_ID") aml_sp_username = os.environ.get("AML_SP_USERNAME") auth = ServicePrincipalAuthentication(tenant_id=aml_sp_tennant_id, username=aml_sp_username, password=aml_sp_password) else: logger.debug("Trying to create Workspace with CLI Authentication") try: auth = AzureCliAuthentication() auth.get_authentication_header() except AuthenticationException: logger.debug("Trying to create Workspace with Interactive login") auth = InteractiveLoginAuthentication() return auth
def _get_auth(): """Returns authentication to Azure Machine Learning workspace.""" logger = logging.getLogger(__name__) if os.environ.get("AML_SP_PASSWORD", None): logger.debug("Trying to authenticate with Service Principal") aml_sp_password = os.environ.get("AML_SP_PASSWORD") aml_sp_tennant_id = os.environ.get("AML_SP_TENNANT_ID") aml_sp_username = os.environ.get("AML_SP_USERNAME") auth = ServicePrincipalAuthentication(aml_sp_tennant_id, aml_sp_username, aml_sp_password) else: logger.debug("Trying to authenticate with CLI Authentication") try: auth = AzureCliAuthentication() auth.get_authentication_header() except AuthenticationException: logger.debug("Trying to authenticate with Interactive login") auth = InteractiveLoginAuthentication() return auth
def trigger_env_prep(): # Define Vars < Change the vars>. # In a production situation, don't put secrets in source code, but as secret variables, # see https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch#secret-variables workspace = sys.argv[1] subscription_id = sys.argv[2] resource_grp = sys.argv[3] domain = sys.argv[4] dbr_pat_token_raw = sys.argv[5] DBR_PAT_TOKEN = bytes(dbr_pat_token_raw, encoding='utf-8') # adding b' dataset = "AdultCensusIncome.csv" notebook = "3_IncomeNotebookDevops.py" experiment_name = "experiment_model_release" # Print AML Version print("Azure ML SDK Version: ", azureml.core.VERSION) # Point file to conf directory containing details for the aml service cli_auth = AzureCliAuthentication() ws = Workspace(workspace_name=workspace, subscription_id=subscription_id, resource_group=resource_grp, auth=cli_auth) print(ws.name, ws._workspace_name, ws.resource_group, ws.location, sep='\t') # Create a new experiment print("Starting to create new experiment") Experiment(workspace=ws, name=experiment_name) # Upload notebook to Databricks print("Upload notebook to databricks") upload_notebook(domain, DBR_PAT_TOKEN, notebook)
def get_auth(): """Get an auth object for use with Workspace objects.""" if os.environ.get("AML_SP_PASSWORD", None): print("Trying to create Workspace with Service Principal") aml_sp_password = os.environ.get("AML_SP_PASSWORD") aml_sp_tenant_id = os.environ.get("AML_SP_TENANT_ID") aml_sp_username = os.environ.get("AML_SP_USERNAME") auth = ServicePrincipalAuthentication( tenant_id=aml_sp_tenant_id, service_principal_id=aml_sp_username, service_principal_password=aml_sp_password) else: print("Trying to create Workspace with CLI Authentication") try: auth = AzureCliAuthentication() auth.get_authentication_header() except AuthenticationException: print("Trying to create Workspace with Interactive login") auth = InteractiveLoginAuthentication() return auth
def open_azure_ml_workspace(self): logging.info( "connecting to workspace name:{0} subscription ID:{1}, resource group:{2}" .format( self.args.workspace_name, self.args.subscription_id, self.args.resource_group, )) if self.args.authentication == AUTH_CLI: logging.info("using Azure CLI authentication") auth = AzureCliAuthentication() elif self.args.authentication == AUTH_MI: logging.info("using managed identity authentication") auth = MsiAuthentication() elif self.args.authentication == AUTH_SP: logging.info( "using service principal authentication tenant ID:{0} service principal ID:{1}" .format( self.args.tenant_id, self.args.service_principal_id, )) auth = ServicePrincipalAuthentication( tenant_id=self.args.tenant_id, service_principal_id=self.args.service_principal_id, service_principal_password=self.args. service_principal_password, ) else: ex = Exception('Invalid authentication type.') logging.error(ex) raise ex workspace = Workspace( subscription_id=self.args.subscription_id, resource_group=self.args.resource_group, workspace_name=self.args.workspace_name, auth=auth, ) return workspace
def get_auth(env_path): logger = logging.getLogger(__name__) if get_key(env_path, 'password') != "YOUR_SERVICE_PRINCIPAL_PASSWORD": logger.debug("Trying to create Workspace with Service Principal") aml_sp_password = get_key(env_path, 'password') aml_sp_tennant_id = get_key(env_path, 'tenant_id') aml_sp_username = get_key(env_path, 'username') auth = ServicePrincipalAuthentication( tenant_id=aml_sp_tennant_id, username=aml_sp_username, password=aml_sp_password ) else: logger.debug("Trying to create Workspace with CLI Authentication") try: auth = AzureCliAuthentication() auth.get_authentication_header() except AuthenticationException: logger.debug("Trying to create Workspace with Interactive login") auth = InteractiveLoginAuthentication() return auth
def trigger_training_job(): # Define Vars < Change the vars>. # In a production situation, don't put secrets in source code, but as secret variables, # see https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch#secret-variables workspace = "<Name of your workspace>" subscription_id = "<Subscription id>" resource_grp = "<Name of your resource group where aml service is created>" domain = "westeurope.azuredatabricks.net" # change location in case databricks instance is not in westeurope dbr_pat_token_raw = "<<your Databricks Personal Access Token>>" DBR_PAT_TOKEN = bytes(dbr_pat_token_raw, encoding='utf-8') # adding b' notebookRemote = "/3_IncomeNotebookDevops" experiment_name = "experiment_model_release" model_name_run = datetime.datetime.now().strftime( "%Y%m%d%H%M%S" ) + "_dbrmod.mml" # in case you want to change the name, keep the .mml extension model_name = "databricksmodel.mml" # in case you want to change the name, keep the .mml extension db_compute_name = "dbr-amls-comp" # # Step 1: Run notebook using Databricks Compute in AML SDK # cli_auth = AzureCliAuthentication() ws = Workspace(workspace_name=workspace, subscription_id=subscription_id, resource_group=resource_grp, auth=cli_auth) ws.get_details() # # Step 2: Create job and attach it to cluster # # In this steps, secret are added as parameters (spn_tenant, spn_clientid, spn_clientsecret) # Never do this in a production situation, but use secret scope backed by key vault instead # See https://docs.azuredatabricks.net/user-guide/secrets/secret-scopes.html#azure-key-vault-backed-scopes response = requests.post( 'https://%s/api/2.0/jobs/create' % domain, headers={'Authorization': b"Bearer " + DBR_PAT_TOKEN}, json={ "name": "Run AzureDevopsNotebook Job", "new_cluster": { "spark_version": "4.0.x-scala2.11", "node_type_id": "Standard_D3_v2", "spark_env_vars": { 'PYSPARK_PYTHON': '/databricks/python3/bin/python3', }, "autoscale": { "min_workers": 1, "max_workers": 2 } }, "libraries": [{ "pypi": { "package": "azureml-sdk[databricks]" } }], "notebook_task": { "notebook_path": notebookRemote, "base_parameters": [{ "key": "subscription_id", "value": subscription_id }, { "key": "resource_group", "value": resource_grp }, { "key": "workspace_name", "value": workspace }, { "key": "model_name", "value": model_name_run }] } }) if response.status_code != 200: print("Error launching cluster: %s: %s" % (response.json()["error_code"], response.json()["message"])) exit(2) # # Step 3: Start job # databricks_job_id = response.json()['job_id'] response = requests.post( 'https://%s/api/2.0/jobs/run-now' % domain, headers={'Authorization': b"Bearer " + DBR_PAT_TOKEN}, json={"job_id": +databricks_job_id}) if response.status_code != 200: print("Error launching cluster: %s: %s" % (response.json()["error_code"], response.json()["message"])) exit(3) print(response.json()['run_id']) # # Step 4: Wait until job is finished # databricks_run_id = response.json()['run_id'] scriptRun = 1 count = 0 while scriptRun == 1: response = requests.get( 'https://%s/api/2.0/jobs/runs/get?run_id=%s' % (domain, databricks_run_id), headers={'Authorization': b"Bearer " + DBR_PAT_TOKEN}, ) state = response.json()['state'] life_cycle_state = state['life_cycle_state'] print(state) if life_cycle_state in ["TERMINATED", "SKIPPED", "INTERNAL_ERROR"]: result_state = state['result_state'] if result_state == "SUCCESS": print("run ok") scriptRun = 0 #exit(0) else: exit(4) elif count > 180: print("time out occurred after 30 minutes") exit(5) else: count += 1 time.sleep(30) # wait 30 seconds before next status update # # Step 5: Retrieve model from dbfs # mdl, ext = model_name_run.split(".") model_zip_run = mdl + ".zip" response = requests.get( 'https://%s/api/2.0/dbfs/read?path=/%s' % (domain, model_zip_run), headers={'Authorization': b"Bearer " + DBR_PAT_TOKEN}) if response.status_code != 200: print("Error copying dbfs results: %s: %s" % (response.json()["error_code"], response.json()["message"])) exit(1) model_output = base64.b64decode(response.json()['data']) # download model in deploy folder os.chdir("deploy") with open(model_zip_run, "wb") as outfile: outfile.write(model_output) print("Downloaded model {} to Project root directory".format(model_name)) # # Step 6: Retrieve model metrics from dbfs # mdl, ext = model_name_run.split(".") model_metrics_json_run = mdl + "_metrics.json" response = requests.get( 'https://%s/api/2.0/dbfs/read?path=/%s' % (domain, model_metrics_json_run), headers={'Authorization': b"Bearer " + DBR_PAT_TOKEN}) if response.status_code != 200: print("Error copying dbfs results: %s: %s" % (response.json()["error_code"], response.json()["message"])) exit(2) model_metrics_output = json.loads(base64.b64decode( response.json()['data'])) # # Step 7: Put model and metrics to Azure ML Service # # start a training run by defining an experiment myexperiment = Experiment(ws, experiment_name) run = myexperiment.start_logging() run.upload_file("outputs/" + model_zip_run, model_zip_run) #run.log("pipeline_run", pipeline_run.id) run.log("au_roc", model_metrics_output["Area_Under_ROC"]) run.log("au_prc", model_metrics_output["Area_Under_PR"]) run.log("truePostive", model_metrics_output["True_Positives"]) run.log("falsePostive", model_metrics_output["False_Positives"]) run.log("trueNegative", model_metrics_output["True_Negatives"]) run.log("falseNegative", model_metrics_output["False_Negatives"]) run.complete() run_id = run.id print("run id:", run_id) # unzip file to model_name_run shutil.unpack_archive(model_zip_run, model_name_run) model = Model.register( model_path=model_name_run, # this points to a local file model_name=model_name, # this is the name the model is registered as tags={ "area": "spar", "type": "regression", "run_id": run_id }, description="Medium blog test model", workspace=ws, ) print("Model registered: {} \nModel Description: {} \nModel Version: {}". format(model.name, model.description, model.version)) # Step 8. Finally, writing the registered model details to conf/model.json model_json = {} model_json["model_name"] = model.name model_json["model_version"] = model.version model_json["run_id"] = run_id model_json["model_name_run"] = model_name_run with open("../conf/model.json", "w") as outfile: json.dump(model_json, outfile)
import pandas as pd from sklearn.model_selection import train_test_split from sklearn.linear_model import Ridge from sklearn.metrics import mean_squared_error from sklearn.externals import joblib import math from azureml.core import Workspace from azureml.core.authentication import AzureCliAuthentication from azureml.core import Experiment from azureml.opendatasets import Diabetes from azureml.core import Run #Logon to Azure cli_auth = AzureCliAuthentication() ws = Workspace(subscription_id="affeac10-5929-43aa-9abd-4ada85acb943", resource_group="mlexp", workspace_name="ml_workspace", auth=cli_auth) # Create the workspace using the specified parameters # ws = Workspace.create(name = "ml_workspace", # subscription_id = "affeac10-5929-43aa-9abd-4ada85acb943", # resource_group = 'mlexp', # location = "eastus2", # create_resource_group = False, # sku = 'basic', # exist_ok = True)
from azureml.core.compute import ComputeTarget, AmlCompute from azureml.core.compute_target import ComputeTargetException from azureml.core import Workspace from azureml.core.authentication import AzureCliAuthentication # load Azure ML workspace workspace = Workspace.from_config(auth=AzureCliAuthentication()) # Create compute target if not present # Choose a name for your GPU cluster gpu_cluster_name = "fullcomputegpu" # Verify that cluster does not exist already try: gpu_cluster = ComputeTarget(workspace=workspace, name=gpu_cluster_name) print('Found existing cluster, use it.') except ComputeTargetException: compute_config = AmlCompute.provisioning_configuration( vm_size='Standard_NC12', max_nodes=8) gpu_cluster = ComputeTarget.create(workspace, gpu_cluster_name, compute_config) gpu_cluster.wait_for_completion(show_output=True)
""" Helper to generate an AAD token """ from azureml.core.authentication import AzureCliAuthentication print( AzureCliAuthentication().get_authentication_header().get("Authorization"))
# This software is released under the MIT License. # https://opensource.org/licenses/MIT import sys from typing import Tuple import click from azureml.core import (ComputeTarget, Dataset, Environment, RunConfiguration, Workspace) from azureml.core.authentication import AzureCliAuthentication from azureml.core.experiment import Experiment from azureml.pipeline.core import (Pipeline, PipelineData, PipelineParameter, PublishedPipeline) from azureml.pipeline.steps import DatabricksStep, PythonScriptStep CLI_AUTH = AzureCliAuthentication() # noinspection PyTypeChecker WS = Workspace.from_config(auth=CLI_AUTH) RC = RunConfiguration() RC.environment = Environment.get(WS, "lightgbm") # noinspection PyTypeChecker def create_databricks_step( input_dataset: Dataset, compute: ComputeTarget, debug_run: bool) -> Tuple[DatabricksStep, PipelineData]: output_data = PipelineData(name="ParquetFiles", datastore=WS.get_default_datastore(), is_directory=True) node_size = 'Standard_DS4_v2'