import os parse = argparse.ArgumentParser() parse.add_argument("--tenantid") parse.add_argument("--acclientid") parse.add_argument("--accsecret") args = parse.parse_args() sp = ServicePrincipalAuthentication( tenant_id=args.tenantid, # tenantID service_principal_id=args.acclientid, # clientId service_principal_password=args.accsecret) # clientSecret ws = Workspace.get(name="mlopsdev", auth=sp, subscription_id="c46a9435-c957-4e6c-a0f4-b9a597984773", resource_group="mlops") #ws = Workspace.from_config() # keyvault = ws.get_default_keyvault() # tenantid = keyvault.get_secret(name="tenantid") # acclientid = keyvault.get_secret(name="acclientid") # accsvcname = keyvault.get_secret(name="accsvcname") # accsecret = keyvault.get_secret(name="accsecret") # print(accsvcname) #sp = ServicePrincipalAuthentication(tenant_id=tenantid, # tenantID # service_principal_id=acclientid, # clientId # service_principal_password=accsecret) # clientSecret
def main(): e = Env() # Get Azure machine learning workspace aml_workspace = Workspace.get( name=e.workspace_name, subscription_id=e.subscription_id, resource_group=e.resource_group, ) print("get_workspace:") print(aml_workspace) # Get Azure machine learning cluster aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size) if aml_compute is not None: print("aml_compute:") print(aml_compute) # Create a reusable Azure ML environment environment = get_environment( aml_workspace, e.aml_env_name, conda_dependencies_file=e.aml_env_train_conda_dep_file, create_new=e.rebuild_env, ) # run_config = RunConfiguration() run_config.environment = environment if e.datastore_name: datastore_name = e.datastore_name else: datastore_name = aml_workspace.get_default_datastore().name run_config.environment.environment_variables[ "DATASTORE_NAME"] = datastore_name # NOQA: E501 model_name_param = PipelineParameter( name="model_name", default_value=e.model_name) # NOQA: E501 dataset_version_param = PipelineParameter(name="dataset_version", default_value=e.dataset_version) data_file_path_param = PipelineParameter(name="data_file_path", default_value="none") caller_run_id_param = PipelineParameter(name="caller_run_id", default_value="none") # NOQA: E501 # Get dataset name dataset_name = e.dataset_name # Check to see if dataset exists if dataset_name not in aml_workspace.datasets: create_sample_data_csv() # Use a CSV to read in the data set. file_name = "COVID19Articles.csv" if not os.path.exists(file_name): raise Exception( 'Could not find CSV dataset at "%s". If you have bootstrapped your project, you will need to provide a CSV.' # NOQA: E501 % file_name) # NOQA: E501 # Upload file to default datastore in workspace datatstore = Datastore.get(aml_workspace, datastore_name) target_path = "training-data/" datatstore.upload_files( files=[file_name], target_path=target_path, overwrite=True, show_progress=False, ) # Register dataset path_on_datastore = os.path.join(target_path, file_name) dataset = Dataset.Tabular.from_delimited_files( path=(datatstore, path_on_datastore)) dataset = dataset.register( workspace=aml_workspace, name=dataset_name, description="COVID19Articles training data", tags={"format": "CSV"}, create_new_version=True, ) # Create a PipelineData to pass data between steps pipeline_data = PipelineData( "pipeline_data", datastore=aml_workspace.get_default_datastore()) train_step = PythonScriptStep( name="Train Model", script_name=e.train_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, outputs=[pipeline_data], arguments=[ "--model_name", model_name_param, "--step_output", pipeline_data, "--dataset_version", dataset_version_param, "--data_file_path", data_file_path_param, "--caller_run_id", caller_run_id_param, "--dataset_name", dataset_name, ], runconfig=run_config, allow_reuse=True, ) print("Step Train created") evaluate_step = PythonScriptStep( name="Evaluate Model ", script_name=e.evaluate_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, arguments=[ "--model_name", model_name_param, "--allow_run_cancel", e.allow_run_cancel, ], runconfig=run_config, allow_reuse=False, ) print("Step Evaluate created") register_step = PythonScriptStep( name="Register Model ", script_name=e.register_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, inputs=[pipeline_data], arguments=[ "--model_name", model_name_param, "--step_input", pipeline_data, ], # NOQA: E501 runconfig=run_config, allow_reuse=False, ) print("Step Register created") # Check run_evaluation flag to include or exclude evaluation step. if (e.run_evaluation).lower() == "true": print("Include evaluation step before register step.") evaluate_step.run_after(train_step) register_step.run_after(evaluate_step) steps = [train_step, evaluate_step, register_step] else: print("Exclude evaluation step and directly run register step.") register_step.run_after(train_step) steps = [train_step, register_step] train_pipeline = Pipeline(workspace=aml_workspace, steps=steps) train_pipeline._set_experiment_name train_pipeline.validate() published_pipeline = train_pipeline.publish( name=e.pipeline_name, description="Model training/retraining pipeline", version=e.build_id, ) print(f"Published pipeline: {published_pipeline.name}") print(f"for build {published_pipeline.version}")
from azureml.core.authentication import AzureCliAuthentication with open("./configuration/config.json") as f: config = json.load(f) workspace_name = config["workspace_name"] resource_group = config["resource_group"] subscription_id = config["subscription_id"] location = config["location"] cli_auth = AzureCliAuthentication() # Get workspace #ws = Workspace.from_config(auth=cli_auth) ws = Workspace.get(name=workspace_name, subscription_id=subscription_id, resource_group=resource_group, auth=cli_auth) # Get workspace #ws = Workspace.from_config(auth=cli_auth)# Get the Image to deploy details try: with open("./configuration/image.json") as f: config = json.load(f) except: print("No new model, thus no deployment on ACI") # raise Exception('No new model to register as production model perform better') sys.exit(0) image_name = config["image_name"] image_version = config["image_version"]
import mlflow.azureml from azureml.core import Workspace from azureml.core.webservice import AciWebservice, Webservice # Create or load an existing Azure ML workspace. You can also load an existing workspace using # Workspace.get(name="<workspace_name>") workspace_name = "example_mlflow" subscription_id = "<>" resource_group = "example_mlflow_resgrp" location = "South Central US" workspace = True if workspace: azure_workspace = Workspace.get(name=workspace_name, subscription_id=subscription_id, resource_group=resource_group) else: azure_workspace = Workspace.create( name=workspace_name, subscription_id=subscription_id, resource_group=resource_group, location=location, create_resource_group=False, ) print("Fetched workspace") # Build an Azure ML container image for deployment azure_image, azure_model = mlflow.azureml.build_image( model_uri="examplemodel", workspace=azure_workspace,
def main(): # Loading azure credentials print("::debug::Loading azure credentials") azure_credentials = os.environ.get("INPUT_AZURE_CREDENTIALS", default="{}") try: azure_credentials = json.loads(azure_credentials) except JSONDecodeError: print( "::error::Please paste output of `az ad sp create-for-rbac --name <your-sp-name> --role contributor --scopes /subscriptions/<your-subscriptionId>/resourceGroups/<your-rg> --sdk-auth` as value of secret variable: AZURE_CREDENTIALS. The JSON should include the following keys: 'tenantId', 'clientId', 'clientSecret' and 'subscriptionId'." ) raise AMLConfigurationException( "Incorrect or poorly formed output from azure credentials saved in AZURE_CREDENTIALS secret. See setup in https://github.com/Azure/aml-workspace/blob/master/README.md" ) # Checking provided parameters print("::debug::Checking provided parameters") validate_json(data=azure_credentials, schema=azure_credentials_schema, input_name="AZURE_CREDENTIALS") # Mask values print("::debug::Masking parameters") mask_parameter(parameter=azure_credentials.get("tenantId", "")) mask_parameter(parameter=azure_credentials.get("clientId", "")) mask_parameter(parameter=azure_credentials.get("clientSecret", "")) mask_parameter(parameter=azure_credentials.get("subscriptionId", "")) # Loading parameters file print("::debug::Loading parameters file") parameters_file = os.environ.get("INPUT_PARAMETERS_FILE", default="workspace.json") parameters_file_path = os.path.join(".cloud", ".azure", parameters_file) try: with open(parameters_file_path) as f: parameters = json.load(f) except FileNotFoundError: print( f"::debug::Could not find parameter file in {parameters_file_path}. Please provide a parameter file in your repository if you do not want to use default settings (e.g. .cloud/.azure/workspace.json)." ) parameters = {} # Checking provided parameters print("::debug::Checking provided parameters") validate_json(data=parameters, schema=parameters_schema, input_name="PARAMETERS_FILE") # Define target cloud if azure_credentials.get( "resourceManagerEndpointUrl", "").startswith("https://management.usgovcloudapi.net"): cloud = "AzureUSGovernment" elif azure_credentials.get( "resourceManagerEndpointUrl", "").startswith("https://management.chinacloudapi.cn"): cloud = "AzureChinaCloud" else: cloud = "AzureCloud" # Loading Workspace sp_auth = ServicePrincipalAuthentication( tenant_id=azure_credentials.get("tenantId", ""), service_principal_id=azure_credentials.get("clientId", ""), service_principal_password=azure_credentials.get("clientSecret", ""), cloud=cloud) try: print("::debug::Loading existing Workspace") # Default workspace and resource group name repository_name = str( os.environ.get("GITHUB_REPOSITORY")).split("/")[-1] ws = Workspace.get( name=parameters.get("name", repository_name), subscription_id=azure_credentials.get("subscriptionId", ""), resource_group=parameters.get("resource_group", repository_name), auth=sp_auth) print("::debug::Successfully loaded existing Workspace") except AuthenticationException as exception: print( f"::error::Could not retrieve user token. Please paste output of `az ad sp create-for-rbac --name <your-sp-name> --role contributor --scopes /subscriptions/<your-subscriptionId>/resourceGroups/<your-rg> --sdk-auth` as value of secret variable: AZURE_CREDENTIALS: {exception}" ) raise AuthenticationException except AuthenticationError as exception: print(f"::error::Microsoft REST Authentication Error: {exception}") raise AuthenticationException except AdalError as exception: print( f"::error::Active Directory Authentication Library Error: {exception}" ) raise AdalError except (WorkspaceException, ProjectSystemException) as exception: print(f"::debug::Loading existing Workspace failed: {exception}") if parameters.get("create_workspace", False): try: print("::debug::Creating new Workspace") ws = Workspace.create( name=parameters.get("name", repository_name), subscription_id=azure_credentials.get( "subscriptionId", ""), resource_group=parameters.get("resource_group", repository_name), location=parameters.get("location", None), create_resource_group=parameters.get( "create_resource_group", True), sku=parameters.get("sku", "basic"), friendly_name=parameters.get("friendly_name", None), storage_account=parameters.get("storage_account", None), key_vault=parameters.get("key_vault", None), app_insights=parameters.get("app_insights", None), container_registry=parameters.get("container_registry", None), cmk_keyvault=parameters.get("cmk_key_vault", None), resource_cmk_uri=parameters.get("resource_cmk_uri", None), hbi_workspace=parameters.get("hbi_workspace", None), auth=sp_auth, exist_ok=True, show_output=True) except WorkspaceException as exception: print(f"::error::Creating new Workspace failed: {exception}") raise AMLConfigurationException( f"Creating new Workspace failed with 'WorkspaceException': {exception}." ) else: print( f"::error::Loading existing Workspace failed with 'WorkspaceException' and new Workspace will not be created because parameter 'create_workspace' was not defined or set to false in your parameter file: {exception}" ) raise AMLConfigurationException( "Loading existing Workspace failed with 'WorkspaceException' and new Workspace will not be created because parameter 'create_workspace' was not defined or set to false in your parameter file." ) # Write Workspace ARM properties to config file print("::debug::Writing Workspace ARM properties to config file") config_file_path = os.environ.get("GITHUB_WORKSPACE", default=".cloud/.azure") config_file_name = "aml_arm_config.json" ws.write_config(path=config_file_path, file_name=config_file_name) print( "::debug::Successfully finished Azure Machine Learning Workspace Action" )
def get_azure_workspace(): # pylint: disable=import-error from azureml.core import Workspace return Workspace.get("test_workspace")
parser.add_argument("--datastore", dest="datastore", ) parser.add_argument("--dataset", dest="dataset", ) args = parser.parse_args() ws = Workspace.get(args.ws, ServicePrincipalAuthentication( tenant_id=os.getenv('tenant_id'), service_principal_id=os.getenv('service_principal_id'), service_principal_password=os.getenv('service_principal_password') ), subscription_id=os.getenv('subscription_id'), resource_group=args.rg ) ds = Datastore.get(ws, args.datastore) ds.download("assets", args.dataset, overwrite=False) # Load dataset url = "assets/"+args.dataset names = ['sepal-length', 'sepal-width', 'petal-length', 'petal-width', 'class'] dataset = read_csv(url, names=names) # Split-out validation dataset
def main(): e = Env() # Get Azure machine learning workspace aml_workspace = Workspace.get(name=e.workspace_name, subscription_id=e.subscription_id, resource_group=e.resource_group) print("get_workspace:") print(aml_workspace) # Get Azure machine learning cluster aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size) if aml_compute is not None: print("aml_compute:") print(aml_compute) run_config = RunConfiguration(conda_dependencies=CondaDependencies.create( conda_packages=[ 'numpy', 'pandas', 'scikit-learn', 'tensorflow', 'keras' ], pip_packages=[ 'azure', 'azureml-core', 'azure-storage', 'azure-storage-blob' ])) run_config.environment.docker.enabled = True config_envvar = {} if (e.collection_uri is not None and e.teamproject_name is not None): builduri_base = e.collection_uri + e.teamproject_name builduri_base = builduri_base + "/_build/results?buildId=" config_envvar["BUILDURI_BASE"] = builduri_base run_config.environment.environment_variables = config_envvar model_name_param = PipelineParameter(name="model_name", default_value=e.model_name) build_id_param = PipelineParameter(name="build_id", default_value=e.build_id) hyperparameter_alpha_param = PipelineParameter(name="hyperparameter_alpha", default_value=0.5) train_step = PythonScriptStep( name="Train Model", script_name=e.train_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, arguments=[ "--build_id", build_id_param, "--model_name", model_name_param, "--alpha", hyperparameter_alpha_param, ], runconfig=run_config, allow_reuse=False, ) print("Step Train created") evaluate_step = PythonScriptStep( name="Evaluate Model ", script_name=e.evaluate_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, arguments=[ "--build_id", build_id_param, "--model_name", model_name_param, ], runconfig=run_config, allow_reuse=False, ) print("Step Evaluate created") register_step = PythonScriptStep( name="Register Model ", script_name=e.register_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, arguments=[ "--build_id", build_id_param, "--model_name", model_name_param, ], runconfig=run_config, allow_reuse=False, ) print("Step Register created") evaluate_step.run_after(train_step) register_step.run_after(evaluate_step) steps = [train_step, evaluate_step, register_step] train_pipeline = Pipeline(workspace=aml_workspace, steps=steps) train_pipeline._set_experiment_name train_pipeline.validate() published_pipeline = train_pipeline.publish( name=e.pipeline_name, description="Model training/retraining pipeline", version=e.build_id) print(f'Published pipeline: {published_pipeline.name}') print(f'for build {published_pipeline.version}')
"train": { "X": X_train, "y": y_train }, "test": { "X": X_test, "y": y_test } } spa = ServicePrincipalAuthentication("9d396536-b01d-424e-ace8-2a339bc8e502", "e9311799-43b1-4a27-9e08-266cc96a2c05", "K_?XRu6bSX/DV3.WYsuS3cwxjgOg4DzO", _enable_caching=False) ws = Workspace.get(name='acc_ws', auth=spa, subscription_id='f30951ea-6926-40a7-9c19-adeba1c67ec4', resource_group='acc_ws_rg') exp = Experiment(workspace=ws, name="exp") run = exp.start_logging() clf = RandomForestClassifier(n_estimators=100, max_depth=2, random_state=0) clf.fit(data["train"]["X"], data["train"]["y"]) preds = clf.predict(data["test"]["X"]) run.log("accuracy", accuracy_score(data["test"]["y"], preds)) filename = 'outputs/propensity_model.pkl' joblib.dump(clf, filename) # Registering model with "Model" class. model = Model.register(model_name='propensity_model',
import json import numpy as np from utils import load_data from azureml.core.webservice import Webservice from azureml.core import Workspace as ws #from azureml.core.model import Model #from azureml.core.webservice import AciWebservice import os #import matplotlib import matplotlib.pyplot as plt # find 30 random samples from test set n = 30 myws = ws.get( name='mnist1', subscription_id='a383ccde-05bc-45bf-b3ae-f51bd72644df', resource_group='mnist4', ) #model = Model(myws, 'sklearn-mnist') #web = 'https://mlworkspace.azure.ai/portal/subscriptions/bcbc4e01-e5d6-42b0-95af-06286341e6ca/resourceGroups/mnist3/providers/Microsoft.MachineLearningServices/workspaces/mnist1/deployments/mnist' print(Webservice.list(myws)[0].name) service = Webservice(myws, 'sklearn-mnist-image') #print(type(model)) data_folder = os.path.join(os.getcwd(), 'data') X_test = load_data(os.path.join(data_folder, 'test-images.gz'), False) / 255.0 y_test = load_data(os.path.join(data_folder, 'test-labels.gz'), True).reshape(-1) sample_indices = np.random.permutation(X_test.shape[0])[0:n] test_samples = json.dumps({"data": X_test[sample_indices].tolist()}) test_samples = bytes(test_samples, encoding='utf8') #result = service.run(input_data=test_samples)
def main(): e = Env() # Get Azure machine learning workspace aml_workspace = Workspace.get( name=e.workspace_name, subscription_id=e.subscription_id, resource_group=e.resource_group, ) print(f"get_workspace:{aml_workspace}") # Get Azure machine learning cluster aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size) if aml_compute is not None: print(f"aml_compute:{aml_compute}") # Create a reusable Azure ML environment environment = get_environment( aml_workspace, e.aml_env_name, create_new=e.rebuild_env, enable_docker=True, dockerfile='ml_model/preprocess/Dockerfile' ) # run_config = RunConfiguration() run_config.environment = environment if e.datastore_name: datastore_name = e.datastore_name else: datastore_name = aml_workspace.get_default_datastore().name run_config.environment.environment_variables["DATASTORE_NAME"] = datastore_name # NOQA: E501 datastore = Datastore(aml_workspace, name=datastore_name) data_file_path_param = PipelineParameter(name="data_file_path", default_value=e.dataset_name) # NOQA: E501 # The version of the input/output dataset can't be determined at pipeline publish time, only run time. # NOQA: E501 # Options to store output data: # Option 1: Use blob API to write output data. Otherwise, no way to dynamically change the output dataset based on PipelineParameter, # NOQA: E501 # The following will not work. It generate a path like "PipelineParameter_Name:data_file_path_Default:gear_images" # NOQA: E501 # output_ds = OutputFileDatasetConfig(destination=(datastore, data_file_path_param)) # NOQA: E501 # This option means writing a file locally and upload to the datastore. Fewer dataset, more code. # NOQA: E501 # Option 2: Use a dynamic path in OutputFileDatasetConfig, and register a new dataset at completion # NOQA: E501 # Output dataset can be mounted, so more dataset to maintain, less code. # NOQA: E501 # Using Option 2 below. output_dataset = OutputFileDatasetConfig( name=e.processed_dataset_name, destination=(datastore, "/dataset/{output-name}/{run-id}") ).register_on_complete( name=e.processed_dataset_name) preprocess_step = PythonScriptStep( name="Preprocess Data with OS cmd", script_name='preprocess/preprocess_os_cmd_aml.py', compute_target=aml_compute, source_directory=e.sources_directory_train, arguments=[ "--dataset_name", e.dataset_name, "--datastore_name", datastore_name, "--data_file_path", data_file_path_param, "--output_dataset", output_dataset, ], runconfig=run_config, allow_reuse=False, ) print("Step Preprocess OS cmd created") steps = [preprocess_step] preprocess_pipeline = Pipeline(workspace=aml_workspace, steps=steps) preprocess_pipeline._set_experiment_name preprocess_pipeline.validate() published_pipeline = preprocess_pipeline.publish( name=e.preprocessing_pipeline_name, description="Data preprocessing OS cmd pipeline", version=e.build_id, ) print(f"Published pipeline: {published_pipeline.name}") print(f"for build {published_pipeline.version}")
WORKSPACE_NAME = os.environ.get("BASE_NAME")+"-aml" SUBSCRIPTION_ID = os.environ.get('SUBSCRIPTION_ID') RESOURCE_GROUP = os.environ.get("RESOURCE_GROUP_NAME") if os.path.isfile(MODEL_PATH) is False: print("The given model path %s is invalid" % (MODEL_PATH)) sys.exit(1) SP_AUTH = ServicePrincipalAuthentication( tenant_id=TENANT_ID, service_principal_id=APP_ID, service_principal_password=APP_SECRET) WORKSPACE = Workspace.get( WORKSPACE_NAME, SP_AUTH, SUBSCRIPTION_ID, RESOURCE_GROUP ) try: MODEL = Model.register( model_path=MODEL_PATH, model_name=MODEL_NAME, description="EPIS Model", workspace=WORKSPACE) print("Model registered successfully. ID: " + MODEL.id) except Exception as caught_error: print("Error while registering the model: " + str(caught_error)) sys.exit(1)
from azureml.core import Environment, Experiment, Workspace, Dataset from azureml.core.conda_dependencies import CondaDependencies from azureml.core.runconfig import RunConfiguration from azureml.pipeline.core import PipelineData, Pipeline from azureml.pipeline.steps import PythonScriptStep, EstimatorStep from azureml.train.estimator import Estimator from azureml.core.datastore import Datastore from azureml.data.data_reference import DataReference import os cluster_name = "azuremlscluster" # # Get Azure machine learning workspace ws = Workspace.get(name=os.environ.get("WORKSPACE_NAME"), subscription_id=os.environ.get("SUBSCRIPTION_ID"), resource_group=os.environ.get("RESOURCE_GROUP")) # Set default Datastore # ----> Uncomment the following line if there already is a datastore and name it correctly # ws.set_default_datastore('workspaceblobstore') # if creditcard is not registered if 'creditcard' not in ws.datasets: #Set blobdatastore blob_datastore_name = 'MyBlobDatastore' account_name = os.getenv(
def main(): run = Run.get_context() if (run.id.startswith('OfflineRun')): from dotenv import load_dotenv # For local development, set values in this section load_dotenv() workspace_name = os.environ.get("WORKSPACE_NAME") experiment_name = os.environ.get("EXPERIMENT_NAME") resource_group = os.environ.get("RESOURCE_GROUP") subscription_id = os.environ.get("SUBSCRIPTION_ID") build_id = os.environ.get('BUILD_BUILDID') # run_id useful to query previous runs run_id = "bd184a18-2ac8-4951-8e78-e290bef3b012" aml_workspace = Workspace.get(name=workspace_name, subscription_id=subscription_id, resource_group=resource_group) ws = aml_workspace exp = Experiment(ws, experiment_name) else: ws = run.experiment.workspace exp = run.experiment run_id = 'amlcompute' parser = argparse.ArgumentParser("register") parser.add_argument( "--build_id", type=str, help="The Build ID of the build triggering this pipeline run", ) parser.add_argument( "--run_id", type=str, help="Training run ID", ) parser.add_argument( "--model_name", type=str, help="Name of the Model", default="sklearn_regression_model.pkl", ) parser.add_argument("--step_input", type=str, help=("input from previous steps")) args = parser.parse_args() if (args.build_id is not None): build_id = args.build_id if (args.run_id is not None): run_id = args.run_id if (run_id == 'amlcompute'): run_id = run.parent.id model_name = args.model_name model_path = args.step_input # load the model print("Loading model from " + model_path) model_file = os.path.join(model_path, model_name) model = joblib.load(model_file) model_mse = run.parent.get_metrics()["mse"] if (model is not None): if (build_id is None): register_aml_model(model_file, model_name, exp, run_id) else: run.tag("BuildId", value=build_id) builduri_base = os.environ.get("BUILDURI_BASE") if (builduri_base is not None): build_uri = builduri_base + build_id run.tag("BuildUri", value=build_uri) register_aml_model(model_file, model_name, model_mse, exp, run_id, build_id, build_uri) else: register_aml_model(model_file, model_name, model_mse, exp, run_id, build_id) else: print("Model not found. Skipping model registration.") sys.exit(0)
def main(): e = Env() # Get Azure machine learning workspace aml_workspace = Workspace.get(name=e.workspace_name, subscription_id=e.subscription_id, resource_group=e.resource_group) print("get_workspace:") print(aml_workspace) # Get Azure machine learning cluster aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size) if aml_compute is not None: print("aml_compute:") print(aml_compute) # Create a reusable Azure ML environment environment = get_environment(aml_workspace, e.aml_env_name, create_new=False) # NOQA: E501 run_config = RunConfiguration() run_config.environment = environment if (e.datastore_name): datastore_name = e.datastore_name else: datastore_name = aml_workspace.get_default_datastore().name run_config.environment.environment_variables[ "DATASTORE_NAME"] = datastore_name # NOQA: E501 model_name_param = PipelineParameter(name="model_name", default_value=e.model_name) dataset_version_param = PipelineParameter(name="dataset_version", default_value=e.dataset_version) data_file_path_param = PipelineParameter(name="data_file_path", default_value="none") caller_run_id_param = PipelineParameter(name="caller_run_id", default_value="none") # Get dataset name dataset_name = e.dataset_name # # Check to see if dataset exists # if (dataset_name not in aml_workspace.datasets): # # Create dataset from lacemlops sample data # sample_data = load_lacemlops() # df = pd.DataFrame( # data=sample_data.data, # columns=sample_data.feature_names) # df['Y'] = sample_data.target # file_name = 'lacemlops.csv' # df.to_csv(file_name, index=False) # # Upload file to default datastore in workspace # datatstore = Datastore.get(aml_workspace, datastore_name) # target_path = 'training-data/' # datatstore.upload_files( # files=[file_name], # target_path=target_path, # overwrite=True, # show_progress=False) # # Register dataset # path_on_datastore = os.path.join(target_path, file_name) # dataset = Dataset.Tabular.from_delimited_files( # path=(datatstore, path_on_datastore)) # dataset = dataset.register( # workspace=aml_workspace, # name=dataset_name, # description='lacemlops training data', # tags={'format': 'CSV'}, # create_new_version=True) # Create a PipelineData to pass data between steps pipeline_data = PipelineData( 'pipeline_data', datastore=aml_workspace.get_default_datastore()) train_step = PythonScriptStep( name="Train Model", script_name=e.train_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, outputs=[pipeline_data], arguments=[ "--model_name", model_name_param, "--step_output", pipeline_data, "--dataset_version", dataset_version_param, "--data_file_path", data_file_path_param, "--caller_run_id", caller_run_id_param, "--dataset_name", dataset_name, ], runconfig=run_config, allow_reuse=False, ) print("Step Train created") evaluate_step = PythonScriptStep( name="Evaluate Model ", script_name=e.evaluate_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, arguments=[ "--model_name", model_name_param, "--allow_run_cancel", e.allow_run_cancel, ], runconfig=run_config, allow_reuse=False, ) print("Step Evaluate created") register_step = PythonScriptStep( name="Register Model ", script_name=e.register_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, inputs=[pipeline_data], arguments=[ "--model_name", model_name_param, "--step_input", pipeline_data, ], runconfig=run_config, allow_reuse=False, ) print("Step Register created") # Check run_evaluation flag to include or exclude evaluation step. if ((e.run_evaluation).lower() == 'true'): print("Include evaluation step before register step.") evaluate_step.run_after(train_step) register_step.run_after(evaluate_step) steps = [train_step, evaluate_step, register_step] else: print("Exclude evaluation step and directly run register step.") register_step.run_after(train_step) steps = [train_step, register_step] train_pipeline = Pipeline(workspace=aml_workspace, steps=steps) train_pipeline._set_experiment_name train_pipeline.validate() published_pipeline = train_pipeline.publish( name=e.pipeline_name, description="Model training/retraining pipeline", version=e.build_id) print(f'Published pipeline: {published_pipeline.name}') print(f'for build {published_pipeline.version}')
#!/usr/bin/env python # coding: utf-8 # In[ ]: # Connecting to existing Azure Machine Learning workspace import os from azureml.core import Workspace from azureml.core.authentication import ServicePrincipalAuthentication ws = Workspace.get(name='ws_employee_churn', auth=None, subscription_id='9c43f403-ce9b-4e3a-8f7f-21b1688e2873', resource_group='rg_datascience') cluster_type = os.environ.get("AML_COMPUTE_CLUSTER_TYPE", "CPU") compute_target = ws.get_default_compute_target(cluster_type) # In[ ]: #Read and pre-process data import pandas as pd import numpy as np hr = pd.read_csv('HR_Data.csv') col_names = hr.columns.tolist() col_names cat_vars=['Age Bucket', 'Decade', 'Gender', 'Marital Status', 'Cost Center', 'Managed By', 'Designation', 'Employee Band', 'Experience'] for var in cat_vars: cat_list='var'+'_'+var cat_list = pd.get_dummies(hr[var], prefix=var,drop_first=False)
def main(): run = Run.get_context() if (run.id.startswith('OfflineRun')): from dotenv import load_dotenv # For local development, set values in this section load_dotenv() sources_dir = os.environ.get("SOURCES_DIR_TRAIN") if (sources_dir is None): sources_dir = 'diabetes_regression' path_to_util = os.path.join(".", sources_dir, "util") sys.path.append(os.path.abspath(path_to_util)) # NOQA: E402 from model_helper import get_model_by_tag workspace_name = os.environ.get("WORKSPACE_NAME") experiment_name = os.environ.get("EXPERIMENT_NAME") resource_group = os.environ.get("RESOURCE_GROUP") subscription_id = os.environ.get("SUBSCRIPTION_ID") build_id = os.environ.get('BUILD_BUILDID') aml_workspace = Workspace.get(name=workspace_name, subscription_id=subscription_id, resource_group=resource_group) ws = aml_workspace exp = Experiment(ws, experiment_name) else: sys.path.append(os.path.abspath("./util")) # NOQA: E402 from model_helper import get_model_by_tag ws = run.experiment.workspace exp = run.experiment e = Env() parser = argparse.ArgumentParser("register") parser.add_argument( "--build_id", type=str, help="The Build ID of the build triggering this pipeline run", ) parser.add_argument("--output_model_version_file", type=str, default="model_version.txt", help="Name of a file to write model version to") args = parser.parse_args() if (args.build_id is not None): build_id = args.build_id model_name = e.model_name try: tag_name = 'BuildId' model = get_model_by_tag(model_name, tag_name, build_id, exp.workspace) if (model is not None): print("Model was registered for this build.") if (model is None): print("Model was not registered for this run.") sys.exit(1) except Exception as e: print(e) print("Model was not registered for this run.") sys.exit(1) # Save the Model Version for other AzDO jobs after script is complete if args.output_model_version_file is not None: with open(args.output_model_version_file, "w") as out_file: out_file.write(str(model.version))
route = nx.shortest_path(G, orig, dest, weight='travel_time') edge_lengths = ox.utils_graph.get_route_edge_attributes( G, route, 'length') return sum(edge_lengths) else: raise ValueError('Path not foun') GRAPH_FILE_PATH = "https://grab5033896937.blob.core.windows.net/azureml/Dataset/grab/singapore.graphml" try: # load workspace configuration from the config.json file in the current folder. #ws = Workspace.from_config() ws = Workspace.get(name="<<Insert Name>>", subscription_id="<<Insert Subscription Id>>", resource_group="<<Insert Resource Group>>") dataset = Dataset.get_by_name(ws, 'sg_graphml') # list the files referenced by sg_graphml dataset GRAPH_FILE_PATH = dataset.to_path() G = ox.load_graphml(GRAPH_FILE_PATH) except: G = ox.graph_from_place('Singapore', network_type='drive') ox.save_graphml(G, filepath=GRAPH_FILE_PATH) def init(): global model
def get_workspace(name, subscription_id, resource_group): return Workspace.get(name=name, subscription_id=subscription_id, resource_group=resource_group)
from azureml.core.authentication import InteractiveLoginAuthentication from azureml.core import Workspace interactive_auth = InteractiveLoginAuthentication( tenant_id="99e1e721-7184-498e-8aff-b2ad4e53c1c2") ws = Workspace.get(name='mlw-udea-mon', subscription_id='80986424-c91c-4671-9d12-d7de41662014', resource_group='rg-udea-mon', location='eastus', auth=interactive_auth) ws.write_config(path='.azureml')
def main(gpu_num, class_path, data_dir, data_path, num_clusters, batch_size, learning_rate): # Define an Azure ML run run = Run.get_context() # Folder for persistent assets log_dir = 'outputs/' # Conver to int num_clusters = int(num_clusters) # Define workspace object try: sp = ServicePrincipalAuthentication( tenant_id=os.getenv('AML_TENANT_ID'), service_principal_id=os.getenv('AML_PRINCIPAL_ID'), service_principal_password=os.getenv('AML_PRINCIPAL_PASS')) ws = Workspace.get(name=os.getenv('WORKSPACE_NAME'), auth=sp, subscription_id=os.getenv('SUBSCRIPTION_ID')) # Need to create the workspace except Exception as err: print('Issue with service principal or workspace creation: ', err) assert False # Get classes class_names = get_classes(class_path) num_classes = len(class_names) # Create annot file (file with image path and bounding boxes - one image per line) image_ids = [ os.path.basename(x) for x in glob.glob(os.path.join(data_path, data_dir, 'JPEGImages/*.*')) ] annot_filename = './outputs/train_val.txt' annot_file = open(annot_filename, 'w') # output file for image_id in image_ids: annot_file.write( os.path.join(data_path, data_dir, 'JPEGImages/{}'.format(image_id))) convert_annotation(image_id='.'.join(image_id.split('.')[0:-1]), list_file=annot_file, classes=class_names, data_dir=data_dir, data_path=data_path) annot_file.write('\n') annot_file.close() # Calculate anchor boxes out_file = './outputs/custom_anchors.txt' kmeans = YOLO_Kmeans(args.num_clusters, annot_filename, out_file) kmeans.txt2clusters() # Anchors and filters anchors_orig, anchors = get_anchors(out_file) filter_num = (num_classes + 5) * 3 # Update config file template # replace $FILTERS, $CLASSES, and $ANCHORS w/ correct value from above if num_clusters == 9: with open('yolov3-custom_template.cfg', 'r') as fptr: config_str = fptr.read() config_str = config_str.replace('$FILTERS', str(filter_num)) config_str = config_str.replace('$CLASSES', str(num_classes)) config_str = config_str.replace('$ANCHORS', anchors_orig) else: # num_clusters == 6 with open('yolov3-tiny-custom_template.cfg', 'r') as fptr: config_str = fptr.read() config_str = config_str.replace('$FILTERS', str(filter_num)) config_str = config_str.replace('$CLASSES', str(num_classes)) config_str = config_str.replace('$ANCHORS', anchors_orig) with open('./outputs/yolov3-custom.cfg', 'w') as outptr: outptr.write(config_str) # Download Darknet model (from Azure ML Workspace) and convert keras_custom_model = './outputs/yolov3_custom.h5' if num_clusters == 9: AzureMLModel(ws, name='yolov3.weights').download(target_dir='.', exist_ok=True) convert_to_keras(config_path='./outputs/yolov3-custom.cfg', weights_path='yolov3.weights', output_path=keras_custom_model) else: AzureMLModel(ws, name='yolov3-tiny.weights').download(target_dir='.', exist_ok=True) convert_to_keras(config_path='./outputs/yolov3-custom.cfg', weights_path='yolov3-tiny.weights', output_path=keras_custom_model) initial_lr = learning_rate input_shape = (608, 608) # default setting is_tiny_version = (num_clusters == 6) # Use transfer learning - all layers frozen except last "freeze_body" number if is_tiny_version: input_shape = (416, 416) #Change default anchor box number model = create_tiny_model(input_shape, anchors, num_classes, freeze_body=4, weights_path=keras_custom_model, gpu_num=gpu_num) else: model = create_model(input_shape, anchors, num_classes, freeze_body=4, weights_path=keras_custom_model ) # make sure you know what you freeze logging = TensorBoard(log_dir=log_dir) checkpoint = ModelCheckpoint( log_dir + 'ep{epoch:03d}-loss{loss:.3f}-val_loss{val_loss:.3f}.h5', monitor='val_loss', save_weights_only=True, save_best_only=True, period=3) reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3, verbose=1) early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=10, verbose=1) val_split = 0.2 with open(annot_filename) as f: lines = f.readlines() np.random.seed(10101) np.random.shuffle(lines) np.random.seed(None) num_val = int(len(lines) * val_split) num_train = len(lines) - num_val class LogRunMetrics(Callback): # callback at the end of every epoch def on_epoch_end(self, epoch, log): # log a value repeated which creates a list for Azure ML tracking run.log('Loss', log['loss']) # Train with frozen layers first, to get a stable loss. # Adjust num epochs to your dataset. This step is enough to obtain a not bad model. if True: model.compile( optimizer=Adam(lr=initial_lr), loss={ # use custom yolo_loss Lambda layer. 'yolo_loss': lambda y_true, y_pred: y_pred }) print('Train on {} samples, val on {} samples, with batch size {}.'. format(num_train, num_val, batch_size)) model.fit_generator(data_generator_wrapper(lines[:num_train], batch_size, input_shape, anchors, num_classes), steps_per_epoch=max(1, num_train // batch_size), validation_data=data_generator_wrapper( lines[num_train:], batch_size, input_shape, anchors, num_classes), validation_steps=max(1, num_val // batch_size), epochs=30, initial_epoch=0, callbacks=[logging, LogRunMetrics()]) model.save_weights(log_dir + 'trained_weights_intermediate.h5') # Save architecture, too with open(log_dir + 'trained_weights_intermediate.json', 'w') as f: f.write(model.to_json()) # Unfreeze and continue training, to fine-tune. # Train longer if the result is not good. if True: for i in range(len(model.layers)): model.layers[i].trainable = True model.compile(optimizer=Adam(lr=initial_lr / 100), loss={ 'yolo_loss': lambda y_true, y_pred: y_pred }) # recompile to apply the change print('Unfreeze all of the layers.') print('Train on {} samples, val on {} samples, with batch size {}.'. format(num_train, num_val, batch_size)) model.fit_generator( data_generator_wrapper(lines[:num_train], batch_size, input_shape, anchors, num_classes), steps_per_epoch=max(1, num_train // batch_size), validation_data=data_generator_wrapper(lines[num_train:], batch_size, input_shape, anchors, num_classes), validation_steps=max(1, num_val // batch_size), epochs=60, initial_epoch=30, callbacks=[logging, early_stopping, reduce_lr, LogRunMetrics()]) model.save_weights(log_dir + 'trained_weights_final.h5') # Save architecture, too with open(log_dir + 'trained_weights_final.json', 'w') as f: f.write(model.to_json())
from azureml.core import Workspace from azureml.core.compute import ComputeTarget, AmlCompute from azureml.core.compute_target import ComputeTargetException from ml_service.util.env_variables import Env # Environment variables env = Env() # Azure ML workspace aml_workspace = Workspace.get( name=env.workspace_name, subscription_id=env.subscription_id, resource_group=env.resource_group, ) # Verify that the cluster does not exist already try: cpu_cluster = ComputeTarget(workspace=aml_workspace, name=env.compute_name) print('Found existing cluster, use it.') except ComputeTargetException: compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_D2_V2', idle_seconds_before_scaledown=2400, min_nodes=0, max_nodes=1) cpu_cluster = ComputeTarget.create(aml_workspace, env.compute_name, compute_config) cpu_cluster.wait_for_completion(show_output=True)
def main(): parser = argparse.ArgumentParser("register") parser.add_argument("--output_pipeline_id_file", type=str, default="pipeline_id.txt", help="Name of a file to write pipeline ID to") parser.add_argument( "--skip_train_execution", action="store_true", help=("Do not trigger the execution. " "Use this in Azure DevOps when using a server job to trigger"), ) args = parser.parse_args() e = Env() aml_workspace = Workspace.get(name=e.workspace_name, subscription_id=e.subscription_id, resource_group=e.resource_group) # Find the pipeline that was published by the specified build ID pipelines = PublishedPipeline.list(aml_workspace) matched_pipes = [] for p in pipelines: if p.name == e.pipeline_name: if p.version == e.build_id: matched_pipes.append(p) if len(matched_pipes) > 1: published_pipeline = None raise Exception( f"Multiple active pipelines are published for build {e.build_id}." ) # NOQA: E501 elif len(matched_pipes) == 0: published_pipeline = None raise KeyError( f"Unable to find a published pipeline for this build {e.build_id}" ) # NOQA: E501 else: published_pipeline = matched_pipes[0] print("published pipeline id is", published_pipeline.id) # Save the Pipeline ID for other AzDO jobs after script is complete if args.output_pipeline_id_file is not None: with open(args.output_pipeline_id_file, "w") as out_file: out_file.write(published_pipeline.id) if args.skip_train_execution is False: pipeline_parameters = {"model_name": e.model_name} tags = {"BuildId": e.build_id} if e.build_uri is not None: tags["BuildUri"] = e.build_uri experiment = Experiment(workspace=aml_workspace, name=e.experiment_name) run = experiment.submit(published_pipeline, tags=tags, pipeline_parameters=pipeline_parameters) print("Pipeline run initiated ", run.id)
model_bin = bz2.decompress(model_compressed) with open(MODEL_PATH, "wb") as f: model = f.write(model_bin) #pickle.dump(model_bin,open(MODEL_PATH,"wb")) print('model size (original): ', sys.getsizeof(MODEL_PATH), " bytes") MODEL_PATH = os.path.join(os.getcwd(), MODEL_PATH) else: MODEL_PATH = os.path.join(os.getcwd(), MODEL_PATH) wget.download(MODEL_URL, MODEL_PATH) #Set the inetractive authentication interactive_auth = InteractiveLoginAuthentication() #Get the chosen workspace ws = Workspace.get(name=AZURE_WORKSPACE_NAME, auth=interactive_auth, subscription_id=AZURE_SUBSCRIPTION_ID, resource_group=AZURE_RESOURCE_GROUP) print(ws.name, ws.location, ws.resource_group, ws.location, sep='\t') # register model model = Model.register(model_name=MODEL_NAME, model_path=MODEL_PATH, workspace=ws, description=MODEL_DESCRIPTION) print(model.name, model.id, model.version, sep='\t') #Set the image aciconfig = AciWebservice.deploy_configuration(cpu_cores=CPU_CORES, memory_gb=MEMORY_GB, description=SERVICE_DESCRIPTION,
import os import argparse from azureml.core import Workspace from azureml.core.environment import Environment from azureml.core.model import Model, InferenceConfig import shutil from ml_service.util.env_variables import Env e = Env() # Get Azure machine learning workspace ws = Workspace.get(name=e.workspace_name, subscription_id=e.subscription_id, resource_group=e.resource_group) parser = argparse.ArgumentParser("create scoring image") parser.add_argument( "--output_image_location_file", type=str, help=("Name of a file to write image location to, " "in format REGISTRY.azurecr.io/IMAGE_NAME:IMAGE_VERSION")) args = parser.parse_args() model = Model(ws, name=e.model_name, version=e.model_version) sources_dir = e.sources_directory_train if (sources_dir is None): sources_dir = 'COVID19Articles' score_script = os.path.join(".", sources_dir, e.score_script) score_file = os.path.basename(score_script) path_to_scoring = os.path.dirname(score_script) cwd = os.getcwd()
from azureml.core.environment import Environment from azureml.core.model import InferenceConfig, Model from azureml.core.webservice import LocalWebservice from azureml.core import Workspace ws = Workspace.get(name="QCM", subscription_id='ed4fc9bc-f386-4f01-a8b4-2077312476f3', resource_group='appsvc_linux_centralus') models = Model.register( model_path="model_scaler.joblib", model_name="mymodel", tags={ 'area': "diabetes", 'type': "regression" }, description="Ridge regression model to predict diabetes", workspace=ws) # Create inference configuration based on the environment definition and the entry script myenv = Environment.from_conda_specification(name="env", file_path="myenv.yml") inference_config = InferenceConfig(entry_script="score.py", environment=myenv) # Create a local deployment, using port 8890 for the web service endpoint deployment_config = LocalWebservice.deploy_configuration(port=8890) # Deploy the service service = Model.deploy(ws, "mymodel", [models], inference_config, deployment_config) # Wait for the deployment to complete
def main(): run = Run.get_context() if (run.id.startswith('OfflineRun')): from dotenv import load_dotenv # For local development, set values in this section load_dotenv() workspace_name = os.environ.get("WORKSPACE_NAME") experiment_name = os.environ.get("EXPERIMENT_NAME") resource_group = os.environ.get("RESOURCE_GROUP") subscription_id = os.environ.get("SUBSCRIPTION_ID") # run_id useful to query previous runs run_id = "bd184a18-2ac8-4951-8e78-e290bef3b012" aml_workspace = Workspace.get( name=workspace_name, subscription_id=subscription_id, resource_group=resource_group ) ws = aml_workspace exp = Experiment(ws, experiment_name) else: ws = run.experiment.workspace exp = run.experiment run_id = 'amlcompute' parser = argparse.ArgumentParser("register") parser.add_argument( "--run_id", type=str, help="Training run ID", ) parser.add_argument( "--model_name", type=str, help="Name of the Model", default="sales_forecast_model.pkl", ) parser.add_argument( "--step_input", type=str, help=("input from previous steps") ) args = parser.parse_args() if (args.run_id is not None): run_id = args.run_id if (run_id == 'amlcompute'): run_id = run.parent.id model_name = args.model_name model_path = args.step_input print("Getting registration parameters") # Load the registration parameters from the parameters file with open("parameters.json") as f: pars = json.load(f) try: register_args = pars["registration"] except KeyError: print("Could not load registration values from file") register_args = {"tags": []} model_tags = {} for tag in register_args["tags"]: try: mtag = run.parent.get_metrics()[tag] model_tags[tag] = mtag except KeyError: print(f"Could not find {tag} metric on parent run.") #--------------------------------------- # later... # load json and create model #json_file = open('model.json', 'r') #loaded_model_json = json_file.read() #json_file.close() #model = model_from_json(loaded_model_json) # load weights into new model #model.load_weights("model.h5") #print("Loaded model from disk") # load the model print("Loading model from " + model_path) model_file = os.path.join(model_path, model_name) print("Printing model file") print(model_file) model = keras.models.load_model(model_file) print("Loading the model----:)") #model = load_model(model_path) #model = load_model(model_file) # model = joblib.load(model_file) parent_tags = run.parent.get_tags() try: build_id = parent_tags["BuildId"] except KeyError: build_id = None print("BuildId tag not found on parent run.") print(f"Tags present: {parent_tags}") try: build_uri = parent_tags["BuildUri"] except KeyError: build_uri = None print("BuildUri tag not found on parent run.") print(f"Tags present: {parent_tags}") #if (model is not None): dataset_id = parent_tags["dataset_id"] if (build_id is None): register_aml_model( model_file, model_name, model_tags, exp, run_id, dataset_id) elif (build_uri is None): register_aml_model( model_file, model_name, model_tags, exp, run_id, dataset_id, build_id) else: register_aml_model( model_file, model_name, model_tags, exp, run_id, dataset_id, build_id, build_uri)
type=str, dest="friendly_name", help="Friendly name of the Azure Machine Learning Workspace") args = parser.parse_args() # Mask values print("Masking values") print(f"::add-mask::{args.subscription_id}") # Use Azure CLI authentication cli_auth = AzureCliAuthentication() try: print("Loading existing Workspace") ws = Workspace.get(name=args.workspace_name, subscription_id=args.subscription_id, resource_group=args.resource_group, auth=cli_auth) print("Found existing Workspace") except WorkspaceException: print("Loading failed") print("Creating new Workspace") ws = Workspace.create(name=args.workspace_name, auth=cli_auth, subscription_id=args.subscription_id, resource_group=args.resource_group, location=args.location, create_resource_group=True, friendly_name=args.friendly_name, show_output=True) # Write out the Workspace ARM properties to a config file
resource_group = args.resource_group workspace_name = args.workspace_name workspace_region = args.workspace_region model_name = args.model_name cluster_name_cpu = args.cluster_name_cpu cluster_name_gpu = args.cluster_name_gpu pipeline_experiment_name = args.pipeline_experiment_name pipeline_name = args.pipeline_name service_principal = ServicePrincipalAuthentication( tenant_id=tenant_id, service_principal_id=application_id, service_principal_password=app_secret) ws = Workspace.get(name=workspace_name, subscription_id=subscription_id, resource_group=resource_group, auth=service_principal) # Retrieve the pointer to the default Blob storage. def_blob_store = Datastore(ws, "workspaceblobstore") print("Blobstore's name: {}".format(def_blob_store.name)) blob_input_data = DataReference(datastore=def_blob_store, data_reference_name="mnist_datainput", path_on_datastore="mnist_datainput") print("DataReference object created") # Create a CPU cluster of type D2 V2 with 1 node. (due to subscription's limitations we stick to 1 node) try:
def main(): run = Run.get_context() if run.id.startswith("OfflineRun"): from dotenv import load_dotenv load_dotenv() sources_dir = os.environ.get("SOURCES_DIR_TRAIN") if sources_dir is None: sources_dir = "oilwells" workspace_name = os.environ.get("WORKSPACE_NAME") experiment_name = os.environ.get("EXPERIMENT_NAME") resource_group = os.environ.get("RESOURCE_GROUP") subscription_id = os.environ.get("SUBSCRIPTION_ID") build_id = os.environ.get("BUILD_BUILDID") aml_workspace = Workspace.get(name=workspace_name, subscription_id=subscription_id, resource_group=resource_group) ws = aml_workspace exp = Experiment(ws, experiment_name) else: exp = run.experiment e = Env() parser = argparse.ArgumentParser("register") parser.add_argument( "--build_id", type=str, help="The Build ID of the build triggering this pipeline run", ) parser.add_argument( "--output_model_version_file", type=str, default="model_version.txt", help="Name of a file to write model version to", ) args = parser.parse_args() if args.build_id is not None: build_id = args.build_id model_name = e.model_name try: tag_name = "BuildId" model = get_latest_model(model_name, tag_name, build_id, exp.workspace) if model is not None: print("Model was registered for this build.") if model is None: print("Model was not registered for this run.") sys.exit(1) except Exception as e: print(e) print("Model was not registered for this run.") sys.exit(1) # Save the Model Version for other AzDO jobs after script is complete if args.output_model_version_file is not None: with open(args.output_model_version_file, "w") as out_file: out_file.write(str(model.version))