# tutorial/01-create-workspace.py from azureml.core.authentication import InteractiveLoginAuthentication from azureml.core import Workspace interactive_auth = InteractiveLoginAuthentication(tenant_id="99e1e721-7184-498e-8aff-b2ad4e53c1c2") ws = Workspace.create(name='azure-ml', subscription_id='59e1d56a-8a2d-48a7-9cd3-a52c1a268c55', resource_group='cloud-ml', create_resource_group=True, location='eastus2', auth=interactive_auth ) # write out the workspace details to a configuration file: .azureml/config.json ws.write_config(path='.azureml')
full_X = df_affordability[["Age", "KM"]] full_Y = df_affordability[["Affordable"]] # Create a Workspace #Provide the Subscription ID of your existing Azure subscription subscription_id = "e223f1b3-d19b-4cfa-98e9-bc9be62717bc" #Provide values for the Resource Group and Workspace that will be created resource_group = "aml-workspace-z" workspace_name = "aml-workspace-z" workspace_region = 'westcentralus' # eastus, westcentralus, southeastasia, australiaeast, westeurope # By using the exist_ok param, if the worskpace already exists we get a reference to the existing workspace instead of an error ws = Workspace.create(name=workspace_name, subscription_id=subscription_id, resource_group=resource_group, location=workspace_region, exist_ok=True) print("Workspace Provisioning complete.") # Step 2 - Define a helper method that will use AutoML to train multiple models and pick the best one ##################################################################################################### def auto_train_model(ws, experiment_name, model_name, full_X, full_Y, training_set_percentage, training_target_accuracy): # start a training run by defining an experiment experiment = Experiment(ws, experiment_name) train_X, test_X, train_Y, test_Y = train_test_split(
from azureml.core import Workspace from azureml.core.webservice import Webservice # Requires the config to be downloaded first to the current working directory ws = Workspace.from_config() # Set with the deployment name name = "votingensemblemodels" # load existing web service service = Webservice(name=name, workspace=ws) # Enable app insight service.update(enable_app_insights=True) # Get service log logs = service.get_logs() for line in logs.split('\n'): print(line)
from azureml.core.compute import ComputeTarget, AmlCompute from azureml.core.compute_target import ComputeTargetException from azureml.core import Workspace from azureml.core.authentication import AzureCliAuthentication # load Azure ML workspace workspace = Workspace.from_config(auth=AzureCliAuthentication()) # Create compute target if not present # Choose a name for your CPU cluster cpu_cluster_name = "hypercomputecpu" # Verify that cluster does not exist already try: cu_cluster = ComputeTarget(workspace=workspace, name=cpu_cluster_name) print('Found existing cluster, use it.') except ComputeTargetException: compute_config = AmlCompute.provisioning_configuration( vm_size='STANDARD_D3_V2', max_nodes=4) cpu_cluster = ComputeTarget.create(workspace, cpu_cluster_name, compute_config) cpu_cluster.wait_for_completion(show_output=True)
def main(): # Ger our configs with open("ptgnn/authentication.json") as jsonFile: authData = json.load(jsonFile)[args.auth_cluster] # Copy the convertCorpus script here. Done so we don't upload the corpus to Azure, or keep a copy of the script in here. # (It's weird, I know. It works and has a purpose though) convertCorpusLocation = Path("../convertCorpusForML.py") convertCorpusAzureLocation = Path("./convertCorpusForML.py") shutil.copy(convertCorpusLocation, convertCorpusAzureLocation) # Grab the authentication data from the JSON file subID = authData["subID"] # Get from Azure Portal; used for billing resGroup = authData["resGroup"] # Name for the resource group wsName = authData["wsName"] # Name for the workspace, which is the collection of compute clusters + experiments computeName = authData["computeName"] # Name for computer cluster datastoreName = authData["datastoreName"] # Get the workspace, the compute target and the datastore ws = Workspace.get(wsName, subscription_id=subID, resource_group=resGroup) computeTarget = ComputeTarget(ws, computeName) datastore = Datastore(ws, name=datastoreName) # Download the entire corpus to the compute target. Save the DataReference obj here # as_mount is also possible, but slows things down due to network opening of files corpus_location = datastore.path(args.aml_location).as_download() output_location = "./" # The files that will be uploaded for usage by our script (everything in the azure folder) source_directory = "." # params for the script params = { "--corpus_location": corpus_location, "--output_folder": output_location, "--aml": "", "--training_percent": args.training_percent, "--validation_percent": args.validation_percent, "-c": "" } if args.log_num is not None: params["-l"] = args.log_num tags = { "logs": str(args.log_num) } else: tags = { "logs": "MAX" } if args.statement_generation: params["-s"] = "" tags["generationType"] = "Statement" else: tags["generationType"] = "Severity" # Set up the estimator object. Note the inputs element, it tells azure that corpus_location in params # will be a DataReference Object. est = Estimator(source_directory=source_directory, compute_target=computeTarget, entry_script='convertCorpusForML.py', script_params=params, inputs=[corpus_location], conda_packages=["pip"], pip_packages=["azureml-core", "tqdm", "numpy", "protobuf"], use_docker=True, use_gpu=False) # Start the experiment run = Experiment(ws, args.exp_name).submit(config=est, tags=tags) # remove the copy of convertCorpus (Remember, don't question this) convertCorpusAzureLocation.unlink() # print out the portral URL # print("Portal URL: ", run.get_portal_url()) # this will stream everything that the compute target does. print("Experiment Started. Remember you can exit out of this program but the experiment will still run on Azure!") run.wait_for_completion(show_output=True)
def main(): run = Run.get_context() if (run.id.startswith('OfflineRun')): from dotenv import load_dotenv # For local development, set values in this section load_dotenv() workspace_name = os.environ.get("WORKSPACE_NAME") experiment_name = os.environ.get("EXPERIMENT_NAME") resource_group = os.environ.get("RESOURCE_GROUP") subscription_id = os.environ.get("SUBSCRIPTION_ID") # run_id useful to query previous runs run_id = "bd184a18-2ac8-4951-8e78-e290bef3b012" aml_workspace = Workspace.get(name=workspace_name, subscription_id=subscription_id, resource_group=resource_group) ws = aml_workspace exp = Experiment(ws, experiment_name) else: ws = run.experiment.workspace exp = run.experiment run_id = 'amlcompute' parser = argparse.ArgumentParser("register") parser.add_argument( "--run_id", type=str, help="Training run ID", ) parser.add_argument( "--model_name", type=str, help="Name of the Model", default="fusion_model.pkl", ) parser.add_argument("--step_input", type=str, help=("input from previous steps")) args = parser.parse_args() if (args.run_id is not None): run_id = args.run_id if (run_id == 'amlcompute'): run_id = run.parent.id model_name = args.model_name model_path = args.step_input print("Getting registration parameters") # Load the registration parameters from the parameters file with open("parameters.json") as f: pars = json.load(f) try: register_args = pars["registration"] except KeyError: print("Could not load registration values from file") register_args = {"tags": []} model_tags = {} for tag in register_args["tags"]: try: mtag = run.parent.get_metrics()[tag] model_tags[tag] = mtag except KeyError: print(f"Could not find {tag} metric on parent run.") # load the model print("Loading model from " + model_path) model_file = os.path.join(model_path, model_name) model = joblib.load(model_file) parent_tags = run.parent.get_tags() try: build_id = parent_tags["BuildId"] except KeyError: build_id = None print("BuildId tag not found on parent run.") print(f"Tags present: {parent_tags}") try: build_uri = parent_tags["BuildUri"] except KeyError: build_uri = None print("BuildUri tag not found on parent run.") print(f"Tags present: {parent_tags}") if (model is not None): dataset_id = parent_tags["dataset_id"] if (build_id is None): register_aml_model(model_file, model_name, model_tags, exp, run_id, dataset_id) elif (build_uri is None): register_aml_model(model_file, model_name, model_tags, exp, run_id, dataset_id, build_id) else: register_aml_model(model_file, model_name, model_tags, exp, run_id, dataset_id, build_id, build_uri) else: print("Model not found. Skipping model registration.") sys.exit(0)
def __init__(self, request_id, input_container_sas, internal_datastore): try: aml_config = api_config.AML_CONFIG self.ws = Workspace(subscription_id=aml_config['subscription_id'], resource_group=aml_config['resource_group'], workspace_name=aml_config['workspace_name'], auth=svc_pr) print('AMLCompute constructor, AML workspace obtained.') internal_dir, output_dir = self._get_data_references( request_id, internal_datastore) compute_target = self.ws.compute_targets[ aml_config['aml_compute_name']] dependencies = CondaDependencies.create(pip_packages=[ 'tensorflow-gpu==1.9.0', 'pillow', 'numpy', 'azure', 'azure-storage-blob', 'azureml-defaults' ]) amlcompute_run_config = RunConfiguration( conda_dependencies=dependencies) amlcompute_run_config.environment.docker.enabled = True amlcompute_run_config.environment.docker.gpu_support = True amlcompute_run_config.environment.docker.base_image = DEFAULT_GPU_IMAGE amlcompute_run_config.environment.spark.precache_packages = False # default values are required and need to be literal values or data references as JSON param_job_id = PipelineParameter(name='param_job_id', default_value='default_job_id') param_begin_index = PipelineParameter(name='param_begin_index', default_value=0) param_end_index = PipelineParameter(name='param_end_index', default_value=0) param_detection_threshold = PipelineParameter( name='param_detection_threshold', default_value=0.05) param_batch_size = PipelineParameter(name='param_batch_size', default_value=8) batch_score_step = PythonScriptStep( aml_config['script_name'], source_directory=aml_config['source_dir'], name='batch_scoring', arguments=[ '--job_id', param_job_id, '--model_name', aml_config['model_name'], '--input_container_sas', input_container_sas, '--internal_dir', internal_dir, '--begin_index', param_begin_index, # inclusive '--end_index', param_end_index, # exclusive '--output_dir', output_dir, '--detection_threshold', param_detection_threshold, '--batch_size', param_batch_size ], compute_target=compute_target, inputs=[internal_dir], outputs=[output_dir], runconfig=amlcompute_run_config) self.pipeline = Pipeline(workspace=self.ws, steps=[batch_score_step]) self.aml_config = aml_config print('AMLCompute constructor all good.') except Exception as e: raise RuntimeError( 'Error in setting up AML Compute resource: {}.'.format(str(e)))
def setUpClass(cls) -> None: cls.workspace = Workspace.from_config( str(Path(__file__).parent.parent / 'config.json'))
from azureml.core import Workspace, Experiment, Environment, ScriptRunConfig ws = Workspace.from_config(path='./.azureml', _file_name='config.json') experiment = Experiment(workspace=ws, name='day1-experiment-testing-workspace') config = ScriptRunConfig(source_directory='./src', script='test-workspace.py', compute_target='cpu-cluster') run = experiment.submit(config) aml_url = run.get_portal_url() print(aml_url)
# imports import argparse from azureml.core import Workspace # setup argparse parser = argparse.ArgumentParser() parser.add_argument("--config", type=str, default="") args = parser.parse_args() # get workspace ws = Workspace.from_config(args.config) # delete all webservices for webservice in ws.webservices: ws.webservices[webservice].delete()
print("Argument 1: %s" % args.service_name) print("Argument 2: %s" % args.aks_name) print("Argument 3: %s" % args.aks_region) print("Argument 4: %s" % args.description) print('..4.completed') print('') print('') print('5. Authenticating with AzureCliAuthentication...') clientAuthn = AzureCliAuthentication() print('..5.completed') print('') print('') print('6. Instantiate AML workspace') amlWs = Workspace.from_config(auth=clientAuthn) print('..6.completed') print('') print('') print('7. Instantiate image') containerImage = Image(amlWs, id=image_id) print(containerImage) print('..7.completed') print('') print('') print('8. Check for and delete any existing web service instance') aksName = args.aks_name aksRegion = args.aks_region
from pyspark.ml.evaluation import RegressionEvaluator from pyspark.ml import Pipeline import datetime as dt from pyspark.ml.feature import OneHotEncoder, VectorAssembler from pyspark.ml.tuning import CrossValidator, ParamGridBuilder # COMMAND ---------- # set aml workspace parameters here. subscription_id = "" resource_group = "" workspace_name = "" workspace_region = "" ws = Workspace(subscription_id=subscription_id, resource_group=resource_group, workspace_name=workspace_name) # COMMAND ---------- mlflow.set_tracking_uri(ws.get_mlflow_tracking_uri()) # COMMAND ---------- # create experiment experiment_name = 'bikeSharingDemandMLFlowAML' exp = Experiment(workspace=ws, name=experiment_name) # COMMAND ---------- spark.conf.set("spark.databricks.mlflow.trackMLlib.enabled", "true")
parser.add_argument("--image_name", type=str, help="image name", dest="image_name", required=True) parser.add_argument("--path", type=str, help="path", dest="path", required=True) args = parser.parse_args() print("Argument 1: %s" % args.aml_compute_target) print("Argument 2: %s" % args.model_name) print("Argument 3: %s" % args.build_number) print("Argument 4: %s" % args.image_name) print("Argument 5: %s" % args.path) print('creating AzureCliAuthentication...') cli_auth = AzureCliAuthentication() print('done creating AzureCliAuthentication!') print('get workspace...') ws = Workspace.from_config(path=args.path, auth=cli_auth) print('done getting workspace!') print("looking for existing compute target.") aml_compute = AmlCompute(ws, args.aml_compute_target) print("found existing compute target.") # Create a new runconfig object run_amlcompute = RunConfiguration() # Use the cpu_cluster you created above. run_amlcompute.target = args.aml_compute_target # Enable Docker run_amlcompute.environment.docker.enabled = True
# Get workspace # ws = Workspace.from_config() svc_pr_password = os.environ.get("AZUREML_PASSWORD") tenant = os.environ.get("TENANT_ID") serviceprin = os.environ.get("APPID") sub = os.environ.get("SUBSCRIPTION") rg = os.environ.get("RESOURCE_GROUP") wrkspc = os.environ.get("WORKSPACE_NAME") svc_pr = ServicePrincipalAuthentication( tenant_id=tenant, service_principal_id=serviceprin, service_principal_password=svc_pr_password) ws = Workspace(subscription_id=sub, resource_group=rg, workspace_name=wrkspc, auth=svc_pr) # Get the Image to deploy details try: with open("./aml_config/image.json") as f: config = json.load(f) except: print('No new model, thus no deployment on ACI') #raise Exception('No new model to register as production model perform better') sys.exit(0) image_name = config['image_name'] image_version = config['image_version'] images = Image.list(workspace=ws)
#!/usr/bin/env python # coding: utf-8 # In[6]: from azureml.core import Workspace subscription_id ='ca64cab5-4953-4dec-b4c9-3a934974109d' resource_group ='ML1' workspace_name = 'MachineLearning' try: ws = Workspace(subscription_id = subscription_id, resource_group = resource_group, workspace_name = workspace_name) ws.write_config() print('Library configuration succeeded') except: print('Workspace not found') from azureml.core import Workspace ws = Workspace.from_config() from azureml.core.model import Model model_name = "Model_hurrthreat" model = Model.register(model_path="model.pkl", model_name=model_name, tags={"data": "IRENE", "model": "classification"}, description="hurrthreat prediction", workspace=ws)
def setAutomatedMLWorkspace(create_workspace=False, create_resource_group=False, workspace_region=None, *, subscription_id=None, resource_group=None, workspace_name=None, auth=None): """Set configuration file for AutomatedML actions with the EconML library. If ``create_workspace`` is set true, a new workspace is created for the user. If ``create_workspace`` is set true, a new workspace is created for the user. Parameters ---------- create_workspace: Boolean, optional, default False If set to true, a new workspace will be created if the specified workspace does not exist. create_resource_group: Boolean, optional, default False If set to true, a new resource_group will be created if the specified resource_group does not exist. workspace_region: String, optional Region of workspace, only necessary if create_new is set to true and a new workspace is being created. auth: azureml.core.authentication.AbstractAuthentication, optional If set EconML will use auth object for handling Azure Authentication. Otherwise, EconML will use interactive automation, opening an authentication portal in the browser. subscription_id: String, required Definition of a class that will serve as the parent class of the AutomatedMLMixin. This class must inherit from _BaseDML. resource_group: String, required Name of resource group of workspace to be created or set. workspace_name: String, required Name of workspace of workspace to be created or set. """ try: ws = Workspace(subscription_id=subscription_id, resource_group=resource_group, workspace_name=workspace_name, auth=auth) # write the details of the workspace to a configuration file to the notebook library ws.write_config() print("Workspace configuration has succeeded.") except ProjectSystemException: if (create_workspace): if (create_resource_group): print("Workspace not accessible. Creating a new workspace and \ resource group.") ws = Workspace.create( name=workspace_name, subscription_id=subscription_id, resource_group=resource_group, location=workspace_region, create_resource_group=create_resource_group, sku='basic', auth=auth, exist_ok=True) ws.get_details() else: print("Workspace not accessible. Set \ create_resource_group = True and run again to create a new \ workspace and resource group.") else: print("Workspace not accessible. Set create_workspace = True \ to create a new workspace.")
# aml imports import azureml.core from azureml.core import Workspace from azureml.core import Experiment from azureml.core.dataset import Dataset from azureml.train.sklearn import SKLearn from azureml.core.compute import AmlCompute from azureml.core.compute import ComputeTarget from azureml.core.environment import Environment from azureml.core.conda_dependencies import CondaDependencies # check core sdk version number print('Azure ML SDK Version: ', azureml.core.VERSION) # load workspace instance ws = Workspace.from_config(path='.azureml/config.json') print(ws.name, ws.location, ws.resource_group, sep='\t') # create classification experiment experiment_name = 'sklearn-classification' exp = Experiment(workspace=ws, name=experiment_name) # choose a name for your cluster and node limits compute_min_nodes = os.environ.get('AML_COMPUTE_CLUSTER_MIN_NODES', 0) compute_max_nodes = os.environ.get('AML_COMPUTE_CLUSTER_MAX_NODES', 4) compute_name = os.environ.get('AML_COMPUTE_CLUSTER_NAME', 'sklearn-classify') # setup cpu based vm for computation vm_size = os.environ.get('AML_COMPUTE_CLUSTER_SKU', 'STANDARD_D2_V2') if compute_name in ws.compute_targets:
# Create workspace ############################################################################### print(f"Setting up workspace:") subscription_id = os.getenv("SUBSCRIPTION_ID", default=amlsetup["Environment"]["subscription_id"]) resource_group = os.getenv("RESOURCE_GROUP", default=amlsetup["Environment"]["resource_group"]) workspace_name = os.getenv("WORKSPACE_NAME", default=amlsetup["Environment"]["workspace_name"]) workspace_region = os.getenv( "WORKSPACE_REGION", default=amlsetup["Environment"]["workspace_region"]) try: ws = Workspace(subscription_id=subscription_id, resource_group=resource_group, workspace_name=workspace_name) # write the details of the workspace to a configuration file to the notebook library ws.write_config() print( "\tWorkspace configuration succeeded. Skip the workspace creation steps below" ) except: print(f"\tWorkspace {workspace_name} doesn't exist - creating...") # Create the workspace using the specified parameters ws = Workspace.create(name=workspace_name, subscription_id=subscription_id, resource_group=resource_group, location=workspace_region, create_resource_group=True, exist_ok=False)
from azureml.core import Experiment import random import pandas as pd from sklearn.ensemble import RandomForestClassifier as rfc from sklearn.ensemble import RandomForestRegressor as rfr # O azureml-core da versão 1.0.72 ou superior é requerido # é necessário azureml-dataprep[pandas] na versão 1.1.34 ou superior from azureml.core import Workspace, Dataset #Trocar os códigos abaixo pelos da sua instância! subscription_id = '5298dbd8-b7e6-432a-b8fc-828328490c29' resource_group = '14IA' workspace_name = 'WSML14IA' workspace = Workspace(subscription_id, resource_group, workspace_name) def get_a_funnyName(): f = open("../../datasets/names/funnynames.txt", "r") nomes = f.readlines() return random.choice(nomes).rstrip('\n') if __name__ == "__main__": # Carrega os dados mydf = pd.read_csv('../../datasets/statistical/BaseDefault01.csv') #mydf = pd.read_csv('datasets/statistical/BaseDefault01.csv') # Identifica no dataset as variáveis independentes e a variavel alvo targetcol = 'default'
from azureml.core import Workspace from azureml.core.model import Model import os from azureml.core.runconfig import RunConfiguration from azureml.core.authentication import AzureCliAuthentication cli_auth = AzureCliAuthentication() # Get workspace ws = Workspace.from_config(auth=cli_auth) model = Model(ws, name='mn.MargeOrHomer', version=10) print("Name", model.name) print("Version", model.version) result = model.download(target_dir=os.getcwd(), exist_ok=True) print(result)
def main(): e = Env() # Get Azure machine learning workspace aml_workspace = Workspace.get(name=e.workspace_name, subscription_id=e.subscription_id, resource_group=e.resource_group) print("get_workspace:") print(aml_workspace) # Get Azure machine learning cluster aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size) if aml_compute is not None: print("aml_compute:") print(aml_compute) # Create a reusable Azure ML environment environment = get_environment(aml_workspace, e.aml_env_name, create_new=e.rebuild_env) # run_config = RunConfiguration() run_config.environment = environment if (e.datastore_name): datastore_name = e.datastore_name else: datastore_name = aml_workspace.get_default_datastore().name run_config.environment.environment_variables[ "DATASTORE_NAME"] = datastore_name # NOQA: E501 model_name_param = PipelineParameter(name="model_name", default_value=e.model_name) dataset_version_param = PipelineParameter(name="dataset_version", default_value=e.dataset_version) data_file_path_param = PipelineParameter(name="data_file_path", default_value="none") caller_run_id_param = PipelineParameter(name="caller_run_id", default_value="none") # Get dataset name dataset_name = e.dataset_name # Check to see if dataset exists if (dataset_name not in aml_workspace.datasets): # This call creates an example CSV from sklearn sample data. If you # have already bootstrapped your project, you can comment this line # out and use your own CSV. create_sample_data_csv() # Use a CSV to read in the data set. file_name = 'sales_forecast.csv' if (not os.path.exists(file_name)): raise Exception( "Could not find CSV dataset at \"%s\". If you have bootstrapped your project, you will need to provide a CSV." % file_name) # NOQA: E501 # Upload file to default datastore in workspace datatstore = Datastore.get(aml_workspace, datastore_name) target_path = 'training-data/' datatstore.upload_files(files=[file_name], target_path=target_path, overwrite=True, show_progress=False) # Register dataset path_on_datastore = os.path.join(target_path, file_name) dataset = Dataset.Tabular.from_delimited_files( path=(datatstore, path_on_datastore)) dataset = dataset.register(workspace=aml_workspace, name=dataset_name, description='sales_forecast training data', tags={'format': 'CSV'}, create_new_version=True) # Create a PipelineData to pass data between steps pipeline_data = PipelineData( 'pipeline_data', datastore=aml_workspace.get_default_datastore()) train_step = PythonScriptStep( name="Train Model", script_name=e.train_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, outputs=[pipeline_data], arguments=[ "--model_name", model_name_param, "--step_output", pipeline_data, "--dataset_version", dataset_version_param, "--data_file_path", data_file_path_param, "--caller_run_id", caller_run_id_param, "--dataset_name", dataset_name, ], runconfig=run_config, allow_reuse=True, ) print("Step Train created") evaluate_step = PythonScriptStep( name="Evaluate Model ", script_name=e.evaluate_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, arguments=[ "--model_name", model_name_param, "--allow_run_cancel", e.allow_run_cancel, ], runconfig=run_config, allow_reuse=False, ) print("Step Evaluate created") register_step = PythonScriptStep( name="Register Model ", script_name=e.register_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, inputs=[pipeline_data], arguments=[ "--model_name", model_name_param, "--step_input", pipeline_data, ], runconfig=run_config, allow_reuse=False, ) print("Step Register created") # Check run_evaluation flag to include or exclude evaluation step. if ((e.run_evaluation).lower() == 'true'): print("Include evaluation step before register step.") evaluate_step.run_after(train_step) register_step.run_after(evaluate_step) steps = [train_step, evaluate_step, register_step] else: print("Exclude evaluation step and directly run register step.") register_step.run_after(train_step) steps = [train_step, register_step] train_pipeline = Pipeline(workspace=aml_workspace, steps=steps) train_pipeline._set_experiment_name train_pipeline.validate() published_pipeline = train_pipeline.publish( name=e.pipeline_name, description="Model training/retraining pipeline", version=e.build_id) print(f'Published pipeline: {published_pipeline.name}') print(f'for build {published_pipeline.version}')
from azureml.core import Workspace from ml_service.util.env_variables import Env if __name__ == "__main__": # Environment variables env = Env() ws = Workspace.create(name=env.aml_workspace_name, subscription_id=env.subscription_id, resource_group=env.resource_group, create_resource_group=True, location=env.aml_workspace_location, exist_ok=True) # .azureml/config.json file will be created. ws.write_config()
def main(): # Replace this with the Use of Service Principal for authenticating with the Workspace ws = Workspace.from_config() # Choose a name for your CPU cluster cpu_cluster_name = "cpu-cluster" # Verify that cluster does not exist already try: cpu_cluster = ComputeTarget(workspace=ws, name=cpu_cluster_name) print('Found existing cluster, use it.') except ComputeTargetException: compute_config = AmlCompute.provisioning_configuration( vm_size='STANDARD_D2_V2', max_nodes=4) cpu_cluster = ComputeTarget.create(ws, cpu_cluster_name, compute_config) cpu_cluster.wait_for_completion(show_output=True) # Run configuration for R rc = RunConfiguration() rc.framework = "R" # Run configuration for python py_rc = RunConfiguration() py_rc.framework = "Python" py_rc.environment.docker.enabled = True # Combine GitHub and Cran packages for R env rc.environment.r = RSection() rc.environment.docker.enabled = True # Upload iris data to the datastore # target_path = "iris_data" # upload_files_to_datastore(ds, # list("./iris.csv"), # target_path = target_path, # overwrite = TRUE) training_data = DataReference( datastore=ws.get_default_datastore(), data_reference_name="iris_data", path_on_datastore="iris_data/iris.csv", ) print('Succesfull') print(training_data) # PipelineData object for newly trained model # trained_model_dir = PipelineData( # name="trained_model", datastore=ws.get_default_datastore(), is_directory=True # ) # Train and Register the model train_step = RScriptStep( script_name="train.R", arguments=[training_data], inputs=[training_data], compute_target=cpu_cluster_name, source_directory=".", runconfig=rc, allow_reuse=True, ) # Deploy the trained model print("Step Train created") steps = [train_step] train_pipeline = Pipeline(workspace=ws, steps=steps) train_pipeline.validate() pipeline_run = Experiment(ws, 'iris_training').submit(train_pipeline) pipeline_run.wait_for_completion(show_output=True) published_pipeline = train_pipeline.publish( name="iris-train", description="Model training/retraining pipeline", ) print(f"Published pipeline: {published_pipeline.name}") print(f"for build {published_pipeline.version}")
def main(): parser = argparse.ArgumentParser("register") parser.add_argument( "--output_pipeline_id_file", type=str, default="pipeline_id.txt", help="Name of a file to write pipeline ID to" ) parser.add_argument( "--skip_train_execution", action="store_true", help=("Do not trigger the execution. " "Use this in Azure DevOps when using a server job to trigger") ) args = parser.parse_args() e = Env() aml_workspace = Workspace.get( name=e.workspace_name, subscription_id=e.subscription_id, resource_group=e.resource_group ) # Find the pipeline that was published by the specified build ID pipelines = PublishedPipeline.list(aml_workspace) matched_pipes = [] for p in pipelines: if p.name == e.pipeline_name: if p.version == e.build_id: matched_pipes.append(p) if(len(matched_pipes) > 1): published_pipeline = None raise Exception(f"Multiple active pipelines are published for build {e.build_id}.") # NOQA: E501 elif(len(matched_pipes) == 0): published_pipeline = None raise KeyError(f"Unable to find a published pipeline for this build {e.build_id}") # NOQA: E501 else: published_pipeline = matched_pipes[0] print("published pipeline id is", published_pipeline.id) # Save the Pipeline ID for other AzDO jobs after script is complete if args.output_pipeline_id_file is not None: with open(args.output_pipeline_id_file, "w") as out_file: out_file.write(published_pipeline.id) if(args.skip_train_execution is False): pipeline_parameters = {"model_name": e.model_name} tags = {"BuildId": e.build_id} if (e.build_uri is not None): tags["BuildUri"] = e.build_uri experiment = Experiment( workspace=aml_workspace, name=e.experiment_name) run = experiment.submit( published_pipeline, tags=tags, pipeline_parameters=pipeline_parameters) print("Pipeline run initiated ", run.id)
from azureml.core import Workspace, Datastore, Dataset from azureml.core.experiment import Experiment from azureml.pipeline.core import Pipeline, PipelineData from azureml.pipeline.steps import PythonScriptStep ws = Workspace.from_config(path="./file-path/ws_config.json") experiment = Experiment(workspace=ws, name='BrainStar') def_blob_store = Datastore(ws, "workspaceblobstore") compute_target = ws.compute_targets["BrainStar1"] input_data = Dataset.get(ws, name="Absence data") output_data1 = PipelineData("output_data1", datastore=def_blob_store, output_name="output_data1") source_directory = './process' step1 = PythonScriptStep(name="process_step", script_name="process.py", inputs=[input_data], outputs=[output_data1], compute_target=compute_target, source_directory=source_directory, allow_reuse=True) steps = step1 pipeline1 = Pipeline(workspace=ws, steps=steps) pipeline1.validate() pipeline_run1 = Experiment(ws, 'Hello_World1').submit(pipeline1, regenerate_outputs=False)
def deploy(ws_name,model_name,path_to_model, environment_name,register_environment,pip_packages,conda_packages, cpu_cores , memory_gb, path_to_entry_script,service_name): ''' Get Workspace ''' ws = Workspace.from_config() print("Got Workspace {}".format(ws_name)) ''' Register Model ''' model = Model.register(workspace = ws, model_path =path_to_model, model_name = model_name, ) print("Registered Model {}".format(model_name)) ''' Register Environment ''' # to install required packages if register_environment: env = Environment(environment_name) cd = CondaDependencies.create(pip_packages=pip_packages, conda_packages = conda_packages) env.python.conda_dependencies = cd # Register environment to re-use later env.register(workspace = ws) print("Registered Environment") myenv = Environment.get(workspace=ws, name=environment_name) # Uncomment to save environment # myenv.save_to_directory('./environ', overwrite=True) ''' Config Objects ''' aciconfig = AciWebservice.deploy_configuration( cpu_cores=cpu_cores, memory_gb=memory_gb, ) inference_config = InferenceConfig(entry_script=path_to_entry_script, environment=myenv) ''' Deploying ''' print("Deploying....... This may take a few mins, check the status in MLS after the function finishes executing") service = Model.deploy(workspace=ws, name=ws_name, models=[model], inference_config=inference_config, deployment_config=aciconfig, overwrite = True) service.wait_for_deployment(show_output=True) url = service.scoring_uri print(url) service = Webservice(ws,ws_name) print(service.get_logs()) return url
} akscomputes = { "aks-cpu-deploy": { "vm_size": "STANDARD_DS3_V2", "agent_count": 3, }, "aks-gpu-deploy": { "vm_size": "STANDARD_NC6S_V3", "agent_count": 3, }, } # create or get Workspace try: ws = Workspace.from_config(args.config) except: ws = Workspace.create( args.workspace_name, subscription_id=args.subscription_id, resource_group=args.resource_group, location=args.location, create_resource_group=True, exist_ok=True, show_output=True, ) ws.write_config() # create aml compute targets for ct_name in amlcomputes: if ct_name not in ws.compute_targets:
from azureml.core.authentication import ServicePrincipalAuthentication workspace_name = dbutils.secrets.get(scope="azureml", key="workspace_name") workspace_location = "westeurope" resource_group = dbutils.secrets.get(scope="azureml", key="resource_group") subscription_id = dbutils.secrets.get(scope="azureml", key="subscription_id") svc_pr = ServicePrincipalAuthentication( tenant_id=dbutils.secrets.get(scope="azureml", key="tenant_id"), service_principal_id=dbutils.secrets.get(scope="azureml", key="client_id"), service_principal_password=dbutils.secrets.get(scope="azureml", key="client_secret")) workspace = Workspace.create(name=workspace_name, location=workspace_location, resource_group=resource_group, subscription_id=subscription_id, auth=svc_pr, exist_ok=True) # COMMAND ---------- # MAGIC %md ## Build an Azure Container Image for model deployment # COMMAND ---------- # MAGIC %md ### Use MLflow to build a Container Image for the trained model # MAGIC # MAGIC Use the `mlflow.azuereml.build_image` function to build an Azure Container Image for the trained MLflow model. This function also registers the MLflow model with a specified Azure ML workspace. The resulting image can be deployed to Azure Container Instances (ACI) or Azure Kubernetes Service (AKS) for real-time serving. # COMMAND ----------
config = json.load(f) except: print("No new model to register thus no need to create new scoring image") # raise Exception('No new model to register as production model perform better') sys.exit(0) model_name = config["model_name"] model_version = config["model_version"] print("Starting to download the model file") # Start creating # Point file to conf directory containing details for the aml service cli_auth = AzureCliAuthentication() ws = Workspace(workspace_name=workspace, subscription_id=subscription_id, resource_group=resource_grp, auth=cli_auth) model_list = Model.list(workspace=ws) model, = (m for m in model_list if m.version == model_version and m.name == model_name) print("Model picked: {} \nModel Description: {} \nModel Version: {}".format( model.name, model.description, model.version)) #print("Writing Conda File") #myenv = CondaDependencies() #myenv.add_conda_package("scikit-learn") #with open("myenv.yml","w") as f: # f.write(myenv.serialize_to_string()) #print("Finished Writing Conda File")
subscription_id = os.environ.get("SUBSCRIPTION_ID") tenant_id = os.environ.get("TENANT_ID") model_name = os.environ.get("MODEL_NAME") app_id = os.environ.get('SP_APP_ID') app_secret = os.environ.get('SP_APP_SECRET') build_id = os.environ.get('BUILD_BUILDID') # run_id useful to query previous runs run_id = "57fee47f-5ae8-441c-bc0c-d4c371f32d70" service_principal = ServicePrincipalAuthentication( tenant_id=tenant_id, service_principal_id=app_id, service_principal_password=app_secret) aml_workspace = Workspace.get( name=workspace_name, subscription_id=subscription_id, resource_group=resource_group, auth=service_principal ) ws = aml_workspace exp = Experiment(ws, experiment_name) else: sys.path.append(os.path.abspath("./util")) # NOQA: E402 from model_helper import get_model_by_tag exp = run.experiment ws = run.experiment.workspace run_id = 'amlcompute' parser = argparse.ArgumentParser("evaluate") parser.add_argument( "--build_id", type=str,