# tutorial/01-create-workspace.py
from azureml.core.authentication import InteractiveLoginAuthentication
from azureml.core import Workspace
interactive_auth = InteractiveLoginAuthentication(tenant_id="99e1e721-7184-498e-8aff-b2ad4e53c1c2")
ws = Workspace.create(name='azure-ml',
subscription_id='59e1d56a-8a2d-48a7-9cd3-a52c1a268c55',
resource_group='cloud-ml',
create_resource_group=True,
location='eastus2',
auth=interactive_auth
)
# write out the workspace details to a configuration file: .azureml/config.json
ws.write_config(path='.azureml')
full_X = df_affordability[["Age", "KM"]]
full_Y = df_affordability[["Affordable"]]
# Create a Workspace
#Provide the Subscription ID of your existing Azure subscription
subscription_id = "e223f1b3-d19b-4cfa-98e9-bc9be62717bc"
#Provide values for the Resource Group and Workspace that will be created
resource_group = "aml-workspace-z"
workspace_name = "aml-workspace-z"
workspace_region = 'westcentralus' # eastus, westcentralus, southeastasia, australiaeast, westeurope
# By using the exist_ok param, if the worskpace already exists we get a reference to the existing workspace instead of an error
ws = Workspace.create(name=workspace_name,
subscription_id=subscription_id,
resource_group=resource_group,
location=workspace_region,
exist_ok=True)
print("Workspace Provisioning complete.")
# Step 2 - Define a helper method that will use AutoML to train multiple models and pick the best one
#####################################################################################################
def auto_train_model(ws, experiment_name, model_name, full_X, full_Y,
training_set_percentage, training_target_accuracy):
# start a training run by defining an experiment
experiment = Experiment(ws, experiment_name)
train_X, test_X, train_Y, test_Y = train_test_split(
from azureml.core import Workspace
from azureml.core.webservice import Webservice
# Requires the config to be downloaded first to the current working directory
ws = Workspace.from_config()
# Set with the deployment name
name = "votingensemblemodels"
# load existing web service
service = Webservice(name=name, workspace=ws)
# Enable app insight
service.update(enable_app_insights=True)
# Get service log
logs = service.get_logs()
for line in logs.split('\n'):
print(line)
from azureml.core.compute import ComputeTarget, AmlCompute
from azureml.core.compute_target import ComputeTargetException
from azureml.core import Workspace
from azureml.core.authentication import AzureCliAuthentication
# load Azure ML workspace
workspace = Workspace.from_config(auth=AzureCliAuthentication())
# Create compute target if not present
# Choose a name for your CPU cluster
cpu_cluster_name = "hypercomputecpu"
# Verify that cluster does not exist already
try:
cu_cluster = ComputeTarget(workspace=workspace, name=cpu_cluster_name)
print('Found existing cluster, use it.')
except ComputeTargetException:
compute_config = AmlCompute.provisioning_configuration(
vm_size='STANDARD_D3_V2', max_nodes=4)
cpu_cluster = ComputeTarget.create(workspace, cpu_cluster_name,
compute_config)
cpu_cluster.wait_for_completion(show_output=True)
def main():
# Ger our configs
with open("ptgnn/authentication.json") as jsonFile:
authData = json.load(jsonFile)[args.auth_cluster]
# Copy the convertCorpus script here. Done so we don't upload the corpus to Azure, or keep a copy of the script in here.
# (It's weird, I know. It works and has a purpose though)
convertCorpusLocation = Path("../convertCorpusForML.py")
convertCorpusAzureLocation = Path("./convertCorpusForML.py")
shutil.copy(convertCorpusLocation, convertCorpusAzureLocation)
# Grab the authentication data from the JSON file
subID = authData["subID"] # Get from Azure Portal; used for billing
resGroup = authData["resGroup"] # Name for the resource group
wsName = authData["wsName"] # Name for the workspace, which is the collection of compute clusters + experiments
computeName = authData["computeName"] # Name for computer cluster
datastoreName = authData["datastoreName"]
# Get the workspace, the compute target and the datastore
ws = Workspace.get(wsName, subscription_id=subID, resource_group=resGroup)
computeTarget = ComputeTarget(ws, computeName)
datastore = Datastore(ws, name=datastoreName)
# Download the entire corpus to the compute target. Save the DataReference obj here
# as_mount is also possible, but slows things down due to network opening of files
corpus_location = datastore.path(args.aml_location).as_download()
output_location = "./"
# The files that will be uploaded for usage by our script (everything in the azure folder)
source_directory = "."
# params for the script
params = {
"--corpus_location": corpus_location,
"--output_folder": output_location,
"--aml": "",
"--training_percent": args.training_percent,
"--validation_percent": args.validation_percent,
"-c": ""
}
if args.log_num is not None:
params["-l"] = args.log_num
tags = {
"logs": str(args.log_num)
}
else:
tags = {
"logs": "MAX"
}
if args.statement_generation:
params["-s"] = ""
tags["generationType"] = "Statement"
else:
tags["generationType"] = "Severity"
# Set up the estimator object. Note the inputs element, it tells azure that corpus_location in params
# will be a DataReference Object.
est = Estimator(source_directory=source_directory,
compute_target=computeTarget,
entry_script='convertCorpusForML.py',
script_params=params,
inputs=[corpus_location],
conda_packages=["pip"],
pip_packages=["azureml-core", "tqdm", "numpy", "protobuf"],
use_docker=True,
use_gpu=False)
# Start the experiment
run = Experiment(ws, args.exp_name).submit(config=est, tags=tags)
# remove the copy of convertCorpus (Remember, don't question this)
convertCorpusAzureLocation.unlink()
# print out the portral URL
# print("Portal URL: ", run.get_portal_url())
# this will stream everything that the compute target does.
print("Experiment Started. Remember you can exit out of this program but the experiment will still run on Azure!")
run.wait_for_completion(show_output=True)
def main():
run = Run.get_context()
if (run.id.startswith('OfflineRun')):
from dotenv import load_dotenv
# For local development, set values in this section
load_dotenv()
workspace_name = os.environ.get("WORKSPACE_NAME")
experiment_name = os.environ.get("EXPERIMENT_NAME")
resource_group = os.environ.get("RESOURCE_GROUP")
subscription_id = os.environ.get("SUBSCRIPTION_ID")
# run_id useful to query previous runs
run_id = "bd184a18-2ac8-4951-8e78-e290bef3b012"
aml_workspace = Workspace.get(name=workspace_name,
subscription_id=subscription_id,
resource_group=resource_group)
ws = aml_workspace
exp = Experiment(ws, experiment_name)
else:
ws = run.experiment.workspace
exp = run.experiment
run_id = 'amlcompute'
parser = argparse.ArgumentParser("register")
parser.add_argument(
"--run_id",
type=str,
help="Training run ID",
)
parser.add_argument(
"--model_name",
type=str,
help="Name of the Model",
default="fusion_model.pkl",
)
parser.add_argument("--step_input",
type=str,
help=("input from previous steps"))
args = parser.parse_args()
if (args.run_id is not None):
run_id = args.run_id
if (run_id == 'amlcompute'):
run_id = run.parent.id
model_name = args.model_name
model_path = args.step_input
print("Getting registration parameters")
# Load the registration parameters from the parameters file
with open("parameters.json") as f:
pars = json.load(f)
try:
register_args = pars["registration"]
except KeyError:
print("Could not load registration values from file")
register_args = {"tags": []}
model_tags = {}
for tag in register_args["tags"]:
try:
mtag = run.parent.get_metrics()[tag]
model_tags[tag] = mtag
except KeyError:
print(f"Could not find {tag} metric on parent run.")
# load the model
print("Loading model from " + model_path)
model_file = os.path.join(model_path, model_name)
model = joblib.load(model_file)
parent_tags = run.parent.get_tags()
try:
build_id = parent_tags["BuildId"]
except KeyError:
build_id = None
print("BuildId tag not found on parent run.")
print(f"Tags present: {parent_tags}")
try:
build_uri = parent_tags["BuildUri"]
except KeyError:
build_uri = None
print("BuildUri tag not found on parent run.")
print(f"Tags present: {parent_tags}")
if (model is not None):
dataset_id = parent_tags["dataset_id"]
if (build_id is None):
register_aml_model(model_file, model_name, model_tags, exp, run_id,
dataset_id)
elif (build_uri is None):
register_aml_model(model_file, model_name, model_tags, exp, run_id,
dataset_id, build_id)
else:
register_aml_model(model_file, model_name, model_tags, exp, run_id,
dataset_id, build_id, build_uri)
else:
print("Model not found. Skipping model registration.")
sys.exit(0)
def __init__(self, request_id, input_container_sas, internal_datastore):
try:
aml_config = api_config.AML_CONFIG
self.ws = Workspace(subscription_id=aml_config['subscription_id'],
resource_group=aml_config['resource_group'],
workspace_name=aml_config['workspace_name'],
auth=svc_pr)
print('AMLCompute constructor, AML workspace obtained.')
internal_dir, output_dir = self._get_data_references(
request_id, internal_datastore)
compute_target = self.ws.compute_targets[
aml_config['aml_compute_name']]
dependencies = CondaDependencies.create(pip_packages=[
'tensorflow-gpu==1.9.0', 'pillow', 'numpy', 'azure',
'azure-storage-blob', 'azureml-defaults'
])
amlcompute_run_config = RunConfiguration(
conda_dependencies=dependencies)
amlcompute_run_config.environment.docker.enabled = True
amlcompute_run_config.environment.docker.gpu_support = True
amlcompute_run_config.environment.docker.base_image = DEFAULT_GPU_IMAGE
amlcompute_run_config.environment.spark.precache_packages = False
# default values are required and need to be literal values or data references as JSON
param_job_id = PipelineParameter(name='param_job_id',
default_value='default_job_id')
param_begin_index = PipelineParameter(name='param_begin_index',
default_value=0)
param_end_index = PipelineParameter(name='param_end_index',
default_value=0)
param_detection_threshold = PipelineParameter(
name='param_detection_threshold', default_value=0.05)
param_batch_size = PipelineParameter(name='param_batch_size',
default_value=8)
batch_score_step = PythonScriptStep(
aml_config['script_name'],
source_directory=aml_config['source_dir'],
name='batch_scoring',
arguments=[
'--job_id',
param_job_id,
'--model_name',
aml_config['model_name'],
'--input_container_sas',
input_container_sas,
'--internal_dir',
internal_dir,
'--begin_index',
param_begin_index, # inclusive
'--end_index',
param_end_index, # exclusive
'--output_dir',
output_dir,
'--detection_threshold',
param_detection_threshold,
'--batch_size',
param_batch_size
],
compute_target=compute_target,
inputs=[internal_dir],
outputs=[output_dir],
runconfig=amlcompute_run_config)
self.pipeline = Pipeline(workspace=self.ws,
steps=[batch_score_step])
self.aml_config = aml_config
print('AMLCompute constructor all good.')
except Exception as e:
raise RuntimeError(
'Error in setting up AML Compute resource: {}.'.format(str(e)))
def setUpClass(cls) -> None:
cls.workspace = Workspace.from_config(
str(Path(__file__).parent.parent / 'config.json'))
from azureml.core import Workspace, Experiment, Environment, ScriptRunConfig
ws = Workspace.from_config(path='./.azureml', _file_name='config.json')
experiment = Experiment(workspace=ws, name='day1-experiment-testing-workspace')
config = ScriptRunConfig(source_directory='./src',
script='test-workspace.py',
compute_target='cpu-cluster')
run = experiment.submit(config)
aml_url = run.get_portal_url()
print(aml_url)
# imports
import argparse
from azureml.core import Workspace
# setup argparse
parser = argparse.ArgumentParser()
parser.add_argument("--config", type=str, default="")
args = parser.parse_args()
# get workspace
ws = Workspace.from_config(args.config)
# delete all webservices
for webservice in ws.webservices:
ws.webservices[webservice].delete()
print("Argument 1: %s" % args.service_name)
print("Argument 2: %s" % args.aks_name)
print("Argument 3: %s" % args.aks_region)
print("Argument 4: %s" % args.description)
print('..4.completed')
print('')
print('')
print('5. Authenticating with AzureCliAuthentication...')
clientAuthn = AzureCliAuthentication()
print('..5.completed')
print('')
print('')
print('6. Instantiate AML workspace')
amlWs = Workspace.from_config(auth=clientAuthn)
print('..6.completed')
print('')
print('')
print('7. Instantiate image')
containerImage = Image(amlWs, id=image_id)
print(containerImage)
print('..7.completed')
print('')
print('')
print('8. Check for and delete any existing web service instance')
aksName = args.aks_name
aksRegion = args.aks_region
from pyspark.ml.evaluation import RegressionEvaluator
from pyspark.ml import Pipeline
import datetime as dt
from pyspark.ml.feature import OneHotEncoder, VectorAssembler
from pyspark.ml.tuning import CrossValidator, ParamGridBuilder
# COMMAND ----------
# set aml workspace parameters here.
subscription_id = ""
resource_group = ""
workspace_name = ""
workspace_region = ""
ws = Workspace(subscription_id=subscription_id,
resource_group=resource_group,
workspace_name=workspace_name)
# COMMAND ----------
mlflow.set_tracking_uri(ws.get_mlflow_tracking_uri())
# COMMAND ----------
# create experiment
experiment_name = 'bikeSharingDemandMLFlowAML'
exp = Experiment(workspace=ws, name=experiment_name)
# COMMAND ----------
spark.conf.set("spark.databricks.mlflow.trackMLlib.enabled", "true")
parser.add_argument("--image_name", type=str, help="image name", dest="image_name", required=True)
parser.add_argument("--path", type=str, help="path", dest="path", required=True)
args = parser.parse_args()
print("Argument 1: %s" % args.aml_compute_target)
print("Argument 2: %s" % args.model_name)
print("Argument 3: %s" % args.build_number)
print("Argument 4: %s" % args.image_name)
print("Argument 5: %s" % args.path)
print('creating AzureCliAuthentication...')
cli_auth = AzureCliAuthentication()
print('done creating AzureCliAuthentication!')
print('get workspace...')
ws = Workspace.from_config(path=args.path, auth=cli_auth)
print('done getting workspace!')
print("looking for existing compute target.")
aml_compute = AmlCompute(ws, args.aml_compute_target)
print("found existing compute target.")
# Create a new runconfig object
run_amlcompute = RunConfiguration()
# Use the cpu_cluster you created above.
run_amlcompute.target = args.aml_compute_target
# Enable Docker
run_amlcompute.environment.docker.enabled = True
# Get workspace
# ws = Workspace.from_config()
svc_pr_password = os.environ.get("AZUREML_PASSWORD")
tenant = os.environ.get("TENANT_ID")
serviceprin = os.environ.get("APPID")
sub = os.environ.get("SUBSCRIPTION")
rg = os.environ.get("RESOURCE_GROUP")
wrkspc = os.environ.get("WORKSPACE_NAME")
svc_pr = ServicePrincipalAuthentication(
tenant_id=tenant,
service_principal_id=serviceprin,
service_principal_password=svc_pr_password)
ws = Workspace(subscription_id=sub,
resource_group=rg,
workspace_name=wrkspc,
auth=svc_pr)
# Get the Image to deploy details
try:
with open("./aml_config/image.json") as f:
config = json.load(f)
except:
print('No new model, thus no deployment on ACI')
#raise Exception('No new model to register as production model perform better')
sys.exit(0)
image_name = config['image_name']
image_version = config['image_version']
images = Image.list(workspace=ws)
#!/usr/bin/env python
# coding: utf-8
# In[6]:
from azureml.core import Workspace
subscription_id ='ca64cab5-4953-4dec-b4c9-3a934974109d'
resource_group ='ML1'
workspace_name = 'MachineLearning'
try:
ws = Workspace(subscription_id = subscription_id, resource_group = resource_group, workspace_name = workspace_name)
ws.write_config()
print('Library configuration succeeded')
except:
print('Workspace not found')
from azureml.core import Workspace
ws = Workspace.from_config()
from azureml.core.model import Model
model_name = "Model_hurrthreat"
model = Model.register(model_path="model.pkl",
model_name=model_name,
tags={"data": "IRENE", "model": "classification"},
description="hurrthreat prediction",
workspace=ws)
def setAutomatedMLWorkspace(create_workspace=False,
create_resource_group=False,
workspace_region=None,
*,
subscription_id=None,
resource_group=None,
workspace_name=None,
auth=None):
"""Set configuration file for AutomatedML actions with the EconML library. If
``create_workspace`` is set true, a new workspace is created
for the user. If ``create_workspace`` is set true, a new workspace is
created for the user.
Parameters
----------
create_workspace: Boolean, optional, default False
If set to true, a new workspace will be created if the specified
workspace does not exist.
create_resource_group: Boolean, optional, default False
If set to true, a new resource_group will be created if the specified
resource_group does not exist.
workspace_region: String, optional
Region of workspace, only necessary if create_new is set to true and a
new workspace is being created.
auth: azureml.core.authentication.AbstractAuthentication, optional
If set EconML will use auth object for handling Azure Authentication.
Otherwise, EconML will use interactive automation, opening an
authentication portal in the browser.
subscription_id: String, required
Definition of a class that will serve as the parent class of the
AutomatedMLMixin. This class must inherit from _BaseDML.
resource_group: String, required
Name of resource group of workspace to be created or set.
workspace_name: String, required
Name of workspace of workspace to be created or set.
"""
try:
ws = Workspace(subscription_id=subscription_id,
resource_group=resource_group,
workspace_name=workspace_name,
auth=auth)
# write the details of the workspace to a configuration file to the notebook library
ws.write_config()
print("Workspace configuration has succeeded.")
except ProjectSystemException:
if (create_workspace):
if (create_resource_group):
print("Workspace not accessible. Creating a new workspace and \
resource group.")
ws = Workspace.create(
name=workspace_name,
subscription_id=subscription_id,
resource_group=resource_group,
location=workspace_region,
create_resource_group=create_resource_group,
sku='basic',
auth=auth,
exist_ok=True)
ws.get_details()
else:
print("Workspace not accessible. Set \
create_resource_group = True and run again to create a new \
workspace and resource group.")
else:
print("Workspace not accessible. Set create_workspace = True \
to create a new workspace.")
# aml imports
import azureml.core
from azureml.core import Workspace
from azureml.core import Experiment
from azureml.core.dataset import Dataset
from azureml.train.sklearn import SKLearn
from azureml.core.compute import AmlCompute
from azureml.core.compute import ComputeTarget
from azureml.core.environment import Environment
from azureml.core.conda_dependencies import CondaDependencies
# check core sdk version number
print('Azure ML SDK Version: ', azureml.core.VERSION)
# load workspace instance
ws = Workspace.from_config(path='.azureml/config.json')
print(ws.name, ws.location, ws.resource_group, sep='\t')
# create classification experiment
experiment_name = 'sklearn-classification'
exp = Experiment(workspace=ws, name=experiment_name)
# choose a name for your cluster and node limits
compute_min_nodes = os.environ.get('AML_COMPUTE_CLUSTER_MIN_NODES', 0)
compute_max_nodes = os.environ.get('AML_COMPUTE_CLUSTER_MAX_NODES', 4)
compute_name = os.environ.get('AML_COMPUTE_CLUSTER_NAME', 'sklearn-classify')
# setup cpu based vm for computation
vm_size = os.environ.get('AML_COMPUTE_CLUSTER_SKU', 'STANDARD_D2_V2')
if compute_name in ws.compute_targets:
# Create workspace
###############################################################################
print(f"Setting up workspace:")
subscription_id = os.getenv("SUBSCRIPTION_ID",
default=amlsetup["Environment"]["subscription_id"])
resource_group = os.getenv("RESOURCE_GROUP",
default=amlsetup["Environment"]["resource_group"])
workspace_name = os.getenv("WORKSPACE_NAME",
default=amlsetup["Environment"]["workspace_name"])
workspace_region = os.getenv(
"WORKSPACE_REGION", default=amlsetup["Environment"]["workspace_region"])
try:
ws = Workspace(subscription_id=subscription_id,
resource_group=resource_group,
workspace_name=workspace_name)
# write the details of the workspace to a configuration file to the notebook library
ws.write_config()
print(
"\tWorkspace configuration succeeded. Skip the workspace creation steps below"
)
except:
print(f"\tWorkspace {workspace_name} doesn't exist - creating...")
# Create the workspace using the specified parameters
ws = Workspace.create(name=workspace_name,
subscription_id=subscription_id,
resource_group=resource_group,
location=workspace_region,
create_resource_group=True,
exist_ok=False)
from azureml.core import Experiment
import random
import pandas as pd
from sklearn.ensemble import RandomForestClassifier as rfc
from sklearn.ensemble import RandomForestRegressor as rfr
# O azureml-core da versão 1.0.72 ou superior é requerido
# é necessário azureml-dataprep[pandas] na versão 1.1.34 ou superior
from azureml.core import Workspace, Dataset
#Trocar os códigos abaixo pelos da sua instância!
subscription_id = '5298dbd8-b7e6-432a-b8fc-828328490c29'
resource_group = '14IA'
workspace_name = 'WSML14IA'
workspace = Workspace(subscription_id, resource_group, workspace_name)
def get_a_funnyName():
f = open("../../datasets/names/funnynames.txt", "r")
nomes = f.readlines()
return random.choice(nomes).rstrip('\n')
if __name__ == "__main__":
# Carrega os dados
mydf = pd.read_csv('../../datasets/statistical/BaseDefault01.csv')
#mydf = pd.read_csv('datasets/statistical/BaseDefault01.csv')
# Identifica no dataset as variáveis independentes e a variavel alvo
targetcol = 'default'
from azureml.core import Workspace
from azureml.core.model import Model
import os
from azureml.core.runconfig import RunConfiguration
from azureml.core.authentication import AzureCliAuthentication
cli_auth = AzureCliAuthentication()
# Get workspace
ws = Workspace.from_config(auth=cli_auth)
model = Model(ws, name='mn.MargeOrHomer', version=10)
print("Name", model.name)
print("Version", model.version)
result = model.download(target_dir=os.getcwd(), exist_ok=True)
print(result)
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable Azure ML environment
environment = get_environment(aml_workspace,
e.aml_env_name,
create_new=e.rebuild_env) #
run_config = RunConfiguration()
run_config.environment = environment
if (e.datastore_name):
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables[
"DATASTORE_NAME"] = datastore_name # NOQA: E501
model_name_param = PipelineParameter(name="model_name",
default_value=e.model_name)
dataset_version_param = PipelineParameter(name="dataset_version",
default_value=e.dataset_version)
data_file_path_param = PipelineParameter(name="data_file_path",
default_value="none")
caller_run_id_param = PipelineParameter(name="caller_run_id",
default_value="none")
# Get dataset name
dataset_name = e.dataset_name
# Check to see if dataset exists
if (dataset_name not in aml_workspace.datasets):
# This call creates an example CSV from sklearn sample data. If you
# have already bootstrapped your project, you can comment this line
# out and use your own CSV.
create_sample_data_csv()
# Use a CSV to read in the data set.
file_name = 'sales_forecast.csv'
if (not os.path.exists(file_name)):
raise Exception(
"Could not find CSV dataset at \"%s\". If you have bootstrapped your project, you will need to provide a CSV."
% file_name) # NOQA: E501
# Upload file to default datastore in workspace
datatstore = Datastore.get(aml_workspace, datastore_name)
target_path = 'training-data/'
datatstore.upload_files(files=[file_name],
target_path=target_path,
overwrite=True,
show_progress=False)
# Register dataset
path_on_datastore = os.path.join(target_path, file_name)
dataset = Dataset.Tabular.from_delimited_files(
path=(datatstore, path_on_datastore))
dataset = dataset.register(workspace=aml_workspace,
name=dataset_name,
description='sales_forecast training data',
tags={'format': 'CSV'},
create_new_version=True)
# Create a PipelineData to pass data between steps
pipeline_data = PipelineData(
'pipeline_data', datastore=aml_workspace.get_default_datastore())
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
outputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_output",
pipeline_data,
"--dataset_version",
dataset_version_param,
"--data_file_path",
data_file_path_param,
"--caller_run_id",
caller_run_id_param,
"--dataset_name",
dataset_name,
],
runconfig=run_config,
allow_reuse=True,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--model_name",
model_name_param,
"--allow_run_cancel",
e.allow_run_cancel,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_input",
pipeline_data,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
# Check run_evaluation flag to include or exclude evaluation step.
if ((e.run_evaluation).lower() == 'true'):
print("Include evaluation step before register step.")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
else:
print("Exclude evaluation step and directly run register step.")
register_step.run_after(train_step)
steps = [train_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
from azureml.core import Workspace
from ml_service.util.env_variables import Env
if __name__ == "__main__":
# Environment variables
env = Env()
ws = Workspace.create(name=env.aml_workspace_name,
subscription_id=env.subscription_id,
resource_group=env.resource_group,
create_resource_group=True,
location=env.aml_workspace_location,
exist_ok=True)
# .azureml/config.json file will be created.
ws.write_config()
def main():
# Replace this with the Use of Service Principal for authenticating with the Workspace
ws = Workspace.from_config()
# Choose a name for your CPU cluster
cpu_cluster_name = "cpu-cluster"
# Verify that cluster does not exist already
try:
cpu_cluster = ComputeTarget(workspace=ws, name=cpu_cluster_name)
print('Found existing cluster, use it.')
except ComputeTargetException:
compute_config = AmlCompute.provisioning_configuration(
vm_size='STANDARD_D2_V2', max_nodes=4)
cpu_cluster = ComputeTarget.create(ws, cpu_cluster_name,
compute_config)
cpu_cluster.wait_for_completion(show_output=True)
# Run configuration for R
rc = RunConfiguration()
rc.framework = "R"
# Run configuration for python
py_rc = RunConfiguration()
py_rc.framework = "Python"
py_rc.environment.docker.enabled = True
# Combine GitHub and Cran packages for R env
rc.environment.r = RSection()
rc.environment.docker.enabled = True
# Upload iris data to the datastore
# target_path = "iris_data"
# upload_files_to_datastore(ds,
# list("./iris.csv"),
# target_path = target_path,
# overwrite = TRUE)
training_data = DataReference(
datastore=ws.get_default_datastore(),
data_reference_name="iris_data",
path_on_datastore="iris_data/iris.csv",
)
print('Succesfull')
print(training_data)
# PipelineData object for newly trained model
# trained_model_dir = PipelineData(
# name="trained_model", datastore=ws.get_default_datastore(), is_directory=True
# )
# Train and Register the model
train_step = RScriptStep(
script_name="train.R",
arguments=[training_data],
inputs=[training_data],
compute_target=cpu_cluster_name,
source_directory=".",
runconfig=rc,
allow_reuse=True,
)
# Deploy the trained model
print("Step Train created")
steps = [train_step]
train_pipeline = Pipeline(workspace=ws, steps=steps)
train_pipeline.validate()
pipeline_run = Experiment(ws, 'iris_training').submit(train_pipeline)
pipeline_run.wait_for_completion(show_output=True)
published_pipeline = train_pipeline.publish(
name="iris-train",
description="Model training/retraining pipeline",
)
print(f"Published pipeline: {published_pipeline.name}")
print(f"for build {published_pipeline.version}")
def main():
parser = argparse.ArgumentParser("register")
parser.add_argument(
"--output_pipeline_id_file",
type=str,
default="pipeline_id.txt",
help="Name of a file to write pipeline ID to"
)
parser.add_argument(
"--skip_train_execution",
action="store_true",
help=("Do not trigger the execution. "
"Use this in Azure DevOps when using a server job to trigger")
)
args = parser.parse_args()
e = Env()
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group
)
# Find the pipeline that was published by the specified build ID
pipelines = PublishedPipeline.list(aml_workspace)
matched_pipes = []
for p in pipelines:
if p.name == e.pipeline_name:
if p.version == e.build_id:
matched_pipes.append(p)
if(len(matched_pipes) > 1):
published_pipeline = None
raise Exception(f"Multiple active pipelines are published for build {e.build_id}.") # NOQA: E501
elif(len(matched_pipes) == 0):
published_pipeline = None
raise KeyError(f"Unable to find a published pipeline for this build {e.build_id}") # NOQA: E501
else:
published_pipeline = matched_pipes[0]
print("published pipeline id is", published_pipeline.id)
# Save the Pipeline ID for other AzDO jobs after script is complete
if args.output_pipeline_id_file is not None:
with open(args.output_pipeline_id_file, "w") as out_file:
out_file.write(published_pipeline.id)
if(args.skip_train_execution is False):
pipeline_parameters = {"model_name": e.model_name}
tags = {"BuildId": e.build_id}
if (e.build_uri is not None):
tags["BuildUri"] = e.build_uri
experiment = Experiment(
workspace=aml_workspace,
name=e.experiment_name)
run = experiment.submit(
published_pipeline,
tags=tags,
pipeline_parameters=pipeline_parameters)
print("Pipeline run initiated ", run.id)
from azureml.core import Workspace, Datastore, Dataset
from azureml.core.experiment import Experiment
from azureml.pipeline.core import Pipeline, PipelineData
from azureml.pipeline.steps import PythonScriptStep
ws = Workspace.from_config(path="./file-path/ws_config.json")
experiment = Experiment(workspace=ws, name='BrainStar')
def_blob_store = Datastore(ws, "workspaceblobstore")
compute_target = ws.compute_targets["BrainStar1"]
input_data = Dataset.get(ws, name="Absence data")
output_data1 = PipelineData("output_data1",
datastore=def_blob_store,
output_name="output_data1")
source_directory = './process'
step1 = PythonScriptStep(name="process_step",
script_name="process.py",
inputs=[input_data],
outputs=[output_data1],
compute_target=compute_target,
source_directory=source_directory,
allow_reuse=True)
steps = step1
pipeline1 = Pipeline(workspace=ws, steps=steps)
pipeline1.validate()
pipeline_run1 = Experiment(ws, 'Hello_World1').submit(pipeline1,
regenerate_outputs=False)
def deploy(ws_name,model_name,path_to_model,
environment_name,register_environment,pip_packages,conda_packages,
cpu_cores , memory_gb, path_to_entry_script,service_name):
'''
Get Workspace
'''
ws = Workspace.from_config()
print("Got Workspace {}".format(ws_name))
'''
Register Model
'''
model = Model.register(workspace = ws,
model_path =path_to_model,
model_name = model_name,
)
print("Registered Model {}".format(model_name))
'''
Register Environment
'''
# to install required packages
if register_environment:
env = Environment(environment_name)
cd = CondaDependencies.create(pip_packages=pip_packages, conda_packages = conda_packages)
env.python.conda_dependencies = cd
# Register environment to re-use later
env.register(workspace = ws)
print("Registered Environment")
myenv = Environment.get(workspace=ws, name=environment_name)
# Uncomment to save environment
# myenv.save_to_directory('./environ', overwrite=True)
'''
Config Objects
'''
aciconfig = AciWebservice.deploy_configuration(
cpu_cores=cpu_cores,
memory_gb=memory_gb,
)
inference_config = InferenceConfig(entry_script=path_to_entry_script, environment=myenv)
'''
Deploying
'''
print("Deploying....... This may take a few mins, check the status in MLS after the function finishes executing")
service = Model.deploy(workspace=ws,
name=ws_name,
models=[model],
inference_config=inference_config,
deployment_config=aciconfig, overwrite = True)
service.wait_for_deployment(show_output=True)
url = service.scoring_uri
print(url)
service = Webservice(ws,ws_name)
print(service.get_logs())
return url
}
akscomputes = {
"aks-cpu-deploy": {
"vm_size": "STANDARD_DS3_V2",
"agent_count": 3,
},
"aks-gpu-deploy": {
"vm_size": "STANDARD_NC6S_V3",
"agent_count": 3,
},
}
# create or get Workspace
try:
ws = Workspace.from_config(args.config)
except:
ws = Workspace.create(
args.workspace_name,
subscription_id=args.subscription_id,
resource_group=args.resource_group,
location=args.location,
create_resource_group=True,
exist_ok=True,
show_output=True,
)
ws.write_config()
# create aml compute targets
for ct_name in amlcomputes:
if ct_name not in ws.compute_targets:
from azureml.core.authentication import ServicePrincipalAuthentication
workspace_name = dbutils.secrets.get(scope="azureml", key="workspace_name")
workspace_location = "westeurope"
resource_group = dbutils.secrets.get(scope="azureml", key="resource_group")
subscription_id = dbutils.secrets.get(scope="azureml", key="subscription_id")
svc_pr = ServicePrincipalAuthentication(
tenant_id=dbutils.secrets.get(scope="azureml", key="tenant_id"),
service_principal_id=dbutils.secrets.get(scope="azureml", key="client_id"),
service_principal_password=dbutils.secrets.get(scope="azureml",
key="client_secret"))
workspace = Workspace.create(name=workspace_name,
location=workspace_location,
resource_group=resource_group,
subscription_id=subscription_id,
auth=svc_pr,
exist_ok=True)
# COMMAND ----------
# MAGIC %md ## Build an Azure Container Image for model deployment
# COMMAND ----------
# MAGIC %md ### Use MLflow to build a Container Image for the trained model
# MAGIC
# MAGIC Use the `mlflow.azuereml.build_image` function to build an Azure Container Image for the trained MLflow model. This function also registers the MLflow model with a specified Azure ML workspace. The resulting image can be deployed to Azure Container Instances (ACI) or Azure Kubernetes Service (AKS) for real-time serving.
# COMMAND ----------
config = json.load(f)
except:
print("No new model to register thus no need to create new scoring image")
# raise Exception('No new model to register as production model perform better')
sys.exit(0)
model_name = config["model_name"]
model_version = config["model_version"]
print("Starting to download the model file")
# Start creating
# Point file to conf directory containing details for the aml service
cli_auth = AzureCliAuthentication()
ws = Workspace(workspace_name=workspace,
subscription_id=subscription_id,
resource_group=resource_grp,
auth=cli_auth)
model_list = Model.list(workspace=ws)
model, = (m for m in model_list
if m.version == model_version and m.name == model_name)
print("Model picked: {} \nModel Description: {} \nModel Version: {}".format(
model.name, model.description, model.version))
#print("Writing Conda File")
#myenv = CondaDependencies()
#myenv.add_conda_package("scikit-learn")
#with open("myenv.yml","w") as f:
# f.write(myenv.serialize_to_string())
#print("Finished Writing Conda File")
subscription_id = os.environ.get("SUBSCRIPTION_ID")
tenant_id = os.environ.get("TENANT_ID")
model_name = os.environ.get("MODEL_NAME")
app_id = os.environ.get('SP_APP_ID')
app_secret = os.environ.get('SP_APP_SECRET')
build_id = os.environ.get('BUILD_BUILDID')
# run_id useful to query previous runs
run_id = "57fee47f-5ae8-441c-bc0c-d4c371f32d70"
service_principal = ServicePrincipalAuthentication(
tenant_id=tenant_id,
service_principal_id=app_id,
service_principal_password=app_secret)
aml_workspace = Workspace.get(
name=workspace_name,
subscription_id=subscription_id,
resource_group=resource_group,
auth=service_principal
)
ws = aml_workspace
exp = Experiment(ws, experiment_name)
else:
sys.path.append(os.path.abspath("./util")) # NOQA: E402
from model_helper import get_model_by_tag
exp = run.experiment
ws = run.experiment.workspace
run_id = 'amlcompute'
parser = argparse.ArgumentParser("evaluate")
parser.add_argument(
"--build_id",
type=str,
