import os
parse = argparse.ArgumentParser()
parse.add_argument("--tenantid")
parse.add_argument("--acclientid")
parse.add_argument("--accsecret")
args = parse.parse_args()
sp = ServicePrincipalAuthentication(
tenant_id=args.tenantid, # tenantID
service_principal_id=args.acclientid, # clientId
service_principal_password=args.accsecret) # clientSecret
ws = Workspace.get(name="mlopsdev",
auth=sp,
subscription_id="c46a9435-c957-4e6c-a0f4-b9a597984773",
resource_group="mlops")
#ws = Workspace.from_config()
# keyvault = ws.get_default_keyvault()
# tenantid = keyvault.get_secret(name="tenantid")
# acclientid = keyvault.get_secret(name="acclientid")
# accsvcname = keyvault.get_secret(name="accsvcname")
# accsecret = keyvault.get_secret(name="accsecret")
# print(accsvcname)
#sp = ServicePrincipalAuthentication(tenant_id=tenantid, # tenantID
# service_principal_id=acclientid, # clientId
# service_principal_password=accsecret) # clientSecret
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group,
)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable Azure ML environment
environment = get_environment(
aml_workspace,
e.aml_env_name,
conda_dependencies_file=e.aml_env_train_conda_dep_file,
create_new=e.rebuild_env,
) #
run_config = RunConfiguration()
run_config.environment = environment
if e.datastore_name:
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables[
"DATASTORE_NAME"] = datastore_name # NOQA: E501
model_name_param = PipelineParameter(
name="model_name", default_value=e.model_name) # NOQA: E501
dataset_version_param = PipelineParameter(name="dataset_version",
default_value=e.dataset_version)
data_file_path_param = PipelineParameter(name="data_file_path",
default_value="none")
caller_run_id_param = PipelineParameter(name="caller_run_id",
default_value="none") # NOQA: E501
# Get dataset name
dataset_name = e.dataset_name
# Check to see if dataset exists
if dataset_name not in aml_workspace.datasets:
create_sample_data_csv()
# Use a CSV to read in the data set.
file_name = "COVID19Articles.csv"
if not os.path.exists(file_name):
raise Exception(
'Could not find CSV dataset at "%s". If you have bootstrapped your project, you will need to provide a CSV.' # NOQA: E501
% file_name) # NOQA: E501
# Upload file to default datastore in workspace
datatstore = Datastore.get(aml_workspace, datastore_name)
target_path = "training-data/"
datatstore.upload_files(
files=[file_name],
target_path=target_path,
overwrite=True,
show_progress=False,
)
# Register dataset
path_on_datastore = os.path.join(target_path, file_name)
dataset = Dataset.Tabular.from_delimited_files(
path=(datatstore, path_on_datastore))
dataset = dataset.register(
workspace=aml_workspace,
name=dataset_name,
description="COVID19Articles training data",
tags={"format": "CSV"},
create_new_version=True,
)
# Create a PipelineData to pass data between steps
pipeline_data = PipelineData(
"pipeline_data", datastore=aml_workspace.get_default_datastore())
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
outputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_output",
pipeline_data,
"--dataset_version",
dataset_version_param,
"--data_file_path",
data_file_path_param,
"--caller_run_id",
caller_run_id_param,
"--dataset_name",
dataset_name,
],
runconfig=run_config,
allow_reuse=True,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--model_name",
model_name_param,
"--allow_run_cancel",
e.allow_run_cancel,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_input",
pipeline_data,
], # NOQA: E501
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
# Check run_evaluation flag to include or exclude evaluation step.
if (e.run_evaluation).lower() == "true":
print("Include evaluation step before register step.")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
else:
print("Exclude evaluation step and directly run register step.")
register_step.run_after(train_step)
steps = [train_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id,
)
print(f"Published pipeline: {published_pipeline.name}")
print(f"for build {published_pipeline.version}")
from azureml.core.authentication import AzureCliAuthentication
with open("./configuration/config.json") as f:
config = json.load(f)
workspace_name = config["workspace_name"]
resource_group = config["resource_group"]
subscription_id = config["subscription_id"]
location = config["location"]
cli_auth = AzureCliAuthentication()
# Get workspace
#ws = Workspace.from_config(auth=cli_auth)
ws = Workspace.get(name=workspace_name,
subscription_id=subscription_id,
resource_group=resource_group,
auth=cli_auth)
# Get workspace
#ws = Workspace.from_config(auth=cli_auth)# Get the Image to deploy details
try:
with open("./configuration/image.json") as f:
config = json.load(f)
except:
print("No new model, thus no deployment on ACI")
# raise Exception('No new model to register as production model perform better')
sys.exit(0)
image_name = config["image_name"]
image_version = config["image_version"]
import mlflow.azureml
from azureml.core import Workspace
from azureml.core.webservice import AciWebservice, Webservice
# Create or load an existing Azure ML workspace. You can also load an existing workspace using
# Workspace.get(name="<workspace_name>")
workspace_name = "example_mlflow"
subscription_id = "<>"
resource_group = "example_mlflow_resgrp"
location = "South Central US"
workspace = True
if workspace:
azure_workspace = Workspace.get(name=workspace_name,
subscription_id=subscription_id,
resource_group=resource_group)
else:
azure_workspace = Workspace.create(
name=workspace_name,
subscription_id=subscription_id,
resource_group=resource_group,
location=location,
create_resource_group=False,
)
print("Fetched workspace")
# Build an Azure ML container image for deployment
azure_image, azure_model = mlflow.azureml.build_image(
model_uri="examplemodel",
workspace=azure_workspace,
def main():
# Loading azure credentials
print("::debug::Loading azure credentials")
azure_credentials = os.environ.get("INPUT_AZURE_CREDENTIALS", default="{}")
try:
azure_credentials = json.loads(azure_credentials)
except JSONDecodeError:
print(
"::error::Please paste output of `az ad sp create-for-rbac --name <your-sp-name> --role contributor --scopes /subscriptions/<your-subscriptionId>/resourceGroups/<your-rg> --sdk-auth` as value of secret variable: AZURE_CREDENTIALS. The JSON should include the following keys: 'tenantId', 'clientId', 'clientSecret' and 'subscriptionId'."
)
raise AMLConfigurationException(
"Incorrect or poorly formed output from azure credentials saved in AZURE_CREDENTIALS secret. See setup in https://github.com/Azure/aml-workspace/blob/master/README.md"
)
# Checking provided parameters
print("::debug::Checking provided parameters")
validate_json(data=azure_credentials,
schema=azure_credentials_schema,
input_name="AZURE_CREDENTIALS")
# Mask values
print("::debug::Masking parameters")
mask_parameter(parameter=azure_credentials.get("tenantId", ""))
mask_parameter(parameter=azure_credentials.get("clientId", ""))
mask_parameter(parameter=azure_credentials.get("clientSecret", ""))
mask_parameter(parameter=azure_credentials.get("subscriptionId", ""))
# Loading parameters file
print("::debug::Loading parameters file")
parameters_file = os.environ.get("INPUT_PARAMETERS_FILE",
default="workspace.json")
parameters_file_path = os.path.join(".cloud", ".azure", parameters_file)
try:
with open(parameters_file_path) as f:
parameters = json.load(f)
except FileNotFoundError:
print(
f"::debug::Could not find parameter file in {parameters_file_path}. Please provide a parameter file in your repository if you do not want to use default settings (e.g. .cloud/.azure/workspace.json)."
)
parameters = {}
# Checking provided parameters
print("::debug::Checking provided parameters")
validate_json(data=parameters,
schema=parameters_schema,
input_name="PARAMETERS_FILE")
# Define target cloud
if azure_credentials.get(
"resourceManagerEndpointUrl",
"").startswith("https://management.usgovcloudapi.net"):
cloud = "AzureUSGovernment"
elif azure_credentials.get(
"resourceManagerEndpointUrl",
"").startswith("https://management.chinacloudapi.cn"):
cloud = "AzureChinaCloud"
else:
cloud = "AzureCloud"
# Loading Workspace
sp_auth = ServicePrincipalAuthentication(
tenant_id=azure_credentials.get("tenantId", ""),
service_principal_id=azure_credentials.get("clientId", ""),
service_principal_password=azure_credentials.get("clientSecret", ""),
cloud=cloud)
try:
print("::debug::Loading existing Workspace")
# Default workspace and resource group name
repository_name = str(
os.environ.get("GITHUB_REPOSITORY")).split("/")[-1]
ws = Workspace.get(
name=parameters.get("name", repository_name),
subscription_id=azure_credentials.get("subscriptionId", ""),
resource_group=parameters.get("resource_group", repository_name),
auth=sp_auth)
print("::debug::Successfully loaded existing Workspace")
except AuthenticationException as exception:
print(
f"::error::Could not retrieve user token. Please paste output of `az ad sp create-for-rbac --name <your-sp-name> --role contributor --scopes /subscriptions/<your-subscriptionId>/resourceGroups/<your-rg> --sdk-auth` as value of secret variable: AZURE_CREDENTIALS: {exception}"
)
raise AuthenticationException
except AuthenticationError as exception:
print(f"::error::Microsoft REST Authentication Error: {exception}")
raise AuthenticationException
except AdalError as exception:
print(
f"::error::Active Directory Authentication Library Error: {exception}"
)
raise AdalError
except (WorkspaceException, ProjectSystemException) as exception:
print(f"::debug::Loading existing Workspace failed: {exception}")
if parameters.get("create_workspace", False):
try:
print("::debug::Creating new Workspace")
ws = Workspace.create(
name=parameters.get("name", repository_name),
subscription_id=azure_credentials.get(
"subscriptionId", ""),
resource_group=parameters.get("resource_group",
repository_name),
location=parameters.get("location", None),
create_resource_group=parameters.get(
"create_resource_group", True),
sku=parameters.get("sku", "basic"),
friendly_name=parameters.get("friendly_name", None),
storage_account=parameters.get("storage_account", None),
key_vault=parameters.get("key_vault", None),
app_insights=parameters.get("app_insights", None),
container_registry=parameters.get("container_registry",
None),
cmk_keyvault=parameters.get("cmk_key_vault", None),
resource_cmk_uri=parameters.get("resource_cmk_uri", None),
hbi_workspace=parameters.get("hbi_workspace", None),
auth=sp_auth,
exist_ok=True,
show_output=True)
except WorkspaceException as exception:
print(f"::error::Creating new Workspace failed: {exception}")
raise AMLConfigurationException(
f"Creating new Workspace failed with 'WorkspaceException': {exception}."
)
else:
print(
f"::error::Loading existing Workspace failed with 'WorkspaceException' and new Workspace will not be created because parameter 'create_workspace' was not defined or set to false in your parameter file: {exception}"
)
raise AMLConfigurationException(
"Loading existing Workspace failed with 'WorkspaceException' and new Workspace will not be created because parameter 'create_workspace' was not defined or set to false in your parameter file."
)
# Write Workspace ARM properties to config file
print("::debug::Writing Workspace ARM properties to config file")
config_file_path = os.environ.get("GITHUB_WORKSPACE",
default=".cloud/.azure")
config_file_name = "aml_arm_config.json"
ws.write_config(path=config_file_path, file_name=config_file_name)
print(
"::debug::Successfully finished Azure Machine Learning Workspace Action"
)
def get_azure_workspace():
# pylint: disable=import-error
from azureml.core import Workspace
return Workspace.get("test_workspace")
parser.add_argument("--datastore",
dest="datastore",
)
parser.add_argument("--dataset",
dest="dataset",
)
args = parser.parse_args()
ws = Workspace.get(args.ws, ServicePrincipalAuthentication(
tenant_id=os.getenv('tenant_id'),
service_principal_id=os.getenv('service_principal_id'),
service_principal_password=os.getenv('service_principal_password')
),
subscription_id=os.getenv('subscription_id'),
resource_group=args.rg
)
ds = Datastore.get(ws, args.datastore)
ds.download("assets", args.dataset, overwrite=False)
# Load dataset
url = "assets/"+args.dataset
names = ['sepal-length', 'sepal-width', 'petal-length', 'petal-width', 'class']
dataset = read_csv(url, names=names)
# Split-out validation dataset
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
run_config = RunConfiguration(conda_dependencies=CondaDependencies.create(
conda_packages=[
'numpy', 'pandas', 'scikit-learn', 'tensorflow', 'keras'
],
pip_packages=[
'azure', 'azureml-core', 'azure-storage', 'azure-storage-blob'
]))
run_config.environment.docker.enabled = True
config_envvar = {}
if (e.collection_uri is not None and e.teamproject_name is not None):
builduri_base = e.collection_uri + e.teamproject_name
builduri_base = builduri_base + "/_build/results?buildId="
config_envvar["BUILDURI_BASE"] = builduri_base
run_config.environment.environment_variables = config_envvar
model_name_param = PipelineParameter(name="model_name",
default_value=e.model_name)
build_id_param = PipelineParameter(name="build_id",
default_value=e.build_id)
hyperparameter_alpha_param = PipelineParameter(name="hyperparameter_alpha",
default_value=0.5)
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
"--alpha",
hyperparameter_alpha_param,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
"train": {
"X": X_train,
"y": y_train
},
"test": {
"X": X_test,
"y": y_test
}
}
spa = ServicePrincipalAuthentication("9d396536-b01d-424e-ace8-2a339bc8e502",
"e9311799-43b1-4a27-9e08-266cc96a2c05",
"K_?XRu6bSX/DV3.WYsuS3cwxjgOg4DzO",
_enable_caching=False)
ws = Workspace.get(name='acc_ws',
auth=spa,
subscription_id='f30951ea-6926-40a7-9c19-adeba1c67ec4',
resource_group='acc_ws_rg')
exp = Experiment(workspace=ws, name="exp")
run = exp.start_logging()
clf = RandomForestClassifier(n_estimators=100, max_depth=2, random_state=0)
clf.fit(data["train"]["X"], data["train"]["y"])
preds = clf.predict(data["test"]["X"])
run.log("accuracy", accuracy_score(data["test"]["y"], preds))
filename = 'outputs/propensity_model.pkl'
joblib.dump(clf, filename)
# Registering model with "Model" class.
model = Model.register(model_name='propensity_model',
import json
import numpy as np
from utils import load_data
from azureml.core.webservice import Webservice
from azureml.core import Workspace as ws
#from azureml.core.model import Model
#from azureml.core.webservice import AciWebservice
import os
#import matplotlib
import matplotlib.pyplot as plt
# find 30 random samples from test set
n = 30
myws = ws.get(
name='mnist1',
subscription_id='a383ccde-05bc-45bf-b3ae-f51bd72644df',
resource_group='mnist4',
)
#model = Model(myws, 'sklearn-mnist')
#web = 'https://mlworkspace.azure.ai/portal/subscriptions/bcbc4e01-e5d6-42b0-95af-06286341e6ca/resourceGroups/mnist3/providers/Microsoft.MachineLearningServices/workspaces/mnist1/deployments/mnist'
print(Webservice.list(myws)[0].name)
service = Webservice(myws, 'sklearn-mnist-image')
#print(type(model))
data_folder = os.path.join(os.getcwd(), 'data')
X_test = load_data(os.path.join(data_folder, 'test-images.gz'), False) / 255.0
y_test = load_data(os.path.join(data_folder, 'test-labels.gz'),
True).reshape(-1)
sample_indices = np.random.permutation(X_test.shape[0])[0:n]
test_samples = json.dumps({"data": X_test[sample_indices].tolist()})
test_samples = bytes(test_samples, encoding='utf8')
#result = service.run(input_data=test_samples)
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group,
)
print(f"get_workspace:{aml_workspace}")
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print(f"aml_compute:{aml_compute}")
# Create a reusable Azure ML environment
environment = get_environment(
aml_workspace,
e.aml_env_name,
create_new=e.rebuild_env,
enable_docker=True,
dockerfile='ml_model/preprocess/Dockerfile'
) #
run_config = RunConfiguration()
run_config.environment = environment
if e.datastore_name:
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables["DATASTORE_NAME"] = datastore_name # NOQA: E501
datastore = Datastore(aml_workspace, name=datastore_name)
data_file_path_param = PipelineParameter(name="data_file_path", default_value=e.dataset_name) # NOQA: E501
# The version of the input/output dataset can't be determined at pipeline publish time, only run time. # NOQA: E501
# Options to store output data:
# Option 1: Use blob API to write output data. Otherwise, no way to dynamically change the output dataset based on PipelineParameter, # NOQA: E501
# The following will not work. It generate a path like "PipelineParameter_Name:data_file_path_Default:gear_images" # NOQA: E501
# output_ds = OutputFileDatasetConfig(destination=(datastore, data_file_path_param)) # NOQA: E501
# This option means writing a file locally and upload to the datastore. Fewer dataset, more code. # NOQA: E501
# Option 2: Use a dynamic path in OutputFileDatasetConfig, and register a new dataset at completion # NOQA: E501
# Output dataset can be mounted, so more dataset to maintain, less code. # NOQA: E501
# Using Option 2 below.
output_dataset = OutputFileDatasetConfig(
name=e.processed_dataset_name,
destination=(datastore, "/dataset/{output-name}/{run-id}")
).register_on_complete(
name=e.processed_dataset_name)
preprocess_step = PythonScriptStep(
name="Preprocess Data with OS cmd",
script_name='preprocess/preprocess_os_cmd_aml.py',
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--dataset_name", e.dataset_name,
"--datastore_name", datastore_name,
"--data_file_path", data_file_path_param,
"--output_dataset", output_dataset,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Preprocess OS cmd created")
steps = [preprocess_step]
preprocess_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
preprocess_pipeline._set_experiment_name
preprocess_pipeline.validate()
published_pipeline = preprocess_pipeline.publish(
name=e.preprocessing_pipeline_name,
description="Data preprocessing OS cmd pipeline",
version=e.build_id,
)
print(f"Published pipeline: {published_pipeline.name}")
print(f"for build {published_pipeline.version}")
WORKSPACE_NAME = os.environ.get("BASE_NAME")+"-aml"
SUBSCRIPTION_ID = os.environ.get('SUBSCRIPTION_ID')
RESOURCE_GROUP = os.environ.get("RESOURCE_GROUP_NAME")
if os.path.isfile(MODEL_PATH) is False:
print("The given model path %s is invalid" % (MODEL_PATH))
sys.exit(1)
SP_AUTH = ServicePrincipalAuthentication(
tenant_id=TENANT_ID,
service_principal_id=APP_ID,
service_principal_password=APP_SECRET)
WORKSPACE = Workspace.get(
WORKSPACE_NAME,
SP_AUTH,
SUBSCRIPTION_ID,
RESOURCE_GROUP
)
try:
MODEL = Model.register(
model_path=MODEL_PATH,
model_name=MODEL_NAME,
description="EPIS Model",
workspace=WORKSPACE)
print("Model registered successfully. ID: " + MODEL.id)
except Exception as caught_error:
print("Error while registering the model: " + str(caught_error))
sys.exit(1)
from azureml.core import Environment, Experiment, Workspace, Dataset
from azureml.core.conda_dependencies import CondaDependencies
from azureml.core.runconfig import RunConfiguration
from azureml.pipeline.core import PipelineData, Pipeline
from azureml.pipeline.steps import PythonScriptStep, EstimatorStep
from azureml.train.estimator import Estimator
from azureml.core.datastore import Datastore
from azureml.data.data_reference import DataReference
import os
cluster_name = "azuremlscluster"
#
# Get Azure machine learning workspace
ws = Workspace.get(name=os.environ.get("WORKSPACE_NAME"),
subscription_id=os.environ.get("SUBSCRIPTION_ID"),
resource_group=os.environ.get("RESOURCE_GROUP"))
# Set default Datastore
# ----> Uncomment the following line if there already is a datastore and name it correctly
# ws.set_default_datastore('workspaceblobstore')
# if creditcard is not registered
if 'creditcard' not in ws.datasets:
#Set blobdatastore
blob_datastore_name = 'MyBlobDatastore'
account_name = os.getenv(
def main():
run = Run.get_context()
if (run.id.startswith('OfflineRun')):
from dotenv import load_dotenv
# For local development, set values in this section
load_dotenv()
workspace_name = os.environ.get("WORKSPACE_NAME")
experiment_name = os.environ.get("EXPERIMENT_NAME")
resource_group = os.environ.get("RESOURCE_GROUP")
subscription_id = os.environ.get("SUBSCRIPTION_ID")
build_id = os.environ.get('BUILD_BUILDID')
# run_id useful to query previous runs
run_id = "bd184a18-2ac8-4951-8e78-e290bef3b012"
aml_workspace = Workspace.get(name=workspace_name,
subscription_id=subscription_id,
resource_group=resource_group)
ws = aml_workspace
exp = Experiment(ws, experiment_name)
else:
ws = run.experiment.workspace
exp = run.experiment
run_id = 'amlcompute'
parser = argparse.ArgumentParser("register")
parser.add_argument(
"--build_id",
type=str,
help="The Build ID of the build triggering this pipeline run",
)
parser.add_argument(
"--run_id",
type=str,
help="Training run ID",
)
parser.add_argument(
"--model_name",
type=str,
help="Name of the Model",
default="sklearn_regression_model.pkl",
)
parser.add_argument("--step_input",
type=str,
help=("input from previous steps"))
args = parser.parse_args()
if (args.build_id is not None):
build_id = args.build_id
if (args.run_id is not None):
run_id = args.run_id
if (run_id == 'amlcompute'):
run_id = run.parent.id
model_name = args.model_name
model_path = args.step_input
# load the model
print("Loading model from " + model_path)
model_file = os.path.join(model_path, model_name)
model = joblib.load(model_file)
model_mse = run.parent.get_metrics()["mse"]
if (model is not None):
if (build_id is None):
register_aml_model(model_file, model_name, exp, run_id)
else:
run.tag("BuildId", value=build_id)
builduri_base = os.environ.get("BUILDURI_BASE")
if (builduri_base is not None):
build_uri = builduri_base + build_id
run.tag("BuildUri", value=build_uri)
register_aml_model(model_file, model_name, model_mse, exp,
run_id, build_id, build_uri)
else:
register_aml_model(model_file, model_name, model_mse, exp,
run_id, build_id)
else:
print("Model not found. Skipping model registration.")
sys.exit(0)
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable Azure ML environment
environment = get_environment(aml_workspace,
e.aml_env_name,
create_new=False) # NOQA: E501
run_config = RunConfiguration()
run_config.environment = environment
if (e.datastore_name):
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables[
"DATASTORE_NAME"] = datastore_name # NOQA: E501
model_name_param = PipelineParameter(name="model_name",
default_value=e.model_name)
dataset_version_param = PipelineParameter(name="dataset_version",
default_value=e.dataset_version)
data_file_path_param = PipelineParameter(name="data_file_path",
default_value="none")
caller_run_id_param = PipelineParameter(name="caller_run_id",
default_value="none")
# Get dataset name
dataset_name = e.dataset_name
# # Check to see if dataset exists
# if (dataset_name not in aml_workspace.datasets):
# # Create dataset from lacemlops sample data
# sample_data = load_lacemlops()
# df = pd.DataFrame(
# data=sample_data.data,
# columns=sample_data.feature_names)
# df['Y'] = sample_data.target
# file_name = 'lacemlops.csv'
# df.to_csv(file_name, index=False)
# # Upload file to default datastore in workspace
# datatstore = Datastore.get(aml_workspace, datastore_name)
# target_path = 'training-data/'
# datatstore.upload_files(
# files=[file_name],
# target_path=target_path,
# overwrite=True,
# show_progress=False)
# # Register dataset
# path_on_datastore = os.path.join(target_path, file_name)
# dataset = Dataset.Tabular.from_delimited_files(
# path=(datatstore, path_on_datastore))
# dataset = dataset.register(
# workspace=aml_workspace,
# name=dataset_name,
# description='lacemlops training data',
# tags={'format': 'CSV'},
# create_new_version=True)
# Create a PipelineData to pass data between steps
pipeline_data = PipelineData(
'pipeline_data', datastore=aml_workspace.get_default_datastore())
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
outputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_output",
pipeline_data,
"--dataset_version",
dataset_version_param,
"--data_file_path",
data_file_path_param,
"--caller_run_id",
caller_run_id_param,
"--dataset_name",
dataset_name,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--model_name",
model_name_param,
"--allow_run_cancel",
e.allow_run_cancel,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_input",
pipeline_data,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
# Check run_evaluation flag to include or exclude evaluation step.
if ((e.run_evaluation).lower() == 'true'):
print("Include evaluation step before register step.")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
else:
print("Exclude evaluation step and directly run register step.")
register_step.run_after(train_step)
steps = [train_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
#!/usr/bin/env python
# coding: utf-8
# In[ ]:
# Connecting to existing Azure Machine Learning workspace
import os
from azureml.core import Workspace
from azureml.core.authentication import ServicePrincipalAuthentication
ws = Workspace.get(name='ws_employee_churn', auth=None, subscription_id='9c43f403-ce9b-4e3a-8f7f-21b1688e2873', resource_group='rg_datascience')
cluster_type = os.environ.get("AML_COMPUTE_CLUSTER_TYPE", "CPU")
compute_target = ws.get_default_compute_target(cluster_type)
# In[ ]:
#Read and pre-process data
import pandas as pd
import numpy as np
hr = pd.read_csv('HR_Data.csv')
col_names = hr.columns.tolist()
col_names
cat_vars=['Age Bucket', 'Decade', 'Gender', 'Marital Status', 'Cost Center', 'Managed By', 'Designation', 'Employee Band', 'Experience']
for var in cat_vars:
cat_list='var'+'_'+var
cat_list = pd.get_dummies(hr[var], prefix=var,drop_first=False)
def main():
run = Run.get_context()
if (run.id.startswith('OfflineRun')):
from dotenv import load_dotenv
# For local development, set values in this section
load_dotenv()
sources_dir = os.environ.get("SOURCES_DIR_TRAIN")
if (sources_dir is None):
sources_dir = 'diabetes_regression'
path_to_util = os.path.join(".", sources_dir, "util")
sys.path.append(os.path.abspath(path_to_util)) # NOQA: E402
from model_helper import get_model_by_tag
workspace_name = os.environ.get("WORKSPACE_NAME")
experiment_name = os.environ.get("EXPERIMENT_NAME")
resource_group = os.environ.get("RESOURCE_GROUP")
subscription_id = os.environ.get("SUBSCRIPTION_ID")
build_id = os.environ.get('BUILD_BUILDID')
aml_workspace = Workspace.get(name=workspace_name,
subscription_id=subscription_id,
resource_group=resource_group)
ws = aml_workspace
exp = Experiment(ws, experiment_name)
else:
sys.path.append(os.path.abspath("./util")) # NOQA: E402
from model_helper import get_model_by_tag
ws = run.experiment.workspace
exp = run.experiment
e = Env()
parser = argparse.ArgumentParser("register")
parser.add_argument(
"--build_id",
type=str,
help="The Build ID of the build triggering this pipeline run",
)
parser.add_argument("--output_model_version_file",
type=str,
default="model_version.txt",
help="Name of a file to write model version to")
args = parser.parse_args()
if (args.build_id is not None):
build_id = args.build_id
model_name = e.model_name
try:
tag_name = 'BuildId'
model = get_model_by_tag(model_name, tag_name, build_id, exp.workspace)
if (model is not None):
print("Model was registered for this build.")
if (model is None):
print("Model was not registered for this run.")
sys.exit(1)
except Exception as e:
print(e)
print("Model was not registered for this run.")
sys.exit(1)
# Save the Model Version for other AzDO jobs after script is complete
if args.output_model_version_file is not None:
with open(args.output_model_version_file, "w") as out_file:
out_file.write(str(model.version))
route = nx.shortest_path(G, orig, dest, weight='travel_time')
edge_lengths = ox.utils_graph.get_route_edge_attributes(
G, route, 'length')
return sum(edge_lengths)
else:
raise ValueError('Path not foun')
GRAPH_FILE_PATH = "https://grab5033896937.blob.core.windows.net/azureml/Dataset/grab/singapore.graphml"
try:
# load workspace configuration from the config.json file in the current folder.
#ws = Workspace.from_config()
ws = Workspace.get(name="<<Insert Name>>",
subscription_id="<<Insert Subscription Id>>",
resource_group="<<Insert Resource Group>>")
dataset = Dataset.get_by_name(ws, 'sg_graphml')
# list the files referenced by sg_graphml dataset
GRAPH_FILE_PATH = dataset.to_path()
G = ox.load_graphml(GRAPH_FILE_PATH)
except:
G = ox.graph_from_place('Singapore', network_type='drive')
ox.save_graphml(G, filepath=GRAPH_FILE_PATH)
def init():
global model
def get_workspace(name, subscription_id, resource_group):
return Workspace.get(name=name,
subscription_id=subscription_id,
resource_group=resource_group)
from azureml.core.authentication import InteractiveLoginAuthentication
from azureml.core import Workspace
interactive_auth = InteractiveLoginAuthentication(
tenant_id="99e1e721-7184-498e-8aff-b2ad4e53c1c2")
ws = Workspace.get(name='mlw-udea-mon',
subscription_id='80986424-c91c-4671-9d12-d7de41662014',
resource_group='rg-udea-mon',
location='eastus',
auth=interactive_auth)
ws.write_config(path='.azureml')
def main(gpu_num, class_path, data_dir, data_path, num_clusters, batch_size,
learning_rate):
# Define an Azure ML run
run = Run.get_context()
# Folder for persistent assets
log_dir = 'outputs/'
# Conver to int
num_clusters = int(num_clusters)
# Define workspace object
try:
sp = ServicePrincipalAuthentication(
tenant_id=os.getenv('AML_TENANT_ID'),
service_principal_id=os.getenv('AML_PRINCIPAL_ID'),
service_principal_password=os.getenv('AML_PRINCIPAL_PASS'))
ws = Workspace.get(name=os.getenv('WORKSPACE_NAME'),
auth=sp,
subscription_id=os.getenv('SUBSCRIPTION_ID'))
# Need to create the workspace
except Exception as err:
print('Issue with service principal or workspace creation: ', err)
assert False
# Get classes
class_names = get_classes(class_path)
num_classes = len(class_names)
# Create annot file (file with image path and bounding boxes - one image per line)
image_ids = [
os.path.basename(x)
for x in glob.glob(os.path.join(data_path, data_dir, 'JPEGImages/*.*'))
]
annot_filename = './outputs/train_val.txt'
annot_file = open(annot_filename, 'w') # output file
for image_id in image_ids:
annot_file.write(
os.path.join(data_path, data_dir,
'JPEGImages/{}'.format(image_id)))
convert_annotation(image_id='.'.join(image_id.split('.')[0:-1]),
list_file=annot_file,
classes=class_names,
data_dir=data_dir,
data_path=data_path)
annot_file.write('\n')
annot_file.close()
# Calculate anchor boxes
out_file = './outputs/custom_anchors.txt'
kmeans = YOLO_Kmeans(args.num_clusters, annot_filename, out_file)
kmeans.txt2clusters()
# Anchors and filters
anchors_orig, anchors = get_anchors(out_file)
filter_num = (num_classes + 5) * 3
# Update config file template
# replace $FILTERS, $CLASSES, and $ANCHORS w/ correct value from above
if num_clusters == 9:
with open('yolov3-custom_template.cfg', 'r') as fptr:
config_str = fptr.read()
config_str = config_str.replace('$FILTERS', str(filter_num))
config_str = config_str.replace('$CLASSES', str(num_classes))
config_str = config_str.replace('$ANCHORS', anchors_orig)
else: # num_clusters == 6
with open('yolov3-tiny-custom_template.cfg', 'r') as fptr:
config_str = fptr.read()
config_str = config_str.replace('$FILTERS', str(filter_num))
config_str = config_str.replace('$CLASSES', str(num_classes))
config_str = config_str.replace('$ANCHORS', anchors_orig)
with open('./outputs/yolov3-custom.cfg', 'w') as outptr:
outptr.write(config_str)
# Download Darknet model (from Azure ML Workspace) and convert
keras_custom_model = './outputs/yolov3_custom.h5'
if num_clusters == 9:
AzureMLModel(ws, name='yolov3.weights').download(target_dir='.',
exist_ok=True)
convert_to_keras(config_path='./outputs/yolov3-custom.cfg',
weights_path='yolov3.weights',
output_path=keras_custom_model)
else:
AzureMLModel(ws, name='yolov3-tiny.weights').download(target_dir='.',
exist_ok=True)
convert_to_keras(config_path='./outputs/yolov3-custom.cfg',
weights_path='yolov3-tiny.weights',
output_path=keras_custom_model)
initial_lr = learning_rate
input_shape = (608, 608) # default setting
is_tiny_version = (num_clusters == 6)
# Use transfer learning - all layers frozen except last "freeze_body" number
if is_tiny_version:
input_shape = (416, 416) #Change default anchor box number
model = create_tiny_model(input_shape,
anchors,
num_classes,
freeze_body=4,
weights_path=keras_custom_model,
gpu_num=gpu_num)
else:
model = create_model(input_shape,
anchors,
num_classes,
freeze_body=4,
weights_path=keras_custom_model
) # make sure you know what you freeze
logging = TensorBoard(log_dir=log_dir)
checkpoint = ModelCheckpoint(
log_dir + 'ep{epoch:03d}-loss{loss:.3f}-val_loss{val_loss:.3f}.h5',
monitor='val_loss',
save_weights_only=True,
save_best_only=True,
period=3)
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=3,
verbose=1)
early_stopping = EarlyStopping(monitor='val_loss',
min_delta=0,
patience=10,
verbose=1)
val_split = 0.2
with open(annot_filename) as f:
lines = f.readlines()
np.random.seed(10101)
np.random.shuffle(lines)
np.random.seed(None)
num_val = int(len(lines) * val_split)
num_train = len(lines) - num_val
class LogRunMetrics(Callback):
# callback at the end of every epoch
def on_epoch_end(self, epoch, log):
# log a value repeated which creates a list for Azure ML tracking
run.log('Loss', log['loss'])
# Train with frozen layers first, to get a stable loss.
# Adjust num epochs to your dataset. This step is enough to obtain a not bad model.
if True:
model.compile(
optimizer=Adam(lr=initial_lr),
loss={
# use custom yolo_loss Lambda layer.
'yolo_loss': lambda y_true, y_pred: y_pred
})
print('Train on {} samples, val on {} samples, with batch size {}.'.
format(num_train, num_val, batch_size))
model.fit_generator(data_generator_wrapper(lines[:num_train],
batch_size, input_shape,
anchors, num_classes),
steps_per_epoch=max(1, num_train // batch_size),
validation_data=data_generator_wrapper(
lines[num_train:], batch_size, input_shape,
anchors, num_classes),
validation_steps=max(1, num_val // batch_size),
epochs=30,
initial_epoch=0,
callbacks=[logging, LogRunMetrics()])
model.save_weights(log_dir + 'trained_weights_intermediate.h5')
# Save architecture, too
with open(log_dir + 'trained_weights_intermediate.json', 'w') as f:
f.write(model.to_json())
# Unfreeze and continue training, to fine-tune.
# Train longer if the result is not good.
if True:
for i in range(len(model.layers)):
model.layers[i].trainable = True
model.compile(optimizer=Adam(lr=initial_lr / 100),
loss={
'yolo_loss': lambda y_true, y_pred: y_pred
}) # recompile to apply the change
print('Unfreeze all of the layers.')
print('Train on {} samples, val on {} samples, with batch size {}.'.
format(num_train, num_val, batch_size))
model.fit_generator(
data_generator_wrapper(lines[:num_train], batch_size, input_shape,
anchors, num_classes),
steps_per_epoch=max(1, num_train // batch_size),
validation_data=data_generator_wrapper(lines[num_train:],
batch_size, input_shape,
anchors, num_classes),
validation_steps=max(1, num_val // batch_size),
epochs=60,
initial_epoch=30,
callbacks=[logging, early_stopping, reduce_lr,
LogRunMetrics()])
model.save_weights(log_dir + 'trained_weights_final.h5')
# Save architecture, too
with open(log_dir + 'trained_weights_final.json', 'w') as f:
f.write(model.to_json())
from azureml.core import Workspace
from azureml.core.compute import ComputeTarget, AmlCompute
from azureml.core.compute_target import ComputeTargetException
from ml_service.util.env_variables import Env
# Environment variables
env = Env()
# Azure ML workspace
aml_workspace = Workspace.get(
name=env.workspace_name,
subscription_id=env.subscription_id,
resource_group=env.resource_group,
)
# Verify that the cluster does not exist already
try:
cpu_cluster = ComputeTarget(workspace=aml_workspace, name=env.compute_name)
print('Found existing cluster, use it.')
except ComputeTargetException:
compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_D2_V2',
idle_seconds_before_scaledown=2400,
min_nodes=0,
max_nodes=1)
cpu_cluster = ComputeTarget.create(aml_workspace, env.compute_name, compute_config)
cpu_cluster.wait_for_completion(show_output=True)
def main():
parser = argparse.ArgumentParser("register")
parser.add_argument("--output_pipeline_id_file",
type=str,
default="pipeline_id.txt",
help="Name of a file to write pipeline ID to")
parser.add_argument(
"--skip_train_execution",
action="store_true",
help=("Do not trigger the execution. "
"Use this in Azure DevOps when using a server job to trigger"),
)
args = parser.parse_args()
e = Env()
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
# Find the pipeline that was published by the specified build ID
pipelines = PublishedPipeline.list(aml_workspace)
matched_pipes = []
for p in pipelines:
if p.name == e.pipeline_name:
if p.version == e.build_id:
matched_pipes.append(p)
if len(matched_pipes) > 1:
published_pipeline = None
raise Exception(
f"Multiple active pipelines are published for build {e.build_id}."
) # NOQA: E501
elif len(matched_pipes) == 0:
published_pipeline = None
raise KeyError(
f"Unable to find a published pipeline for this build {e.build_id}"
) # NOQA: E501
else:
published_pipeline = matched_pipes[0]
print("published pipeline id is", published_pipeline.id)
# Save the Pipeline ID for other AzDO jobs after script is complete
if args.output_pipeline_id_file is not None:
with open(args.output_pipeline_id_file, "w") as out_file:
out_file.write(published_pipeline.id)
if args.skip_train_execution is False:
pipeline_parameters = {"model_name": e.model_name}
tags = {"BuildId": e.build_id}
if e.build_uri is not None:
tags["BuildUri"] = e.build_uri
experiment = Experiment(workspace=aml_workspace,
name=e.experiment_name)
run = experiment.submit(published_pipeline,
tags=tags,
pipeline_parameters=pipeline_parameters)
print("Pipeline run initiated ", run.id)
model_bin = bz2.decompress(model_compressed)
with open(MODEL_PATH, "wb") as f:
model = f.write(model_bin)
#pickle.dump(model_bin,open(MODEL_PATH,"wb"))
print('model size (original): ', sys.getsizeof(MODEL_PATH), " bytes")
MODEL_PATH = os.path.join(os.getcwd(), MODEL_PATH)
else:
MODEL_PATH = os.path.join(os.getcwd(), MODEL_PATH)
wget.download(MODEL_URL, MODEL_PATH)
#Set the inetractive authentication
interactive_auth = InteractiveLoginAuthentication()
#Get the chosen workspace
ws = Workspace.get(name=AZURE_WORKSPACE_NAME,
auth=interactive_auth,
subscription_id=AZURE_SUBSCRIPTION_ID,
resource_group=AZURE_RESOURCE_GROUP)
print(ws.name, ws.location, ws.resource_group, ws.location, sep='\t')
# register model
model = Model.register(model_name=MODEL_NAME,
model_path=MODEL_PATH,
workspace=ws,
description=MODEL_DESCRIPTION)
print(model.name, model.id, model.version, sep='\t')
#Set the image
aciconfig = AciWebservice.deploy_configuration(cpu_cores=CPU_CORES,
memory_gb=MEMORY_GB,
description=SERVICE_DESCRIPTION,
import os
import argparse
from azureml.core import Workspace
from azureml.core.environment import Environment
from azureml.core.model import Model, InferenceConfig
import shutil
from ml_service.util.env_variables import Env
e = Env()
# Get Azure machine learning workspace
ws = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
parser = argparse.ArgumentParser("create scoring image")
parser.add_argument(
"--output_image_location_file",
type=str,
help=("Name of a file to write image location to, "
"in format REGISTRY.azurecr.io/IMAGE_NAME:IMAGE_VERSION"))
args = parser.parse_args()
model = Model(ws, name=e.model_name, version=e.model_version)
sources_dir = e.sources_directory_train
if (sources_dir is None):
sources_dir = 'COVID19Articles'
score_script = os.path.join(".", sources_dir, e.score_script)
score_file = os.path.basename(score_script)
path_to_scoring = os.path.dirname(score_script)
cwd = os.getcwd()
from azureml.core.environment import Environment
from azureml.core.model import InferenceConfig, Model
from azureml.core.webservice import LocalWebservice
from azureml.core import Workspace
ws = Workspace.get(name="QCM",
subscription_id='ed4fc9bc-f386-4f01-a8b4-2077312476f3',
resource_group='appsvc_linux_centralus')
models = Model.register(
model_path="model_scaler.joblib",
model_name="mymodel",
tags={
'area': "diabetes",
'type': "regression"
},
description="Ridge regression model to predict diabetes",
workspace=ws)
# Create inference configuration based on the environment definition and the entry script
myenv = Environment.from_conda_specification(name="env", file_path="myenv.yml")
inference_config = InferenceConfig(entry_script="score.py", environment=myenv)
# Create a local deployment, using port 8890 for the web service endpoint
deployment_config = LocalWebservice.deploy_configuration(port=8890)
# Deploy the service
service = Model.deploy(ws, "mymodel", [models], inference_config,
deployment_config)
# Wait for the deployment to complete
def main():
run = Run.get_context()
if (run.id.startswith('OfflineRun')):
from dotenv import load_dotenv
# For local development, set values in this section
load_dotenv()
workspace_name = os.environ.get("WORKSPACE_NAME")
experiment_name = os.environ.get("EXPERIMENT_NAME")
resource_group = os.environ.get("RESOURCE_GROUP")
subscription_id = os.environ.get("SUBSCRIPTION_ID")
# run_id useful to query previous runs
run_id = "bd184a18-2ac8-4951-8e78-e290bef3b012"
aml_workspace = Workspace.get(
name=workspace_name,
subscription_id=subscription_id,
resource_group=resource_group
)
ws = aml_workspace
exp = Experiment(ws, experiment_name)
else:
ws = run.experiment.workspace
exp = run.experiment
run_id = 'amlcompute'
parser = argparse.ArgumentParser("register")
parser.add_argument(
"--run_id",
type=str,
help="Training run ID",
)
parser.add_argument(
"--model_name",
type=str,
help="Name of the Model",
default="sales_forecast_model.pkl",
)
parser.add_argument(
"--step_input",
type=str,
help=("input from previous steps")
)
args = parser.parse_args()
if (args.run_id is not None):
run_id = args.run_id
if (run_id == 'amlcompute'):
run_id = run.parent.id
model_name = args.model_name
model_path = args.step_input
print("Getting registration parameters")
# Load the registration parameters from the parameters file
with open("parameters.json") as f:
pars = json.load(f)
try:
register_args = pars["registration"]
except KeyError:
print("Could not load registration values from file")
register_args = {"tags": []}
model_tags = {}
for tag in register_args["tags"]:
try:
mtag = run.parent.get_metrics()[tag]
model_tags[tag] = mtag
except KeyError:
print(f"Could not find {tag} metric on parent run.")
#---------------------------------------
# later...
# load json and create model
#json_file = open('model.json', 'r')
#loaded_model_json = json_file.read()
#json_file.close()
#model = model_from_json(loaded_model_json)
# load weights into new model
#model.load_weights("model.h5")
#print("Loaded model from disk")
# load the model
print("Loading model from " + model_path)
model_file = os.path.join(model_path, model_name)
print("Printing model file")
print(model_file)
model = keras.models.load_model(model_file)
print("Loading the model----:)")
#model = load_model(model_path)
#model = load_model(model_file)
# model = joblib.load(model_file)
parent_tags = run.parent.get_tags()
try:
build_id = parent_tags["BuildId"]
except KeyError:
build_id = None
print("BuildId tag not found on parent run.")
print(f"Tags present: {parent_tags}")
try:
build_uri = parent_tags["BuildUri"]
except KeyError:
build_uri = None
print("BuildUri tag not found on parent run.")
print(f"Tags present: {parent_tags}")
#if (model is not None):
dataset_id = parent_tags["dataset_id"]
if (build_id is None):
register_aml_model(
model_file,
model_name,
model_tags,
exp,
run_id,
dataset_id)
elif (build_uri is None):
register_aml_model(
model_file,
model_name,
model_tags,
exp,
run_id,
dataset_id,
build_id)
else:
register_aml_model(
model_file,
model_name,
model_tags,
exp,
run_id,
dataset_id,
build_id,
build_uri)
type=str,
dest="friendly_name",
help="Friendly name of the Azure Machine Learning Workspace")
args = parser.parse_args()
# Mask values
print("Masking values")
print(f"::add-mask::{args.subscription_id}")
# Use Azure CLI authentication
cli_auth = AzureCliAuthentication()
try:
print("Loading existing Workspace")
ws = Workspace.get(name=args.workspace_name,
subscription_id=args.subscription_id,
resource_group=args.resource_group,
auth=cli_auth)
print("Found existing Workspace")
except WorkspaceException:
print("Loading failed")
print("Creating new Workspace")
ws = Workspace.create(name=args.workspace_name,
auth=cli_auth,
subscription_id=args.subscription_id,
resource_group=args.resource_group,
location=args.location,
create_resource_group=True,
friendly_name=args.friendly_name,
show_output=True)
# Write out the Workspace ARM properties to a config file
resource_group = args.resource_group
workspace_name = args.workspace_name
workspace_region = args.workspace_region
model_name = args.model_name
cluster_name_cpu = args.cluster_name_cpu
cluster_name_gpu = args.cluster_name_gpu
pipeline_experiment_name = args.pipeline_experiment_name
pipeline_name = args.pipeline_name
service_principal = ServicePrincipalAuthentication(
tenant_id=tenant_id,
service_principal_id=application_id,
service_principal_password=app_secret)
ws = Workspace.get(name=workspace_name,
subscription_id=subscription_id,
resource_group=resource_group,
auth=service_principal)
# Retrieve the pointer to the default Blob storage.
def_blob_store = Datastore(ws, "workspaceblobstore")
print("Blobstore's name: {}".format(def_blob_store.name))
blob_input_data = DataReference(datastore=def_blob_store,
data_reference_name="mnist_datainput",
path_on_datastore="mnist_datainput")
print("DataReference object created")
# Create a CPU cluster of type D2 V2 with 1 node. (due to subscription's limitations we stick to 1 node)
try:
def main():
run = Run.get_context()
if run.id.startswith("OfflineRun"):
from dotenv import load_dotenv
load_dotenv()
sources_dir = os.environ.get("SOURCES_DIR_TRAIN")
if sources_dir is None:
sources_dir = "oilwells"
workspace_name = os.environ.get("WORKSPACE_NAME")
experiment_name = os.environ.get("EXPERIMENT_NAME")
resource_group = os.environ.get("RESOURCE_GROUP")
subscription_id = os.environ.get("SUBSCRIPTION_ID")
build_id = os.environ.get("BUILD_BUILDID")
aml_workspace = Workspace.get(name=workspace_name,
subscription_id=subscription_id,
resource_group=resource_group)
ws = aml_workspace
exp = Experiment(ws, experiment_name)
else:
exp = run.experiment
e = Env()
parser = argparse.ArgumentParser("register")
parser.add_argument(
"--build_id",
type=str,
help="The Build ID of the build triggering this pipeline run",
)
parser.add_argument(
"--output_model_version_file",
type=str,
default="model_version.txt",
help="Name of a file to write model version to",
)
args = parser.parse_args()
if args.build_id is not None:
build_id = args.build_id
model_name = e.model_name
try:
tag_name = "BuildId"
model = get_latest_model(model_name, tag_name, build_id, exp.workspace)
if model is not None:
print("Model was registered for this build.")
if model is None:
print("Model was not registered for this run.")
sys.exit(1)
except Exception as e:
print(e)
print("Model was not registered for this run.")
sys.exit(1)
# Save the Model Version for other AzDO jobs after script is complete
if args.output_model_version_file is not None:
with open(args.output_model_version_file, "w") as out_file:
out_file.write(str(model.version))
