def register_model(model_name, model_path): model_config = next( iter( filter(lambda x: x["name"] == model_name, self.output_reg_models))) tags = model_config.get("tags") description = model_config.get("description") Model.register(workspace=ws, model_path=model_path, model_name=model_name, tags=tags, description=description)
def upload_model(ws: Workspace, config: MoveModelConfig) -> Model: """ Uploads an InnerEye model to an AzureML workspace :param ws: The AzureML workspace :param config: move config :return: imported Model """ model_path, environment_path = config.get_paths() with open(model_path / MODEL_JSON, 'r') as f: model_dict = json.load(f) # Find the folder containing the final model. final_model_path = model_path / FINAL_MODEL_FOLDER full_model_path = final_model_path if final_model_path.exists( ) else model_path / FINAL_ENSEMBLE_MODEL_FOLDER new_model = Model.register(ws, model_path=str(full_model_path), model_name=model_dict['name'], tags=model_dict['tags'], properties=model_dict['properties'], description=model_dict['description']) env = Environment.load_from_directory(str(environment_path)) env.register(workspace=ws) print(f"Environment {env.name} registered") return new_model
def _register(self, run_id, asset_name, asset_label=None): """ This method register a new model as a AML reference. :param run_id: run's identifier :param asset_name: name of asset of the current run :param asset_label: name of label of the current run :return: None """ tags = {"run_id": run_id, "asset_name": asset_name} if asset_label is not None: tags["asset_label"] = asset_label register_path = os.path.join(OUTPUTS_FOLDER, AML_MLAPP_FOLDER) Model.register(self.ws, model_path=register_path, model_name=get_model_register_name(run_id), tags=tags, description=asset_name)
def register(self, validated_model_folder, registered_model_folder, azure_ml_logs_provider, web_service_deployer): IGNORE_TRAIN_STEP = azure_ml_logs_provider.get_tag_from_brother_run( "prep_data.py", "IGNORE_TRAIN_STEP") if IGNORE_TRAIN_STEP == True: print("Ignore register step") self._execute_sampling_pipeline() print("launch sampling state") return _, classifier = disc_network() classifier_name = "classifier.hdf5" validated_model_file = os.path.join(validated_model_folder, classifier_name) classifier.load_weights(validated_model_file) self.run.upload_file(name=self.config.MODEL_NAME, path_or_stream=validated_model_file) #_ = self.run.register_model(model_name=self.config.MODEL_NAME, # tags={'Training context':'Pipeline'}, # model_path=validated_model_file) Model.register(workspace=self.run.experiment.workspace, model_path=validated_model_file, model_name=self.config.MODEL_NAME, tags={'Training context': 'Pipeline'}) acc = azure_ml_logs_provider.get_log_from_brother_run( "eval_model.py", "acc") print("acc :", acc) #deploy model if web_service_deployer.to_deploy(acc): print("deploying...") web_service_deployer.deploy() print("model deployed") #pas si import à part pour le test registered_model_file = os.path.join(registered_model_folder, classifier_name) os.makedirs(registered_model_folder) _ = shutil.copy(validated_model_file, registered_model_file)
def log_results(model_accuracy, output_folder, model_name, matrix, labels, run): run.log('model_accuracy', model_accuracy) run.log_image('Confusion Matrix Plot', plot=create_confusion_matrix_plot(matrix, labels)) model = Model.register(workspace=run.experiment.workspace, model_path=output_folder, model_name=model_name, model_framework=Model.Framework.SCIKITLEARN, model_framework_version=sklearn.__version__, tags={'Training context': 'Pipeline'}) run.log('model_name', model.name) run.log('model_version', model.version)
def deploy_pickled_model(amls_config, workspace): """ Publish a pickled model to AMLS model repository :param amls_config: :param workspace: :return: """ model_path = '../model/model.pkl' logger.info(f"Deploying model.") model = Model.register(model_path=model_path, model_name='model', tags=amls_config['tags'], description=amls_config['description'], workspace=workspace) return model
def register_model(**parameters): zsh('az ml model list --output json > az_ml_model_list.json') model_name = parameters['model_name'] # model_id = parameters['id'] # model_version = parameters['version'] ws = parameters['workspace'] with open('az_ml_model_list.json') as models_json: models = json.load(models_json) matched_models = [ model for model in models if model['name'] == model_name ] if len(matched_models) == 1: print('Found the model\n') model = Model(workspace=ws, name=model_name) elif len(matched_models) == 0: print('Provided model {} has not been found\n'.format(model_name)) print('registering new model in Azure ...\n') model = Model.register(**parameters) elif len(matched_models) > 1: model_name = matched_models[0]['name'] model = Model(workspace=ws, name=model_name) zsh('rm az_ml_model_list.json') return model
plt.plot(history.history["val_loss"], label="validation") plt.title("Loss Plot") plt.xlabel("Epochs") plt.ylabel("Loss") plt.legend() plt.subplot(1, 2, 2) plt.plot(history.history["acc"], label="training") plt.plot(history.history["val_acc"], label="validation") plt.title("Accuracy Plot") plt.xlabel("Epochs") plt.ylabel("Accuracy") plt.legend() plt.savefig("/tmp/learning-curves.png") mlflow.log_artifact("/tmp/learning-curves.png") if run._run_id.startswith("OfflineRun"): print("This appears to be an offline run. I will not register the model") else: with open("conf.yaml", "r") as f: metadata = yaml.load(f)["metadata"] Model.register( run.experiment.workspace, model_path="./outputs/model", model_name=metadata["model_name"], description=metadata["description"], tags=metadata["tags"], )
# "run" is a reference to a completed experiment run # List the files generated by the experiment for file in run.get_file_names(): print(file) # Download a named file run.download_file(name='outputs/model.pkl', output_file_path='model.pkl') from azureml.core import Model model = Model.register(workspace=ws, model_name='classification_model', model_path='model.pkl', # local path description='A classification model', tags={'dept': 'sales'}, model_framework=Model.Framework.SCIKITLEARN, model_framework_version='0.20.3') #alternatively run.register_model( model_name='classification_model', model_path='outputs/model.pkl', # run outputs path description='A classification model', tags={'dept': 'sales'}, model_framework=Model.Framework.SCIKITLEARN, model_framework_version='0.20.3') #view registered models from azureml.core import Model
# Import libraries import argparse import joblib from azureml.core import Workspace, Model, Run # Get parameters parser = argparse.ArgumentParser() parser.add_argument('--model_folder', type=str, dest='model_folder', default="diabetes_model", help='model location') args = parser.parse_args() model_folder = args.model_folder # Get the experiment run context run = Run.get_context() # load the model print("Loading model from " + model_folder) model_file = model_folder + "/model.pkl" model = joblib.load(model_file) Model.register(workspace=run.experiment.workspace, model_path=model_file, model_name='diabetes_model', tags={'Training context': 'Pipeline'}) run.complete()
import logging import os from azureml.core import Run, Model import argparse logging.basicConfig(level=logging.INFO) log: logging.Logger = logging.getLogger(__name__) parser = argparse.ArgumentParser("register") parser.add_argument("--input", type=str, required=True) args = parser.parse_args() run = Run.get_context() ws = run.experiment.workspace Model.register(model_path=os.path.join(args.input, "parallel_run_step.txt"), model_name="parallel_model", workspace=ws)
from azureml.core import Workspace from azureml.core import Model if __name__ == "__main__": ws = Workspace.from_config(path='./.azureml', _file_name='config.json') model = Model.register(model_name='model', tags={'area': 'trabajo_cluod'}, model_path='outputs/model.pkl', workspace=ws) print(model.name, model.id, model.version, sep='\t')
import logging import os from azureml.core import Run, Model import argparse logging.basicConfig(level=logging.INFO) log: logging.Logger = logging.getLogger(__name__) parser = argparse.ArgumentParser("register") parser.add_argument("--input", type=str, required=True) args = parser.parse_args() run = Run.get_context() ws = run.experiment.workspace Model.register(model_path=os.path.join(args.input, "model.json"), model_name="multifolder_model", workspace=ws)
AML_RESOURCE_GROUP = "" AML_WORKSPACE_NAME = "ajagarw-demo-aml-ws" svc_pr = ServicePrincipalAuthentication( tenant_id=AZURE_TENANT_ID, service_principal_id=AZURE_CLIENT_ID, service_principal_password=AZURE_CLIENT_SECRET) ws = Workspace(workspace_name=AML_WORKSPACE_NAME, subscription_id=AML_SUBSCRIPTION_ID, resource_group=AML_RESOURCE_GROUP, auth=svc_pr) model = Model.register( model_path="./artifacts/output", model_name="mlflow_sklearn_e2e", workspace=ws, ) # ### Import SynapseML Predict # In[4]: from pyspark.sql.functions import col, pandas_udf, udf, lit import azure.synapse.ml.predict as pcontext # ### Set some input parameters # <p>Data is stored on ADLS, model is stored on AML<p> # <p>Return type is int<p> # In[5]:
from azureml.train.hyperdrive import HyperDriveRun from shutil import copy2 parser = argparse.ArgumentParser() parser.add_argument('--saved-model', type=str, dest='saved_model', help='path to saved model file') parser.add_argument('--metrics_data', type=str, dest='metrics_data', help='metrics') parser.add_argument('--model_name', type=str, dest='model_name', help='model name') args = parser.parse_args() saved_model = args.saved_model print('saved_model') print(saved_model) metrics_data = args.metrics_data print('metrics_data') print(metrics_data) model_name = args.model_name print('model_name') print(model_name) model_output_dir = './model/' os.makedirs(model_output_dir, exist_ok=True) copy2(args.saved_model, model_output_dir) ws = Run.get_context().experiment.workspace model = Model.register(workspace=ws, model_name=model_name, model_path=model_output_dir)
# Copyright (c) Microsoft. All rights reserved. # Licensed under the MIT license. See LICENSE file in the project root for # full license information. import json from azureml.core import Workspace, Model, VERSION from azureml.core.authentication import ServicePrincipalAuthentication print(VERSION) with open("aml/config.json", "r") as f: config = json.load(f) auth = ServicePrincipalAuthentication( config["tenant_id"], config["service_principal_id"], config["service_principal_password"] ) ws = Workspace.create( name=config["workspace_name"], auth=auth, subscription_id=config['subscription_id'], resource_group=config['resource_group'], location=config['location'] exist_ok=True, show_output=True, ) Model.register(ws, 'models/TinyYOLO.onnx', 'TinyYOLO')
y, test_size=0.2, random_state=1) knn = KNeighborsClassifier() knn.fit(X_train, y_train) score = knn.predict(X_test) predictions = X_test.copy(deep=True) predictions["Prediction"] = score predictions["Actual"] = y_test print(predictions) if not os.path.isdir('outputs'): os.mkdir('outputs') model_path = os.path.join('outputs', 'highspender.pkl') dump(knn, model_path) # Register model from azureml.core import Model from azureml.core.run import Run run = Run.get_context() workspace = run.experiment.workspace model = Model.register(workspace=workspace, model_name='highspender', model_path=model_path)
import numpy as np import pandas as pd #we can deploy model as a real-time web service to several kinds of compute target like local compute, ACI(Azure Container instance), AKS, an azure function, IOT modules #AzureML uses containers as a deployment mechanism, packaging the model and the code to use it as an image that can be deployed to a container in your chosen compute target. #for the deployment of the model these are the steps: # 1. Register a trained model from azureml.core import Model classi_model = Model.register( workspace=ws, model_name='classi_model', model_path='model.pkl', # localpath description="classification model") #if we have a reference to the Run used to train the model run.register_model(model_name='classi_model', model_path='outputs/model.pkl', description='A classification Model') #Now we need inference configuration in which 2 things will be there 1. script which will return the prediction and 2. Environment script in which 1 will run. # Need to create Entry Script import json import joblib
from azureml.core.run import Run, _OfflineRun from azureml.core import Workspace, Model import joblib import argparse # Get context run = Run.get_context() ws = Workspace.from_config() if type(run) == _OfflineRun else run.experiment.workspace if __name__ == "__main__": # Parse args parser = argparse.ArgumentParser() parser.add_argument('--model_name', type=str) parser.add_argument('--metrics_data', type=str) parser.add_argument('--saved_model', type=str) args = parser.parse_args() print(f'--model_name={args.model_name}') print(f'--metrics_data={args.metrics_data}') print(f'--saved_model={args.saved_model}') # load model loaded_model = joblib.load(args.saved_model) print(loaded_model) # Register model model = Model.register(workspace=ws, model_path=args.saved_model, model_name=args.model_name) print(model)
blob_store.upload(src_dir=local_path, target_path=blob_path, overwrite=True, show_progress=True) #%% # ** Register the data as a dataset ** # %% now that the data is up on the blobstore we can register it as a dataset # to keep track of its versions and make it easily acessible dataset = Dataset.File.from_files( blob_store.path(blob_path + "/data.csv") ) dataset.register(ws, name="Campus_Recruitment_PCA_Training_Data", create_new_version=True) #%% # ** Upload and register the model as a Model ** #%% model = Model.register(workspace=ws, model_name='Campus_Recruitment_PCA', # Name of the registered model in your workspace. model_path='./Upload/Model/model.pkl', # Local file to upload and register as a model. sample_input_dataset=dataset, sample_output_dataset=None, description='PCA model for dimention reduction of the Campus Recruitment Dataset', ) print('Name:', model.name) print('Version:', model.version)
modelPath = args.model_path modelName = args.model_name modelDescription = args.model_description print('Model Path: ', modelPath) print('Model Name: ', modelName) print('Model Description: ', modelDescription) print('getting run context') run_context = Run.get_context() run_context.log('Author', 'Sam was here!') ws = run_context.experiment.workspace print('Model Registeration began') Model.register(workspace=ws, model_path=modelPath, model_name=modelName, tags=None, properties=None, description=modelDescription, datasets=None, model_framework=None, model_framework_version=None, child_paths=None, sample_input_dataset=None, sample_output_dataset=None, resource_configuration=None) print('Model Registeration completed')
print(trainData.shape) X = trainData[allfeatures[1:len(allfeatures)]].astype(float).values # exclude first feature y = trainData[allfeatures[0]].astype(float).values # use first feature X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42) from sklearn.ensemble import GradientBoostingClassifier model_GBM = GradientBoostingClassifier(random_state=42, verbose=1) model_GBM.fit(X_train, y_train) printScores(model_GBM.predict(X_train), y_train) printScores(model_GBM.predict(X_test), y_test) ##------------- Serialize the Model with open(args.modelName,'wb') as f: pickle.dump(model_GBM, f) ##------------- Upload the Model file explicitly into artifacts run.upload_file(name="./outputs/" + args.modelName, path_or_stream=args.modelName) print("Uploaded the model {} to experiment {}".format(args.modelName, run.experiment.name)) ##------------- Register the Model model = Model.register(model_path = args.modelName, model_name = args.modelName, tags = {'area': "QualityPrediction", 'type': "GBM"}, description = "Quality prediction model", workspace = ws)
# 07-model-registration-azure.py from azureml.core import Workspace from azureml.core import Model if __name__ == "__main__": ws = Workspace.from_config(path='./.azureml', _file_name='config.json') model = Model.register(model_name='digits_model', tags={'area': 'udea_training'}, model_path='outputs/digits_model.pkl', workspace=ws) print(model.name, model.id, model.version, sep='\t')
) ds = Datastore.get(ws, args.datastore) ds.download("assets", args.dataset, overwrite=False) # Load dataset url = "assets/"+args.dataset names = ['sepal-length', 'sepal-width', 'petal-length', 'petal-width', 'class'] dataset = read_csv(url, names=names) # Split-out validation dataset array = dataset.values X = array[:,0:4] y = array[:,4] X_train, X_validation, Y_train, Y_validation = train_test_split(X, y, test_size=0.20, random_state=1) # Make predictions on validation dataset model = SVC(gamma='auto') model.fit(X_train, Y_train) predictions = model.predict(X_validation) # Evaluate predictions print(accuracy_score(Y_validation, predictions)) print(confusion_matrix(Y_validation, predictions)) print(classification_report(Y_validation, predictions)) joblib.dump(model, "trained_model1.pkl") Model.register(ws, "trained_model1.pkl", "new_model")
# Reference: "Fully Convolutional Networks for Semantic Segmentation" layer_names = ["block3_pool", "block4_pool", "block5_pool"] layers = [base_model.get_layer(layer) for layer in layer_names] decoder_output = build_decoder(*layers) model = keras.Model(base_model.inputs, decoder_output) model.compile(optimizer="adam", loss=keras.losses.BinaryCrossentropy(from_logits=True), metrics=["accuracy"]) if args.plot_model: keras.utils.plot_model(model, to_file="vgg16_segmentation.png", show_shapes=True, show_layer_names=True) model.save(args.path_model) # Register the model run = Run.get_context() ws = run.experiment.workspace Model.register(workspace=ws, model_path=args.path_model, model_name='segmentation_new', description='Instance of untrained model', model_framework=Model.Framework.TENSORFLOW, model_framework_version='2.3')
# 07-model-registration-azure.py from azureml.core import Workspace from azureml.core import Model if __name__ == "__main__": ws = Workspace.from_config() model = Model.register(model_name='cifar_10', tags={'area': 'udea_training'}, model_path='outputs/cifar_10_model.pkl', workspace=ws) print(model.name, model.id, model.version, sep='\t')
lr=args.learning_rate) criterion = nn.CrossEntropyLoss() if args.cuda: decoder.cuda() start = time.time() all_losses = [] loss_avg = 0 print("Start training for {} epochs...".format(args.n_epochs)) for epoch in tqdm(range(1, args.n_epochs + 1)): loss = train(*random_training_set(args.chunk_len, args.batch_size)) loss_avg += loss if epoch % args.print_every == 0: print('[%s (%d %d%%) %.4f]' % (time_since(start), epoch, epoch / args.n_epochs * 100, loss)) print(generate(decoder, 'Wh', 100, cuda=args.cuda), '\n') # Saving model to outputs/ in Azure ML save_filename = "outputs/" + args.modelname + ".pt" torch.save(decoder.state_dict(), save_filename) # TODO: Use the Model class and the 'register' method to upload the model to Azure ML # HINT: # https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.core.model.model?view=azure-ml-py model = Model.register(workspace=ws, model_name=args.modelname, model_path="outputs/") # Complete the run run.complete()
# Initialize a workspace ws = Workspace.from_config( "C:/Users/Danilo.Bento/Icon Dropbox/DEVDATA/RO/DEVELOPMENT/SIB2/dev/.azureml/config.json" ) print('Workspace name: ' + ws.name, 'Azure region: ' + ws.location, 'Subscription id: ' + ws.subscription_id, 'Resource group: ' + ws.resource_group, 'Workspace connected', sep='\n') model = Model.register( workspace=ws, model_name='mod5_test', model_path= 'C:/Users/Danilo.Bento/Icon Dropbox/DEVDATA/RO/DEVELOPMENT/SIB2/tutorials/model5/mod5_deploy/mod5_azure', # local path #child_paths=['C:/Users/Danilo.Bento/Icon Dropbox/DEVDATA/RO/DEVELOPMENT/SIB2/tutorials/model5/mod5_deploy/mod5_azure/yolov5/runs/exp0/weights/last.pt'], description= 'Test model based on model 5 done in phase 1. It trained for 250 epochs. Contain original folder structure', #tags={'dept': 'sales'}, model_framework=Model.Framework.PYTORCH, model_framework_version='1.6.0') for model in Model.list(ws): # Get model name and auto-generated version print(model.name, 'version:', model.version) # import logging # logging.basicConfig(level=logging.DEBUG) # print(Model.get_model_path(model_name='mod5_test'))
np.array(df[label], dtype=int), test_size=0.9) # train model mod = DecisionTreeClassifier(max_depth=5) mod.fit(x_train, y_train) # make sure we can predict a value preds = mod.predict(x_test) cm = confusion_matrix(y_test, preds) common.plot_confusion_matrix(cm, ['0', '1']) plt.savefig("outputs/confusion_matrix.png") precision = precision_score(y_test, preds) accuracy = accuracy_score(y_test, preds) run.log('precision', precision) run.log('accuracy', accuracy) with open('outputs/decision_tree_model.pkl', 'wb') as f: pickle.dump(mod, f) Model.register(run.experiment.workspace, 'outputs/decision_tree_model.pkl', 'decision_tree_model', tags={ 'precision': precision, 'accuracy': accuracy })
# 07-model-registration-azure.py from azureml.core import Workspace from azureml.core import Model if __name__ == "__main__": ws = Workspace.from_config(path='./.azureml',_file_name='config.json') model = Model.register(model_name='titanic_model', tags={'area': 'udea_project' , 'scoring' : 0.70}, model_path='outputs/clf.pkl', workspace = ws) print(model.name, model.id, model.version, sep='\t')