def init():
global model
global username
global git_repo_url
global gitAPI
global jenkins_key
global jenkins_url
global sql_server
global sql_database
global sql_username
global sql_password
global used_file_extensions
# TODO: Change model name
model_name = 'TestAutoModel'
# TODO: Update list of file extensions used in your project
used_file_extensions = ['swift', 'pdf']
model_path = Model.get_model_path(model_name=model_name)
model = joblib.load(model_path)
load_dotenv(verbose=True, dotenv_path=Path('.') / 'server_files' / 'env')
username = os.getenv("USERNAME")
git_repo_url = os.getenv("GIT_REPO_URL")
gitAPI = os.getenv("GIT_KEY")
jenkins_key = os.getenv("JENKINS_KEY")
jenkins_url = os.getenv("JENKINS_URL")
sql_server = os.getenv("SQL_SERVER")
sql_database = os.getenv("SQL_DATABASE")
sql_username = os.getenv("SQL_USERNAME")
sql_password = os.getenv("SQL_PASSWORD")
def init():
global g_tf_sess
svc_pr_password = "******"
svc_pr = ServicePrincipalAuthentication(
tenant_id="72f988bf-86f1-41af-91ab-2d7cd011db47",
service_principal_id="8a3ddafe-6dd6-48af-867e-d745232a1833",
service_principal_password="******")
ws = Workspace(
subscription_id="c46a9435-c957-4e6c-a0f4-b9a597984773",
resource_group="mlops",
workspace_name="gputraining",
auth=svc_pr
)
model_root = os.getenv('AZUREML_MODEL_DIR')
# Pull down the model from the workspace
model_path = Model.get_model_path("tf-dnn-mnist")
tf_model_folder = 'model'
# Create a model folder in the current directory
os.makedirs('./outputs', exist_ok=True)
os.makedirs('./outputs/model', exist_ok=True)
# Construct a graph to execute
tf.reset_default_graph()
saver = tf.train.import_meta_graph(os.path.join(model_path, 'tf-dnn-mnist.meta'))
g_tf_sess = tf.Session()
#saver.restore(g_tf_sess, os.path.join(model_path, tf_model_folder, 'tf-dnn-mnist.model'))
saver.restore(g_tf_sess, os.path.join(model_path, 'tf-dnn-mnist'))
def init():
# Runs when the pipeline step is initialized
global model
# load the model
model_path = Model.get_model_path('classification_model')
model = joblib.load(model_path)
def init():
global model_p
print("Executed")
model_path = Model.get_model_path('optimal_model.joblib')
#model_path = Model.get_model_path('model_automl')
print(model_path)
model_p = joblib.load(model_path)
def init():
# Runs when the pipeline step is initialized
global model
# load the model
model_path = Model.get_model_path('diabetes_model')
model = joblib.load(model_path)
def init():
global model
global inputs_dc, prediction_dc
# The AZUREML_MODEL_DIR environment variable indicates
# a directory containing the model file you registered.
model_path = Model.get_model_path(model_name="model")
model = joblib.load(model_path)
inputs_dc = ModelDataCollector("sample-model", designation="inputs", feature_names=["feat1", "feat2", "feat3", "feat4"])
prediction_dc = ModelDataCollector("sample-model", designation="predictions", feature_names=["prediction"])
def init():
# retrieve the path to the model file using the model name
model_name = 'automl_best_model'
global model
model_path = Model.get_model_path(model_name)
model = joblib.load(model_path)
def init():
global model
# AZUREML_MODEL_DIR is an environment variable created during deployment. Join this path with the filename of the model file.
# It holds the path to the directory that contains the deployed model (./azureml-models/$MODEL_NAME/$VERSION).
# If there are multiple models, this value is the path to the directory containing all deployed models (./azureml-models).
model_path = Model.get_model_path('new_model')
#model_path = os.path.join(os.getenv('AZUREML_MODEL_DIR'), 'trained_model.pkl')
# Deserialize the model file back into a sklearn model
model = joblib.load(model_path)
def init():
global g_tf_sess
# pull down model from workspace
model_path = Model.get_model_path("mnist")
# contruct graph to execute
tf.reset_default_graph()
saver = tf.train.import_meta_graph(os.path.join(model_path, 'mnist-tf.model.meta'))
g_tf_sess = tf.Session(config=tf.ConfigProto(device_count={'GPU': 0}))
saver.restore(g_tf_sess, os.path.join(model_path, 'mnist-tf.model'))
def init():
global g_tf_sess
# Pull down the model from the workspace
model_path = Model.get_model_path("mnist")
# Construct a graph to execute
tf.reset_default_graph()
saver = tf.train.import_meta_graph(os.path.join(model_path, 'mnist-tf.model.meta'))
g_tf_sess = tf.Session()
saver.restore(g_tf_sess, os.path.join(model_path, 'mnist-tf.model'))
def main(args):
# Load model
print("Loading model")
model_path = Model.get_model_path(model_name=args.model_name,
version=args.model_version)
with open(model_path, "rb") as model_file:
my_model = joblib.load(filename=model_file)
# Load mounted dataset path
print("Get file dataset mount paths")
input_path = os.environ.get(args.input_dataset, None)
output_path = os.environ.get(
f"AZUREML_DATAREFERENCE_{args.output_dataset}", None)
print(f"Input mount path: {input_path}")
print(f"Output mount path: {output_path}")
# Create output path
print("Creating output path and /outputs folder")
os.makedirs(name=output_path, exist_ok=True)
# Get input file paths list
print("Get input file paths")
paths = []
for root, dirs, files in os.walk(input_path):
for filename in files:
if ".parquet" in filename:
path = os.path.join(root, filename)
paths.append(path)
print(f"Path List: {paths}")
# Create one large dataframe from all files
if len(paths) > 0:
print("Creating one large pandas dataframe")
df = pd.read_parquet(path=paths.pop(), engine="auto")
for path in paths:
df_temp = pd.read_parquet(path=path, engine="auto")
df.append(df_temp)
else:
print("File dataset does not include any files")
return
# Score data
print("Scoring data in dataframe")
num_rows = df.shape
predictions = my_model.predict(df) #.reshape((num_rows, 1))
result = df
result["predictions"] = predictions
# Save parquet for training
print("Saving Parquet file for training")
result.to_parquet(path=os.path.join(output_path, "result.parquet"))
def init():
global model
model_path = Model.get_model_path('qubvel-segmentation_models-u-net')
BACKBONE = 'efficientnetb3'
CLASSES = ['car']
n_classes = 1 if len(CLASSES) == 1 else (
len(CLASSES) + 1) # case for binary and multiclass segmentation
activation = 'sigmoid' if n_classes == 1 else 'softmax'
model = sm.Unet(BACKBONE, classes=n_classes, activation=activation)
model.load_weights(os.path.join(model_path, 'best_model.h5'))
def init():
"""
This function loads the bert model, bert tokenizer, and class names from the model
directory. The contents of the folder can also be seen under Artifacts in the model registry.
"""
global model
global tokenizer
global classes
model_dir = Model.get_model_path(model_name='page_binary_bert', version=2)
model = BertForSequenceClassification.from_pretrained(model_dir)
tokenizer = BertTokenizer.from_pretrained(model_dir)
def init():
global model
model_path = Model.get_model_path("mask_rcnn_horovod")
# initialize the inference configuration
config = LesionBoundaryInferenceConfig()
# initialize the Mask R-CNN model for inference
model = modellib.MaskRCNN(mode="inference",
config=config,
model_dir=LOGS_AND_MODEL_DIR)
model.load_weights(os.path.join(model_path, 'mask_rcnn_lesion_0020.h5'),
by_name=True)
def init():
try:
print('Loading Model')
model_params = parse_args()
aml_model = get_model(model_name=model_params[0],
model_version=model_params[1],
tag_name=model_params[2],
tag_value=model_params[3])
global model
model_path = Model.get_model_path(model_name=aml_model.name,
version=aml_model.version)
model = joblib.load(model_path)
print(f'model:{aml_model.name} downloading is successful')
except Exception as ex:
print(ex)
def init():
"""
Initializer called once per node that runs the scoring job. Parse command
line arguments and get the right model to use for scoring.
"""
try:
print("Initializing batch scoring script...")
# Get the model using name/version/tags filter
model_filter = parse_args()
amlmodel = get_model(model_name=model_filter[0],
model_version=model_filter[1],
tag_name=model_filter[2],
tag_value=model_filter[3])
# Load the model using name/version found
global model
modelpath = Model.get_model_path(model_name=amlmodel.name,
version=amlmodel.version)
model = joblib.load(modelpath)
print("Loaded model {}".format(model_filter[0]))
except Exception as ex:
print("Error: {}".format(ex))
preprocessing_function=binarize_mask)
image_datagen = (keras.preprocessing.image.ImageDataGenerator(
**generator_config_images))
mask_datagen = (keras.preprocessing.image.ImageDataGenerator(
**generator_config_masks))
image_generator = (image_datagen.flow_from_directory(PATH_IMAGES,
target_size=(224, 224),
class_mode=None,
seed=SEED))
mask_generator = (mask_datagen.flow_from_directory(PATH_MASKS,
target_size=(224, 224),
class_mode=None,
seed=SEED))
# Download the model architecture from AzureML
run = Run.get_context()
ws = run.experiment.workspace
path_model = Model.get_model_path("segmentation_new", version=2, _workspace=ws)
# Load and train the model
loaded_model = keras.models.load_model(path_model)
train_generator = zip(image_generator, mask_generator)
loaded_model.fit(train_generator, epochs=3, steps_per_epoch=100)
# Save the model in the path specified of the compute target
loaded_model.save(args.trained_model)
def init():
global model
model_path = Model.get_model_path("moa_prediction_model")
print("Model Path is ", model_path)
model = joblib.load(model_path)
]
OPTIMIZER = tf.keras.optimizers.Adam(learning_rate=LR,
beta_1=0.9,
beta_2=0.999)
m = model_tools.get_multiclass_model(depth=DEPTH,
nclasses=NCLASSES,
optim=OPTIMIZER,
loss=get_gen_dice,
mets=METRICS,
bias=BIAS)
# if a model directory provided we will reload previously trained model and weights
if args.model_id:
# we will package the 'models' directory within the 'azure' dirrectory submitted with experiment run
model_dir = Model.get_model_path(args.model_id, _workspace=ws)
# model_dir = os.path.join('./models', args.model_id, '1', 'outputs')
# load our previously trained model and weights
model_file = glob.glob(os.path.join(model_dir, '*.h5'))[0]
weights_file = glob.glob(os.path.join(model_dir, '*.hdf5'))[0]
m, checkpoint = model_tools.retrain_model(
model_file,
checkpoint,
evaluation,
'mean_iou',
weights_file,
custom_objects={'get_gen_dice': get_gen_dice},
lr=LR)
# TODO: make this dynamic
initial_epoch = 100
def init():
global tokenizer, model
tokenizer = BertTokenizer.from_pretrained('bert-base-cased')
model_dir = Model.get_model_path('bert-mrpc')
model = TFBertForSequenceClassification.from_pretrained(model_dir)
def init():
global model
model_path = Model.get_model_path("classi_model")
model = joblib.load(model_path)
def init():
global model
model = joblib.load(Model.get_model_path('classi_model'))
import pickle
import json
import numpy
from azureml.core import Workspace
from azureml.core import Model, Run
from sklearn.externals import joblib
import argparse
##------------- Get Workspace
run = Run.get_context()
exp = run.experiment
ws = run.experiment.workspace
##------------- Get Arguments
parser = argparse.ArgumentParser("train")
parser.add_argument("--modelName", type=str)
args = parser.parse_args()
##------------- Model Scoring
model_path = Model.get_model_path(model_name=args.modelName)
model = joblib.load(model_path)
rawdata = '{ "data" : [4.996352,41.68612,41.79799,4.998839,5.051471,27.01976,28.5,36,39.6,22.7,921.2494,2081.41,2170.84,1017.489,24.68081,29.16544,29.65642,29.15765,21.35513,140.4473,133.4049,4.998026,63.68074,25.13597,32.08001,5.075305,27.029,7.516817,54.95351,4.998026,60,40,13,4.933381,11,5.000257,5.001157,28283.33,26892.26,18333.06641]}'
data = json.loads(rawdata)['data']
data = [numpy.array(data)]
result = model.predict(data)
print("Result =>", result.tolist())
def load_model(modelName):
modelPath = Model.get_model_path(modelName)
model = joblib.load(modelPath)
return model
def init():
global model
# Get the path to the registered model file and load it
model_path = Model.get_model_path('classification_model')
model = joblib.load(model_path)
def inner(*args, **kwargs):
run = kwargs.get("run")
ws = run.experiment.workspace
def register_model(model_name, model_path):
model_config = next(
iter(
filter(lambda x: x["name"] == model_name,
self.output_reg_models)))
tags = model_config.get("tags")
description = model_config.get("description")
Model.register(workspace=ws,
model_path=model_path,
model_name=model_name,
tags=tags,
description=description)
def register_dataset(dataset_name, dataframe):
dataset_config = next(
iter(
filter(lambda x: x["name"] == dataset_name,
self.output_reg_datasets)))
datastore = dataset_config.get("datastore") or "default"
description = dataset_config.get("description")
tags = dataset_config.get("tags")
if datastore == "default":
ds = ws.get_default_datastore()
else:
ds = Datastore.get(workspace=ws, datastore_name=datastore)
target_path = f'experiment/{run.experiment.name}/run/{run.number}/out/{dataset_name}'
default_output_dataset_tags = {
"format": self.
OUTPUT_FORMAT, # Dataset.Tabular.register_pandas_dataframe always writes a parquet
"experiment": run.experiment.name,
"run": run.number
}
output_dataset_tags = {**default_output_dataset_tags, **tags}
Dataset.Tabular.register_pandas_dataframe(
dataframe,
target=(ds, target_path),
name=dataset_name,
description=description,
tags=output_dataset_tags)
dataframes = {}
for i, d in enumerate(self.input_datasets or []):
dkey = self.named_input_keys[i]
dataframes[dkey] = run.input_datasets[
dkey].to_pandas_dataframe()
for d in self.input_reg_datasets or []:
dname = d["name"]
dver = d.get("version")
if dver == "latest":
dver = None
dataframes[dname] = Dataset.get_by_name(
ws, name=dname, version=dver).to_pandas_dataframe()
kwargs["dataframes"] = dataframes
models = {}
for m in self.input_reg_models or []:
mname = m["name"]
mver = m.get("version")
if mver == "latest":
mver = None
models[mname] = Model.get_model_path(model_name=mname,
version=mver,
_workspace=ws)
kwargs["models"] = models
pipeline_data_dirs = []
for p in self.input_pipeline_data_dirs or []:
pipeline_data_dirs.append(p["name"])
kwargs["pipeline_data_dirs"] = pipeline_data_dirs
register_dataframes, register_models = func(*args, **kwargs)
for k in register_dataframes or {}:
v = register_dataframes[k]
register_dataset(dataset_name=k, dataframe=v)
for k in register_models or {}:
v = register_models[k]
register_model(model_name=k, model_path=v)
if __name__ == "__main__":
run = Run.get_context()
args = get_args()
model_name = args.model_name
ouput_dataset_name = args.ouput_dataset_name
test_dataset_name = args.test_dataset_name
target_column_name = args.target_column_name
print("args passed are: ")
print(model_name)
print(test_dataset_name)
print(ouput_dataset_name)
print(target_column_name)
model_path = Model.get_model_path(model_name)
model_file_name = get_model_filename(run, model_name, model_path)
print(model_file_name)
fitted_model = get_model(model_path, model_file_name)
X_test_df, y_test = get_data(
run,
fitted_model,
target_column_name,
test_dataset_name,
)
infer_forecasting_dataset_tcn(X_test_df, y_test, fitted_model,
args.output_path, ouput_dataset_name)
def init():
global model
model_path = Model.get_model_path('bestHpModel')
model = joblib.load(model_path)
def init():
global model
# Get the path to the deployed model file and load it
model_path = Model.get_model_path('diabetes_model')
model = joblib.load(model_path)
def init():
global model
model_path = Model.get_model_path('best-model-machine-cpu-hd')
model = joblib.load(model_path)
