def train_step(train_dir, compute_target):
max_depth = PipelineParameter(name='max_depth', default_value=5)
n_estimators = PipelineParameter(name='n_estimators', default_value=500)
model_dir = PipelineData(name='model_dir',
pipeline_output_name='model_dir',
datastore=train_dir.datastore,
output_mode='mount',
is_directory=True)
outputs = [model_dir]
outputs_map = {'model_dir': model_dir}
estimator = SKLearn(source_directory=os.path.dirname(
os.path.abspath(__file__)),
entry_script='train.py',
compute_target=compute_target)
step = EstimatorStep(estimator=estimator,
estimator_entry_script_arguments=[
'--train_dir', train_dir, '--output_dir',
model_dir, '--max_depth', max_depth,
'--n_estimators', n_estimators
],
inputs=[train_dir],
compute_target=compute_target,
outputs=outputs,
allow_reuse=False)
return step, outputs_map
def evaluate_step(model_dir, test_dir, compute_target):
accuracy_file = PipelineData(name='accuracy_file',
pipeline_output_name='accuracy_file',
datastore=test_dir.datastore,
output_mode='mount',
is_directory=False)
outputs = [accuracy_file]
outputs_map = {'accuracy_file': accuracy_file}
estimator = SKLearn(source_directory=os.path.dirname(
os.path.abspath(__file__)),
entry_script='evaluate.py',
compute_target=compute_target)
step = EstimatorStep(estimator=estimator,
estimator_entry_script_arguments=[
'--test_dir', test_dir, '--model_dir', model_dir,
'--accuracy_file', accuracy_file
],
inputs=[model_dir, test_dir],
outputs=outputs,
compute_target=compute_target,
allow_reuse=True)
return step, outputs_map
def process_step(datastore: Datastore, compute: ComputeTarget,
path_on_datastore: str) -> (PipelineData, EstimatorStep):
datapath = DataPath(datastore=datastore,
path_on_datastore=path_on_datastore)
data_path_pipeline_param = (PipelineParameter(name="data",
default_value=datapath),
DataPathComputeBinding(mode='mount'))
seer_tfrecords = PipelineData("tfrecords_set",
datastore=datastore,
is_directory=True)
prep = Estimator(source_directory='.',
compute_target=compute,
entry_script='prep.py',
pip_requirements_file='requirements.txt')
prepStep = EstimatorStep(name='Data Preparation',
estimator=prep,
estimator_entry_script_arguments=[
"--source_path", data_path_pipeline_param,
"--target_path", seer_tfrecords
],
inputs=[data_path_pipeline_param],
outputs=[seer_tfrecords],
compute_target=compute)
return seer_tfrecords, prepStep
def train_step(train_dir, valid_dir, compute_target):
'''
This step will fine-tune a RESNET-18 model on our dataset using PyTorch.
It will use the corresponding input image directories as training and validation data.
:param train_dir: The reference to the directory containing the training data
:type train_dir: DataReference
:param valid_dir: The reference to the directory containing the validation data
:type valid_dir: DataReference
:param compute_target: The compute target to run the step on
:type compute_target: ComputeTarget
:return: The training step, step outputs dictionary (keys: model_dir)
:rtype: EstimatorStep, dict
'''
num_epochs = PipelineParameter(name='num_epochs', default_value=25)
batch_size = PipelineParameter(name='batch_size', default_value=16)
learning_rate = PipelineParameter(name='learning_rate', default_value=0.001)
momentum = PipelineParameter(name='momentum', default_value=0.9)
model_dir = PipelineData(
name='model_dir',
pipeline_output_name='model_dir',
datastore=train_dir.datastore,
output_mode='mount',
is_directory=True)
outputs = [model_dir]
outputs_map = { 'model_dir': model_dir }
estimator = PyTorch(
source_directory=os.path.dirname(os.path.abspath(__file__)),
entry_script='train.py',
framework_version='1.3',
compute_target=compute_target,
use_gpu=True)
step = EstimatorStep(
name="Train Model",
estimator=estimator,
estimator_entry_script_arguments=[
'--train_dir', train_dir,
'--valid_dir', valid_dir,
'--output_dir', model_dir,
'--num_epochs', num_epochs,
'--batch_size', batch_size,
'--learning_rate', learning_rate,
'--momentum', momentum
],
inputs=[train_dir, valid_dir],
compute_target=compute_target,
outputs=outputs,
allow_reuse=False)
return step, outputs_map
def evaluate_step(model_dir, test_dir, compute_target):
'''
This step evaluates the trained model on the testing data and outputs the accuracy.
:param model_dir: The reference to the directory containing the trained model
:type model_dir: DataReference
:param test_dir: The reference to the directory containing the testing data
:type test_dir: DataReference
:param compute_target: The compute target to run the step on
:type compute_target: ComputeTarget
:return: The evaluate step, step outputs dictionary (keys: accuracy_file)
:rtype: EstimatorStep, dict
'''
accuracy_file = PipelineData(
name='accuracy_file',
pipeline_output_name='accuracy_file',
datastore=test_dir.datastore,
output_mode='mount',
is_directory=False)
outputs = [accuracy_file]
outputs_map = { 'accuracy_file': accuracy_file }
estimator = PyTorch(
source_directory=os.path.dirname(os.path.abspath(__file__)),
entry_script='evaluate.py',
framework_version='1.3',
compute_target=compute_target,
use_gpu=True)
step = EstimatorStep(
name="Evaluate Model",
estimator=estimator,
estimator_entry_script_arguments=[
'--test_dir', test_dir,
'--model_dir', model_dir,
'--accuracy_file', accuracy_file
],
inputs=[model_dir, test_dir],
outputs=outputs,
compute_target=compute_target,
allow_reuse=True)
return step, outputs_map
def estimator(data, store, compute):
estimator = Estimator(source_directory=os.path.dirname(
os.path.abspath(__file__)),
compute_target=compute,
entry_script='train.py',
pip_packages=['azureml-dataprep', 'lightgbm'])
output = PipelineData("output", datastore=store)
step = EstimatorStep(name=os.path.basename(__file__),
estimator=estimator,
estimator_entry_script_arguments=[
'--input_dir', data, '--output_dir', output
],
inputs=[data],
outputs=[output],
compute_target=estimator._compute_target,
allow_reuse=True)
return step, output
def register_step(datastore: Datastore, input_data: PipelineData,
compute: ComputeTarget,
build: str) -> (PipelineData, EstimatorStep):
seer_model = PipelineData("model", datastore=datastore, is_directory=True)
register = Estimator(source_directory='.',
compute_target=compute,
entry_script='register.py')
registerStep = EstimatorStep(name='Model Registration',
estimator=register,
estimator_entry_script_arguments=[
"--source_path", input_data,
"--target_path", seer_model, "--build",
build
],
inputs=[input_data],
outputs=[seer_model],
compute_target=compute)
return seer_model, registerStep
def train_step(datastore: Datastore, input_data: PipelineData,
compute: ComputeTarget) -> (PipelineData, EstimatorStep):
seer_training = PipelineData("train",
datastore=datastore,
is_directory=True)
train = Estimator(source_directory='.',
compute_target=compute,
entry_script='train.py',
use_gpu=True,
pip_requirements_file='requirements.txt')
trainStep = EstimatorStep(name='Model Training',
estimator=train,
estimator_entry_script_arguments=[
"--source_path", input_data, "--target_path",
seer_training, "--epochs", 15, "--batch", 10,
"--lr", 0.001
],
inputs=[input_data],
outputs=[seer_training],
compute_target=compute)
return seer_training, trainStep
source_directory = 'Training'
est = TensorFlow(source_directory=source_directory,
compute_target=compute_target_cpu,
entry_script='train.py',
use_gpu=False,
framework_version='1.13')
from azureml.pipeline.steps import EstimatorStep
trainingStep = EstimatorStep(name="Training-Step",
estimator=est,
estimator_entry_script_arguments=[
"--input_data_location", processed_mnist_data,
'--batch-size', 50, '--first-layer-neurons',
300, '--second-layer-neurons', 100,
'--learning-rate', 0.01, "--release_id", 0,
'--model_name', model_name
],
runconfig_pipeline_params=None,
inputs=[processed_mnist_data],
compute_target=compute_target_cpu)
print("Model Training Step is Completed")
# source directory
source_directory = 'RegisterModel'
modelEvalReg = PythonScriptStep(
name="Evaluate and Register Model",
script_name="evaluate_model.py",
arguments=["--release_id", 0, '--model_name', model_name],
def main():
load_dotenv()
workspace_name = os.environ.get("BASE_NAME") + "-AML-WS"
resource_group = "AML-RG-" + os.environ.get("BASE_NAME")
subscription_id = os.environ.get("SUBSCRIPTION_ID")
tenant_id = os.environ.get("TENANT_ID")
app_id = os.environ.get("SP_APP_ID")
app_secret = os.environ.get("SP_APP_SECRET")
sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN")
train_script_path = os.environ.get("TRAIN_SCRIPT_PATH")
evaluate_script_path = os.environ.get("EVALUATE_SCRIPT_PATH")
vm_size = os.environ.get("AML_COMPUTE_CLUSTER_SKU")
compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME")
aks_name = os.environ.get("AKS_CLUSTER_NAME")
model_name = os.environ.get("MODEL_NAME")
build_id = os.environ.get("BUILD_BUILDID")
pipeline_name = os.environ.get("TRAINING_PIPELINE_NAME")
experiment_name = os.environ.get("EXPERIMENT_NAME")
# Get Azure machine learning workspace
aml_workspace = get_workspace(workspace_name, resource_group,
subscription_id, tenant_id, app_id,
app_secret)
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, compute_name, vm_size)
if aml_compute is not None:
print(aml_compute)
run_config = RunConfiguration(conda_dependencies=CondaDependencies.create(
conda_packages=['numpy', 'pandas', 'scikit-learn', 'keras'],
pip_packages=[
'azure', 'azureml-sdk', 'azure-storage', 'azure-storage-blob',
'transformers>=2.1.1', 'tensorflow>=2.0.0', 'tensorflow-gpu>=2.0.0'
]))
run_config.environment.docker.enabled = True
datastore_name = 'tfworld'
container_name = 'azure-service-classifier'
account_name = 'johndatasets'
sas_token = '?sv=2019-02-02&ss=bfqt&srt=sco&sp=rl&se=2021-06-02T03:40:25Z&st=2020-03-09T19:40:25Z&spr=https&sig=bUwK7AJUj2c%2Fr90Qf8O1sojF0w6wRFgL2c9zMVCWNPA%3D'
try:
existing_datastore = Datastore.get(aml_workspace, datastore_name)
except: # noqa: E722
existing_datastore = Datastore \
.register_azure_blob_container(workspace=aml_workspace,
datastore_name=datastore_name,
container_name=container_name,
account_name=account_name,
sas_token=sas_token
)
azure_dataset = Dataset.File.from_files(path=(existing_datastore, 'data'))
azure_dataset = azure_dataset.register(
workspace=aml_workspace,
name='Azure Services Dataset',
description='Dataset containing azure related posts on Stackoverflow',
create_new_version=True)
azure_dataset.to_path()
input_data = azure_dataset.as_named_input('input_data1').as_mount(
'/tmp/data')
model_name = PipelineParameter(name="model_name", default_value=model_name)
max_seq_length = PipelineParameter(name="max_seq_length",
default_value=128)
learning_rate = PipelineParameter(name="learning_rate", default_value=3e-5)
num_epochs = PipelineParameter(name="num_epochs", default_value=3)
export_dir = PipelineParameter(name="export_dir",
default_value="./outputs/exports")
batch_size = PipelineParameter(name="batch_size", default_value=32)
steps_per_epoch = PipelineParameter(name="steps_per_epoch",
default_value=100)
# initialize the TensorFlow estimator
estimator = TensorFlow(source_directory=sources_directory_train,
entry_script=train_script_path,
compute_target=aml_compute,
framework_version='2.0',
use_gpu=True,
pip_packages=[
'transformers==2.0.0',
'azureml-dataprep[fuse,pandas]==1.3.0'
])
train_step = EstimatorStep(
name="Train Model",
estimator=estimator,
estimator_entry_script_arguments=[
"--data_dir", input_data, "--max_seq_length", max_seq_length,
"--learning_rate", learning_rate, "--num_epochs", num_epochs,
"--export_dir", export_dir, "--batch_size", batch_size,
"--steps_per_epoch", steps_per_epoch
],
compute_target=aml_compute,
inputs=[input_data],
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=evaluate_script_path,
compute_target=aml_compute,
source_directory=sources_directory_train,
arguments=[
"--model_name",
model_name,
"--build_id",
build_id,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
# Currently, the Evaluate step will automatically register
# the model if it performs better. This step is based on a
# previous version of the repo which utilized JSON files to
# track evaluation results.
evaluate_step.run_after(train_step)
steps = [evaluate_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=pipeline_name,
description="Model training/retraining pipeline",
version=build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
response = published_pipeline.submit( # noqa: F841
workspace=aml_workspace,
experiment_name=experiment_name)
# Get AKS cluster for deployment
aks_compute = get_aks(aml_workspace, aks_name)
if aks_compute is not None:
print(aks_compute)
# environment = Environment.from_conda_specification(name="sentiment-env", file_path="experiment-env.yml")
# environment.register(ws)
environment = Environment.get(ws, "sentiment-env")
estimator = TensorFlow(
source_directory="imdb",
entry_script="experiment.py",
framework_version="2.1",
conda_packages=["python=3.7.4", "tensorflow", "tensorflow-datasets"],
pip_packages=["azureml-sdk[notebooks,automl,explain]"],
compute_target="archi-trainer")
model_step = EstimatorStep(
name="training model",
estimator=estimator,
compute_target="archi-trainer",
estimator_entry_script_arguments=['--n-words', 5000, '--epochs', 2])
# register_step = PythonScriptStep(name="register pipeline", source_directory="sentiment_analysis", script_name="registration.py", compute_target="dummy", runconfig=run_config)
# register_step.run_after(model_step)
sentiment_pipe = Pipeline(workspace=ws, steps=[model_step])
sentiment_pipe.validate()
experiment = Experiment(workspace=ws, name="sentiment-analysis")
run = experiment.submit(config=sentiment_pipe)
run.wait_for_completion(show_output=True)
ds.upload('outputs/sentiment_model.h5',
'models',
data_ref = DataReference(datastore=default_ds,
data_reference_name='data_ref',
path_on_datastore="config/")
estimator = Estimator(source_directory=experiment_folder,
compute_target=pipeline_cluster,
environment_definition=pipeline_run_config.environment,
entry_script='train.py')
# Step 1, run the estimator to train the model
train_step = EstimatorStep(
name="Train Model",
estimator=estimator,
estimator_entry_script_arguments=[
'--output_folder', model_folder, '--data_dir', data_ref
],
inputs=[fraud_ds.as_named_input('fraud_train'), data_ref],
outputs=[model_folder],
compute_target=pipeline_cluster,
allow_reuse=True)
# Step 2, run the model registration script
register_step = PythonScriptStep(name="Register Model",
source_directory=experiment_folder,
script_name="register.py",
arguments=['--model_folder', model_folder],
inputs=[model_folder],
compute_target=pipeline_cluster,
runconfig=pipeline_run_config,
allow_reuse=True)
#######################################################################################################
est = TensorFlow(source_directory='./scripts/train',
compute_target=GPU_compute_target,
entry_script="estimator_training.py",
pip_packages=[
'keras<=2.3.1', 'matplotlib', 'opencv-python',
'azure-storage-blob==2.1.0', 'tensorflow-gpu==2.0.0'
],
conda_packages=['scikit-learn==0.22.1'],
use_gpu=True)
est_step = EstimatorStep(name="Estimator_Train",
estimator=est,
estimator_entry_script_arguments=[
'--PreProcessingData', PreProcessingData
],
inputs=[PreProcessingData],
runconfig_pipeline_params=None,
compute_target=GPU_compute_target)
#######################################################################################################
register_step = PythonScriptStep(name="register_step",
script_name="estimator_register.py",
runconfig=run_config_user_managed,
source_directory='./scripts/register',
arguments=['--ModelData', ModelData],
outputs=[ModelData],
compute_target=GPU_compute_target)
#######################################################################################################
def create_experiment_config(workspace):
########################################
### Creating data prep Pipeline Step ###
########################################
# Load settings
print("Loading settings")
data_prep_step_path = os.path.join("steps", "data_prep")
with open(os.path.join(data_prep_step_path, "step.json")) as f:
data_prep_settings = json.load(f)
# Setup datasets of first step
print("Setting up datasets")
data_prep_input = Dataset.get_by_name(workspace=workspace,
name=data_prep_settings.get(
"dataset_input_name",
None)).as_named_input(
data_prep_settings.get(
"dataset_input_name",
None)).as_mount()
data_prep_output = PipelineData(
name=data_prep_settings.get("dataset_output_name", None),
datastore=Datastore(workspace=workspace,
name=data_prep_settings.get(
"datastore_output_name",
"workspaceblobstore")),
output_mode="mount").as_dataset()
# Uncomment next lines, if you want to register intermediate dataset
#data_prep_output.register(
# name=data_prep_settings.get("dataset_output_name", None),
# create_new_version=True
#)
# Create conda dependencies
print("Creating conda dependencies")
data_prep_dependencies = CondaDependencies.create(
pip_packages=data_prep_settings.get("pip_packages", []),
conda_packages=data_prep_settings.get("conda_packages", []),
python_version=data_prep_settings.get("python_version", "3.6.2"))
# Create run configuration
print("Creating RunConfiguration")
data_prep_run_config = RunConfiguration(
conda_dependencies=data_prep_dependencies,
framework=data_prep_settings.get("framework", "Python"))
# Loading compute target
print("Loading ComputeTarget")
data_prep_compute_target = ComputeTarget(workspace=workspace,
name=data_prep_settings.get(
"compute_target_name", None))
# Create python step
print("Creating Step")
data_prep = PythonScriptStep(
name=data_prep_settings.get("step_name", None),
script_name=data_prep_settings.get("script_name", None),
arguments=data_prep_settings.get("arguments", []),
compute_target=data_prep_compute_target,
runconfig=data_prep_run_config,
inputs=[data_prep_input],
outputs=[data_prep_output],
params=data_prep_settings.get("parameters", []),
source_directory=data_prep_step_path,
allow_reuse=data_prep_settings.get("allow_reuse", True),
version=data_prep_settings.get("version", None),
)
###############################################
### Creating data model train Pipeline Step ###
###############################################
# Load settings
print("Loading settings")
model_train_step_path = os.path.join("steps", "model_train")
with open(os.path.join(model_train_step_path, "step.json")) as f:
model_train_settings = json.load(f)
hyperparameter_sampling_settings = model_train_settings.get(
"hyperparameter_sampling", {})
# Setup datasets of first step
print("Setting up datasets")
model_train_input = data_prep_output.as_named_input(
name=model_train_settings.get("dataset_input_name", None))
model_train_output = PipelineData(
name=model_train_settings.get("dataset_output_name", None),
datastore=Datastore(workspace=workspace,
name=model_train_settings.get(
"datastore_output_name", None)),
output_mode="mount",
).as_dataset()
# Uncomment next lines, if you want to register intermediate dataset
#model_train_output.register(
# name=model_train_settings.get("dataset_output_name", None),
# create_new_version=True
#)
# Create conda dependencies
print("Creating conda dependencies")
model_train_dependencies = CondaDependencies.create(
pip_packages=model_train_settings.get("pip_packages", []),
conda_packages=model_train_settings.get("conda_packages", []),
python_version=model_train_settings.get("python_version", "3.6.2"))
# Create run configuration
print("Creating RunConfiguration")
model_train_run_config = RunConfiguration(
conda_dependencies=model_train_dependencies,
framework=model_train_settings.get("framework", "Python"))
# Loading compute target
print("Loading ComputeTarget")
model_train_compute_target = ComputeTarget(workspace=workspace,
name=model_train_settings.get(
"compute_target_name",
None))
# Create distributed training backend
print("Creating distributed training backend")
distributed_training_backend = get_distributed_backend(
backend_name=model_train_settings.get("distributed_backend", None))
# Create Estimator for Training
print("Creating Estimator for training")
model_train_estimator = Estimator(
source_directory=model_train_step_path,
entry_script=model_train_settings.get("script_name", None),
environment_variables=model_train_settings.get("parameters", None),
compute_target=model_train_compute_target,
node_count=model_train_settings.get("node_count", None),
distributed_training=distributed_training_backend,
conda_packages=model_train_settings.get("conda_packages", None),
pip_packages=model_train_settings.get("pip_packages", None),
)
try:
# Create parameter sampling
print("Creating Parameter Sampling")
parameter_dict = {}
parameters = hyperparameter_sampling_settings.get(
"parameters",
{}) if "parameters" in hyperparameter_sampling_settings else {}
for parameter_name, parameter_details in parameters.items():
parameter_distr = get_parameter_distribution(
distribution=parameter_details.get("distribution", None),
**parameter_details.get("settings", {}))
parameter_dict[f"--{parameter_name}"] = parameter_distr
model_train_ps = get_parameter_sampling(
sampling_method=hyperparameter_sampling_settings.get(
"method", None),
parameter_dict=parameter_dict)
# Get Policy definition
policy_settings = hyperparameter_sampling_settings.get("policy", {})
kwargs = {
key: value
for key, value in policy_settings.items() if key not in
["policy_method", "evaluation_interval", "delay_evaluation"]
}
# Create termination policy
print("Creating early termination policy")
model_train_policy = get_policy(
policy_method=policy_settings.get("method", ""),
evaluation_interval=policy_settings.get("evaluation_interval",
None),
delay_evaluation=policy_settings.get("delay_evaluation", None),
**kwargs)
# Create HyperDriveConfig
print("Creating HyperDriveConfig")
model_train_hyperdrive_config = HyperDriveConfig(
estimator=model_train_estimator,
hyperparameter_sampling=model_train_ps,
policy=model_train_policy,
primary_metric_name=hyperparameter_sampling_settings.get(
"primary_metric", None),
primary_metric_goal=PrimaryMetricGoal.MINIMIZE
if "min" in hyperparameter_sampling_settings.get(
"primary_metric_goal", None) else PrimaryMetricGoal.MAXIMIZE,
max_total_runs=hyperparameter_sampling_settings.get(
"max_total_runs", 1),
max_concurrent_runs=hyperparameter_sampling_settings.get(
"max_concurrent_runs", 1),
max_duration_minutes=hyperparameter_sampling_settings.get(
"max_duration_minutes", None))
# Create HyperDriveStep
print("Creating HyperDriveStep")
model_train = HyperDriveStep(
name=model_train_settings.get("step_name", None),
hyperdrive_config=model_train_hyperdrive_config,
estimator_entry_script_arguments=model_train_settings.get(
"arguments", None),
inputs=[model_train_input],
outputs=[model_train_output],
allow_reuse=model_train_settings.get("allow_reuse", True),
version=model_train_settings.get("version", True))
except:
print("Not all required parameters specified for HyperDrive step")
# Create EstimatorStep
print("Creating EstimatorStep")
model_train = EstimatorStep(
name=model_train_settings.get("step_name", None),
estimator=model_train_estimator,
estimator_entry_script_arguments=model_train_settings.get(
"arguments", None),
inputs=[model_train_input],
outputs=[model_train_output],
compute_target=model_train_compute_target,
allow_reuse=model_train_settings.get("allow_reuse", True),
version=model_train_settings.get("version", True))
#########################
### Creating Pipeline ###
#########################
# Create Pipeline
print("Creating Pipeline")
pipeline = Pipeline(
workspace=workspace,
steps=[model_train],
description="Training Pipeline",
)
# Validate pipeline
print("Validating pipeline")
pipeline.validate()
return pipeline
train = Estimator(source_directory='.',
compute_target=compute,
entry_script='train.py',
use_gpu=True,
pip_requirements_file='requirements-dataprepandtraining.txt')
trainStep = EstimatorStep(
name='Model Training',
estimator=train,
estimator_entry_script_arguments=[
"--source_path",
seer_tfrecords,
"--target_path",
seer_training,
"--epochs",
5, # Consider transfer learning. See line 111 in train.py file.
"--batch",
10,
"--lr",
0.001
],
inputs=[seer_tfrecords],
outputs=[seer_training],
compute_target=compute)
## Register Model Step ##
# Once training is complete, register.py registers the model with AML #
# Configuration for registration step #
registerEnvironment = Environment.from_pip_requirements(
'registerenv', 'requirements-registration.txt')
print('Error while retrieving compute', e)
sys.exit(-1)
################################
# If you want to use datastore
################################
# from azureml.core import Datastore
# from azureml.data.data_reference import DataReference
# from azureml.pipeline.core import PipelineData
# def_blob_store = Datastore(ws, "workspaceblobstore")
# input_data = DataReference(
# datastore=def_blob_store,
# data_reference_name="input_data",
# path_on_datastore="20newsgroups/20news.pkl")
# output = PipelineData("output", datastore=def_blob_store)
est_step = EstimatorStep(name="Estimator_Train",
estimator=est,
estimator_entry_script_arguments=["--datadir", input_data, "--output", output],
runconfig_pipeline_params=None,
inputs=[input_data],
outputs=[output],
compute_target=compute_target)
pipeline = Pipeline(workspace=ws, steps=[est_step])
pipeline_run = experiment.submit(pipeline)
]
est = Estimator(source_directory='./local_scripts2/',
entry_script='2-train.py',
pip_packages=pip_packages,
compute_target=compute_target)
# 2.C) Create the EstimatorStep object
from azureml.pipeline.steps import EstimatorStep
train_step = EstimatorStep(
name="2: Training",
estimator=est,
estimator_entry_script_arguments=[
"--reg", 0.8, "--datapreparation_output", datapreparation_output,
"--datatrain_output", datatrain_output, "--is_directory",
'aaa' if is_directory else ''
], # All non-empty strings have a True boolean value],
inputs=[datapreparation_output],
outputs=[datatrain_output],
compute_target=compute_target)
#
# ***************** CREATE PythonScriptStep3: MODEL REGISTRATION **************************
#
# 3.A) Create PipelineData Object modelregistration_output
from azureml.pipeline.core import PipelineData
is_directory = False # it's a file where we save the details of the registered model
default_datastore = ws.get_default_datastore()
modelregistration_output = PipelineData('modelregistration_output3',
datastore=default_datastore,
## Training Step ##
# train.py does the training based on the processed data #
seer_training = PipelineData("train", datastore=datastore, is_directory=True)
train = Estimator(source_directory='.',
compute_target=compute,
entry_script='train.py',
pip_requirements_file='requirements-dataprepandtraining.txt')
trainStep = EstimatorStep(name='Model Training',
estimator=train,
estimator_entry_script_arguments=[
"--source_path", seer_tfrecords, "--target_path",
seer_training, "--epochs", 5, "--batch", 10,
"--lr", 0.001
],
inputs=[seer_tfrecords],
outputs=[seer_training],
compute_target=compute)
## Register Model Step ##
# Once training is complete, register.py registers the model with AML #
# Configuration for registration step #
registerEnvironment = Environment.from_pip_requirements(
'registerenv', 'requirements-registration.txt')
registerRunConfig = RunConfiguration()
registerRunConfig.environment = registerEnvironment
seer_model = PipelineData("model", datastore=datastore, is_directory=True)
]
# Create the tensorflow Estimator
trainEstimator = PyTorch(
source_directory = script_folder,
compute_target = cluster,
entry_script = "steps/train.py",
use_gpu = True,
framework_version='1.3'
)
# Create a pipeline step with the TensorFlow Estimator
trainOnGpuStep = EstimatorStep(
name='Train Estimator Step',
estimator=trainEstimator,
inputs=[training_data_location],
outputs=[model],
compute_target=cluster,
estimator_entry_script_arguments = estimator_script_params
)
## Register Model Step ##
# Once training is complete, register.py registers the model with AML #
# Configuration for registration step #
registerModelStep = PythonScriptStep(
name="Register model in Model Management",
script_name="steps/register.py",
compute_target=cluster,
inputs=[model],
arguments=['--model_name', model_name,
'--model_assets_path', model
def main():
e = Env()
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group
)
print("get_workspace:")
print(aml_workspace)
aml_compute = get_compute(
aml_workspace,
e.compute_name,
e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
environment = get_environment(
aml_workspace, e.aml_env_name, create_new=e.rebuild_env)
run_config = RunConfiguration()
run_config.environment = environment
if (e.datastore_name):
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables["DATASTORE_NAME"] \
= datastore_name
dataset_name = e.dataset_name
file_name = e.file_name
datastore = Datastore.get(aml_workspace, datastore_name)
if (dataset_name not in aml_workspace.datasets):
raise Exception("Could not find dataset at \"%s\"." % dataset_name)
else:
dataset = Dataset.get_by_name(aml_workspace, name=dataset_name)
dataset.download(target_path='.', overwrite=True)
datastore.upload_files([file_name],
target_path=dataset_name,
overwrite=True)
raw_data_file = DataReference(datastore=datastore,
data_reference_name="Raw_Data_File",
path_on_datastore=dataset_name + '/'
+ file_name)
clean_data_file = PipelineParameter(name="clean_data_file",
default_value="/clean_data.csv")
clean_data_folder = PipelineData("clean_data_folder",
datastore=datastore)
prepDataStep = PythonScriptStep(name="Prepare Data",
source_directory=e.sources_directory_train,
script_name=e.data_prep_script_path,
arguments=["--raw_data_file",
raw_data_file,
"--clean_data_folder",
clean_data_folder,
"--clean_data_file",
clean_data_file],
inputs=[raw_data_file],
outputs=[clean_data_folder],
compute_target=aml_compute,
allow_reuse=False)
print("Step Prepare Data created")
new_model_file = PipelineParameter(name="new_model_file ",
default_value='/' + e.model_name
+ '.pkl')
new_model_folder = PipelineData("new_model_folder", datastore=datastore)
est = SKLearn(source_directory=e.sources_directory_train,
entry_script=e.train_script_path,
pip_packages=['azureml-sdk', 'scikit-learn==0.20.3',
'azureml-dataprep[pandas,fuse]>=1.1.14'],
compute_target=aml_compute)
trainingStep = EstimatorStep(
name="Model Training",
estimator=est,
estimator_entry_script_arguments=["--clean_data_folder",
clean_data_folder,
"--new_model_folder",
new_model_folder,
"--clean_data_file",
clean_data_file.default_value,
"--new_model_file",
new_model_file.default_value],
runconfig_pipeline_params=None,
inputs=[clean_data_folder],
outputs=[new_model_folder],
compute_target=aml_compute,
allow_reuse=False)
print("Step Train created")
model_name_param = PipelineParameter(name="model_name",
default_value=e.model_name)
evaluateStep = PythonScriptStep(
name="Evaluate Model",
source_directory=e.sources_directory_train,
script_name=e.evaluate_script_path,
arguments=["--model_name", model_name_param],
compute_target=aml_compute,
allow_reuse=False)
print("Step Evaluate created")
registerStep = PythonScriptStep(
name="Register Model",
source_directory=e.sources_directory_train,
script_name=e.register_script_path,
arguments=["--new_model_folder", new_model_folder,
"--new_model_file", new_model_file,
"--model_name", model_name_param],
inputs=[new_model_folder],
compute_target=aml_compute,
allow_reuse=False)
print("Step Register created")
if ((e.run_evaluation).lower() == 'true'):
print("Include evaluation step before register step.")
trainingStep.run_after(prepDataStep)
evaluateStep.run_after(trainingStep)
registerStep.run_after(evaluateStep)
else:
print("Exclude evaluation step and directly run register step.")
trainingStep.run_after(prepDataStep)
registerStep.run_after(trainingStep)
pipeline = Pipeline(workspace=aml_workspace, steps=[registerStep])
pipeline.validate()
print("Pipeline is built")
pipeline._set_experiment_name
published_pipeline = pipeline.publish(
name=e.pipeline_name,
description="Predict Employee Retention Model training pipeline",
version=e.build_id
)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
# Create a PipelineData (Data Reference) for the model folder
model_folder = PipelineData("model_folder",
datastore=ws.get_default_datastore())
estimator = Estimator(source_directory=experiment_folder,
compute_target=gpu_cluster,
environment_definition=pipeline_run_config.environment,
entry_script='train_diabetes.py')
#Step 1
train_step = EstimatorStep(name="Train Model",
estimator=estimator,
estimator_entry_script_arguments=[
'--regularization', 0.1, '--output_folder',
model_folder
],
inputs=[diabetes_ds.as_named_input('diabetes')],
outputs=[model_folder],
compute_target=gpu_cluster,
allow_reuse=False)
# Step 2, run the model registration script
register_step = PythonScriptStep(name="Register Model",
source_directory=experiment_folder,
script_name="register_diabetes.py",
arguments=['--model_folder', model_folder],
inputs=[model_folder],
compute_target=gpu_cluster,
runconfig=pipeline_run_config,
allow_reuse=False)
source_directory="preprocess",
runconfig=run_config,
allow_reuse=True,
)
estimator_step = EstimatorStep(
name="Train Model",
estimator=Estimator(
source_directory="src/train",
entry_script="train_pipeline.py",
compute_target=compute_target,
environment_definition=freezer_environment,
),
estimator_entry_script_arguments=[
"--input",
output_df_nested,
"--n_estimators",
n_estimators_param,
"--train_data_split",
train_data_split_param,
],
runconfig_pipeline_params=None,
inputs=[output_df_nested],
compute_target=compute_target,
allow_reuse=True,
)
deploy_step = PythonScriptStep(
name="Deploy Model",
script_name="deploy.py",
arguments=[
print("Blobstore's name: {}".format(def_blob_store.name))
blob_output_data = DataReference(datastore=def_blob_store,
data_reference_name="data",
path_on_datastore="data")
print("DataReference object created")
est = Estimator(source_directory='.',
compute_target=ws.compute_targets['cpu'],
entry_script='azureml-issues.py',
pip_packages=['azure-devops', 'pandas'])
data_processing = EstimatorStep(estimator=est,
estimator_entry_script_arguments=[
'--data_path', blob_output_data,
'--analyze', '--load_open', '--load_closed'
],
inputs=[blob_output_data],
compute_target=ws.compute_targets['cpu'],
allow_reuse=False)
pipeline = Pipeline(workspace=ws, steps=[data_processing])
print("Pipeline is built")
pipeline.validate()
print("Simple validation complete")
pipeline_run = Experiment(ws, 'issues_pipeline').submit(pipeline)
print("Pipeline is submitted for execution")
published_pipeline = pipeline.publish(
name="Issues_Stats",
bootstrap_args = [r_script]
estimator = Estimator(source_directory='src',
entry_script='bootstrapper.py',
compute_target=aml_compute_target,
custom_docker_image=acr_image,
image_registry_details=acr_details,
user_managed=True)
inputs = []
step = EstimatorStep(name='execute-r',
estimator=estimator,
estimator_entry_script_arguments=bootstrap_args,
inputs=inputs,
outputs=None,
compute_target=aml_compute_target,
allow_reuse=False)
aml_pipeline = AmlPipeline(workspace=aml_workspace,
steps=AmlStepSequence([step]),
description='Run R Workloads')
published_pipeline = aml_pipeline.publish(description='Execute R Workload',
name='pipeline-r')
aml_run = published_pipeline.submit(workspace=aml_workspace,
experiment_name=aml_experiment_name)
if (aml_run):
preprocessing_est = SKLearn(
source_directory='010-preprocessing',
compute_target=cpu_cluster,
entry_script='dataprep.py',
conda_packages=['pandas'],
pip_packages=['fastavro'],
)
output = PipelineData("output", datastore=telemetry_ds)
preprocessing_step = EstimatorStep(
name="Preprocessing_Train",
estimator=preprocessing_est,
estimator_entry_script_arguments=[
"--data_dir", input_data, "--output_data_dir", output
],
inputs=[input_data],
outputs=[output],
compute_target=cpu_cluster,
allow_reuse=True,
)
pytorch_est = PyTorch(
source_directory='020-ann',
compute_target=cpu_cluster,
entry_script='pytorch_train.py',
use_gpu=False,
framework_version='1.1',
conda_packages=['pandas'],
)
source_directory=script_folder,
compute_target=compute_target,
environment_definition=pipeline_run_config.environment,
entry_script='train.py')
model_candidate_folder = PipelineData('model_candidate_folder',
datastore=data_store)
# Step to run an estimator
train_step = EstimatorStep(
name='Train model',
estimator=estimator_train,
compute_target=compute_target,
# Specify PipelineData as input
inputs=[input_data],
outputs=[model_candidate_folder],
# Pass as data reference to estimator script
estimator_entry_script_arguments=[
'--input_data', input_data, '--model_candidate_folder',
model_candidate_folder, '--mode', pipeline_mode_param
],
allow_reuse=False)
estimator_evaluate = Estimator(
source_directory=script_folder,
compute_target=compute_target,
environment_definition=pipeline_run_config.environment,
entry_script='eval_model.py')
validated_model_folder = PipelineData('validated_model_folder',
datastore=data_store)
min_node_count=None,
timeout_in_minutes=0)
script_params = {}
estimator = PyTorch(source_directory='./',
script_params=script_params,
compute_target=compute_target_gpu,
entry_script='train.py',
use_gpu=True,
pip_packages=[],
framework_version='1.2')
est_step = EstimatorStep(name="Train_Step",
estimator=estimator,
estimator_entry_script_arguments=auth_params,
runconfig_pipeline_params=None,
inputs=[],
outputs=[],
compute_target=compute_target_gpu)
est_step.run_after(process_step)
# step 3
pipeline = Pipeline(workspace=ws, steps=[process_step, est_step])
pipeline_run_first = Experiment(
ws, project_config['experiment_name']).submit(pipeline)
#https://docs.microsoft.com/en-us/python/api/azureml-pipeline-core/azureml.pipeline.core.pipelinerun?view=azure-ml-py
pipeline_run_first.wait_for_completion(show_output=True,
timeout_seconds=9223372036854775807,
raise_on_error=True)
def main():
load_dotenv()
workspace_name = os.environ.get("BASE_NAME") + "-AML-WS"
resource_group = os.environ.get("BASE_NAME") + "-AML-RG"
subscription_id = os.environ.get("SUBSCRIPTION_ID")
tenant_id = os.environ.get("TENANT_ID")
app_id = os.environ.get("SP_APP_ID")
app_secret = os.environ.get("SP_APP_SECRET")
sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN")
train_script_path = os.environ.get("TRAIN_SCRIPT_PATH")
vm_size = os.environ.get("AML_COMPUTE_CLUSTER_CPU_SKU")
compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME")
model_name = os.environ.get("MODEL_NAME")
build_id = os.environ.get("BUILD_BUILDID")
pipeline_name = os.environ.get("TRAINING_PIPELINE_NAME")
data_path = os.environ.get("DATA_PATH_DATASTORE")
model_data_path = os.environ.get("MODEL_DATA_PATH_DATASTORE")
# Get Azure machine learning workspace
aml_workspace = get_workspace(workspace_name, resource_group,
subscription_id, tenant_id, app_id,
app_secret)
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, compute_name, vm_size)
if aml_compute is not None:
print(aml_compute)
model_name = PipelineParameter(name="model_name", default_value=model_name)
release_id = PipelineParameter(name="release_id", default_value="0")
ds = aml_workspace.get_default_datastore()
dataref_folder = ds.path(data_path).as_mount()
model_dataref = ds.path(model_data_path).as_mount()
# NEED those two folders mounted on datastore and env variables specified in variable groups
#ds.upload(src_dir='./VOCdevkit', target_path='VOCdevkit', overwrite=True, show_progress=True)
#ds.upload(src_dir='./model_data', target_path='VOCmodel_data', overwrite=True, show_progress=True)
yoloEstimator = TensorFlow(
source_directory=sources_directory_train + '/training',
compute_target=aml_compute,
entry_script=train_script_path,
pip_packages=[
'keras', 'pillow', 'matplotlib', 'onnxmltools', 'keras2onnx==1.5.1'
], # recent versions of keras2onnx give conversion issues
use_gpu=True,
framework_version='1.13')
train_step = EstimatorStep(name="Train & Convert Model",
estimator=yoloEstimator,
estimator_entry_script_arguments=[
"--release_id", release_id, "--model_name",
model_name, "--data_folder", dataref_folder,
"--model_path", model_dataref
],
runconfig_pipeline_params=None,
inputs=[dataref_folder, model_dataref],
compute_target=aml_compute,
allow_reuse=False)
print("Step Train & Convert created")
train_pipeline = Pipeline(workspace=aml_workspace, steps=[train_step])
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=pipeline_name,
description="Model training/retraining pipeline",
version=build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
train_step = EstimatorStep(
name="Training Step",
estimator=estimator,
estimator_entry_script_arguments=[
"--model_name_or_path",
model_name_param,
"--task",
task_param,
"--max_seq_length",
max_seq_len_param,
"--max_epochs",
max_epochs_param,
"--learning_rate",
learning_rate_param,
"--seed",
seed_param,
"--gpus",
num_gpus_param,
"--num_workers",
num_workers_param,
"--train_batch_size",
train_batch_size_param,
"--eval_batch_size",
eval_batch_size_param,
"--output_dir",
"./outputs",
"--do_train",
"--do_predict",
],
inputs=[prepared_dataset.as_mount()],
compute_target=compute_target,
)
from azureml.pipeline.steps import PythonScriptStep, EstimatorStep
from azureml.train.estimator import Estimator
# Get the training dataset
diabetes_ds = ws.datasets.get("diabetes dataset")
# Create a PipelineData (temporary Data Reference) for the model folder
model_folder = PipelineData("model_folder", datastore=ws.get_default_datastore())
estimator = Estimator(source_directory=experiment_folder,
compute_target = pipeline_cluster,
environment_definition=pipeline_run_config.environment,
entry_script='train_diabetes.py')
# Step 1, run the estimator to train the model
train_step = EstimatorStep(name = "Train Model",
estimator=estimator,
estimator_entry_script_arguments=['--output_folder', model_folder],
inputs=[diabetes_ds.as_named_input('diabetes_train')],
outputs=[model_folder],
compute_target = pipeline_cluster,
allow_reuse = True)
# Step 2, run the model registration script
register_step = PythonScriptStep(name = "Register Model",
source_directory = experiment_folder,
script_name = "register_diabetes.py",
arguments = ['--model_folder', model_folder],
inputs=[model_folder],
compute_target = pipeline_cluster,