name='Train Estimator Step',
estimator=trainEstimator,
inputs=[training_data_location],
outputs=[model],
compute_target=computeCluster,
estimator_entry_script_arguments = script_params
)
# == Step 3 ==
model_name = "MargeOrHomer"
registerStep = PythonScriptStep(name="Register model for deployment",
script_name="register.py",
compute_target=computeCluster,
inputs=[model],
arguments=['--dataset_name', model_name,
'--model_assets_path', model
],
source_directory=script_folder)
# Create the pipeline
prep_train_register = [preProcessDataStep,trainOnGpuStep,registerStep]
pipeline = Pipeline(workspace=ws, steps=[prep_train_register])
pipeline.validate()
# Publish the pipeline
mlpipeline = pipeline.publish(name="Marge Or Homer - Training pipeline")
print("Pipeline Published ID:"+mlpipeline.id)
# Submit the pipeline to be run
mlpipeline.submit(ws,"Marge-or-Homer" ,pipeline_parameters={"source_dataset":DataPath(datastore=source_ds, path_on_datastore="trainingdata")})
def build_pipeline(dataset, ws, config):
print("building pipeline for dataset %s in workspace %s" %
(dataset, ws.name))
base_dir = '.'
def_blob_store = ws.get_default_datastore()
# folder for scripts that need to be uploaded to Aml compute target
script_folder = './scripts'
os.makedirs(script_folder, exist_ok=True)
shutil.copy(os.path.join(base_dir, 'video_decoding.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'pipelines_submit.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'pipelines_create.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'train.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'data_utils.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'prednet.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'keras_utils.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'data_preparation.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'register_prednet.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'register_classification_model.py'),
script_folder)
shutil.copy(os.path.join(base_dir, 'config.json'), script_folder)
os.makedirs(os.path.join(script_folder, 'models/logistic_regression'))
shutil.copy(os.path.join(base_dir, 'models/logistic_regression/model.pkl'),
os.path.join(script_folder, 'models/logistic_regression/'))
cpu_compute_name = config['cpu_compute']
try:
cpu_compute_target = AmlCompute(ws, cpu_compute_name)
print("found existing compute target: %s" % cpu_compute_name)
except: # ComputeTargetException:
print("creating new compute target")
provisioning_config = AmlCompute.provisioning_configuration(
vm_size='STANDARD_D2_V2',
max_nodes=4,
idle_seconds_before_scaledown=1800)
cpu_compute_target = ComputeTarget.create(ws, cpu_compute_name,
provisioning_config)
cpu_compute_target.wait_for_completion(show_output=True,
min_node_count=None,
timeout_in_minutes=20)
# use get_status() to get a detailed status for the current cluster.
print(cpu_compute_target.get_status().serialize())
# choose a name for your cluster
gpu_compute_name = config['gpu_compute']
try:
gpu_compute_target = AmlCompute(workspace=ws, name=gpu_compute_name)
print("found existing compute target: %s" % gpu_compute_name)
except:
print('Creating a new compute target...')
provisioning_config = AmlCompute.provisioning_configuration(
vm_size='STANDARD_NC6',
max_nodes=10,
idle_seconds_before_scaledown=1800)
# create the cluster
gpu_compute_target = ComputeTarget.create(ws, gpu_compute_name,
provisioning_config)
# can poll for a minimum number of nodes and for a specific timeout.
# if no min node count is provided it uses the scale settings for the cluster
gpu_compute_target.wait_for_completion(show_output=True,
min_node_count=None,
timeout_in_minutes=20)
# use get_status() to get a detailed status for the current cluster.
try:
print(gpu_compute_target.get_status().serialize())
except BaseException as e:
print("Could not get status of compute target.")
print(e)
# conda dependencies for compute targets
cpu_cd = CondaDependencies.create(conda_packages=["py-opencv=3.4.2"],
pip_packages=[
"azure-storage-blob==2.1.0",
"hickle==3.4.3", "requests==2.21.0",
"sklearn", "pandas==0.24.2",
"azureml-sdk", "numpy==1.16.2",
"pillow==6.0.0"
])
# Runconfigs
cpu_compute_run_config = RunConfiguration(conda_dependencies=cpu_cd)
cpu_compute_run_config.environment.docker.enabled = True
cpu_compute_run_config.environment.docker.gpu_support = False
cpu_compute_run_config.environment.docker.base_image = DEFAULT_CPU_IMAGE
cpu_compute_run_config.environment.spark.precache_packages = False
print("PipelineData object created")
# DataReference to where video data is stored.
video_data = DataReference(datastore=def_blob_store,
data_reference_name="video_data",
path_on_datastore=os.path.join(
"prednet", "data", "video", dataset))
print("DataReference object created")
# Naming the intermediate data as processed_data1 and assigning it to the variable processed_data1.
raw_data = PipelineData("raw_video_fames", datastore=def_blob_store)
preprocessed_data = PipelineData("preprocessed_video_frames",
datastore=def_blob_store)
data_metrics = PipelineData("data_metrics", datastore=def_blob_store)
data_output = PipelineData("output_data", datastore=def_blob_store)
# prepare dataset for training/testing prednet
video_decoding = PythonScriptStep(
name='decode_videos',
script_name="video_decoding.py",
arguments=["--input_data", video_data, "--output_data", raw_data],
inputs=[video_data],
outputs=[raw_data],
compute_target=cpu_compute_target,
source_directory=script_folder,
runconfig=cpu_compute_run_config,
allow_reuse=True,
hash_paths=['.'])
print("video_decode step created")
# prepare dataset for training/testing recurrent neural network
data_prep = PythonScriptStep(name='prepare_data',
script_name="data_preparation.py",
arguments=[
"--input_data", raw_data, "--output_data",
preprocessed_data
],
inputs=[raw_data],
outputs=[preprocessed_data],
compute_target=cpu_compute_target,
source_directory=script_folder,
runconfig=cpu_compute_run_config,
allow_reuse=True,
hash_paths=['.'])
data_prep.run_after(video_decoding)
print("data_prep step created")
# configure access to ACR for pulling our custom docker image
acr = ContainerRegistry()
acr.address = config['acr_address']
acr.username = config['acr_username']
acr.password = config['acr_password']
est = Estimator(source_directory=script_folder,
compute_target=gpu_compute_target,
entry_script='train.py',
use_gpu=True,
node_count=1,
custom_docker_image="wopauli_1.8-gpu:1",
image_registry_details=acr,
user_managed=True)
ps = RandomParameterSampling({
'--batch_size':
choice(1, 2, 4, 8),
'--filter_sizes':
choice("3, 3, 3", "4, 4, 4", "5, 5, 5"),
'--stack_sizes':
choice("48, 96, 192", "36, 72, 144", "12, 24, 48"), #, "48, 96"),
'--learning_rate':
loguniform(-6, -1),
'--lr_decay':
loguniform(-9, -1),
'--freeze_layers':
choice("0, 1, 2", "1, 2, 3", "0, 1", "1, 2", "2, 3", "0", "3"),
'--transfer_learning':
choice("True", "False")
})
policy = BanditPolicy(evaluation_interval=2,
slack_factor=0.1,
delay_evaluation=10)
hdc = HyperDriveConfig(estimator=est,
hyperparameter_sampling=ps,
policy=policy,
primary_metric_name='val_loss',
primary_metric_goal=PrimaryMetricGoal.MINIMIZE,
max_total_runs=2,
max_concurrent_runs=2,
max_duration_minutes=60 * 6)
hd_step = HyperDriveStep("train_w_hyperdrive",
hdc,
estimator_entry_script_arguments=[
'--data-folder', preprocessed_data,
'--remote_execution', '--dataset', dataset
],
inputs=[preprocessed_data],
metrics_output=data_metrics,
allow_reuse=True)
hd_step.run_after(data_prep)
register_prednet = PythonScriptStep(
name='register_prednet',
script_name='register_prednet.py',
arguments=['--input_dir', data_metrics, '--output_dir', data_output],
compute_target=cpu_compute_target,
inputs=[data_metrics],
outputs=[data_output],
source_directory=script_folder,
allow_reuse=True,
hash_paths=['.'])
register_prednet.run_after(hd_step)
register_classification_model = PythonScriptStep(
name='register_classification_model',
script_name='register_classification_model.py',
arguments=
[], # ['--input_dir', data_metrics, '--output_dir', data_output],
compute_target=cpu_compute_target,
# inputs=[data_metrics],
# outputs=[data_output],
source_directory=script_folder,
allow_reuse=True,
hash_paths=['.'])
register_classification_model.run_after(register_prednet)
pipeline = Pipeline(workspace=ws,
steps=[
video_decoding, data_prep, hd_step,
register_prednet, register_classification_model
])
print("Pipeline is built")
pipeline.validate()
print("Simple validation complete")
pipeline_name = 'prednet_' + dataset
published_pipeline = pipeline.publish(name=pipeline_name)
_ = Schedule.create(
workspace=ws,
name=pipeline_name + "_sch",
pipeline_id=published_pipeline.id,
experiment_name=pipeline_name,
datastore=def_blob_store,
wait_for_provisioning=True,
description="Datastore scheduler for Pipeline" + pipeline_name,
path_on_datastore=os.path.join('prednet/data/video', dataset, 'Train'),
polling_interval=60 * 24)
return pipeline_name
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group,
)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable Azure ML environment
environment = get_environment(
aml_workspace,
e.aml_env_name,
conda_dependencies_file=e.aml_env_train_conda_dep_file,
create_new=e.rebuild_env,
) #
run_config = RunConfiguration()
run_config.environment = environment
if e.datastore_name:
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables[
"DATASTORE_NAME"] = datastore_name # NOQA: E501
model_name_param = PipelineParameter(
name="model_name", default_value=e.model_name) # NOQA: E501
dataset_version_param = PipelineParameter(name="dataset_version",
default_value=e.dataset_version)
data_file_path_param = PipelineParameter(name="data_file_path",
default_value="none")
caller_run_id_param = PipelineParameter(name="caller_run_id",
default_value="none") # NOQA: E501
# Get dataset name
dataset_name = e.dataset_name
# Check to see if dataset exists
if dataset_name not in aml_workspace.datasets:
# This call creates an example CSV from sklearn sample data. If you
# have already bootstrapped your project, you can comment this line
# out and use your own CSV.
create_sample_data_csv()
# Use a CSV to read in the data set.
file_name = "diabetes.csv"
if not os.path.exists(file_name):
raise Exception(
'Could not find CSV dataset at "%s". If you have bootstrapped your project, you will need to provide a CSV.' # NOQA: E501
% file_name) # NOQA: E501
# Upload file to default datastore in workspace
datatstore = Datastore.get(aml_workspace, datastore_name)
target_path = "training-data/"
datatstore.upload_files(
files=[file_name],
target_path=target_path,
overwrite=True,
show_progress=False,
)
# Register dataset
path_on_datastore = os.path.join(target_path, file_name)
dataset = Dataset.Tabular.from_delimited_files(
path=(datatstore, path_on_datastore))
dataset = dataset.register(
workspace=aml_workspace,
name=dataset_name,
description="diabetes training data",
tags={"format": "CSV"},
create_new_version=True,
)
# Create a PipelineData to pass data between steps
pipeline_data = PipelineData(
"pipeline_data", datastore=aml_workspace.get_default_datastore())
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
outputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_output",
pipeline_data,
"--dataset_version",
dataset_version_param,
"--data_file_path",
data_file_path_param,
"--caller_run_id",
caller_run_id_param,
"--dataset_name",
dataset_name,
],
runconfig=run_config,
allow_reuse=True,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--model_name",
model_name_param,
"--allow_run_cancel",
e.allow_run_cancel,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_input",
pipeline_data,
], # NOQA: E501
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
# Check run_evaluation flag to include or exclude evaluation step.
if (e.run_evaluation).lower() == "true":
print("Include evaluation step before register step.")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
else:
print("Exclude evaluation step and directly run register step.")
register_step.run_after(train_step)
steps = [train_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id,
)
print(f"Published pipeline: {published_pipeline.name}")
print(f"for build {published_pipeline.version}")
# Step 2, run the model registration script
register_step = PythonScriptStep(name = "Register Model",
source_directory = experiment_folder,
script_name = "register.py",
arguments = ['--model_folder', model_folder, '--modelname', modelname],
inputs=[model_folder],
compute_target = pipeline_cluster,
runconfig = pipeline_run_config,
allow_reuse = True)
print("Pipeline steps defined")
# Construct the pipeline
pipeline_steps = [train_step, register_step]
pipeline = Pipeline(workspace = ws, steps=pipeline_steps)
print("Pipeline is built.")
# Create an experiment and run the pipeline
experiment = Experiment(workspace = ws, name = pipelinename)
pipeline_run = experiment.submit(pipeline, regenerate_outputs=True)
print("Pipeline submitted for execution.")
# RunDetails(pipeline_run).show()
pipeline_run.wait_for_completion()
published_pipeline = pipeline.publish(name=pipelinename,
description="Trains " + modelname + " model",
version="1.0")
rest_endpoint = published_pipeline.endpoint
print(rest_endpoint)
def build_prednet_pipeline(dataset, ws):
print(
"building pipeline for dataset %s in workspace %s" % (dataset, ws.name)
)
base_dir = "."
def_blob_store = ws.get_default_datastore()
# folder for scripts that need to be uploaded to Aml compute target
script_folder = "./scripts"
os.makedirs(script_folder)
shutil.copytree(
os.path.join(base_dir, "models"),
os.path.join(base_dir, script_folder, "models"))
shutil.copy(os.path.join(base_dir, "train.py"), script_folder)
shutil.copy(os.path.join(base_dir, "data_preparation.py"), script_folder)
shutil.copy(os.path.join(base_dir, "register_prednet.py"), script_folder)
shutil.copy(os.path.join(base_dir, "batch_scoring.py"), script_folder)
shutil.copy(os.path.join(base_dir, "train_clf.py"), script_folder)
shutil.copy(os.path.join(base_dir, "register_clf.py"), script_folder)
cpu_compute_name = args.cpu_compute_name
cpu_compute_target = AmlCompute(ws, cpu_compute_name)
print("found existing compute target: %s" % cpu_compute_name)
# use get_status() to get a detailed status for the current cluster.
print(cpu_compute_target.get_status().serialize())
# choose a name for your cluster
gpu_compute_name = args.gpu_compute_name
gpu_compute_target = AmlCompute(workspace=ws, name=gpu_compute_name)
print(gpu_compute_target.get_status().serialize())
env = Environment.get(ws, "prednet")
# Runconfigs
runconfig = RunConfiguration()
runconfig.environment = env
print("PipelineData object created")
# DataReference to where raw data is stored.
raw_data = DataReference(
datastore=def_blob_store,
data_reference_name="raw_data",
path_on_datastore=os.path.join("prednet", "data", "raw_data"),
)
print("DataReference object created")
# Naming the intermediate data as processed_data and assigning it to the
# variable processed_data.
preprocessed_data = PipelineData(
"preprocessed_data", datastore=def_blob_store
)
data_metrics = PipelineData("data_metrics", datastore=def_blob_store)
hd_child_cwd = PipelineData(
"prednet_model_path",
datastore=def_blob_store)
prednet_path = PipelineData("output_data", datastore=def_blob_store)
scored_data = PipelineData("scored_data", datastore=def_blob_store)
model_path = PipelineData("model_path", datastore=def_blob_store)
# prepare dataset for training/testing recurrent neural network
data_prep = PythonScriptStep(
name="prepare_data",
script_name="data_preparation.py",
arguments=[
"--raw_data",
raw_data,
"--preprocessed_data",
preprocessed_data,
"--dataset",
dataset,
],
inputs=[raw_data],
outputs=[preprocessed_data],
compute_target=cpu_compute_target,
source_directory=script_folder,
runconfig=runconfig,
allow_reuse=True,
)
# data_prep.run_after(video_decoding)
print("data_prep step created")
est = Estimator(
source_directory=script_folder,
compute_target=gpu_compute_target,
entry_script="train.py",
node_count=1,
environment_definition=env,
)
ps = BayesianParameterSampling(
{
"--batch_size": choice(1, 2, 4, 10),
"--filter_sizes": choice("3, 3, 3", "4, 4, 4", "5, 5, 5"),
"--stack_sizes": choice(
"48, 96, 192", "36, 72, 144", "12, 24, 48"
),
"--learning_rate": uniform(1e-6, 1e-3),
"--lr_decay": uniform(1e-9, 1e-2),
"--freeze_layers": choice(
"0, 1, 2", "1, 2, 3", "0, 1", "1, 2", "2, 3", "0", "3"
),
"--fine_tuning": choice("True", "False"),
}
)
hdc = HyperDriveConfig(
estimator=est,
hyperparameter_sampling=ps,
primary_metric_name="val_loss",
primary_metric_goal=PrimaryMetricGoal.MINIMIZE,
max_total_runs=1,
max_concurrent_runs=1,
max_duration_minutes=60 * 6,
)
train_prednet = HyperDriveStep(
"train_w_hyperdrive",
hdc,
estimator_entry_script_arguments=[
"--preprocessed_data",
preprocessed_data,
"--remote_execution",
"--dataset",
dataset,
# "--hd_child_cwd",
# hd_child_cwd
],
inputs=[preprocessed_data],
outputs=[hd_child_cwd],
metrics_output=data_metrics,
allow_reuse=True,
)
train_prednet.run_after(data_prep)
register_prednet = PythonScriptStep(
name="register_prednet",
script_name="register_prednet.py",
arguments=[
"--data_metrics",
data_metrics,
# "--hd_child_cwd",
# hd_child_cwd,
"--prednet_path",
prednet_path],
compute_target=cpu_compute_target,
inputs=[data_metrics, hd_child_cwd],
outputs=[prednet_path],
source_directory=script_folder,
allow_reuse=True,
)
register_prednet.run_after(train_prednet)
batch_scoring = PythonScriptStep(
name="batch_scoring",
script_name="batch_scoring.py",
arguments=[
"--preprocessed_data",
preprocessed_data,
"--scored_data",
scored_data,
"--dataset",
dataset,
"--prednet_path",
prednet_path],
compute_target=gpu_compute_target,
inputs=[preprocessed_data, prednet_path],
outputs=[scored_data],
source_directory=script_folder,
runconfig=runconfig,
allow_reuse=True,
)
batch_scoring.run_after(train_prednet)
train_clf = PythonScriptStep(
name="train_clf",
script_name="train_clf.py",
arguments=[
"--preprocessed_data",
preprocessed_data,
"--scored_data",
scored_data,
"--model_path",
model_path],
compute_target=cpu_compute_target,
inputs=[preprocessed_data, scored_data],
outputs=[model_path],
source_directory=script_folder,
runconfig=runconfig,
allow_reuse=True,
)
train_clf.run_after(batch_scoring)
register_clf = PythonScriptStep(
name="register_clf",
script_name="register_clf.py",
arguments=[
"--model_path",
model_path],
inputs=[model_path],
compute_target=cpu_compute_target,
source_directory=script_folder,
allow_reuse=True,
runconfig=runconfig,
)
register_clf.run_after(train_clf)
pipeline = Pipeline(
workspace=ws,
steps=[
data_prep,
train_prednet,
register_prednet,
train_clf,
register_clf,
],
)
print("Pipeline is built")
pipeline.validate()
print("Simple validation complete")
pipeline_name = "prednet_" + dataset
published_pipeline = pipeline.publish(name=pipeline_name)
_ = Schedule.create(
workspace=ws,
name=pipeline_name + "_sch",
pipeline_id=published_pipeline.id,
experiment_name=pipeline_name,
datastore=def_blob_store,
wait_for_provisioning=True,
description="Datastore scheduler for Pipeline" + pipeline_name,
path_on_datastore=os.path.join(
"prednet/data/raw_data", dataset, "Train"
),
polling_interval=60 * 24,
)
published_pipeline.submit(ws, pipeline_name)
def main():
load_dotenv()
workspace_name = os.environ.get("BASE_NAME") + "-AML-WS"
resource_group = os.environ.get("BASE_NAME") + "-AML-RG"
subscription_id = os.environ.get("SUBSCRIPTION_ID")
tenant_id = os.environ.get("TENANT_ID")
app_id = os.environ.get("SP_APP_ID")
app_secret = os.environ.get("SP_APP_SECRET")
sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN")
train_script_path = os.environ.get("TRAIN_SCRIPT_PATH")
evaluate_script_path = os.environ.get("EVALUATE_SCRIPT_PATH")
# register_script_path = os.environ.get("REGISTER_SCRIPT_PATH")
vm_size = os.environ.get("AML_COMPUTE_CLUSTER_CPU_SKU")
compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME")
model_name = os.environ.get("MODEL_NAME")
build_id = os.environ.get("BUILD_BUILDID")
pipeline_name = os.environ.get("TRAINING_PIPELINE_NAME")
# Get Azure machine learning workspace
aml_workspace = get_workspace(workspace_name, resource_group,
subscription_id, tenant_id, app_id,
app_secret)
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, compute_name, vm_size)
if aml_compute is not None:
print(aml_compute)
run_config = RunConfiguration(conda_dependencies=CondaDependencies.create(
conda_packages=[
'numpy', 'pandas', 'scikit-learn', 'tensorflow', 'keras'
],
pip_packages=[
'azure', 'azureml-core', 'azure-storage', 'azure-storage-blob'
]))
run_config.environment.docker.enabled = True
model_name = PipelineParameter(name="model_name", default_value=model_name)
release_id = PipelineParameter(name="release_id", default_value="0")
train_step = PythonScriptStep(
name="Train Model",
script_name=train_script_path,
compute_target=aml_compute,
source_directory=sources_directory_train,
arguments=[
"--release_id",
release_id,
"--model_name",
model_name,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=evaluate_script_path,
compute_target=aml_compute,
source_directory=sources_directory_train,
arguments=[
"--release_id",
release_id,
"--model_name",
model_name,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
# Currently, the Evaluate step will automatically register
# the model if it performs better. This step is based on a
# previous version of the repo which utilized JSON files to
# track evaluation results.
# register_model_step = PythonScriptStep(
# name="Register New Trained Model",
# script_name=register_script_path,
# compute_target=aml_compute,
# source_directory=sources_directory_train,
# arguments=[
# "--release_id", release_id,
# "--model_name", model_name,
# ],
# runconfig=run_config,
# allow_reuse=False,
# )
# print("Step register model created")
evaluate_step.run_after(train_step)
# register_model_step.run_after(evaluate_step)
steps = [evaluate_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=pipeline_name,
description="Model training/retraining pipeline",
version=build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group,
)
print(f"get_workspace:{aml_workspace}")
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print(f"aml_compute:{aml_compute}")
# Create a reusable Azure ML environment
environment = get_environment(
aml_workspace,
e.aml_env_name,
conda_dependencies_file=e.aml_env_train_conda_dep_file,
create_new=e.rebuild_env,
) #
run_config = RunConfiguration()
run_config.environment = environment
if e.datastore_name:
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables[
"DATASTORE_NAME"] = datastore_name # NOQA: E501
# datastore and dataset names are fixed for this pipeline, however
# data_file_path can be specified for registering new versions of dataset
# Note that AML pipeline parameters don't take empty string as default, "" won't work # NOQA: E501
model_name_param = PipelineParameter(
name="model_name", default_value=e.model_name) # NOQA: E501
data_file_path_param = PipelineParameter(
name="data_file_path", default_value="nopath") # NOQA: E501
ml_params = PipelineParameter(name="ml_params",
default_value="default") # NOQA: E501
# Create a PipelineData to pass data between steps
pipeline_data = PipelineData(
"pipeline_data", datastore=aml_workspace.get_default_datastore())
train_step = PythonScriptStep(
name="Train Model",
script_name="train/train_aml.py",
compute_target=aml_compute,
source_directory=e.sources_directory_train,
outputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_output",
pipeline_data,
"--data_file_path",
data_file_path_param,
"--dataset_name",
e.processed_dataset_name,
"--datastore_name",
datastore_name,
"--ml_params",
ml_params,
],
runconfig=run_config,
allow_reuse=True,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name="evaluate/evaluate_model.py",
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--model_name",
model_name_param,
"--ml_params",
ml_params,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name="register/register_model.py",
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_input",
pipeline_data,
"--ml_params",
ml_params,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.training_pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id,
)
print(f"Published pipeline: {published_pipeline.name}")
print(f"for build {published_pipeline.version}")
compute_target=compute_target,
source_directory='.',
arguments=[
'--model_name',
'category_{}_model.pkl'.format(cat),
'--model_path', model_data
],
inputs=[model_data],
allow_reuse=False)
# And chain them together so they run sequentially
if current:
automl_step.run_after(current)
current = register_step
steps.append(automl_step)
steps.append(register_step)
pipeline = Pipeline(description='Generate recommendation models',
workspace=ws,
steps=steps)
pipeline.validate()
# Once published, we can invoke on demand via the SDK or via a REST endpoint
published_pipeline = pipeline.publish(
name='category-based-propensity-pipeline')
# Run it on demand
published_pipeline.submit(ws, published_pipeline.name)
inputs=[model_outpath],
compute_target=compute_target,
source_directory=".",
runconfig=py_rc,
allow_reuse=False,
)
pipeline = Pipeline(
workspace=ws,
steps=[download_step, train_step, register_model],
description="Builds R model for iris dataset",
)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--publish", action="store_true")
args = parser.parse_args()
if args.publish:
p = pipeline.publish(
name="iris-classifier-train-r",
description="train a classifer on iris dataset and register model",
)
print(f"Published Train Pipeline ID: {p.id}")
else:
Experiment(ws, "fit-iris-model").submit(pipeline).wait_for_completion(
show_output=True)
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print(f"get_workspace: {aml_workspace}")
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print(f"aml_compute: {aml_compute}")
# Prepare the dataset input
data_store = aml_workspace.get_default_datastore()
print("data_store: %s" % data_store.name)
# Parameters
sources_directory_train = e.sources_directory_train
build_id = e.build_id
pipeline_name = 'Prepare Data Pipeline'
train_ds_name = e.dataset_name
train_data_path = e.datafile_path
# Register the train dataset
if (train_ds_name not in aml_workspace.datasets):
train_path_on_datastore = train_data_path # +'/*.csv'
train_ds_data_path = [(data_store, train_path_on_datastore)]
train_ds = Dataset.File.from_files(path=train_ds_data_path,
validate=False)
train_ds = train_ds.register(workspace=aml_workspace,
name=train_ds_name,
description='train data',
tags={'format': 'CSV'},
create_new_version=True)
else:
train_ds = Dataset.get_by_name(aml_workspace, train_ds_name)
# Conda environment
environment = Environment.from_conda_specification(
"myenv", os.path.join(sources_directory_train,
"conda_dependencies.yml"))
run_config = RunConfiguration()
run_config.environment = environment
with open(os.path.join(sources_directory_train,
'pipeline_config.json')) as json_file:
pipe_param = json.load(json_file)
for param in pipe_param['pipeline_parameter']:
print(param)
# Prepare pipeline parameters
source_blob_url_param = PipelineParameter(name="source_blob_url",
default_value="url")
data_file_param = PipelineParameter(name="data_file",
default_value="data_file")
target_column_param = PipelineParameter(name="target_column",
default_value="target_column")
features_param = PipelineParameter(name="features", default_value="")
# train_storage_connection_string = "DefaultEndpointsProtocol=https;AccountName=forecastingml8724233808;AccountKey=9o2ZH/5cLtmYmNyoHpoeKEA7Xjw0zi1fHLjI0Z0CZeQL5i4Ky2FZ9Wa6VpSYgK6uwLaHC3eamwnfEAscNTcgYw==;EndpointSuffix=core.windows.net"
# Copy data step
copy_step = PythonScriptStep(
name="Copy Data",
script_name="copy_data.py",
arguments=[
"--source_blob_url", source_blob_url_param,
"--train_storage_connection_string",
e.train_storage_connection_string, "--train_storage_container",
e.train_storage_container, "--data_file", data_file_param,
"--data_file_path", train_data_path
],
runconfig=run_config,
compute_target=aml_compute,
source_directory=sources_directory_train)
print("Step Copy Data created")
# Prepare data step
prepare_step = PythonScriptStep(name="Prepare Data",
script_name="prepare.py",
arguments=[
"--data_file_path", train_data_path,
"--data_file", data_file_param,
"--target_column", target_column_param,
"--features", features_param
],
runconfig=run_config,
compute_target=aml_compute,
source_directory=sources_directory_train)
print("Step Prepare created")
# Publish the pipeline
prepare_step.run_after(copy_step)
pipeline_steps = [copy_step, prepare_step]
pipeline = Pipeline(workspace=aml_workspace, steps=pipeline_steps)
pipeline._set_experiment_name
pipeline.validate()
published_pipeline = pipeline.publish(name=pipeline_name,
description="Prepare Data pipeline",
version=build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
# Run the pipelines
runs = []
for param in pipe_param['pipeline_parameter']:
# pipeline_parameters = {"model_name": "nyc_energy_model", "build_id": build_id}
target_column = param['automl_settings']['label_column_name']
param.pop('automl_settings')
param.update({"target_column":
target_column}) # Special process target_column
print(param)
pipeline_run = published_pipeline.submit(aml_workspace,
e.experiment_name, param)
runs.append(pipeline_run)
print("Pipeline run initiated ", pipeline_run.id)
# Wait for all runs to finish
wait(lambda: are_all_runs_finished(runs),
timeout_seconds=3600,
sleep_seconds=5,
waiting_for="all runs are finished")
print("All prepare data pipeline runs done")
def main():
e = Env()
from azureml.core.authentication import InteractiveLoginAuthentication
myten=os.environ.get("AZURE_TENANT_ID")
interactive_auth = InteractiveLoginAuthentication(tenant_id=os.environ.get("AZURE_TENANT_ID"))
subscription=os.environ.get("CSUBSCRIPTION")
workspace_name=e.workspace_name
resource_group=e.resource_group
aml_workspace = Workspace.get(
name = workspace_name,
subscription_id = subscription,
resource_group=resource_group,
auth=interactive_auth
)
from ml_service.util.attach_compute import get_compute
# Get Azure machine learning cluster
# If not present then get_compute will create a compute based on environment variables
aml_compute = get_compute(
aml_workspace,
e.compute_name,
e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
print("SDK version: ", azureml.core.VERSION)
## Variable names that can be passed in as parameter values
from azureml.pipeline.core.graph import PipelineParameter
from azureml.core import Datastore
model_name_param = PipelineParameter(
name="model_name", default_value=e.model_name)
dataset_version_param = PipelineParameter(
name="dataset_version", default_value=e.dataset_version)
data_file_path_param = PipelineParameter(
name="data_file_path", default_value="none")
caller_run_id_param = PipelineParameter(
name="caller_run_id", default_value="none")
#model_path = PipelineParameter(
# name="model_path", default_value=e.model_path)
if (e.datastore_name):
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
# Get the datastore whether it is the default or named store
datastore = Datastore.get(aml_workspace, datastore_name)
dataset_name = e.dataset_name
# Create a reusable Azure ML environment
from ml_service.util.manage_environment import get_environment
from azureml.core import Environment
# RUN Configuration
## Must have this process to work with AzureML-SDK 1.0.85
from azureml.core.runconfig import RunConfiguration, DEFAULT_CPU_IMAGE
from azureml.core.conda_dependencies import CondaDependencies
try:
app_env=Environment(name="smartschedule_env")
app_env.register(workspace=aml_workspace)
except:
print("Environment not found")
# Create a new runconfig object
aml_run_config = RunConfiguration()
aml_run_config.environment.environment_variables["DATASTORE_NAME"] = e.datastore_name # NOQA: E501
# Use the aml_compute you created above.
aml_run_config.target = aml_compute
# Enable Docker
aml_run_config.environment.docker.enabled = True
# Set Docker base image to the default CPU-based image
aml_run_config.environment.docker.base_image = DEFAULT_CPU_IMAGE
#aml_run_config.environment.docker.base_image = "mcr.microsoft.com/azureml/base:0.2.1"
# Use conda_dependencies.yml to create a conda environment in the Docker image for execution
aml_run_config.environment.python.user_managed_dependencies = False
app_conda_deps=CondaDependencies.create(
conda_packages=['pandas','scikit-learn', 'libgcc','pyodbc', 'sqlalchemy', 'py-xgboost==0.90'],
pip_packages=['azureml-sdk[automl,explain,contrib,interpret]==1.4.0', 'xgboost==0.90', 'azureml-dataprep==1.4.6', 'pyarrow', 'azureml-defaults==1.4.0', 'azureml-train-automl-runtime==1.4.0'], pin_sdk_version=False)
# Specify CondaDependencies obj, add necessary packages
aml_run_config.environment.python.conda_dependencies = app_conda_deps
print ("Run configuration created.")
from azure.common.credentials import ServicePrincipalCredentials
#from azure.keyvault import KeyVaultClient, KeyVaultAuthentication
from azure.keyvault.secrets import SecretClient
from azure.identity import DefaultAzureCredential
import pandas as pd
#import sqlalchemy as sql
import pyodbc
def get_data(sql_string, columns):
credentials = None
credential = DefaultAzureCredential()
secret_client = SecretClient("https://smrtschd-aml-kv.vault.azure.net", credential=credential)
secret = secret_client.get_secret("database-connection")
#client = KeyVaultClient(KeyVaultAuthentication(auth_callback))
#secret_bundle = client.get_secret("https://smrtschd-aml-kv.vault.azure.net", "database-connection", "")
server = 'starlims-sql.database.windows.net'
database = 'QM12_DATA_AUTOMATION'
username = '******'
password = secret.value
driver= '{ODBC Driver 17 for SQL Server}'
conn = pyodbc.connect('Driver='+driver+';'+
'Server='+server+';'+
'Database='+database+';'+
'PORT=1433;'+
'UID='+username+';'+
'PWD='+password+'; MARS_Connection=Yes'
)
try:
SQL_Query = pd.read_sql_query(sql_string, conn)
df = pd.DataFrame(SQL_Query, columns=columns)
return df
except Exception as e:
print(e)
raise
sql_str = "SELECT " \
" Dept " \
", Method " \
", Servgrp " \
", Runno " \
", TestNo " \
", Testcode " \
", Total_Duration_Min " \
", Total_Duration_Hr " \
", Usrnam " \
", Eqid " \
", Eqtype " \
"FROM dbo.Draft " \
"order by TESTCODE, RUNNO, dept, method;"
columns = ["Dept", "Method", "Servgrp", "Runno", "TestNo", "Testcode", "Total_Duration_Min", "Total_Duration_Hr", "Usrnam", "Eqid","Eqtype"]
from azureml.core import Dataset
from sklearn.model_selection import train_test_split
if (e.train_dataset_name not in aml_workspace.datasets):
df = get_data(sql_str, columns)
train_df, test_df=train_test_split(df, test_size=0.2)
MY_DIR = "data"
CHECK_FOLDER = os.path.isdir(MY_DIR)
if not CHECK_FOLDER:
os.makedirs(MY_DIR)
else:
print("Folder ", MY_DIR, " is already created")
#files = ["data/analyst_tests.csv"]
files = ["data/train_data.csv","data/test_data.csv"]
def_file_store = Datastore(aml_workspace, "workspacefilestore")
dtfrm = df.to_csv(files[0], header=True, index=False)
train_dataframe=train_df.to_csv(files[0], header=True, index=False)
test_dataframe=test_df.to_csv(files[1], header=True, index=False)
datastore.upload_files(
files=files,
target_path='data/',
overwrite=True
)
from azureml.data.data_reference import DataReference
blob_input_data_test=DataReference(
datastore=datastore,
data_reference_name="smartschedulertest",
path_on_datastore="data/test_data.csv"
)
test_data=Dataset.Tabular.from_delimited_files(blob_input_data_test)
test_data.register(aml_workspace, e.test_dataset_name, create_new_version=True)
blob_input_data_train=DataReference(
datastore=datastore,
data_reference_name="smartschedulertrain",
path_on_datastore="data/train_data.csv"
)
train_data=Dataset.Tabular.from_delimited_files(blob_input_data_train)
train_data.register(aml_workspace, e.train_dataset_name, create_new_version=True)
else:
from azureml.data.data_reference import DataReference
print("getting from the datastore instead of uploading")
train_data=Dataset.get_by_name(aml_workspace, name=e.train_dataset_name)
test_data=Dataset.get_by_name(aml_workspace, name=e.test_dataset_name)
# check the training dataset to make sure it has at least 50 records.
tdf=train_data.to_pandas_dataframe().head(5)
print(tdf.shape)
print(tdf)
# display the first five rows of the data
# create a variable that can be used for other purposes
df=train_data.to_pandas_dataframe().head()
label_column="Total_Duration_Min"
import random
import string
def randomString(stringLength=15):
letters = string.ascii_lowercase
return ''.join(random.choice(letters) for i in range(stringLength))
from azureml.core import Experiment
experiment = Experiment(aml_workspace, "SmartScheduler_Pipeline")
import logging
aml_name = 'smart_scheduler_' + randomString(5)
print(aml_name)
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.ticker import StrMethodFormatter
print(df.head(5))
print(df.shape)
print(df.dtypes)
#df.hist(column='Dept')
list(df.columns.values)
# Remove Features that are not necessary.
#df.hist(column="Servgrp", bins=4)
train_data=train_data.drop_columns(["Runno","TestNo","Total_Duration_Hr"])
test_data=test_data.drop_columns(["Runno","TestNo","Total_Duration_Hr"])
print(train_data.to_pandas_dataframe())
print(test_data.to_pandas_dataframe())
from azureml.automl.core.featurization import FeaturizationConfig
# some of the columns could be change to one hot encoding especially if the categorical column
featurization_config=FeaturizationConfig()
featurization_config.blocked_transformers=['LabelEncoder']
featurization_config.add_column_purpose('Dept', 'CategoricalHash')
featurization_config.add_transformer_params('HashOneHotEncoder',['Method'], {"number_of_bits":3})
featurization_config.add_column_purpose('Servgrp', 'CategoricalHash')
featurization_config.add_column_purpose('Testcode', 'Numeric')
featurization_config.add_column_purpose('Usrnam', 'CategoricalHash')
featurization_config.add_column_purpose('Eqid', 'CategoricalHash')
featurization_config.add_column_purpose('Eqtype', 'CategoricalHash')
from azureml.pipeline.core import Pipeline, PipelineData
from azureml.pipeline.steps import PythonScriptStep
#train_model_folder = './scripts/trainmodel'
automl_settings = {
"iteration_timeout_minutes": 5,
"iterations": 5,
"enable_early_stopping": True,
"primary_metric": 'spearman_correlation',
"verbosity": logging.INFO,
"n_cross_validation":5
}
automl_config = AutoMLConfig(task="regression",
debug_log='automated_ml_errors.log',
#path = train_model_folder,
training_data=train_data,
featurization=featurization_config,
blacklist_models=['XGBoostRegressor'],
label_column_name=label_column,
compute_target=aml_compute,
**automl_settings)
from azureml.pipeline.steps import AutoMLStep
from azureml.pipeline.core import TrainingOutput
metrics_output_name = 'metrics_output'
best_model_output_name='best_model_output'
metrics_data = PipelineData(name = 'metrics_data',
datastore = datastore,
pipeline_output_name=metrics_output_name,
training_output=TrainingOutput(type='Metrics'))
model_data = PipelineData(name='model_data',
datastore=datastore,
pipeline_output_name=best_model_output_name,
training_output=TrainingOutput(type='Model'))
trainWithAutomlStep = AutoMLStep(
name=aml_name,
automl_config=automl_config,
passthru_automl_config=False,
outputs=[metrics_data, model_data],
allow_reuse=True
)
evaluate_step = PythonScriptStep(
name="Evaluate Model",
script_name='./evaluate/evaluate_model.py',
# e.evaluate_script_path,
compute_target=aml_compute,
source_directory='../app',
arguments=[
"--model_name", model_name_param,
"--allow_run_cancel", e.allow_run_cancel
]
)
register_step = PythonScriptStep(
name="Register Model ",
script_name='register/register_model2.py', #e.register_script_path,
compute_target=aml_compute,
source_directory='../app',
inputs=[model_data],
arguments=[
"--model_name", model_name_param,
"--model_path", model_data,
"--ds_name", e.train_dataset_name
],
runconfig=aml_run_config,
allow_reuse=False
)
if ((e.run_evaluation).lower() == 'true'):
print("Include evaluation step before register step.")
evaluate_step.run_after(trainWithAutomlStep)
register_step.run_after(evaluate_step)
pipeline_steps = [ trainWithAutomlStep, evaluate_step, register_step ]
else:
print("Exclude the evaluation step and run register step")
register_step.run_after(trainWithAutomlStep)
pipeline_steps = [ trainWithAutomlStep, register_step ]
print( "this is the value for execute pipeline: {}".format(e.execute_pipeline))
if( (e.execute_pipeline).lower() =='true' ):
# Execute the pipe normally during testing and debugging
print("Pipeline submitted for execution.")
pipeline = Pipeline(workspace = aml_workspace, steps=pipeline_steps)
pipeline_run = experiment.submit(pipeline)
pipeline_run.wait_for_completion()
print("Pipeline is built.")
else:
# Generates pipeline that will be called in ML Ops
train_pipeline = Pipeline(workspace=aml_workspace, steps=pipeline_steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id
)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
run_config = RunConfiguration(conda_dependencies=CondaDependencies.create(
conda_packages=[
'numpy', 'pandas', 'scikit-learn', 'tensorflow', 'keras'
],
pip_packages=[
'azure', 'azureml-core', 'azure-storage', 'azure-storage-blob',
'azureml-dataprep'
]))
run_config.environment.docker.enabled = True
config_envvar = {}
if (e.collection_uri is not None and e.teamproject_name is not None):
builduri_base = e.collection_uri + e.teamproject_name
builduri_base = builduri_base + "/_build/results?buildId="
config_envvar["BUILDURI_BASE"] = builduri_base
run_config.environment.environment_variables = config_envvar
model_name_param = PipelineParameter(name="model_name",
default_value=e.model_name)
build_id_param = PipelineParameter(name="build_id",
default_value=e.build_id)
hyperparameter_alpha_param = PipelineParameter(name="hyperparameter_alpha",
default_value=0.5)
dataset_name = ""
if (e.datastore_name is not None and e.datafile_name is not None):
dataset_name = e.dataset_name
datastore = Datastore.get(aml_workspace, e.datastore_name)
data_path = [(datastore, e.datafile_name)]
dataset = Dataset.Tabular.from_delimited_files(path=data_path)
dataset.register(workspace=aml_workspace,
name=e.dataset_name,
description="dataset with training data",
create_new_version=True)
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
"--alpha",
hyperparameter_alpha_param,
"--dataset_name",
dataset_name,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
"--allow_run_cancel",
e.allow_run_cancel,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
# Check run_evaluation flag to include or exclude evaluation step.
if ((e.run_evaluation).lower() == 'true'):
print("Include evaluation step before register step.")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
else:
print("Exclude evaluation step and directly run register step.")
register_step.run_after(train_step)
steps = [train_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
def main():
load_dotenv()
workspace_name = os.environ.get("BASE_NAME") + "-AML-WS"
resource_group = os.environ.get("RESOURCE_GROUP")
subscription_id = os.environ.get("SUBSCRIPTION_ID")
tenant_id = os.environ.get("TENANT_ID")
app_id = os.environ.get("SP_APP_ID")
app_secret = os.environ.get("SP_APP_SECRET")
sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN")
train_script_path = os.environ.get("TRAIN_SCRIPT_PATH")
evaluate_script_path = os.environ.get("EVALUATE_SCRIPT_PATH")
register_script_path = os.environ.get("REGISTER_SCRIPT_PATH")
vm_size_cpu = os.environ.get("AML_COMPUTE_CLUSTER_CPU_SKU")
compute_name_cpu = os.environ.get("AML_COMPUTE_CLUSTER_NAME")
model_name = os.environ.get("MODEL_NAME")
# Get Azure machine learning workspace
aml_workspace = get_workspace(workspace_name, resource_group,
subscription_id, tenant_id, app_id,
app_secret)
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute_cpu = get_compute(aml_workspace, compute_name_cpu, vm_size_cpu)
if aml_compute_cpu is not None:
print(aml_compute_cpu)
run_config = RunConfiguration(conda_dependencies=CondaDependencies.create(
conda_packages=[
'numpy', 'pandas', 'scikit-learn', 'tensorflow', 'keras'
],
pip_packages=[
'azure', 'azureml-core', 'azure-storage', 'azure-storage-blob'
]))
run_config.environment.docker.enabled = True
model_name = PipelineParameter(name="model_name", default_value=model_name)
def_blob_store = Datastore(aml_workspace, "workspaceblobstore")
jsonconfigs = PipelineData("jsonconfigs", datastore=def_blob_store)
config_suffix = datetime.datetime.now().strftime("%Y%m%d%H")
train_step = PythonScriptStep(
name="Train Model",
script_name=train_script_path,
compute_target=aml_compute_cpu,
source_directory=sources_directory_train,
arguments=[
"--config_suffix",
config_suffix,
"--json_config",
jsonconfigs,
"--model_name",
model_name,
],
runconfig=run_config,
# inputs=[jsonconfigs],
outputs=[jsonconfigs],
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=evaluate_script_path,
compute_target=aml_compute_cpu,
source_directory=sources_directory_train,
arguments=[
"--config_suffix",
config_suffix,
"--json_config",
jsonconfigs,
],
runconfig=run_config,
inputs=[jsonconfigs],
# outputs=[jsonconfigs],
allow_reuse=False,
)
print("Step Evaluate created")
register_model_step = PythonScriptStep(
name="Register New Trained Model",
script_name=register_script_path,
compute_target=aml_compute_cpu,
source_directory=sources_directory_train,
arguments=[
"--config_suffix",
config_suffix,
"--json_config",
jsonconfigs,
"--model_name",
model_name,
],
runconfig=run_config,
inputs=[jsonconfigs],
# outputs=[jsonconfigs],
allow_reuse=False,
)
print("Step register model created")
evaluate_step.run_after(train_step)
register_model_step.run_after(evaluate_step)
steps = [register_model_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name="training-pipeline",
description="Model training/retraining pipeline")
train_pipeline_json = {}
train_pipeline_json["rest_endpoint"] = published_pipeline.endpoint
json_file_path = "ml_service/pipelines/train_pipeline.json"
with open(json_file_path, "w") as outfile:
json.dump(train_pipeline_json, outfile)
step3 = PythonScriptStep(name="register model",
compute_target=env.aml_compute_name,
source_directory='src/steps',
script_name='03_reg_model.py',
inputs=[model_dir],
outputs=[],
arguments=[
'--model_dir', model_dir, '--model_name',
env.aml_model_name
],
runconfig=run_config,
allow_reuse=False)
# Build pipeline
pipeline = Pipeline(workspace=ws, steps=[step3])
pipeline.validate()
# Publish pipeline & pipeline_endpoint
published_pipeline = pipeline.publish(name=env.aml_pipeline_name)
try:
pipeline_endpoint = PipelineEndpoint.get(
workspace=ws, name=env.aml_pipeline_endpoint_name)
pipeline_endpoint.add_default(published_pipeline)
except ErrorResponseException:
pipeline_endpoint = PipelineEndpoint.publish(
workspace=ws,
name=env.aml_pipeline_endpoint_name,
description=env.aml_pipeline_endpoint_name,
pipeline=published_pipeline)
def main():
load_dotenv()
workspace_name = os.environ.get("BASE_NAME")+"-AML-WS"
resource_group = os.environ.get("RESOURCE_GROUP")
subscription_id = os.environ.get("SUBSCRIPTION_ID")
tenant_id = os.environ.get("TENANT_ID")
app_id = os.environ.get("SP_APP_ID")
app_secret = os.environ.get("SP_APP_SECRET")
deploy_script_path = os.environ.get("DEPLOY_PROD_SCRIPT_PATH")
vm_size = os.environ.get("AML_COMPUTE_CLUSTER_CPU_SKU")
compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME")
model_name = os.environ.get("MODEL_NAME")
build_id = os.environ.get("BUILD_BUILDID")
pipeline_name = os.environ.get("DEPLOY_PROD_PIPELINE_NAME")
service_name = os.environ.get("DEPLOY_PROD_SERVICE_NAME")
sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN")
# Get Azure machine learning workspace
aml_workspace = get_workspace(
workspace_name,
resource_group,
subscription_id,
tenant_id,
app_id,
app_secret)
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(
aml_workspace,
compute_name,
vm_size)
if aml_compute is not None:
print(aml_compute)
conda_dependencies = CondaDependencies.create(
conda_packages=[
'numpy',
'pandas',
'scikit-learn'
],
pip_packages=[
'azureml-core==1.0.72.*',
'azureml-sdk==1.0.72.*',
'azure-storage',
'azure-storage-blob',
'azureml-dataprep',
'azureml-datadrift==1.0.72.*'
],
pin_sdk_version=False
)
print(conda_dependencies.serialize_to_string())
run_config = RunConfiguration(
framework='Python',
conda_dependencies=conda_dependencies
)
run_config.environment.docker.enabled = True
model_name = PipelineParameter(
name="model_name", default_value=model_name
)
print(model_name)
release_id = PipelineParameter(
name="release_id", default_value="0"
)
print(release_id)
service_name = PipelineParameter(
name="service_name", default_value=service_name
)
print(service_name)
deploy_step = PythonScriptStep(
name="Deploy Prod Model",
script_name=deploy_script_path,
compute_target=aml_compute,
source_directory=sources_directory_train,
arguments=[
"--release_id", release_id,
"--model_name", model_name,
"--service_name", service_name
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Deploy Prod created")
steps = [deploy_step]
deploy_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
deploy_pipeline.validate()
published_pipeline = deploy_pipeline.publish(
name=pipeline_name,
description="Model deploy Prod pipeline",
version=build_id
)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
aml_run_config.target = compute_target
train_model = PythonScriptStep(script_name='src/execute_train.py',
compute_target=compute_target,
runconfig=aml_run_config)
deploy_model = PythonScriptStep(script_name="src/deploy_model.py",
compute_target=compute_target,
runconfig=aml_run_config)
steps = [train_model, deploy_model]
step_seq = StepSequence(steps=steps)
pipeline = Pipeline(workspace=ws, steps=step_seq)
pp = pipeline.publish(
name="TitanicDeploymentPipeline",
description="Training and deployment pipeline for our titanic API.",
version="1.0")
# We can also set up a trigger based schedule using the Datastore class - https://docs.microsoft.com/en-us/azure/machine-learning/how-to-schedule-pipelines#create-a-time-based-schedule
recurrence = ScheduleRecurrence(frequency="Month",
interval=1,
start_time='2020-11-01T00:00:00')
recurring_schedule = Schedule.create(ws,
name="TitanicRetrainingSchedule",
description="Once a month training",
pipeline_id=pp.id,
experiment_name=exp_name,
recurrence=recurrence)
run = pp.submit(ws, experiment_name=exp_name)
def main():
load_dotenv()
workspace_name = os.environ.get("BASE_NAME") + "-AML-WS"
resource_group = "AML-RG-" + os.environ.get("BASE_NAME")
subscription_id = os.environ.get("SUBSCRIPTION_ID")
tenant_id = os.environ.get("TENANT_ID")
app_id = os.environ.get("SP_APP_ID")
app_secret = os.environ.get("SP_APP_SECRET")
sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN")
train_script_path = os.environ.get("TRAIN_SCRIPT_PATH")
evaluate_script_path = os.environ.get("EVALUATE_SCRIPT_PATH")
vm_size = os.environ.get("AML_COMPUTE_CLUSTER_SKU")
compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME")
aks_name = os.environ.get("AKS_CLUSTER_NAME")
model_name = os.environ.get("MODEL_NAME")
build_id = os.environ.get("BUILD_BUILDID")
pipeline_name = os.environ.get("TRAINING_PIPELINE_NAME")
experiment_name = os.environ.get("EXPERIMENT_NAME")
# Get Azure machine learning workspace
aml_workspace = get_workspace(workspace_name, resource_group,
subscription_id, tenant_id, app_id,
app_secret)
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, compute_name, vm_size)
if aml_compute is not None:
print(aml_compute)
run_config = RunConfiguration(conda_dependencies=CondaDependencies.create(
conda_packages=['numpy', 'pandas', 'scikit-learn', 'keras'],
pip_packages=[
'azure', 'azureml-sdk', 'azure-storage', 'azure-storage-blob',
'transformers>=2.1.1', 'tensorflow>=2.0.0', 'tensorflow-gpu>=2.0.0'
]))
run_config.environment.docker.enabled = True
datastore_name = 'tfworld'
container_name = 'azureml-blobstore-7c6bdd88-21fa-453a-9c80-16998f02935f'
account_name = 'tfworld6818510241'
sas_token = '?sv=2019-02-02&ss=bfqt&srt=sco&sp=rl&se=2020-06-01T14:18:31Z&st=2019-11-05T07:18:31Z&spr=https&sig=Z4JmM0V%2FQzoFNlWS3a3vJxoGAx58iCz2HAWtmeLDbGE%3D' # noqa: E501
try:
existing_datastore = Datastore.get(aml_workspace, datastore_name)
except: # noqa: E722
existing_datastore = Datastore \
.register_azure_blob_container(workspace=aml_workspace,
datastore_name=datastore_name,
container_name=container_name,
account_name=account_name,
sas_token=sas_token
)
azure_dataset = Dataset.File.from_files(
path=(existing_datastore, 'azure-service-classifier/data'))
azure_dataset = azure_dataset.register(
workspace=aml_workspace,
name='Azure Services Dataset',
description='Dataset containing azure related posts on Stackoverflow',
create_new_version=True)
azure_dataset.to_path()
input_data = azure_dataset.as_named_input('input_data1').as_mount(
'/tmp/data')
model_name = PipelineParameter(name="model_name", default_value=model_name)
max_seq_length = PipelineParameter(name="max_seq_length",
default_value=128)
learning_rate = PipelineParameter(name="learning_rate", default_value=3e-5)
num_epochs = PipelineParameter(name="num_epochs", default_value=3)
export_dir = PipelineParameter(name="export_dir",
default_value="./outputs/exports")
batch_size = PipelineParameter(name="batch_size", default_value=32)
steps_per_epoch = PipelineParameter(name="steps_per_epoch",
default_value=100)
# initialize the TensorFlow estimator
estimator = TensorFlow(source_directory=sources_directory_train,
entry_script=train_script_path,
compute_target=aml_compute,
framework_version='2.0',
use_gpu=True,
pip_packages=[
'transformers==2.0.0',
'azureml-dataprep[fuse,pandas]==1.1.29'
])
train_step = EstimatorStep(
name="Train Model",
estimator=estimator,
estimator_entry_script_arguments=[
"--data_dir", input_data, "--max_seq_length", max_seq_length,
"--learning_rate", learning_rate, "--num_epochs", num_epochs,
"--export_dir", export_dir, "--batch_size", batch_size,
"--steps_per_epoch", steps_per_epoch
],
compute_target=aml_compute,
inputs=[input_data],
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=evaluate_script_path,
compute_target=aml_compute,
source_directory=sources_directory_train,
arguments=[
"--model_name",
model_name,
"--build_id",
build_id,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
# Currently, the Evaluate step will automatically register
# the model if it performs better. This step is based on a
# previous version of the repo which utilized JSON files to
# track evaluation results.
evaluate_step.run_after(train_step)
steps = [evaluate_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=pipeline_name,
description="Model training/retraining pipeline",
version=build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
response = published_pipeline.submit( # noqa: F841
workspace=aml_workspace,
experiment_name=experiment_name)
# Get AKS cluster for deployment
aks_compute = get_aks(aml_workspace, aks_name)
if aks_compute is not None:
print(aks_compute)
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable Azure ML environment
environment = get_environment(aml_workspace,
e.aml_env_name,
create_new=False) # NOQA: E501
run_config = RunConfiguration()
run_config.environment = environment
if (e.datastore_name):
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables[
"DATASTORE_NAME"] = datastore_name # NOQA: E501
model_name_param = PipelineParameter(name="model_name",
default_value=e.model_name)
dataset_version_param = PipelineParameter(name="dataset_version",
default_value=e.dataset_version)
data_file_path_param = PipelineParameter(name="data_file_path",
default_value="none")
caller_run_id_param = PipelineParameter(name="caller_run_id",
default_value="none")
# Get dataset name
dataset_name = e.dataset_name
# Check to see if dataset exists
if (dataset_name not in aml_workspace.datasets):
# Create dataset from MLOpsAutoML sample data
sample_data = load_MLOpsAutoML()
df = pd.DataFrame(data=sample_data.data,
columns=sample_data.feature_names)
df['Y'] = sample_data.target
file_name = 'MLOpsAutoML.csv'
df.to_csv(file_name, index=False)
# Upload file to default datastore in workspace
datatstore = Datastore.get(aml_workspace, datastore_name)
target_path = 'training-data/'
datatstore.upload_files(files=[file_name],
target_path=target_path,
overwrite=True,
show_progress=False)
# Register dataset
path_on_datastore = os.path.join(target_path, file_name)
dataset = Dataset.Tabular.from_delimited_files(
path=(datatstore, path_on_datastore))
dataset = dataset.register(workspace=aml_workspace,
name=dataset_name,
description='MLOpsAutoML training data',
tags={'format': 'CSV'},
create_new_version=True)
# Create a PipelineData to pass data between steps
pipeline_data = PipelineData(
'pipeline_data', datastore=aml_workspace.get_default_datastore())
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
outputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_output",
pipeline_data,
"--dataset_version",
dataset_version_param,
"--data_file_path",
data_file_path_param,
"--caller_run_id",
caller_run_id_param,
"--dataset_name",
dataset_name,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--model_name",
model_name_param,
"--allow_run_cancel",
e.allow_run_cancel,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_input",
pipeline_data,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
# Check run_evaluation flag to include or exclude evaluation step.
if ((e.run_evaluation).lower() == 'true'):
print("Include evaluation step before register step.")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
else:
print("Exclude evaluation step and directly run register step.")
register_step.run_after(train_step)
steps = [train_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
regenerate_outputs=True)
print("Pipeline is submitted for execution")
pipeline_run1.wait_for_completion(show_output=True)
# RunDetails(pipeline_run1).show()
# Define pipeline parameters
# run_env = PipelineParameter(
# name="dev_flag",
# default_value=True)
# dbname = PipelineParameter(
# name="dbname",
# default_value='opex')
# Publish Pipeline
if args.pipeline_action == "publish":
published_pipeline1 = pipeline1.publish(
name=aml_pipeline_name,
description="Model training/retraining pipeline")
print("Pipeline is published as rest_endpoint {} ".format(
published_pipeline1.endpoint))
# write published pipeline details as build artifact
pipeline_config = {}
pipeline_config["pipeline_name"] = published_pipeline1.name
pipeline_config["rest_endpoint"] = published_pipeline1.endpoint
pipeline_config[
"experiment_name"] = "published-pipeline-exp" # experiment_name
with open("aml_config/pipeline_config.json", "w") as outfile:
json.dump(pipeline_config, outfile)
inference_pipeline = Pipeline(workspace=ws, steps=[inferenceStep])
print ("Inference Pipeline is built")
inference_pipeline.validate()
print("Simple validation complete")
# ### Publish the Inference Pipeline
#
# Note that we are not submitting the pipeline to run, instead we are publishing the pipeline. When you publish a pipeline, it can be submitted to run without the Python code which constructed the Pipeline.
# In[ ]:
pipeline_name = 'Inference Pipeline'
published_pipeline = inference_pipeline.publish(name = pipeline_name)
# ### Schedule the Inference Pipeline
#
# We want to run the Inference Pipeline when a new test or input data is uploaded at the location referenced by the `raw_bulk_test_data` DataReference object. The next cell creates a Schedule to monitor the datastore for changes, and is responsible for running the `Inference Pipeline` when it detects a new file being uploaded.
#
# **Please disable the Schedule after completing the quickstart. To disable the Schedule, run the `Cleanup Resources` section at the end of the notebook.**
# In[ ]:
from azureml.pipeline.core.schedule import Schedule
schedule = Schedule.create(workspace=ws, name=pipeline_name + "_sch",
pipeline_id=published_pipeline.id,
source_directory=script_folder,
runconfig=cpu_compute_run_config,
allow_reuse=False,
hash_paths=["."])
print("pipeline submit step created")
submit_pipelines.run_after(create_pipelines)
pipeline = Pipeline(workspace=ws, steps=[create_pipelines, submit_pipelines])
print("Pipeline created")
pipeline.validate()
print("Validation complete")
pipeline_name = 'prednet_master'
published_pipeline = pipeline.publish(name=pipeline_name)
print("pipeline id: ", published_pipeline.id)
datastore = ws.get_default_datastore()
with open('placeholder.txt', 'w') as f:
f.write(
'This is just a placeholder to ensure that this path exists in the blobstore.\n'
)
datastore.upload_files([os.path.join(os.getcwd(), 'placeholder.txt')],
target_path='prednet/data/video/')
schedule = Schedule.create(workspace=ws,
name=pipeline_name + "_sch",
pipeline_id=published_pipeline.id,
inputs=[training_data_location],
outputs=[model])
## Register Model Step ##
# Once training is complete, register.py registers the model with AML #
# Configuration for registration step #
registerModelStep = PythonScriptStep(
name="Register model in Model Management",
script_name="steps/register.py",
compute_target=cluster,
inputs=[model],
arguments=['--model_name', model_name, '--model_assets_path', model],
source_directory=script_folder)
## Create and publish the Pipeline ##
# We now define and publish the pipeline #
pipeline = Pipeline(
workspace=ws,
steps=[preProcessDataStep, trainOnGpuStep, registerModelStep])
published_pipeline = pipeline.publish(
name="Simpsons-PyTorch-Pipeline - Training pipeline (From DevOps)",
description="Training pipeline (From Azure DevOps)")
## Submit the pipeline to be run ##
# Finally, we submit the pipeline for execution #
pipeline_run = published_pipeline.submit(ws, experiment_name)
print('Run created with ID: ', pipeline_run.id)
register = PythonScriptStep(
name="Register model for deployment",
script_name="register.py",
compute_target=aml_compute,
inputs=[model],
arguments=['--dataset_name', model_variant, '--model_assets_path', model],
outputs=[model_id],
source_directory=project_folder)
steps = [datatransfer, train, register]
pipeline = Pipeline(workspace=ws, steps=steps)
pipeline.validate()
mlpipeline = pipeline.publish(
name="Transfer Learning - Training Pipeline",
description="Retrain a mobilenet.imagenet model.")
print("Pipeline Published ID:" + mlpipeline.id)
mlpipeline.submit(ws,
"sodacanclassifier",
pipeline_parameters={
"training_dataset":
DataPath(datastore=source_ds,
path_on_datastore="soda_cans_training_data"),
"model_variant":
"sodacans"
})
"""
Examples showing you how to train different variants of models, with the same training pipeline.
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group
)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(
aml_workspace,
e.compute_name,
e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable run configuration environment
# Read definition from diabetes_regression/azureml_environment.json
environment = Environment.load_from_directory(e.sources_directory_train)
if (e.collection_uri is not None and e.teamproject_name is not None):
builduri_base = e.collection_uri + e.teamproject_name
builduri_base = builduri_base + "/_build/results?buildId="
environment.environment_variables["BUILDURI_BASE"] = builduri_base
environment.register(aml_workspace)
run_config = RunConfiguration()
run_config.environment = environment
model_name_param = PipelineParameter(
name="model_name", default_value=e.model_name)
build_id_param = PipelineParameter(
name="build_id", default_value=e.build_id)
dataset_name = ""
if (e.datastore_name is not None and e.datafile_name is not None):
dataset_name = e.dataset_name
datastore = Datastore.get(aml_workspace, e.datastore_name)
data_path = [(datastore, e.datafile_name)]
dataset = Dataset.Tabular.from_delimited_files(path=data_path)
dataset.register(workspace=aml_workspace,
name=e.dataset_name,
description="dataset with training data",
create_new_version=True)
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id", build_id_param,
"--model_name", model_name_param,
"--dataset_name", dataset_name,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id", build_id_param,
"--model_name", model_name_param,
"--allow_run_cancel", e.allow_run_cancel,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id", build_id_param,
"--model_name", model_name_param,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
# Check run_evaluation flag to include or exclude evaluation step.
if ((e.run_evaluation).lower() == 'true'):
print("Include evaluation step before register step.")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
else:
print("Exclude evaluation step and directly run register step.")
register_step.run_after(train_step)
steps = [train_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id
)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = get_workspace(e.workspace_name, e.resource_group,
e.subscription_id, e.tenant_id, e.app_id,
e.app_secret)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
run_config = RunConfiguration(conda_dependencies=CondaDependencies.create(
conda_packages=[
'numpy', 'pandas', 'scikit-learn', 'tensorflow', 'keras'
],
pip_packages=[
'azure', 'azureml-core', 'azure-storage', 'azure-storage-blob'
]))
run_config.environment.docker.enabled = True
model_name_param = PipelineParameter(name="model_name",
default_value=e.model_name)
build_id_param = PipelineParameter(name="build_id",
default_value=e.build_id)
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable run configuration environment
# Read definition from diabetes_regression/azureml_environment.json
environment = Environment.load_from_directory(e.sources_directory_train)
if (e.collection_uri is not None and e.teamproject_name is not None):
builduri_base = e.collection_uri + e.teamproject_name
builduri_base = builduri_base + "/_build/results?buildId="
environment.environment_variables["BUILDURI_BASE"] = builduri_base
environment.register(aml_workspace)
run_config = RunConfiguration()
run_config.environment = environment
model_name_param = PipelineParameter(name="model_name",
default_value=e.model_name)
build_id_param = PipelineParameter(name="build_id",
default_value=e.build_id)
# Get dataset name
dataset_name = e.dataset_name
# Check to see if dataset exists
if (dataset_name not in aml_workspace.datasets):
# Create dataset from diabetes sample data
sample_data = load_diabetes()
df = pd.DataFrame(data=sample_data.data,
columns=sample_data.feature_names)
df['Y'] = sample_data.target
file_name = 'diabetes.csv'
df.to_csv(file_name, index=False)
# Upload file to default datastore in workspace
default_ds = aml_workspace.get_default_datastore()
target_path = 'training-data/'
default_ds.upload_files(files=[file_name],
target_path=target_path,
overwrite=True,
show_progress=False)
# Register dataset
path_on_datastore = os.path.join(target_path, file_name)
dataset = Dataset.Tabular.from_delimited_files(
path=(default_ds, path_on_datastore))
dataset = dataset.register(workspace=aml_workspace,
name=dataset_name,
description='diabetes training data',
tags={'format': 'CSV'},
create_new_version=True)
# Get the dataset
dataset = Dataset.get_by_name(aml_workspace, dataset_name)
# Create a PipelineData to pass data between steps
pipeline_data = PipelineData(
'pipeline_data', datastore=aml_workspace.get_default_datastore())
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[dataset.as_named_input('training_data')],
outputs=[pipeline_data],
arguments=[
"--build_id", build_id_param, "--model_name", model_name_param,
"--step_output", pipeline_data
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
"--allow_run_cancel",
e.allow_run_cancel,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[pipeline_data],
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
"--step_input",
pipeline_data,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
# Check run_evaluation flag to include or exclude evaluation step.
if ((e.run_evaluation).lower() == 'true'):
print("Include evaluation step before register step.")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
else:
print("Exclude evaluation step and directly run register step.")
register_step.run_after(train_step)
steps = [train_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
datastore = get_datastore(ws, secrets["datastore_name"],
secrets["container"], secrets["storage_account"],
secrets["storage_key"])
# get compute
compute = get_compute(ws, secrets["compute_target"])
# prep step
pdata, pstep = process_step(datastore, compute, secrets["datastore_path"])
# train step
tdata, tstep = train_step(datastore, pdata, compute)
# register step (tag model with version)
rdata, rstep = register_step(datastore, tdata, compute, args.build)
# create pipeline from steps
seer_pipeline = Pipeline(workspace=ws, steps=[pstep, tstep, rstep])
published_pipeline = seer_pipeline.publish(
name="Seer Pipeline",
description="Transfer learned image classifier. Uses folders as labels."
)
# add pipeline to endpoint
endpoint = add_endpoint(ws, published_pipeline, 'seer-endpoint')
# run pipeline
pipeline_run = endpoint.submit('seer')
pipeline_run.set_tags(tags={'build': args.build})
print(f'Run created with ID: {pipeline_run.id}')
print(f'WS name: {ws.name}')
print(f'Region: {ws.location}')
print(f'Subscription id: {ws.subscription_id}')
print(f'Resource group: {ws.resource_group}')
default_training_dataset = Dataset.get_by_name(ws, default_dataset_name)
# Parametrize dataset input to the pipeline
training_dataset_parameter = PipelineParameter(name='training_dataset', default_value=default_training_dataset)
training_dataset_consumption = DatasetConsumptionConfig('training_dataset', training_dataset_parameter).as_download()
# Load runconfig from earlier exercise and create pipeline
runconfig = RunConfiguration.load(os.path.join(source_directory, 'runconfig.yml'))
train_step = PythonScriptStep(name='train-step',
source_directory=source_directory,
script_name='train.py',
arguments=['--data-path', training_dataset_consumption],
inputs=[training_dataset_consumption],
runconfig=runconfig,
allow_reuse=False)
steps = [train_step]
pipeline = Pipeline(workspace=ws, steps=steps)
pipeline.validate()
published_pipeline = pipeline.publish('training-pipeline')
# Printing the pipeline's id this way will make it available in Azure DevOps for the subsequent tasks
print(f'##vso[task.setvariable variable=pipeline_id]{published_pipeline.id}')
train_model = PythonScriptStep(
script_name="./train.py",
name="fit-nlp-model",
inputs=[inputdata.as_download(path_on_compute="./data")],
runconfig=run_config,
compute_target=compute_target,
)
pipeline = Pipeline(
workspace=ws,
steps=[train_model],
description="Builds Keras model for detecting component defects",
)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--publish", action="store_true")
args = parser.parse_args()
if args.publish:
pipeline.publish(
name="component-defects-model-tf",
description="Builds Keras model for detecting component defects",
)
else:
Experiment(ws, "fit-component-defects-model").submit(
pipeline).wait_for_completion(show_output=True)
],
inputs=[blob_output_data],
compute_target=ws.compute_targets['cpu'],
allow_reuse=False)
pipeline = Pipeline(workspace=ws, steps=[data_processing])
print("Pipeline is built")
pipeline.validate()
print("Simple validation complete")
pipeline_run = Experiment(ws, 'issues_pipeline').submit(pipeline)
print("Pipeline is submitted for execution")
published_pipeline = pipeline.publish(
name="Issues_Stats",
description="Pull data from DevOps and aggregate for PowerBI")
print(published_pipeline.id)
from azureml.pipeline.core.schedule import ScheduleRecurrence, Schedule
recurrence = ScheduleRecurrence(frequency="Hour",
interval=1,
start_time="2020-01-13T12:55:00")
schedule = Schedule.create(workspace=ws,
name="Bug_Stats",
pipeline_id=published_pipeline.id,
experiment_name='Schedule_Run',
recurrence=recurrence,
wait_for_provisioning=True,
