def append_step(self, step_list):
train_step = PythonScriptStep(
name="Train Model",
compute_target=self.compute,
source_directory=self.env.sources_directory_train,
script_name=self.env.train_script_path,
outputs=[self.output_pipelinedata],
arguments=[
"--model_name",
self.pipeline_parameters["model_name"],
"--step_output",
self.output_pipelinedata,
"--dataset_version",
self.pipeline_parameters["dataset_version"],
"--data_file_path",
self.pipeline_parameters["data_file_path"],
"--caller_run_id",
self.pipeline_parameters["caller_run_id"],
"--dataset_name",
self.env.dataset_name,
],
runconfig=self.config,
allow_reuse=True,
)
if len(step_list) > 0:
previous_step = step_list[-1]
train_step.run_after(previous_step)
step_list.append(train_step)
def append_step(self, step_list):
evaluate_step = PythonScriptStep(
name="Evaluate Model",
compute_target=self.compute,
source_directory=self.env.sources_directory_train,
script_name=self.env.evaluate_script_path,
arguments=[
"--model_name",
self.pipeline_parameters["model_name"],
"--allow_run_cancel",
self.env.allow_run_cancel,
],
runconfig=self.config,
allow_reuse=False,
)
if len(step_list) > 0:
previous_step = step_list[-1]
evaluate_step.run_after(previous_step)
step_list.append(evaluate_step)
def append_step(self, step_list):
register_step = PythonScriptStep(
name="Register Model",
compute_target=self.compute,
source_directory=self.env.sources_directory_train,
script_name=self.env.register_script_path,
inputs=[self.input_pipelinedata],
arguments=[
"--model_name",
self.pipeline_parameters["model_name"],
"--step_input",
self.input_pipelinedata,
],
runconfig=self.config,
allow_reuse=False,
)
if len(step_list) > 0:
previous_step = step_list[-1]
register_step.run_after(previous_step)
step_list.append(register_step)
package_model = PythonScriptStep(
name="Package Model as Scoring Image",
script_name="scoring/create_scoring_image.py",
compute_target=aml_compute,
source_directory=source_directory,
arguments=["--config_suffix", config_suffix, "--json_config", jsonconfigs],
runconfig=run_config,
inputs=[jsonconfigs],
# outputs=[jsonconfigs],
allow_reuse=False,
)
print("Packed the model into a Scoring Image")
# Create Steps dependency such that they run in sequence
evaluate.run_after(train)
register_model.run_after(evaluate)
package_model.run_after(register_model)
steps = [package_model]
# Build Pipeline
pipeline1 = Pipeline(workspace=ws, steps=steps)
print("Pipeline is built")
# Validate Pipeline
pipeline1.validate()
print("Pipeline validation complete")
# Submit unpublished pipeline with small data set for test
if args.pipeline_action == "pipeline-test":
def main():
load_dotenv()
workspace_name = os.environ.get("BASE_NAME") + "-AML-WS"
resource_group = os.environ.get("BASE_NAME") + "-AML-RG"
subscription_id = os.environ.get("SUBSCRIPTION_ID")
tenant_id = os.environ.get("TENANT_ID")
app_id = os.environ.get("SP_APP_ID")
app_secret = os.environ.get("SP_APP_SECRET")
sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN")
train_script_path = os.environ.get("TRAIN_SCRIPT_PATH")
evaluate_script_path = os.environ.get("EVALUATE_SCRIPT_PATH")
register_script_path = os.environ.get("REGISTER_SCRIPT_PATH")
vm_size_cpu = os.environ.get("AML_COMPUTE_CLUSTER_CPU_SKU")
compute_name_cpu = os.environ.get("AML_COMPUTE_CLUSTER_NAME")
model_name = os.environ.get("MODEL_NAME")
# Get Azure machine learning workspace
aml_workspace = get_workspace(workspace_name, resource_group,
subscription_id, tenant_id, app_id,
app_secret)
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute_cpu = get_compute(aml_workspace, compute_name_cpu, vm_size_cpu)
if aml_compute_cpu is not None:
print(aml_compute_cpu)
run_config = RunConfiguration(conda_dependencies=CondaDependencies.create(
conda_packages=[
'numpy', 'pandas', 'scikit-learn', 'tensorflow', 'keras'
]))
run_config.environment.docker.enabled = True
model_name = PipelineParameter(name="model_name", default_value=model_name)
def_blob_store = Datastore(aml_workspace, "workspaceblobstore")
jsonconfigs = PipelineData("jsonconfigs", datastore=def_blob_store)
config_suffix = datetime.datetime.now().strftime("%Y%m%d%H")
train_step = PythonScriptStep(
name="Train Model",
script_name=train_script_path,
compute_target=aml_compute_cpu,
source_directory=sources_directory_train,
arguments=[
"--config_suffix",
config_suffix,
"--json_config",
jsonconfigs,
"--model_name",
model_name,
],
runconfig=run_config,
# inputs=[jsonconfigs],
outputs=[jsonconfigs],
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=evaluate_script_path,
compute_target=aml_compute_cpu,
source_directory=sources_directory_train,
arguments=[
"--config_suffix",
config_suffix,
"--json_config",
jsonconfigs,
],
runconfig=run_config,
inputs=[jsonconfigs],
# outputs=[jsonconfigs],
allow_reuse=False,
)
print("Step Evaluate created")
register_model_step = PythonScriptStep(
name="Register New Trained Model",
script_name=register_script_path,
compute_target=aml_compute_cpu,
source_directory=sources_directory_train,
arguments=[
"--config_suffix",
config_suffix,
"--json_config",
jsonconfigs,
"--model_name",
model_name,
],
runconfig=run_config,
inputs=[jsonconfigs],
# outputs=[jsonconfigs],
allow_reuse=False,
)
print("Step register model created")
evaluate_step.run_after(train_step)
register_model_step.run_after(evaluate_step)
steps = [register_model_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name="training-pipeline",
description="Model training/retraining pipeline")
train_pipeline_json = {}
train_pipeline_json["rest_endpoint"] = published_pipeline.endpoint
json_file_path = "ml_service/pipelines/train_pipeline.json"
with open(json_file_path, "w") as outfile:
json.dump(train_pipeline_json, outfile)
script_name="evaluate.py",
arguments=[
"--model_name", args.model_name,
"--metric_threshold",
float(args.metric_threshold),
"--image_name", args.image_name,
"--output", evaluate_output
],
outputs=[evaluate_output],
compute_target=aml_compute,
runconfig=run_amlcompute,
source_directory=scripts_folder,
allow_reuse=False)
print("evaluateStep created")
evaluateStep.run_after(trainStep)
steps = [evaluateStep]
pipeline = Pipeline(workspace=ws, steps=steps)
print("Pipeline is built")
pipeline.validate()
print("Simple validation complete")
run = Run.get_context()
experiment_name = run.experiment.name
pipeline_run = Experiment(ws, experiment_name).submit(pipeline)
print("Pipeline is submitted for execution")
pipeline_run.wait_for_completion(show_output=True)
def get_pipeline(aml_compute: ComputeTarget, blob_ds: Datastore,
batch_env: Environment, tf_env: Environment) -> str:
"""
Creates pipeline steps
Parameters:
aml_compute (ComputeTarget): a reference to a compute
blob_ds (DataStore): a reference to a datastore
batch_env (Environment): a reference to environment object
tf_env (Environment): a horovod/tf environment
Returns:
string: a set of pipeline steps
"""
# We need something to generate data by the way
pipeline_files = PipelineData("pipeline_files",
datastore=blob_ds).as_dataset()
# Pipeline parameters to use with every run
is_debug = PipelineParameter("is_debug", default_value=False)
relay_connection_name = PipelineParameter("debug_relay_connection_name",
default_value="none")
single_step_config = RunConfiguration()
single_step_config.environment = batch_env
single_step = PythonScriptStep(
name=f"single-step",
script_name="samples/azure_ml_advanced/steps/single_step.py",
source_directory=".",
runconfig=single_step_config,
arguments=[
"--pipeline-files", pipeline_files, "--is-debug", is_debug,
"--debug-relay-connection-name", relay_connection_name,
"--debug-port", 5678, "--debug-relay-connection-string-secret",
debug_connection_string_secret_name
],
inputs=[],
outputs=[pipeline_files],
compute_target=aml_compute,
allow_reuse=False)
output_dir = PipelineData("output_dir")
parallel_run_config = ParallelRunConfig(
entry_script="samples/azure_ml_advanced/steps/parallel_step.py",
source_directory=".",
mini_batch_size="5",
output_action="summary_only",
environment=batch_env,
compute_target=aml_compute,
error_threshold=10,
run_invocation_timeout=600, # very important for debugging
node_count=2,
process_count_per_node=1)
parallelrun_step = ParallelRunStep(
name="parallel-run-step",
parallel_run_config=parallel_run_config,
inputs=[pipeline_files],
output=output_dir,
arguments=[
"--is-debug", is_debug, "--debug-relay-connection-name",
relay_connection_name, "--debug-port", 5679,
"--debug-relay-connection-string-secret",
debug_connection_string_secret_name
],
allow_reuse=False)
parallelrun_step.run_after(single_step)
distr_config = MpiConfiguration(process_count_per_node=1, node_count=2)
src = ScriptRunConfig(
source_directory=".",
script="samples/azure_ml_advanced/steps/mpi/mpi_step_starter.py",
arguments=[
"--input-ds", pipeline_files, "--is-debug", is_debug,
"--debug-relay-connection-name", relay_connection_name,
"--debug-port", 5680, "--debug-relay-connection-string-secret",
debug_connection_string_secret_name
],
compute_target=compute_name,
environment=tf_env,
distributed_job_config=distr_config,
)
mpi_step = PythonScriptStep(
name="mpi-step",
script_name="samples/azure_ml_advanced/steps/mpi/mpi_step_starter.py",
arguments=[
"--input-ds", pipeline_files, "--is-debug", is_debug,
"--debug-relay-connection-name", relay_connection_name,
"--debug-port", 5680, "--debug-relay-connection-string-secret",
debug_connection_string_secret_name
],
compute_target=aml_compute,
inputs=[pipeline_files],
outputs=[],
runconfig=src.run_config,
source_directory=".")
mpi_step.run_after(parallelrun_step)
print("Pipeline Steps Created")
steps = [single_step, parallelrun_step, mpi_step]
print(f"Returning {len(steps)} steps")
return steps
def build_pipeline(dataset, ws, config):
print("building pipeline for dataset %s in workspace %s" % (dataset, ws.name))
base_dir = '.'
def_blob_store = ws.get_default_datastore()
# folder for scripts that need to be uploaded to Aml compute target
script_folder = './scripts'
os.makedirs(script_folder, exist_ok=True)
shutil.copy(os.path.join(base_dir, 'video_decoding.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'pipelines_submit.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'pipelines_create.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'train.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'data_utils.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'prednet.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'keras_utils.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'data_preparation.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'model_registration.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'config.json'), script_folder)
cpu_compute_name = config['cpu_compute']
try:
cpu_compute_target = AmlCompute(ws, cpu_compute_name)
print("found existing compute target: %s" % cpu_compute_name)
except:# ComputeTargetException:
print("creating new compute target")
provisioning_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_D2_V2',
max_nodes=4,
idle_seconds_before_scaledown=1800)
cpu_compute_target = ComputeTarget.create(ws, cpu_compute_name, provisioning_config)
cpu_compute_target.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20)
# use get_status() to get a detailed status for the current cluster.
print(cpu_compute_target.get_status().serialize())
# choose a name for your cluster
gpu_compute_name = config['gpu_compute']
try:
gpu_compute_target = AmlCompute(workspace=ws, name=gpu_compute_name)
print("found existing compute target: %s" % gpu_compute_name)
except:
print('Creating a new compute target...')
provisioning_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_NC6',
max_nodes=10,
idle_seconds_before_scaledown=1800)
# create the cluster
gpu_compute_target = ComputeTarget.create(ws, gpu_compute_name, provisioning_config)
# can poll for a minimum number of nodes and for a specific timeout.
# if no min node count is provided it uses the scale settings for the cluster
gpu_compute_target.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20)
# use get_status() to get a detailed status for the current cluster.
try:
print(gpu_compute_target.get_status().serialize())
except BaseException as e:
print("Could not get status of compute target.")
print(e)
# conda dependencies for compute targets
cpu_cd = CondaDependencies.create(conda_packages=["py-opencv=3.4.2"], pip_indexurl='https://azuremlsdktestpypi.azureedge.net/sdk-release/Candidate/604C89A437BA41BD942B4F46D9A3591D', pip_packages=["azure-storage-blob==1.5.0", "hickle==3.4.3", "requests==2.21.0", "sklearn", "pandas==0.24.2", "azureml-sdk", "numpy==1.16.2", "pillow==6.0.0"])
# Runconfigs
cpu_compute_run_config = RunConfiguration(conda_dependencies=cpu_cd)
cpu_compute_run_config.environment.docker.enabled = True
cpu_compute_run_config.environment.docker.gpu_support = False
cpu_compute_run_config.environment.docker.base_image = DEFAULT_CPU_IMAGE
cpu_compute_run_config.environment.spark.precache_packages = False
print("PipelineData object created")
# DataReference to where video data is stored.
video_data = DataReference(
datastore=def_blob_store,
data_reference_name="video_data",
path_on_datastore=os.path.join("prednet", "data", "video", dataset))
print("DataReference object created")
# Naming the intermediate data as processed_data1 and assigning it to the variable processed_data1.
raw_data = PipelineData("raw_video_fames", datastore=def_blob_store)
preprocessed_data = PipelineData("preprocessed_video_frames", datastore=def_blob_store)
data_metrics = PipelineData("data_metrics", datastore=def_blob_store)
data_output = PipelineData("output_data", datastore=def_blob_store)
# prepare dataset for training/testing prednet
video_decoding = PythonScriptStep(
name='decode_videos',
script_name="video_decoding.py",
arguments=["--input_data", video_data, "--output_data", raw_data],
inputs=[video_data],
outputs=[raw_data],
compute_target=cpu_compute_target,
source_directory=script_folder,
runconfig=cpu_compute_run_config,
allow_reuse=True,
hash_paths=['.']
)
print("video_decode step created")
# prepare dataset for training/testing recurrent neural network
data_prep = PythonScriptStep(
name='prepare_data',
script_name="data_preparation.py",
arguments=["--input_data", raw_data, "--output_data", preprocessed_data],
inputs=[raw_data],
outputs=[preprocessed_data],
compute_target=cpu_compute_target,
source_directory=script_folder,
runconfig=cpu_compute_run_config,
allow_reuse=True,
hash_paths=['.']
)
data_prep.run_after(video_decoding)
print("data_prep step created")
# configure access to ACR for pulling our custom docker image
acr = ContainerRegistry()
acr.address = config['acr_address']
acr.username = config['acr_username']
acr.password = config['acr_password']
est = Estimator(source_directory=script_folder,
compute_target=gpu_compute_target,
entry_script='train.py',
use_gpu=True,
node_count=1,
custom_docker_image = "wopauli_1.8-gpu:1",
image_registry_details=acr,
user_managed=True
)
ps = RandomParameterSampling(
{
'--batch_size': choice(1, 2, 4, 8),
'--filter_sizes': choice("3, 3, 3", "4, 4, 4", "5, 5, 5"),
'--stack_sizes': choice("48, 96, 192", "36, 72, 144", "12, 24, 48"), #, "48, 96"),
'--learning_rate': loguniform(-6, -1),
'--lr_decay': loguniform(-9, -1),
'--freeze_layers': choice("0, 1, 2", "1, 2, 3", "0, 1", "1, 2", "2, 3", "0", "3"),
'--transfer_learning': choice("True", "False")
}
)
policy = BanditPolicy(evaluation_interval=2, slack_factor=0.1, delay_evaluation=10)
hdc = HyperDriveConfig(estimator=est,
hyperparameter_sampling=ps,
policy=policy,
primary_metric_name='val_loss',
primary_metric_goal=PrimaryMetricGoal.MINIMIZE,
max_total_runs=10,
max_concurrent_runs=5,
max_duration_minutes=60*6
)
hd_step = HyperDriveStep(
name="train_w_hyperdrive",
hyperdrive_run_config=hdc,
estimator_entry_script_arguments=[
'--data-folder', preprocessed_data,
'--remote_execution',
'--dataset', dataset
],
inputs=[preprocessed_data],
metrics_output = data_metrics,
allow_reuse=True
)
hd_step.run_after(data_prep)
registration_step = PythonScriptStep(
name='register_model',
script_name='model_registration.py',
arguments=['--input_dir', data_metrics, '--output_dir', data_output],
compute_target=cpu_compute_target,
inputs=[data_metrics],
outputs=[data_output],
source_directory=script_folder,
allow_reuse=True,
hash_paths=['.']
)
registration_step.run_after(hd_step)
pipeline = Pipeline(workspace=ws, steps=[video_decoding, data_prep, hd_step, registration_step])
print ("Pipeline is built")
pipeline.validate()
print("Simple validation complete")
pipeline_name = 'prednet_' + dataset
published_pipeline = pipeline.publish(name=pipeline_name)
schedule = Schedule.create(workspace=ws, name=pipeline_name + "_sch",
pipeline_id=published_pipeline.id,
experiment_name=pipeline_name,
datastore=def_blob_store,
wait_for_provisioning=True,
description="Datastore scheduler for Pipeline" + pipeline_name,
path_on_datastore=os.path.join('prednet/data/video', dataset, 'Train'),
polling_interval=1
)
return pipeline_name
metrics_output=data_metrics,
allow_reuse=True)
hd_step.run_after(get_logits_from_xception)
registration_step = PythonScriptStep(
name='register_model',
script_name='model_registration.py',
arguments=['--input_dir', data_metrics, '--output_dir', data_output],
compute_target=gpu_compute_target,
inputs=[data_metrics],
outputs=[data_output],
source_directory=script_folder,
runconfig=gpu_compute_run_config,
allow_reuse=True,
hash_paths=['.'])
registration_step.run_after(hd_step)
pipeline = Pipeline(
workspace=ws, steps=[get_logits_from_xception, hd_step, registration_step])
print("Pipeline is built")
pipeline.validate()
print("Simple validation complete")
pipeline_name = 'kd_teach_the_student'
# We need to disable (delete) previously published pipelines, because we can't have two published pipelines with the same name
from utils.azure import disable_pipeline
disable_pipeline(pipeline_name=pipeline_name, prefix='', dry_run=False)
published_pipeline = pipeline.publish(name=pipeline_name)
hash_paths=["."])
print("pipeline building step created")
# step 2, submit pipelines
submit_pipelines = PythonScriptStep(
name='submit pipelines',
script_name="pipelines_submit.py",
# arguments=["--overwrite_published_pipelines", overwrite_published_pipelines],
compute_target=cpu_compute_target,
source_directory=script_folder,
runconfig=cpu_compute_run_config,
allow_reuse=False,
hash_paths=["."])
print("pipeline submit step created")
submit_pipelines.run_after(create_pipelines)
pipeline = Pipeline(workspace=ws, steps=[create_pipelines, submit_pipelines])
print("Pipeline created")
pipeline.validate()
print("Validation complete")
pipeline_name = 'prednet_master'
published_pipeline = pipeline.publish(name=pipeline_name)
print("pipeline id: ", published_pipeline.id)
datastore = ws.get_default_datastore()
with open('placeholder.txt', 'w') as f:
f.write(
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a run configuration environment
conda_deps_file = "diabetes_regression/training_dependencies.yml"
conda_deps = CondaDependencies(conda_deps_file)
run_config = RunConfiguration(conda_dependencies=conda_deps)
run_config.environment.docker.enabled = True
config_envvar = {}
if (e.collection_uri is not None and e.teamproject_name is not None):
builduri_base = e.collection_uri + e.teamproject_name
builduri_base = builduri_base + "/_build/results?buildId="
config_envvar["BUILDURI_BASE"] = builduri_base
run_config.environment.environment_variables = config_envvar
model_name_param = PipelineParameter(name="model_name",
default_value=e.model_name)
build_id_param = PipelineParameter(name="build_id",
default_value=e.build_id)
dataset_name = ""
if (e.datastore_name is not None and e.datafile_name is not None):
dataset_name = e.dataset_name
datastore = Datastore.get(aml_workspace, e.datastore_name)
data_path = [(datastore, e.datafile_name)]
dataset = Dataset.Tabular.from_delimited_files(path=data_path)
dataset.register(workspace=aml_workspace,
name=e.dataset_name,
description="dataset with training data",
create_new_version=True)
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
"--dataset_name",
dataset_name,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
"--allow_run_cancel",
e.allow_run_cancel,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
# Check run_evaluation flag to include or exclude evaluation step.
if ((e.run_evaluation).lower() == 'true'):
print("Include evaluation step before register step.")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
else:
print("Exclude evaluation step and directly run register step.")
register_step.run_after(train_step)
steps = [train_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
def main():
e = Env()
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group
)
print("get_workspace:")
print(aml_workspace)
aml_compute = get_compute(
aml_workspace,
e.compute_name,
e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
environment = get_environment(
aml_workspace, e.aml_env_name, create_new=e.rebuild_env)
run_config = RunConfiguration()
run_config.environment = environment
if (e.datastore_name):
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables["DATASTORE_NAME"] \
= datastore_name
dataset_name = e.dataset_name
file_name = e.file_name
datastore = Datastore.get(aml_workspace, datastore_name)
if (dataset_name not in aml_workspace.datasets):
raise Exception("Could not find dataset at \"%s\"." % dataset_name)
else:
dataset = Dataset.get_by_name(aml_workspace, name=dataset_name)
dataset.download(target_path='.', overwrite=True)
datastore.upload_files([file_name],
target_path=dataset_name,
overwrite=True)
raw_data_file = DataReference(datastore=datastore,
data_reference_name="Raw_Data_File",
path_on_datastore=dataset_name + '/'
+ file_name)
clean_data_file = PipelineParameter(name="clean_data_file",
default_value="/clean_data.csv")
clean_data_folder = PipelineData("clean_data_folder",
datastore=datastore)
prepDataStep = PythonScriptStep(name="Prepare Data",
source_directory=e.sources_directory_train,
script_name=e.data_prep_script_path,
arguments=["--raw_data_file",
raw_data_file,
"--clean_data_folder",
clean_data_folder,
"--clean_data_file",
clean_data_file],
inputs=[raw_data_file],
outputs=[clean_data_folder],
compute_target=aml_compute,
allow_reuse=False)
print("Step Prepare Data created")
new_model_file = PipelineParameter(name="new_model_file ",
default_value='/' + e.model_name
+ '.pkl')
new_model_folder = PipelineData("new_model_folder", datastore=datastore)
est = SKLearn(source_directory=e.sources_directory_train,
entry_script=e.train_script_path,
pip_packages=['azureml-sdk', 'scikit-learn==0.20.3',
'azureml-dataprep[pandas,fuse]>=1.1.14'],
compute_target=aml_compute)
trainingStep = EstimatorStep(
name="Model Training",
estimator=est,
estimator_entry_script_arguments=["--clean_data_folder",
clean_data_folder,
"--new_model_folder",
new_model_folder,
"--clean_data_file",
clean_data_file.default_value,
"--new_model_file",
new_model_file.default_value],
runconfig_pipeline_params=None,
inputs=[clean_data_folder],
outputs=[new_model_folder],
compute_target=aml_compute,
allow_reuse=False)
print("Step Train created")
model_name_param = PipelineParameter(name="model_name",
default_value=e.model_name)
evaluateStep = PythonScriptStep(
name="Evaluate Model",
source_directory=e.sources_directory_train,
script_name=e.evaluate_script_path,
arguments=["--model_name", model_name_param],
compute_target=aml_compute,
allow_reuse=False)
print("Step Evaluate created")
registerStep = PythonScriptStep(
name="Register Model",
source_directory=e.sources_directory_train,
script_name=e.register_script_path,
arguments=["--new_model_folder", new_model_folder,
"--new_model_file", new_model_file,
"--model_name", model_name_param],
inputs=[new_model_folder],
compute_target=aml_compute,
allow_reuse=False)
print("Step Register created")
if ((e.run_evaluation).lower() == 'true'):
print("Include evaluation step before register step.")
trainingStep.run_after(prepDataStep)
evaluateStep.run_after(trainingStep)
registerStep.run_after(evaluateStep)
else:
print("Exclude evaluation step and directly run register step.")
trainingStep.run_after(prepDataStep)
registerStep.run_after(trainingStep)
pipeline = Pipeline(workspace=aml_workspace, steps=[registerStep])
pipeline.validate()
print("Pipeline is built")
pipeline._set_experiment_name
published_pipeline = pipeline.publish(
name=e.pipeline_name,
description="Predict Employee Retention Model training pipeline",
version=e.build_id
)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
name="train-step",
runconfig=runconfig,
source_directory=args.source_directory,
script_name=runconfig.script,
arguments=['--data_path', training_dataset_consumption],
inputs=[training_dataset_consumption],
allow_reuse=False)
register_step = PythonScriptStep(name="register-step",
runconfig=runconfig_register,
source_directory=args.source_directory,
arguments=['--model_name', args.model_name],
script_name=runconfig_register.script,
allow_reuse=False)
register_step.run_after(train_step)
steps = [train_step, register_step]
print('Creating and validating pipeline')
pipeline = Pipeline(workspace=ws, steps=steps)
pipeline.validate()
#print('Publishing pipeline')
published_pipeline = pipeline.publish(args.pipeline_name)
# Output pipeline_id in specified format which will convert it to a variable in Azure DevOps
print(f'##vso[task.setvariable variable=pipeline_id]{published_pipeline.id}')
pipeline_run = Experiment(ws, 'training-pipeline-mlops').submit(pipeline)
pipeline_run.wait_for_completion()
def main():
load_dotenv()
workspace_name = os.environ.get("WORKSPACE_NAME")
resource_group = os.environ.get("RESOURCE_GROUP_NAME")
subscription_id = os.environ.get("SUBSCRIPTION_ID")
tenant_id = os.environ.get("TENANT_ID")
app_id = os.environ.get("SP_APP_ID")
app_secret = os.environ.get("SP_APP_SECRET")
sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN")
train_script_path = os.environ.get("TRAIN_SCRIPT_PATH")
evaluate_script_path = os.environ.get("EVALUATE_SCRIPT_PATH")
generate_report_path = os.environ.get("GENERATE_REPORT_PATH")
generate_report_name = os.environ.get("GENERATE_REPORT_NAME")
vm_size = os.environ.get("AML_COMPUTE_CLUSTER_GPU_SKU")
compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME")
model_name = os.environ.get("MODEL_NAME")
ckpt_path = os.environ.get("MODEL_CHECKPOINT_PATH")
build_id = os.environ.get("BUILD_BUILDID")
pipeline_name = os.environ.get("TRAINING_PIPELINE_NAME")
epis_datastore = os.environ.get("EPIS_DATASTORE")
epis_container = os.environ.get("EPIS_CONTAINER")
aml_workspace = get_workspace(
workspace_name,
resource_group,
subscription_id,
tenant_id,
app_id,
app_secret)
print(aml_workspace)
aml_compute = get_compute(
aml_workspace,
compute_name,
vm_size)
if aml_compute is not None:
print(aml_compute)
run_config = RunConfiguration(conda_dependencies=CondaDependencies.create(
conda_packages=['numpy==1.18.1', 'pandas', 'tensorflow-gpu==2.0.0'],
pip_packages=['azure', 'azureml-core==1.0.60', 'azureml-tensorboard', 'azure-storage==0.36.0',
'tqdm==4.41.1', 'opencv-python==4.1.2.30', 'easydict==1.9', 'matplotlib==3.1.3'])
)
run_config.environment.docker.enabled = True
run_config.environment.docker.gpu_support = True
run_config.environment.docker.base_image = DEFAULT_GPU_IMAGE
model_name = PipelineParameter(
name="model_name", default_value=model_name)
release_id = PipelineParameter(
name="release_id", default_value=build_id)
train_step = PythonScriptStep(
name="Train Model",
script_name=train_script_path,
compute_target=aml_compute,
source_directory=sources_directory_train,
arguments=[
"--release_id", release_id,
"--model_name", model_name,
"--ckpt_path", ckpt_path,
"--datastore", epis_datastore,
"--storage_container", epis_container,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model",
script_name=evaluate_script_path,
compute_target=aml_compute,
source_directory=sources_directory_train,
arguments=[
"--release_id", release_id,
"--model_name", model_name,
"--ckpt_path", ckpt_path,
"--datastore", epis_datastore,
"--storage_container", epis_container,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
generate_report_step = PythonScriptStep(
name="Generate Report Model",
script_name=generate_report_name,
compute_target=aml_compute,
source_directory=generate_report_path,
arguments=[
"--release_id", release_id,
"--model_name", model_name,
"--ckpt_path", ckpt_path,
"--datastore", epis_datastore,
"--storage_container", epis_container,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step generate report created")
evaluate_step.run_after(train_step)
generate_report_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, generate_report_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=pipeline_name,
description="Model training/retraining pipeline",
version=build_id
)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
validate_deployment_step = PythonScriptStep(
name="Validate Deployment",
script_name="validate.py",
arguments=["--webservicename", webservicename_param],
compute_target=compute_target,
source_directory="deployment",
runconfig=run_config,
allow_reuse=False,
)
logger.info("Pipeline steps prepared")
# DEFINE STEP DEPENDENCIES
first_prepro_step.run_after(update_dataset_step)
deploy_step.run_after(estimator_step)
validate_deployment_step.run_after(deploy_step)
# DEFINE PIPELINE
steps = [
update_dataset_step,
first_prepro_step,
second_prepro_step,
estimator_step,
deploy_step,
validate_deployment_step,
]
pipeline = Pipeline(workspace=ws, steps=[steps])
def main():
load_dotenv()
workspace_name = os.environ.get("WS_NAME")
resource_group = os.environ.get("RG_NAME")
subscription_id = os.environ.get("SUBSCRIPTION_ID")
tenant_id = os.environ.get("TENANT_ID")
app_id = os.environ.get("SP_APP_ID")
app_secret = os.environ.get("SP_APP_SECRET")
sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN")
train_script_path = os.environ.get("TRAIN_SCRIPT_PATH")
evaluate_script_path = os.environ.get("EVALUATE_SCRIPT_PATH")
vm_size = os.environ.get("AML_COMPUTE_CLUSTER_SKU")
compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME")
aks_name = os.environ.get("AKS_CLUSTER_NAME")
model_name = os.environ.get("MODEL_NAME")
build_id = os.environ.get("BUILD_BUILDID")
pipeline_name = os.environ.get("TRAINING_PIPELINE_NAME")
experiment_name = os.environ.get("EXPERIMENT_NAME")
# Get Azure machine learning workspace
aml_workspace = get_workspace(
workspace_name,
resource_group,
subscription_id,
tenant_id,
app_id,
app_secret)
print('Now accessing:')
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(
aml_workspace,
compute_name,
vm_size)
if aml_compute is not None:
print(aml_compute)
run_config = RunConfiguration(conda_dependencies=CondaDependencies.create(
conda_packages=['numpy', 'pandas',
'scikit-learn', 'keras'],
pip_packages=['azureml-core==1.25.0',
'azureml-defaults==1.25.0',
'azureml-telemetry==1.25.0',
'azureml-train-restclients-hyperdrive==1.25.0',
'azureml-train-core==1.25.0',
'azureml-dataprep',
'tensorflow-gpu==2.0.0',
'transformers==2.0.0',
'absl-py',
'azureml-dataprep',
'h5py<3.0.0'])
)
# run_config.environment.docker.enabled = True
datastore_name = 'mtcseattle'
container_name = 'azure-service-classifier'
account_name = 'mtcseattle'
sas_token = '?sv=2020-04-08&st=2021-05-26T04%3A39%3A46Z&se=2022-05-27T04%3A39%3A00Z&sr=c&sp=rl&sig=CTFMEu24bo2X06G%2B%2F2aKiiPZBzvlWHELe15rNFqULUk%3D'
try:
existing_datastore = Datastore.get(aml_workspace, datastore_name)
except: # noqa: E722
existing_datastore = Datastore \
.register_azure_blob_container(workspace=aml_workspace,
datastore_name=datastore_name,
container_name=container_name,
account_name=account_name,
sas_token=sas_token,
overwrite=True)
azure_dataset = Dataset.File.from_files(
path=(existing_datastore, 'data'))
azure_dataset = azure_dataset.register(
workspace=aml_workspace,
name='Azure Services Dataset',
description='Dataset containing azure related posts on Stackoverflow',
create_new_version=True)
azure_dataset.to_path()
input_data = azure_dataset.as_named_input('azureservicedata').as_mount(
'/tmp/data')
model_name = PipelineParameter(
name="model_name", default_value=model_name)
max_seq_length = PipelineParameter(
name="max_seq_length", default_value=128)
learning_rate = PipelineParameter(
name="learning_rate", default_value=3e-5)
num_epochs = PipelineParameter(
name="num_epochs", default_value=1)
export_dir = PipelineParameter(
name="export_dir", default_value="./outputs/model")
batch_size = PipelineParameter(
name="batch_size", default_value=32)
steps_per_epoch = PipelineParameter(
name="steps_per_epoch", default_value=1)
# initialize the PythonScriptStep
train_step = PythonScriptStep(
name='Train Model',
script_name=train_script_path,
arguments=['--data_dir', input_data,
'--max_seq_length', max_seq_length,
'--batch_size', batch_size,
'--learning_rate', learning_rate,
'--steps_per_epoch', steps_per_epoch,
'--num_epochs', num_epochs,
'--export_dir',export_dir],
compute_target=aml_compute,
source_directory=sources_directory_train,
runconfig=run_config,
allow_reuse=True)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=evaluate_script_path,
compute_target=aml_compute,
source_directory=sources_directory_train,
arguments=[
"--model_name", model_name,
"--build_id", build_id,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
# Currently, the Evaluate step will automatically register
# the model if it performs better. This step is based on a
# previous version of the repo which utilized JSON files to
# track evaluation results.
evaluate_step.run_after(train_step)
steps = [evaluate_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=pipeline_name,
description="Model training/retraining pipeline.",
version=build_id
)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
response = published_pipeline.submit( # noqa: F841
workspace=aml_workspace,
experiment_name=experiment_name)
runconfig_pipeline_params=None,
compute_target=GPU_compute_target)
#######################################################################################################
register_step = PythonScriptStep(name="register_step",
script_name="estimator_register.py",
runconfig=run_config_user_managed,
source_directory='./scripts/register',
arguments=['--ModelData', ModelData],
outputs=[ModelData],
compute_target=GPU_compute_target)
#######################################################################################################
est_step.run_after(preprocessing_step)
register_step.run_after(est_step)
#Build Pipeline
pipeline = Pipeline(workspace=ws,
steps=[preprocessing_step, est_step, register_step])
#Validate pipeline
pipeline.validate()
print("Pipeline validation complete")
#Publish the pipeline
published_pipeline = pipeline.publish(
name="MLOps_Pipeline_Estimator",
description="MLOps pipeline for estimator",
continue_on_step_failure=True)
def build_pipeline(dataset, ws, config):
print("building pipeline for dataset %s in workspace %s" %
(dataset, ws.name))
hostname = socket.gethostname()
if hostname == 'wopauliNC6':
base_dir = '.'
else:
base_dir = '.'
def_blob_store = ws.get_default_datastore()
# folder for scripts that need to be uploaded to Aml compute target
script_folder = './scripts'
os.makedirs(script_folder, exist_ok=True)
shutil.copy(os.path.join(base_dir, 'video_decoding.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'pipelines_submit.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'pipelines_build.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'train.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'data_utils.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'prednet.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'keras_utils.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'data_preparation.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'model_registration.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'config.json'), script_folder)
cpu_compute_name = config['cpu_compute']
try:
cpu_compute_target = AmlCompute(ws, cpu_compute_name)
print("found existing compute target: %s" % cpu_compute_name)
except ComputeTargetException:
print("creating new compute target")
provisioning_config = AmlCompute.provisioning_configuration(
vm_size='STANDARD_D2_V2',
max_nodes=4,
idle_seconds_before_scaledown=1800)
cpu_compute_target = ComputeTarget.create(ws, cpu_compute_name,
provisioning_config)
cpu_compute_target.wait_for_completion(show_output=True,
min_node_count=None,
timeout_in_minutes=20)
# use get_status() to get a detailed status for the current cluster.
print(cpu_compute_target.get_status().serialize())
# choose a name for your cluster
gpu_compute_name = config['gpu_compute']
try:
gpu_compute_target = AmlCompute(workspace=ws, name=gpu_compute_name)
print("found existing compute target: %s" % gpu_compute_name)
except ComputeTargetException:
print('Creating a new compute target...')
provisioning_config = AmlCompute.provisioning_configuration(
vm_size='STANDARD_NC6',
max_nodes=5,
idle_seconds_before_scaledown=1800)
# create the cluster
gpu_compute_target = ComputeTarget.create(ws, gpu_compute_name,
provisioning_config)
# can poll for a minimum number of nodes and for a specific timeout.
# if no min node count is provided it uses the scale settings for the cluster
gpu_compute_target.wait_for_completion(show_output=True,
min_node_count=None,
timeout_in_minutes=20)
# use get_status() to get a detailed status for the current cluster.
print(gpu_compute_target.get_status().serialize())
# conda dependencies for compute targets
cpu_cd = CondaDependencies.create(conda_packages=["py-opencv=3.4.2"],
pip_packages=[
"azure-storage-blob==1.5.0",
"hickle==3.4.3", "requests==2.21.0",
"sklearn", "pandas==0.24.2",
"azureml-sdk==1.0.21",
"numpy==1.16.2", "pillow==6.0.0"
])
gpu_cd = CondaDependencies.create(pip_packages=[
"keras==2.0.8", "theano==1.0.4", "tensorflow==1.8.0",
"tensorflow-gpu==1.8.0", "hickle==3.4.3", "matplotlib==3.0.3",
"seaborn==0.9.0", "requests==2.21.0", "bs4==0.0.1", "imageio==2.5.0",
"sklearn", "pandas==0.24.2", "azureml-sdk==1.0.21", "numpy==1.16.2"
])
# Runconfigs
cpu_compute_run_config = RunConfiguration(conda_dependencies=cpu_cd)
cpu_compute_run_config.environment.docker.enabled = True
cpu_compute_run_config.environment.docker.gpu_support = False
cpu_compute_run_config.environment.docker.base_image = DEFAULT_CPU_IMAGE
cpu_compute_run_config.environment.spark.precache_packages = False
gpu_compute_run_config = RunConfiguration(conda_dependencies=gpu_cd)
gpu_compute_run_config.environment.docker.enabled = True
gpu_compute_run_config.environment.docker.gpu_support = True
gpu_compute_run_config.environment.docker.base_image = DEFAULT_GPU_IMAGE
gpu_compute_run_config.environment.spark.precache_packages = False
print("PipelineData object created")
video_data = DataReference(datastore=def_blob_store,
data_reference_name="video_data",
path_on_datastore=os.path.join(
"prednet", "data", "video", dataset))
# Naming the intermediate data as processed_data1 and assigning it to the variable processed_data1.
raw_data = PipelineData("raw_video_fames", datastore=def_blob_store)
preprocessed_data = PipelineData("preprocessed_video_frames",
datastore=def_blob_store)
data_metrics = PipelineData("data_metrics", datastore=def_blob_store)
data_output = PipelineData("output_data", datastore=def_blob_store)
print("DataReference object created")
# prepare dataset for training/testing prednet
video_decoding = PythonScriptStep(
name='decode_videos',
script_name="video_decoding.py",
arguments=["--input_data", video_data, "--output_data", raw_data],
inputs=[video_data],
outputs=[raw_data],
compute_target=cpu_compute_target,
source_directory=script_folder,
runconfig=cpu_compute_run_config,
allow_reuse=True,
hash_paths=['.'])
print("video_decode created")
# prepare dataset for training/testing recurrent neural network
data_prep = PythonScriptStep(name='prepare_data',
script_name="data_preparation.py",
arguments=[
"--input_data", raw_data, "--output_data",
preprocessed_data
],
inputs=[raw_data],
outputs=[preprocessed_data],
compute_target=cpu_compute_target,
source_directory=script_folder,
runconfig=cpu_compute_run_config,
allow_reuse=True,
hash_paths=['.'])
data_prep.run_after(video_decoding)
print("data_prep created")
est = TensorFlow(source_directory=script_folder,
compute_target=gpu_compute_target,
pip_packages=[
'keras==2.0.8', 'theano', 'tensorflow==1.8.0',
'tensorflow-gpu==1.8.0', 'matplotlib', 'horovod',
'hickle'
],
entry_script='train.py',
use_gpu=True,
node_count=1)
ps = RandomParameterSampling({
'--batch_size':
choice(2, 4, 8, 16),
'--filter_sizes':
choice("3, 3, 3", "4, 4, 4", "5, 5, 5"),
'--stack_sizes':
choice("48, 96, 192", "36, 72, 144", "12, 24, 48"), #, "48, 96"),
'--learning_rate':
loguniform(-6, -1),
'--lr_decay':
loguniform(-9, -1),
'--freeze_layers':
choice("0, 1, 2", "1, 2, 3", "0, 1", "1, 2", "2, 3", "0", "1", "2",
"3"),
'--transfer_learning':
choice("True", "False")
})
policy = BanditPolicy(evaluation_interval=2,
slack_factor=0.1,
delay_evaluation=20)
hdc = HyperDriveRunConfig(
estimator=est,
hyperparameter_sampling=ps,
policy=policy,
primary_metric_name='val_loss',
primary_metric_goal=PrimaryMetricGoal.MINIMIZE,
max_total_runs=5, #100,
max_concurrent_runs=5, #10,
max_duration_minutes=60 * 6)
hd_step = HyperDriveStep(name="train_w_hyperdrive",
hyperdrive_run_config=hdc,
estimator_entry_script_arguments=[
'--data-folder', preprocessed_data,
'--remote_execution'
],
inputs=[preprocessed_data],
metrics_output=data_metrics,
allow_reuse=True)
hd_step.run_after(data_prep)
registration_step = PythonScriptStep(
name='register_model',
script_name='model_registration.py',
arguments=['--input_dir', data_metrics, '--output_dir', data_output],
compute_target=gpu_compute_target,
inputs=[data_metrics],
outputs=[data_output],
source_directory=script_folder,
allow_reuse=True,
hash_paths=['.'])
registration_step.run_after(hd_step)
pipeline = Pipeline(
workspace=ws,
steps=[video_decoding, data_prep, hd_step, registration_step])
print("Pipeline is built")
pipeline.validate()
print("Simple validation complete")
pipeline_name = 'prednet_' + dataset
pipeline.publish(name=pipeline_name)
return pipeline_name
def main():
load_dotenv()
workspace_name = os.environ.get("BASE_NAME") + "-AML-WS"
resource_group = "AML-RG-" + os.environ.get("BASE_NAME")
subscription_id = os.environ.get("SUBSCRIPTION_ID")
tenant_id = os.environ.get("TENANT_ID")
app_id = os.environ.get("SP_APP_ID")
app_secret = os.environ.get("SP_APP_SECRET")
sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN")
train_script_path = os.environ.get("TRAIN_SCRIPT_PATH")
evaluate_script_path = os.environ.get("EVALUATE_SCRIPT_PATH")
vm_size = os.environ.get("AML_COMPUTE_CLUSTER_SKU")
compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME")
aks_name = os.environ.get("AKS_CLUSTER_NAME")
model_name = os.environ.get("MODEL_NAME")
build_id = os.environ.get("BUILD_BUILDID")
pipeline_name = os.environ.get("TRAINING_PIPELINE_NAME")
experiment_name = os.environ.get("EXPERIMENT_NAME")
# Get Azure machine learning workspace
aml_workspace = get_workspace(workspace_name, resource_group,
subscription_id, tenant_id, app_id,
app_secret)
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, compute_name, vm_size)
if aml_compute is not None:
print(aml_compute)
run_config = RunConfiguration(conda_dependencies=CondaDependencies.create(
conda_packages=['numpy', 'pandas', 'scikit-learn', 'keras'],
pip_packages=[
'azure', 'azureml-sdk', 'azure-storage', 'azure-storage-blob',
'transformers>=2.1.1', 'tensorflow>=2.0.0', 'tensorflow-gpu>=2.0.0'
]))
run_config.environment.docker.enabled = True
datastore_name = 'tfworld'
container_name = 'azure-service-classifier'
account_name = 'johndatasets'
sas_token = '?sv=2019-02-02&ss=bfqt&srt=sco&sp=rl&se=2021-06-02T03:40:25Z&st=2020-03-09T19:40:25Z&spr=https&sig=bUwK7AJUj2c%2Fr90Qf8O1sojF0w6wRFgL2c9zMVCWNPA%3D'
try:
existing_datastore = Datastore.get(aml_workspace, datastore_name)
except: # noqa: E722
existing_datastore = Datastore \
.register_azure_blob_container(workspace=aml_workspace,
datastore_name=datastore_name,
container_name=container_name,
account_name=account_name,
sas_token=sas_token
)
azure_dataset = Dataset.File.from_files(path=(existing_datastore, 'data'))
azure_dataset = azure_dataset.register(
workspace=aml_workspace,
name='Azure Services Dataset',
description='Dataset containing azure related posts on Stackoverflow',
create_new_version=True)
azure_dataset.to_path()
input_data = azure_dataset.as_named_input('input_data1').as_mount(
'/tmp/data')
model_name = PipelineParameter(name="model_name", default_value=model_name)
max_seq_length = PipelineParameter(name="max_seq_length",
default_value=128)
learning_rate = PipelineParameter(name="learning_rate", default_value=3e-5)
num_epochs = PipelineParameter(name="num_epochs", default_value=3)
export_dir = PipelineParameter(name="export_dir",
default_value="./outputs/exports")
batch_size = PipelineParameter(name="batch_size", default_value=32)
steps_per_epoch = PipelineParameter(name="steps_per_epoch",
default_value=100)
# initialize the TensorFlow estimator
estimator = TensorFlow(source_directory=sources_directory_train,
entry_script=train_script_path,
compute_target=aml_compute,
framework_version='2.0',
use_gpu=True,
pip_packages=[
'transformers==2.0.0',
'azureml-dataprep[fuse,pandas]==1.3.0'
])
train_step = EstimatorStep(
name="Train Model",
estimator=estimator,
estimator_entry_script_arguments=[
"--data_dir", input_data, "--max_seq_length", max_seq_length,
"--learning_rate", learning_rate, "--num_epochs", num_epochs,
"--export_dir", export_dir, "--batch_size", batch_size,
"--steps_per_epoch", steps_per_epoch
],
compute_target=aml_compute,
inputs=[input_data],
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=evaluate_script_path,
compute_target=aml_compute,
source_directory=sources_directory_train,
arguments=[
"--model_name",
model_name,
"--build_id",
build_id,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
# Currently, the Evaluate step will automatically register
# the model if it performs better. This step is based on a
# previous version of the repo which utilized JSON files to
# track evaluation results.
evaluate_step.run_after(train_step)
steps = [evaluate_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=pipeline_name,
description="Model training/retraining pipeline",
version=build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
response = published_pipeline.submit( # noqa: F841
workspace=aml_workspace,
experiment_name=experiment_name)
# Get AKS cluster for deployment
aks_compute = get_aks(aml_workspace, aks_name)
if aks_compute is not None:
print(aks_compute)
# Package model step is moved to Azure DevOps Release Pipeline
# package_model = PythonScriptStep(
# name="Package Model as Scoring Image",
# script_name="scoring/create_scoring_image.py",
# compute_target=aml_compute,
# source_directory=source_directory,
# arguments=["--config_suffix", config_suffix, "--json_config", jsonconfigs],
# runconfig=run_config,
# inputs=[jsonconfigs],
# # outputs=[jsonconfigs],
# allow_reuse=False,
# )
# print("Packed the model into a Scoring Image")
# Create Steps dependency such that they run in sequence
evaluate.run_after(train)
register_model.run_after(evaluate)
#package_model.run_after(register_model)
steps = [register_model]
# Build Pipeline
pipeline1 = Pipeline(workspace=ws, steps=steps)
print("Pipeline is built")
# Validate Pipeline
pipeline1.validate()
print("Pipeline validation complete")
# Submit unpublished pipeline with small data set for test
if args.pipeline_action == "pipeline-test":
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable Azure ML environment
environment = get_environment(aml_workspace,
e.aml_env_name,
create_new=False) # NOQA: E501
run_config = RunConfiguration()
run_config.environment = environment
if (e.datastore_name):
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables[
"DATASTORE_NAME"] = datastore_name # NOQA: E501
model_name_param = PipelineParameter(name="model_name",
default_value=e.model_name)
dataset_version_param = PipelineParameter(name="dataset_version",
default_value=e.dataset_version)
data_file_path_param = PipelineParameter(name="data_file_path",
default_value="none")
caller_run_id_param = PipelineParameter(name="caller_run_id",
default_value="none")
# Get dataset name
dataset_name = e.dataset_name
# # Check to see if dataset exists
# if (dataset_name not in aml_workspace.datasets):
# # Create dataset from lacemlops sample data
# sample_data = load_lacemlops()
# df = pd.DataFrame(
# data=sample_data.data,
# columns=sample_data.feature_names)
# df['Y'] = sample_data.target
# file_name = 'lacemlops.csv'
# df.to_csv(file_name, index=False)
# # Upload file to default datastore in workspace
# datatstore = Datastore.get(aml_workspace, datastore_name)
# target_path = 'training-data/'
# datatstore.upload_files(
# files=[file_name],
# target_path=target_path,
# overwrite=True,
# show_progress=False)
# # Register dataset
# path_on_datastore = os.path.join(target_path, file_name)
# dataset = Dataset.Tabular.from_delimited_files(
# path=(datatstore, path_on_datastore))
# dataset = dataset.register(
# workspace=aml_workspace,
# name=dataset_name,
# description='lacemlops training data',
# tags={'format': 'CSV'},
# create_new_version=True)
# Create a PipelineData to pass data between steps
pipeline_data = PipelineData(
'pipeline_data', datastore=aml_workspace.get_default_datastore())
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
outputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_output",
pipeline_data,
"--dataset_version",
dataset_version_param,
"--data_file_path",
data_file_path_param,
"--caller_run_id",
caller_run_id_param,
"--dataset_name",
dataset_name,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--model_name",
model_name_param,
"--allow_run_cancel",
e.allow_run_cancel,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_input",
pipeline_data,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
# Check run_evaluation flag to include or exclude evaluation step.
if ((e.run_evaluation).lower() == 'true'):
print("Include evaluation step before register step.")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
else:
print("Exclude evaluation step and directly run register step.")
register_step.run_after(train_step)
steps = [train_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
outputs=[sklearnmodelpath],
runconfig=run_config_sklearn,
source_directory=os.path.join(os.path.dirname(os.path.realpath(__file__)),
'..', 'modeling'))
model_training = PythonScriptStep(
name="fullmodel",
script_name="train.py",
arguments=script_params_model_training,
inputs=[metrics_data, train_prepared, test_prepared],
outputs=[modelpath],
runconfig=run_config_full,
source_directory=os.path.join(os.path.dirname(os.path.realpath(__file__)),
'..', 'modeling'))
env_data_preparation.run_after(define_new_dataset)
env_data_preparation_subset.run_after(define_new_dataset)
env_data_profiling.run_after(define_new_dataset)
env_data_validation.run_after(define_new_dataset)
env_data_validation_subset.run_after(define_new_dataset)
env_fullmodel.run_after(define_new_dataset)
env_sklearn_subset.run_after(define_new_dataset)
data_validation.run_after(env_data_validation)
data_validation_subset.run_after(env_data_validation_subset)
data_engineering_subset.run_after(env_data_preparation_subset)
historic_profile.run_after(env_data_profiling)
data_engineering.run_after(env_data_preparation)
model_training.run_after(env_fullmodel)
sklearn_models.run_after(env_sklearn_subset)
def main():
load_dotenv()
workspace_name = os.environ.get("BASE_NAME") + "-AML-WS"
resource_group = os.environ.get("BASE_NAME") + "-AML-RG"
subscription_id = os.environ.get("SUBSCRIPTION_ID")
tenant_id = os.environ.get("TENANT_ID")
app_id = os.environ.get("SP_APP_ID")
app_secret = os.environ.get("SP_APP_SECRET")
sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN")
train_script_path = os.environ.get("TRAIN_SCRIPT_PATH")
evaluate_script_path = os.environ.get("EVALUATE_SCRIPT_PATH")
# register_script_path = os.environ.get("REGISTER_SCRIPT_PATH")
vm_size = os.environ.get("AML_COMPUTE_CLUSTER_CPU_SKU")
compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME")
model_name = os.environ.get("MODEL_NAME")
build_id = os.environ.get("BUILD_BUILDID")
pipeline_name = os.environ.get("TRAINING_PIPELINE_NAME")
print(app_secret)
# Get Azure machine learning workspace
aml_workspace = get_workspace(workspace_name, resource_group,
subscription_id, tenant_id, app_id,
app_secret)
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, compute_name, vm_size)
if aml_compute is not None:
print(aml_compute)
run_config = RunConfiguration(conda_dependencies=CondaDependencies.create(
conda_packages=[
'numpy', 'pandas', 'scikit-learn', 'tensorflow', 'keras'
],
pip_packages=[
'azure', 'azureml-core', 'azure-storage', 'azure-storage-blob'
]))
run_config.environment.docker.enabled = True
model_name = PipelineParameter(name="model_name", default_value=model_name)
release_id = PipelineParameter(name="release_id", default_value="0")
train_step = PythonScriptStep(
name="Train Model",
script_name=train_script_path,
compute_target=aml_compute,
source_directory=sources_directory_train,
arguments=[
"--release_id",
release_id,
"--model_name",
model_name,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=evaluate_script_path,
compute_target=aml_compute,
source_directory=sources_directory_train,
arguments=[
"--release_id",
release_id,
"--model_name",
model_name,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
# Currently, the Evaluate step will automatically register
# the model if it performs better. This step is based on a
# previous version of the repo which utilized JSON files to
# track evaluation results.
# register_model_step = PythonScriptStep(
# name="Register New Trained Model",
# script_name=register_script_path,
# compute_target=aml_compute,
# source_directory=sources_directory_train,
# arguments=[
# "--release_id", release_id,
# "--model_name", model_name,
# ],
# runconfig=run_config,
# allow_reuse=False,
# )
# print("Step register model created")
evaluate_step.run_after(train_step)
# register_model_step.run_after(evaluate_step)
steps = [evaluate_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=pipeline_name,
description="Model training/retraining pipeline",
version=build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
arguments=[
"--model_name", args.model_name, "--image_name", args.image_name,
"--output", containerize_output
],
outputs=[containerize_output],
compute_target=amlTrainingComputeRef,
runconfig=amlComputeRunConf,
source_directory=mlScriptsDir,
allow_reuse=False)
print("..7. completed")
print('')
print('')
print("8. Define pipeline stages sequence, and pipeline itself...")
print('.............................................')
containerizePipelineStep.run_after(trainPipelineStep)
pipeLineSteps = [containerizePipelineStep]
pipeline = Pipeline(workspace=amlWs, steps=pipeLineSteps)
pipeline.validate()
print("..8. completed")
print('')
print('')
print("9. Create run object for the experiment...")
print('.............................................')
run = Run.get_context()
experimentName = run.experiment.name
print("..9. completed")
print('')
print('')
print("Model Training Step is Completed")
# source directory
source_directory = 'RegisterModel'
modelEvalReg = PythonScriptStep(
name="Evaluate and Register Model",
script_name="evaluate_model.py",
arguments=["--release_id", 0, '--model_name', model_name],
compute_target=compute_target_cpu,
source_directory=source_directory,
runconfig=run_config,
allow_reuse=False)
modelEvalReg.run_after(trainingStep)
print("Model Evaluation and Registration Step is Created")
from azureml.pipeline.core import Pipeline
from azureml.core import Experiment
pipeline = Pipeline(workspace=ws,
steps=[extractDataStep, trainingStep, modelEvalReg])
pipeline_run = Experiment(ws, pipeline_experiment_name).submit(pipeline)
pipeline_run.wait_for_completion(show_output=True, raise_on_error=True)
published_pipeline = pipeline_run.publish_pipeline(
name=pipeline_name,
description=
"Steps are: data preparation, training, model validation and model registration",
version="0.1",
def main():
e = Env()
print(e.workspace_name)
svc_pr = ServicePrincipalAuthentication(
tenant_id=os.environ.get("TENANT_ID"),
service_principal_id=os.environ.get("AZURE_SP_ID"),
service_principal_password=os.environ.get("AZURE_SP_PASSWORD"))
# Get Azure machine learning workspace
ws = Workspace.get(name=os.environ.get("WORKSPACE_NAME"),
subscription_id=os.environ.get("SUBSCRIPTION_ID"),
resource_group=os.environ.get("AZURE_RESOURCE_GROUP"),
auth=svc_pr)
#ex = Experiment(ws, 'iris-pipeline')
#ex.archive()
print("get_workspace:")
print(ws)
ws.write_config(path="", file_name="config.json")
print("writing config.json.")
# Get Azure machine learning cluster
aml_compute = get_compute(ws, "train-cluster", "STANDARD_DS2_V2")
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
run_config = RunConfiguration(conda_dependencies=CondaDependencies.create(
conda_packages=[
'numpy', 'pandas', 'scikit-learn', 'tensorflow', 'keras'
],
pip_packages=[
'azure', 'azureml-core', 'azureml-pipeline', 'azure-storage',
'azure-storage-blob', 'azureml-dataprep'
]))
run_config.environment.docker.enabled = True
######### TRAIN ################
train_step = PythonScriptStep(
name="Train",
source_directory="models/python/iris/train",
script_name="train.py",
compute_target=aml_compute,
arguments=[],
runconfig=run_config,
allow_reuse=False,
)
print("Train Step created")
######### EVALUATE ################
evaluate_step = PythonScriptStep(
name="Evaluate",
source_directory="models/python/iris/evaluate",
script_name="evaluate.py",
compute_target=aml_compute,
arguments=[],
runconfig=run_config,
allow_reuse=False,
)
print("Evaluate Step created")
######### REGISTER ################
register_step = PythonScriptStep(
name="Register",
source_directory="models/python/iris/register",
script_name="register.py",
compute_target=aml_compute,
arguments=[],
runconfig=run_config,
allow_reuse=False,
)
print("Register Step created")
#evaluate_step.run_after(train_step)
register_step.run_after(train_step)
steps = [train_step, register_step]
train_pipeline = Pipeline(workspace=ws, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(name="iris-pipeline",
description="")
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
pipeline_parameters = {"model_name": "iris-pipeline-param"}
run = published_pipeline.submit(ws, "iris-pipeline-experiment",
pipeline_parameters)
def main():
e = ENV()
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group
)
print(f'workspace:{aml_workspace}')
aml_compute = get_compute(workspace=aml_workspace,
compute_name=e.compute_name,
vm_size=e.vm_size)
if aml_compute is not None:
print(f'compute target: {aml_compute} is created')
environment = get_environment(workspace=aml_workspace,
env_name=e.aml_env_name,
conda_dependencies=e.aml_conda_train_dependent_files,
create_new=e.rebuild_env)
run_config = RunConfiguration()
run_config.environment = environment
if e.datastore_name:
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables['DATASTORE_NAME'] = datastore_name
model_name_param = PipelineParameter(name='model_name',default_value=e.model_name)
dataset_version_param = PipelineParameter(name='dataset_version',default_value=e.dataset_version)
dataset_file_path = PipelineParameter(name='dataset_file_path',default_value='none')
dataset_name = e.dataset_name
if dataset_name not in aml_workspace.datasets:
create_sample_data_csv()
file_name = 'diabetes.csv'
if not os.path.exists(file_name):
raise Exception(f'{file_name} does not exist!')
datastore = Datastore.get(aml_workspace,datastore_name)
target_path = 'training/'
datastore.upload_files(files=[file_name],
target_path=target_path,
overwrite=True,
show_progress=True)
path_on_datastore = os.path.join(target_path,file_name)
dataset = Dataset.Tabular.from_delimited_files(path=(datastore,path_on_datastore))
dataset.register(workspace=aml_workspace,
name=dataset_name,
description='registered dataset',
create_new_version=True,
tags={'format':'CSV'})
pipeline_data = PipelineData('train',datastore=aml_workspace.get_default_datastore())
train_step = PythonScriptStep(script_name=e.train_script_path,
name='train_step',
arguments=['--model-name',model_name_param,
'--dataset-name',dataset_name,
'--dataset-version',dataset_version_param,
'--dataset-file-path',dataset_file_path,
'--step-output',pipeline_data],
compute_target=aml_compute,
runconfig=run_config,
source_directory=e.source_train_directory,
outputs=[pipeline_data],
allow_reuse=False)
print('Train step created!')
eval_step = PythonScriptStep(name='eval_step',
script_name=e.eval_script_path,
compute_target=aml_compute,
source_directory=e.source_train_directory,
arguments=['--model-name',model_name_param,
'--allow-run-cancel',e.allow_run_cancel],
runconfig=run_config,
allow_reuse=False)
print('EVAL step created!')
register_step = PythonScriptStep(name='register_step',
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.source_train_directory,
inputs=[pipeline_data],
arguments=['--model-name',model_name_param,
'--step-input',pipeline_data],
runconfig=run_config,
allow_reuse=False)
print('Register step created!')
if e.run_evaluation:
print('evaluation step is included')
eval_step.run_after(train_step)
register_step.run_after(eval_step)
steps = [train_step,eval_step,register_step]
else:
print('evaluation step is excluded')
register_step.run_after(train_step)
steps = [train_step,register_step]
train_pipeline = Pipeline(workspace=aml_workspace,
steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(name=e.pipeline_name,
description='model training pipeline',
version=e.build_id)
print(f'{published_pipeline.name} is published1')
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = get_workspace(e.workspace_name, e.resource_group,
e.subscription_id, e.tenant_id, e.app_id,
e.app_secret)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
run_config = RunConfiguration(conda_dependencies=CondaDependencies.create(
conda_packages=[
'numpy', 'pandas', 'scikit-learn', 'tensorflow', 'keras'
],
pip_packages=[
'azure', 'azureml-core', 'azure-storage', 'azure-storage-blob'
]))
run_config.environment.docker.enabled = True
model_name = PipelineParameter(name="model_name",
default_value=e.model_name)
release_id = PipelineParameter(name="release_id", default_value="0")
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--release_id",
release_id,
"--model_name",
model_name,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--release_id",
release_id,
"--model_name",
model_name,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
evaluate_step.run_after(train_step)
steps = [evaluate_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group,
)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable Azure ML environment
environment = get_environment(
aml_workspace,
e.aml_env_name,
conda_dependencies_file=e.aml_env_train_conda_dep_file,
create_new=e.rebuild_env,
) #
run_config = RunConfiguration()
run_config.environment = environment
if e.datastore_name:
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables[
"DATASTORE_NAME"] = datastore_name # NOQA: E501
model_name_param = PipelineParameter(
name="model_name", default_value=e.model_name) # NOQA: E501
dataset_version_param = PipelineParameter(name="dataset_version",
default_value=e.dataset_version)
data_file_path_param = PipelineParameter(name="data_file_path",
default_value="none")
caller_run_id_param = PipelineParameter(name="caller_run_id",
default_value="none") # NOQA: E501
# Get dataset name
dataset_name = e.dataset_name
# Check to see if dataset exists
if dataset_name not in aml_workspace.datasets:
# This call creates an example CSV from sklearn sample data. If you
# have already bootstrapped your project, you can comment this line
# out and use your own CSV.
create_sample_data_csv()
# Use a CSV to read in the data set.
file_name = "safedriver.csv"
if not os.path.exists(file_name):
raise Exception(
'Could not find CSV dataset at "%s". If you have bootstrapped your project, you will need to provide a CSV.' # NOQA: E501
% file_name) # NOQA: E501
# Upload file to default datastore in workspace
datatstore = Datastore.get(aml_workspace, datastore_name)
target_path = "training-data/"
datatstore.upload_files(
files=[file_name],
target_path=target_path,
overwrite=True,
show_progress=False,
)
# Register dataset
path_on_datastore = os.path.join(target_path, file_name)
dataset = Dataset.Tabular.from_delimited_files(
path=(datatstore, path_on_datastore))
dataset = dataset.register(
workspace=aml_workspace,
name=dataset_name,
description="safedriver training data",
tags={"format": "CSV"},
create_new_version=True,
)
# Create a PipelineData to pass data between steps
pipeline_data = PipelineData(
"pipeline_data", datastore=aml_workspace.get_default_datastore())
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
outputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_output",
pipeline_data,
"--dataset_version",
dataset_version_param,
"--data_file_path",
data_file_path_param,
"--caller_run_id",
caller_run_id_param,
"--dataset_name",
dataset_name,
],
runconfig=run_config,
allow_reuse=True,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--model_name",
model_name_param,
"--allow_run_cancel",
e.allow_run_cancel,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_input",
pipeline_data,
], # NOQA: E501
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
# Check run_evaluation flag to include or exclude evaluation step.
if (e.run_evaluation).lower() == "true":
print("Include evaluation step before register step.")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
else:
print("Exclude evaluation step and directly run register step.")
register_step.run_after(train_step)
steps = [train_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id,
)
print(f"Published pipeline: {published_pipeline.name}")
print(f"for build {published_pipeline.version}")
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
run_config = RunConfiguration(conda_dependencies=CondaDependencies.create(
conda_packages=[
'numpy', 'pandas', 'scikit-learn', 'tensorflow', 'keras'
],
pip_packages=[
'azure', 'azureml-core', 'azure-storage', 'azure-storage-blob'
]))
run_config.environment.docker.enabled = True
config_envvar = {}
if (e.collection_uri is not None and e.teamproject_name is not None):
builduri_base = e.collection_uri + e.teamproject_name
builduri_base = builduri_base + "/_build/results?buildId="
config_envvar["BUILDURI_BASE"] = builduri_base
run_config.environment.environment_variables = config_envvar
model_name_param = PipelineParameter(name="model_name",
default_value=e.model_name)
build_id_param = PipelineParameter(name="build_id",
default_value=e.build_id)
hyperparameter_alpha_param = PipelineParameter(name="hyperparameter_alpha",
default_value=0.5)
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
"--alpha",
hyperparameter_alpha_param,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
