def create_runconfig(aml_compute, env=None):
# Create a new runconfig object
aml_run_config = RunConfiguration()
# Use the aml_compute you created above.
aml_run_config.target = aml_compute
if env:
aml_run_config.environment = env
else:
aml_run_config.environment = create_env_from_requirements()
return aml_run_config
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable Azure ML environment
environment = get_environment(aml_workspace,
e.aml_env_name,
create_new=False) # NOQA: E501
run_config = RunConfiguration()
run_config.environment = environment
if (e.datastore_name):
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables[
"DATASTORE_NAME"] = datastore_name # NOQA: E501
def create_run_config(azure_config: AzureConfig,
source_config: SourceConfig,
all_azure_dataset_ids: List[str],
all_dataset_mountpoints: List[str],
environment_name: str = "") -> ScriptRunConfig:
"""
Creates a configuration to run the InnerEye training script in AzureML.
:param azure_config: azure related configurations to use for model scale-out behaviour
:param source_config: configurations for model execution, such as name and execution mode
:param all_azure_dataset_ids: The name of all datasets on blob storage that will be used for this run.
:param all_dataset_mountpoints: When using the datasets in AzureML, these are the per-dataset mount points.
:param environment_name: If specified, try to retrieve the existing Python environment with this name. If that
is not found, create one from the Conda files provided in `source_config`. This parameter is meant to be used
when running inference for an existing model.
:return: The configured script run.
"""
dataset_consumptions = create_dataset_consumptions(
azure_config, all_azure_dataset_ids, all_dataset_mountpoints)
# AzureML seems to sometimes expect the entry script path in Linux format, hence convert to posix path
entry_script_relative_path = source_config.entry_script.relative_to(
source_config.root_folder).as_posix()
logging.info(
f"Entry script {entry_script_relative_path} ({source_config.entry_script} relative to "
f"source directory {source_config.root_folder})")
max_run_duration = None
if azure_config.max_run_duration:
max_run_duration = run_duration_string_to_seconds(
azure_config.max_run_duration)
workspace = azure_config.get_workspace()
run_config = RunConfiguration(
script=entry_script_relative_path,
arguments=source_config.script_params,
)
run_config.environment = get_or_create_python_environment(
azure_config, source_config, environment_name=environment_name)
run_config.target = azure_config.cluster
run_config.max_run_duration_seconds = max_run_duration
if azure_config.num_nodes > 1:
distributed_job_config = MpiConfiguration(
node_count=azure_config.num_nodes)
run_config.mpi = distributed_job_config
run_config.framework = "Python"
run_config.communicator = "IntelMpi"
run_config.node_count = distributed_job_config.node_count
if len(dataset_consumptions) > 0:
run_config.data = {
dataset.name: dataset
for dataset in dataset_consumptions
}
# Use blob storage for storing the source, rather than the FileShares section of the storage account.
run_config.source_directory_data_store = workspace.datastores.get(
WORKSPACE_DEFAULT_BLOB_STORE_NAME).name
script_run_config = ScriptRunConfig(
source_directory=str(source_config.root_folder),
run_config=run_config,
)
if azure_config.hyperdrive:
script_run_config = source_config.hyperdrive_config_func(
script_run_config) # type: ignore
return script_run_config
def create_run_config(azure_config: AzureConfig,
source_config: SourceConfig,
azure_dataset_id: str = "",
environment_name: str = "") -> ScriptRunConfig:
"""
Creates a configuration to run the InnerEye training script in AzureML.
:param azure_config: azure related configurations to use for model scale-out behaviour
:param source_config: configurations for model execution, such as name and execution mode
:param azure_dataset_id: The name of the dataset in blob storage to be used for this run. This can be an empty
string to not use any datasets.
:param environment_name: If specified, try to retrieve the existing Python environment with this name. If that
is not found, create one from the Conda files provided in `source_config`. This parameter is meant to be used
when running inference for an existing model.
:return: The configured script run.
"""
if azure_dataset_id:
azureml_dataset = get_or_create_dataset(azure_config, azure_dataset_id=azure_dataset_id)
if not azureml_dataset:
raise ValueError(f"AzureML dataset {azure_dataset_id} could not be found or created.")
named_input = azureml_dataset.as_named_input(INPUT_DATA_KEY)
dataset_consumption = named_input.as_mount() if azure_config.use_dataset_mount else named_input.as_download()
else:
dataset_consumption = None
# AzureML seems to sometimes expect the entry script path in Linux format, hence convert to posix path
entry_script_relative_path = source_config.entry_script.relative_to(source_config.root_folder).as_posix()
logging.info(f"Entry script {entry_script_relative_path} ({source_config.entry_script} relative to "
f"source directory {source_config.root_folder})")
max_run_duration = None
if azure_config.max_run_duration:
max_run_duration = run_duration_string_to_seconds(azure_config.max_run_duration)
workspace = azure_config.get_workspace()
run_config = RunConfiguration(
script=entry_script_relative_path,
arguments=source_config.script_params,
)
run_config.environment = get_or_create_python_environment(azure_config, source_config,
environment_name=environment_name)
run_config.target = azure_config.cluster
run_config.max_run_duration_seconds = max_run_duration
if azure_config.num_nodes > 1:
distributed_job_config = MpiConfiguration(node_count=azure_config.num_nodes)
run_config.mpi = distributed_job_config
run_config.framework = "Python"
run_config.communicator = "IntelMpi"
run_config.node_count = distributed_job_config.node_count
if dataset_consumption:
run_config.data = {dataset_consumption.name: dataset_consumption}
# Use blob storage for storing the source, rather than the FileShares section of the storage account.
run_config.source_directory_data_store = workspace.datastores.get(WORKSPACE_DEFAULT_BLOB_STORE_NAME).name
script_run_config = ScriptRunConfig(
source_directory=str(source_config.root_folder),
run_config=run_config,
)
if azure_config.hyperdrive:
script_run_config = source_config.hyperdrive_config_func(script_run_config) # type: ignore
return script_run_config
def get_run_config(self, config):
environment_config = config.get("environment")
environment = self.get_environment(environment_config)
cluster_name = config.get("cluster")
cluster = ComputeTarget(workspace=self.workspace, name=cluster_name)
pipeline_run_config = RunConfiguration()
pipeline_run_config.target = cluster
pipeline_run_config.environment = environment
return pipeline_run_config
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group,
)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable Azure ML environment
# Make sure to include `r-essentials'
# in h1c4driver/conda_dependencies.yml
environment = get_environment(
aml_workspace,
e.aml_env_name,
conda_dependencies_file=e.aml_env_train_conda_dep_file,
create_new=e.rebuild_env,
) # NOQA: E501
run_config = RunConfiguration()
run_config.environment = environment
train_step = PythonScriptStep(
name="Train Model",
script_name="train_with_r.py",
compute_target=aml_compute,
source_directory="h1c4driver/training/R",
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
steps = [train_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id,
)
print(f"Published pipeline: {published_pipeline.name}")
print(f"for build {published_pipeline.version}")
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable run configuration environment
# Read definition from diabetes_regression/azureml_environment.json
# Make sure to include `r-essentials'
# in diabetes_regression/conda_dependencies.yml
environment = Environment.load_from_directory(e.sources_directory_train)
if (e.collection_uri is not None and e.teamproject_name is not None):
builduri_base = e.collection_uri + e.teamproject_name
builduri_base = builduri_base + "/_build/results?buildId="
environment.environment_variables["BUILDURI_BASE"] = builduri_base
environment.register(aml_workspace)
run_config = RunConfiguration()
run_config.environment = environment
train_step = PythonScriptStep(
name="Train Model",
script_name="train_with_r.py",
compute_target=aml_compute,
source_directory="diabetes_regression/training/R",
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
steps = [train_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
def create_runconfig(aml_compute, env=None):
# Create a new runconfig object
aml_run_config = RunConfiguration()
# Use the aml_compute you created above.
aml_run_config.target = aml_compute
if env is not None:
aml_run_config.environment = env
else:
# Enable Docker
aml_run_config.environment.docker.enabled = True
# Set Docker base image to the default CPU-based image
aml_run_config.environment.docker.base_image = "mcr.microsoft.com/azureml/base:0.2.1"
# Use conda_dependencies.yml to create a conda environment in the Docker image for execution
aml_run_config.environment.python.user_managed_dependencies = False
return aml_run_config
# Build CPU image for Ray
ray_cpu_env = Environment.from_dockerfile(
name=ray_environment_name, dockerfile=ray_environment_dockerfile_path)
ray_cpu_env.register(workspace=ws)
ray_cpu_build_details = ray_cpu_env.build(workspace=ws)
while ray_cpu_build_details.status not in ["Succeeded", "Failed"]:
print(
f"Awaiting completion of ray CPU environment build. Current status is: {ray_cpu_build_details.status}"
)
time.sleep(10)
command = ["python distribute_automl.py"]
env = Environment.get(workspace=ws, name=ray_environment_name)
compute_target = ws.compute_targets["cpucluster"]
aml_run_config = RunConfiguration(communicator="OpenMpi")
aml_run_config.target = compute_target
aml_run_config.docker = DockerConfiguration(use_docker=True)
aml_run_config.environment = env
aml_run_config.node_count = 2
config = ScriptRunConfig(
source_directory="ray/",
command=command,
run_config=aml_run_config,
)
exp = Experiment(ws, "distribute-automl")
run = exp.submit(config)
print(run.get_portal_url()) # link to ml.azure.com
run.wait_for_completion(show_output=True)
def main():
env = EnvironmentVariables()
args = add_arguments()
workspace = get_workspace()
cpu_cluster_name = env.cpu_cluster_name
compute = get_or_create_compute(workspace, cpu_cluster_name,
env.compute_vm_size, env.max_nodes)
environment = Environment.load_from_directory(env.sources_directory_train)
environment.register(workspace)
run_configuration = RunConfiguration()
run_configuration.environment = environment
model_name_param = PipelineParameter(name="model_name",
default_value=env.model_name)
build_id_param = PipelineParameter(name="build_id",
default_value=env.build_id)
should_tune_hyperparameters_param = PipelineParameter(
name="should_tune_hyperparameters",
default_value=env.should_tune_hyperparameters)
parallelism_level_param = PipelineParameter(
name="parallelism_level", default_value=env.parallelism_level)
force_register_param = PipelineParameter(name="force_register",
default_value=env.force_register)
datastore = get_datastore()
dataset_name = env.dataset_name
dataset_path = env.dataset_path
print(
f"Creating new dataset version for {dataset_name} in datastore {datastore} from file {dataset_path}"
)
temp_dataset = Dataset.Tabular.from_delimited_files(path=[(datastore,
dataset_path)])
dataset = temp_dataset.register(workspace=workspace,
name=dataset_name,
description=dataset_name,
tags={'format': 'CSV'},
create_new_version=True)
train_output = PipelineData('train_output',
output_name='train_output',
datastore=datastore)
train_step = PythonScriptStep(
name="Train model",
compute_target=compute,
script_name=env.train_script_name,
runconfig=run_configuration,
inputs=[dataset.as_named_input('training')],
outputs=[train_output],
arguments=[
"--build_id", build_id_param, "--model_name", model_name_param,
"--parallelism_level", parallelism_level_param,
"--should_tune_hyperparameters", should_tune_hyperparameters_param
],
allow_reuse=False)
evaluate_step = PythonScriptStep(name="Evaluate model",
compute_target=compute,
script_name=env.evaluate_script_name,
runconfig=run_configuration,
inputs=[train_output],
arguments=[
"--build_id", build_id_param,
"--model_name", model_name_param,
"--train_output", train_output,
"--force_register",
force_register_param
],
allow_reuse=False)
register_step = PythonScriptStep(name="Register model",
compute_target=compute,
script_name=env.register_script_name,
runconfig=run_configuration,
inputs=[train_output],
arguments=[
"--build_id", build_id_param,
"--model_name", model_name_param,
"--train_output", train_output
],
allow_reuse=False)
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
train_pipeline = Pipeline(workspace=workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=env.pipeline_name,
description="Train/Eval/Register if better pipeline",
version=env.build_id)
output_file_name = args.output_file_name
if output_file_name:
with open(output_file_name, "w") as output_file:
output_file.write(published_pipeline.id)
print(
f"Published pipeline {published_pipeline.name} for build {published_pipeline.version}"
)
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable Azure ML environment
environment = get_environment(aml_workspace,
e.aml_env_name,
create_new=False) # NOQA: E501
run_config = RunConfiguration()
run_config.environment = environment
if (e.datastore_name):
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables[
"DATASTORE_NAME"] = datastore_name # NOQA: E501
model_name_param = PipelineParameter(name="model_name",
default_value=e.model_name)
dataset_version_param = PipelineParameter(name="dataset_version",
default_value=e.dataset_version)
data_file_path_param = PipelineParameter(name="data_file_path",
default_value="none")
caller_run_id_param = PipelineParameter(name="caller_run_id",
default_value="none")
# Get dataset name
dataset_name = e.dataset_name
# # Check to see if dataset exists
# if (dataset_name not in aml_workspace.datasets):
# # Create dataset from lacemlops sample data
# sample_data = load_lacemlops()
# df = pd.DataFrame(
# data=sample_data.data,
# columns=sample_data.feature_names)
# df['Y'] = sample_data.target
# file_name = 'lacemlops.csv'
# df.to_csv(file_name, index=False)
# # Upload file to default datastore in workspace
# datatstore = Datastore.get(aml_workspace, datastore_name)
# target_path = 'training-data/'
# datatstore.upload_files(
# files=[file_name],
# target_path=target_path,
# overwrite=True,
# show_progress=False)
# # Register dataset
# path_on_datastore = os.path.join(target_path, file_name)
# dataset = Dataset.Tabular.from_delimited_files(
# path=(datatstore, path_on_datastore))
# dataset = dataset.register(
# workspace=aml_workspace,
# name=dataset_name,
# description='lacemlops training data',
# tags={'format': 'CSV'},
# create_new_version=True)
# Create a PipelineData to pass data between steps
pipeline_data = PipelineData(
'pipeline_data', datastore=aml_workspace.get_default_datastore())
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
outputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_output",
pipeline_data,
"--dataset_version",
dataset_version_param,
"--data_file_path",
data_file_path_param,
"--caller_run_id",
caller_run_id_param,
"--dataset_name",
dataset_name,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--model_name",
model_name_param,
"--allow_run_cancel",
e.allow_run_cancel,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_input",
pipeline_data,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
# Check run_evaluation flag to include or exclude evaluation step.
if ((e.run_evaluation).lower() == 'true'):
print("Include evaluation step before register step.")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
else:
print("Exclude evaluation step and directly run register step.")
register_step.run_after(train_step)
steps = [train_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
# Add the dependencies to the environment
fraud_env.python.conda_dependencies = fraud_packages
# Register the environment (just in case you want to use it again)
fraud_env.register(workspace=ws)
registered_env = Environment.get(ws, 'fraud-pipeline-env')
# Create a new runconfig object for the pipeline
pipeline_run_config = RunConfiguration()
# Use the compute you created above.
pipeline_run_config.target = pipeline_cluster
# Assign the environment to the run configuration
pipeline_run_config.environment = registered_env
print("Run configuration created.")
# Get the training dataset
fraud_ds = ws.datasets.get("creditcard")
# Create a PipelineData (temporary Data Reference) for the model folder
model_folder = PipelineData("model_folder",
datastore=ws.get_default_datastore())
#pipeline_data = PipelineData('pipeline_data', datastore=default_ds)
data_ref = DataReference(datastore=default_ds,
data_reference_name='data_ref',
path_on_datastore="config/")
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group,
)
print(f"get_workspace:{aml_workspace}")
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print(f"aml_compute:{aml_compute}")
# Create a reusable Azure ML environment
environment = get_environment(
aml_workspace,
e.aml_env_name,
create_new=e.rebuild_env,
enable_docker=True,
dockerfile='ml_model/preprocess/Dockerfile'
) #
run_config = RunConfiguration()
run_config.environment = environment
if e.datastore_name:
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables["DATASTORE_NAME"] = datastore_name # NOQA: E501
datastore = Datastore(aml_workspace, name=datastore_name)
data_file_path_param = PipelineParameter(name="data_file_path", default_value=e.dataset_name) # NOQA: E501
# The version of the input/output dataset can't be determined at pipeline publish time, only run time. # NOQA: E501
# Options to store output data:
# Option 1: Use blob API to write output data. Otherwise, no way to dynamically change the output dataset based on PipelineParameter, # NOQA: E501
# The following will not work. It generate a path like "PipelineParameter_Name:data_file_path_Default:gear_images" # NOQA: E501
# output_ds = OutputFileDatasetConfig(destination=(datastore, data_file_path_param)) # NOQA: E501
# This option means writing a file locally and upload to the datastore. Fewer dataset, more code. # NOQA: E501
# Option 2: Use a dynamic path in OutputFileDatasetConfig, and register a new dataset at completion # NOQA: E501
# Output dataset can be mounted, so more dataset to maintain, less code. # NOQA: E501
# Using Option 2 below.
output_dataset = OutputFileDatasetConfig(
name=e.processed_dataset_name,
destination=(datastore, "/dataset/{output-name}/{run-id}")
).register_on_complete(
name=e.processed_dataset_name)
preprocess_step = PythonScriptStep(
name="Preprocess Data with OS cmd",
script_name='preprocess/preprocess_os_cmd_aml.py',
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--dataset_name", e.dataset_name,
"--datastore_name", datastore_name,
"--data_file_path", data_file_path_param,
"--output_dataset", output_dataset,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Preprocess OS cmd created")
steps = [preprocess_step]
preprocess_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
preprocess_pipeline._set_experiment_name
preprocess_pipeline.validate()
published_pipeline = preprocess_pipeline.publish(
name=e.preprocessing_pipeline_name,
description="Data preprocessing OS cmd pipeline",
version=e.build_id,
)
print(f"Published pipeline: {published_pipeline.name}")
print(f"for build {published_pipeline.version}")
# 6. Submit the pipeline for execution
## Pipeline Parameters ##
# We need to tell the Pipeline what it needs to learn to see!
datapath = DataPath(datastore=datastore, path_on_datastore=datastorepath)
data_path_pipeline_param = (PipelineParameter(name="data",
default_value=datapath),
DataPathComputeBinding(mode='mount'))
# Configuration for data prep and training steps #
dataprepEnvironment = Environment.from_pip_requirements(
'dataprepenv', 'requirements-dataprepandtraining.txt')
dataprepRunConfig = RunConfiguration()
dataprepRunConfig.environment = dataprepEnvironment
## Data Process Step ##
# parse.py file parses the images in our data source #
seer_tfrecords = PipelineData("tfrecords_set",
datastore=datastore,
is_directory=True)
prepStep = PythonScriptStep('parse.py',
source_directory='.',
name='Data Preparation',
compute_target=compute,
arguments=[
"--source_path", data_path_pipeline_param,
"--target_path", seer_tfrecords
def main():
e = Env()
print('********************')
print(e.source_directory)
files = os.listdir('./aml_pipeline')
for f in files:
print(f)
print('***************')
workspace_name = e.workspace_name
subscription_id = e.subscription_id
resource_group = e.resource_group
#Connect to AML Workspace
print('workspace_name = ' + workspace_name)
print('subscription_id = ' + subscription_id)
print('resource_group = ' + resource_group)
ws = Workspace.get(
name=workspace_name,
subscription_id=subscription_id,
resource_group=resource_group,
)
print('Ready to use Azure ML {} to work with {}'.format(
azureml.core.VERSION, ws.name))
default_ds = ws.get_default_datastore()
if 'diabetes dataset' not in ws.datasets:
default_ds.upload_files(
files=['diabetes.csv',
'diabetes2.csv'], # Upload the diabetes csv files in /data
target_path=
'diabetes-data/', # Put it in a folder path in the datastore
overwrite=True, # Replace existing files of the same name
show_progress=True)
#Create a tabular dataset from the path on the datastore (this may take a short while)
tab_data_set = Dataset.Tabular.from_delimited_files(
path=(default_ds, 'diabetes-data/*.csv'))
# Register the tabular dataset
try:
tab_data_set = tab_data_set.register(workspace=ws,
name='diabetes dataset',
description='diabetes data',
tags={'format': 'CSV'},
create_new_version=True)
print('Dataset registered.')
except Exception as ex:
print(ex)
else:
print('Dataset already registered.')
# Create a folder for the pipeline step files
experiment_folder = 'diabetes_pipeline'
os.makedirs(experiment_folder, exist_ok=True)
print(experiment_folder)
cluster_name = "mmcomputecluster"
try:
# Check for existing compute target
pipeline_cluster = ComputeTarget(workspace=ws, name=cluster_name)
print('Found existing cluster, use it.')
except ComputeTargetException:
# If it doesn't already exist, create it
try:
compute_config = AmlCompute.provisioning_configuration(
vm_size='STANDARD_DS11_V2', max_nodes=2)
pipeline_cluster = ComputeTarget.create(ws, cluster_name,
compute_config)
pipeline_cluster.wait_for_completion(show_output=True)
except Exception as ex:
print(ex)
# Create a Python environment for the experiment
diabetes_env = Environment("diabetes-pipeline-env")
diabetes_env.python.user_managed_dependencies = False # Let Azure ML manage dependencies
diabetes_env.docker.enabled = True # Use a docker container
# Create a set of package dependencies
diabetes_packages = CondaDependencies.create(
conda_packages=[
'scikit-learn', 'ipykernel', 'matplotlib', 'pandas', 'pip'
],
pip_packages=[
'azureml-defaults', 'azureml-dataprep[pandas]', 'pyarrow'
])
# Add the dependencies to the environment
diabetes_env.python.conda_dependencies = diabetes_packages
# Register the environment
diabetes_env.register(workspace=ws)
registered_env = Environment.get(ws, 'diabetes-pipeline-env')
# Create a new runconfig object for the pipeline
pipeline_run_config = RunConfiguration()
# Use the compute you created above.
pipeline_run_config.target = pipeline_cluster
# Assign the environment to the run configuration
pipeline_run_config.environment = registered_env
print("Run configuration created.")
# Get the training dataset
diabetes_ds = ws.datasets.get("diabetes dataset")
# Create a PipelineData (temporary Data Reference) for the model folder
prepped_data_folder = PipelineData("prepped_data_folder",
datastore=ws.get_default_datastore())
# Step 1, Run the data prep script
prep_step = PythonScriptStep(name="Prepare Data",
script_name="prep_diabetes.py",
source_directory='./aml_pipeline',
arguments=[
'--input-data',
diabetes_ds.as_named_input('raw_data'),
'--prepped-data', prepped_data_folder
],
outputs=[prepped_data_folder],
compute_target=pipeline_cluster,
runconfig=pipeline_run_config,
allow_reuse=True)
# Step 2, run the training script
train_step = PythonScriptStep(
name="Train and Register Model",
source_directory='./aml_pipeline',
script_name="train_diabetes.py",
arguments=['--training-folder', prepped_data_folder],
inputs=[prepped_data_folder],
compute_target=pipeline_cluster,
runconfig=pipeline_run_config,
allow_reuse=True)
print("Pipeline steps defined")
pipeline_steps = [prep_step, train_step]
pipeline = Pipeline(workspace=ws, steps=pipeline_steps)
print("Pipeline is built.")
# Create an experiment and run the pipeline
experiment = Experiment(workspace=ws, name='jlg-exp')
pipeline_run = experiment.submit(pipeline, regenerate_outputs=True)
print("Pipeline submitted for execution.")
pipeline_run.wait_for_completion(show_output=True)
for run in pipeline_run.get_children():
print(run.name, ':')
metrics = run.get_metrics()
for metric_name in metrics:
print('\t', metric_name, ":", metrics[metric_name])
for model in Model.list(ws):
print(model.name, 'version:', model.version)
for tag_name in model.tags:
tag = model.tags[tag_name]
print('\t', tag_name, ':', tag)
for prop_name in model.properties:
prop = model.properties[prop_name]
print('\t', prop_name, ':', prop)
print('\n')
# Publish the pipeline from the run
published_pipeline = pipeline_run.publish_pipeline(
name="diabetes-training-pipeline",
description="Trains diabetes model",
version="1.0")
published_pipeline
rest_endpoint = published_pipeline.endpoint
print(rest_endpoint)
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group
)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(
aml_workspace,
e.compute_name,
e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable run configuration environment
# Read definition from diabetes_regression/azureml_environment.json
environment = Environment.load_from_directory(e.sources_directory_train)
if (e.collection_uri is not None and e.teamproject_name is not None):
builduri_base = e.collection_uri + e.teamproject_name
builduri_base = builduri_base + "/_build/results?buildId="
environment.environment_variables["BUILDURI_BASE"] = builduri_base
environment.register(aml_workspace)
run_config = RunConfiguration()
run_config.environment = environment
model_name_param = PipelineParameter(
name="model_name", default_value=e.model_name)
build_id_param = PipelineParameter(
name="build_id", default_value=e.build_id)
dataset_name = ""
if (e.datastore_name is not None and e.datafile_name is not None):
dataset_name = e.dataset_name
datastore = Datastore.get(aml_workspace, e.datastore_name)
data_path = [(datastore, e.datafile_name)]
dataset = Dataset.Tabular.from_delimited_files(path=data_path)
dataset.register(workspace=aml_workspace,
name=e.dataset_name,
description="dataset with training data",
create_new_version=True)
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id", build_id_param,
"--model_name", model_name_param,
"--dataset_name", dataset_name,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id", build_id_param,
"--model_name", model_name_param,
"--allow_run_cancel", e.allow_run_cancel,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id", build_id_param,
"--model_name", model_name_param,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
# Check run_evaluation flag to include or exclude evaluation step.
if ((e.run_evaluation).lower() == 'true'):
print("Include evaluation step before register step.")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
else:
print("Exclude evaluation step and directly run register step.")
register_step.run_after(train_step)
steps = [train_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id
)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group,
)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable Azure ML environment
environment = get_environment(
aml_workspace,
e.aml_env_name,
conda_dependencies_file=e.aml_env_train_conda_dep_file,
create_new=e.rebuild_env,
) #
run_config = RunConfiguration()
run_config.environment = environment
if e.datastore_name:
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables[
"DATASTORE_NAME"] = datastore_name # NOQA: E501
model_name_param = PipelineParameter(
name="model_name", default_value=e.model_name) # NOQA: E501
dataset_version_param = PipelineParameter(name="dataset_version",
default_value=e.dataset_version)
data_file_path_param = PipelineParameter(name="data_file_path",
default_value="none")
caller_run_id_param = PipelineParameter(name="caller_run_id",
default_value="none") # NOQA: E501
# Get dataset name
dataset_name = e.dataset_name
# Check to see if dataset exists
if dataset_name not in aml_workspace.datasets:
# This call creates an example CSV from sklearn sample data. If you
# have already bootstrapped your project, you can comment this line
# out and use your own CSV.
create_sample_data_csv()
# Use a CSV to read in the data set.
file_name = "safedriver.csv"
if not os.path.exists(file_name):
raise Exception(
'Could not find CSV dataset at "%s". If you have bootstrapped your project, you will need to provide a CSV.' # NOQA: E501
% file_name) # NOQA: E501
# Upload file to default datastore in workspace
datatstore = Datastore.get(aml_workspace, datastore_name)
target_path = "training-data/"
datatstore.upload_files(
files=[file_name],
target_path=target_path,
overwrite=True,
show_progress=False,
)
# Register dataset
path_on_datastore = os.path.join(target_path, file_name)
dataset = Dataset.Tabular.from_delimited_files(
path=(datatstore, path_on_datastore))
dataset = dataset.register(
workspace=aml_workspace,
name=dataset_name,
description="safedriver training data",
tags={"format": "CSV"},
create_new_version=True,
)
# Create a PipelineData to pass data between steps
pipeline_data = PipelineData(
"pipeline_data", datastore=aml_workspace.get_default_datastore())
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
outputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_output",
pipeline_data,
"--dataset_version",
dataset_version_param,
"--data_file_path",
data_file_path_param,
"--caller_run_id",
caller_run_id_param,
"--dataset_name",
dataset_name,
],
runconfig=run_config,
allow_reuse=True,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--model_name",
model_name_param,
"--allow_run_cancel",
e.allow_run_cancel,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_input",
pipeline_data,
], # NOQA: E501
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
# Check run_evaluation flag to include or exclude evaluation step.
if (e.run_evaluation).lower() == "true":
print("Include evaluation step before register step.")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
else:
print("Exclude evaluation step and directly run register step.")
register_step.run_after(train_step)
steps = [train_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id,
)
print(f"Published pipeline: {published_pipeline.name}")
print(f"for build {published_pipeline.version}")
"pillow==6.0.0",
"tensorflow-gpu==1.15",
"keras",
"matplotlib",
"seaborn",
])
env = Environment("prednet")
env.python.conda_dependencies = conda_dependencies
env.docker.enabled = True
env.docker.base_image = DEFAULT_GPU_IMAGE
env.register(ws)
# Runconfigs
runconfig = RunConfiguration()
runconfig.environment = env
print("PipelineData object created")
create_pipelines = PythonScriptStep(
name="create pipelines",
script_name="pipelines_slave.py",
compute_target=cpu_compute_target,
arguments=[
"--cpu_compute_name", cpu_compute_name, "--gpu_compute_name",
gpu_compute_name
],
source_directory=script_folder,
runconfig=runconfig,
allow_reuse=False,
)
print("pipeline building step created")
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable run configuration environment
# Read definition from diabetes_regression/azureml_environment.json
environment = Environment.load_from_directory(e.sources_directory_train)
if (e.collection_uri is not None and e.teamproject_name is not None):
builduri_base = e.collection_uri + e.teamproject_name
builduri_base = builduri_base + "/_build/results?buildId="
environment.environment_variables["BUILDURI_BASE"] = builduri_base
environment.register(aml_workspace)
run_config = RunConfiguration()
run_config.environment = environment
model_name_param = PipelineParameter(name="model_name",
default_value=e.model_name)
build_id_param = PipelineParameter(name="build_id",
default_value=e.build_id)
# Get dataset name
dataset_name = e.dataset_name
# Check to see if dataset exists
if (dataset_name not in aml_workspace.datasets):
# Create dataset from diabetes sample data
sample_data = load_diabetes()
df = pd.DataFrame(data=sample_data.data,
columns=sample_data.feature_names)
df['Y'] = sample_data.target
file_name = 'diabetes.csv'
df.to_csv(file_name, index=False)
# Upload file to default datastore in workspace
default_ds = aml_workspace.get_default_datastore()
target_path = 'training-data/'
default_ds.upload_files(files=[file_name],
target_path=target_path,
overwrite=True,
show_progress=False)
# Register dataset
path_on_datastore = os.path.join(target_path, file_name)
dataset = Dataset.Tabular.from_delimited_files(
path=(default_ds, path_on_datastore))
dataset = dataset.register(workspace=aml_workspace,
name=dataset_name,
description='diabetes training data',
tags={'format': 'CSV'},
create_new_version=True)
# Get the dataset
dataset = Dataset.get_by_name(aml_workspace, dataset_name)
# Create a PipelineData to pass data between steps
pipeline_data = PipelineData(
'pipeline_data', datastore=aml_workspace.get_default_datastore())
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[dataset.as_named_input('training_data')],
outputs=[pipeline_data],
arguments=[
"--build_id", build_id_param, "--model_name", model_name_param,
"--step_output", pipeline_data
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
"--allow_run_cancel",
e.allow_run_cancel,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[pipeline_data],
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
"--step_input",
pipeline_data,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
# Check run_evaluation flag to include or exclude evaluation step.
if ((e.run_evaluation).lower() == 'true'):
print("Include evaluation step before register step.")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
else:
print("Exclude evaluation step and directly run register step.")
register_step.run_after(train_step)
steps = [train_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
myenv.docker.base_image = "mcr.microsoft.com/azureml/base-gpu:openmpi3.1.2-cuda10.1-cudnn7-ubuntu18.04"
# comment out this environment variable if you don't have it set!
myenv.environment_variables = {'WANDB_API_KEY': os.environ['WANDB_API_KEY']}
## Environment: python section
conda_dep = CondaDependencies(conda_dependencies_file_path='environment.yml')
myenv.python.conda_dependencies = conda_dep
# create configuration for Run
# Use RunConfiguration to specify compute target / env deps part of run
run_config = RunConfiguration()
# Attach compute target to run config
run_config.framework = 'python'
run_config.target = "gpu-cluster"
# This doesn't actuallly do anything since my target is a persistent compute instead of amlcompute
run_config.amlcompute.vm_size = "Standard_NC24"
run_config.node_count = 1
run_config.environment = myenv
# ScriptRunConfig packaages together environment configuration of
# RunConfiguration with a script for training to create a **script** run.
""" RunConfiguration(script=None, arguments=None, framework=None, communicator=None,
conda_dependencies=None, _history_enabled=None, _path=None, _name=None)
"""
src = ScriptRunConfig(source_directory='.',
script='rl_credit/examples/distractor_delay_expt.py',
run_config=run_config)
get_run_config_from_script_run(src).save(path='.azureml/train.runconfig')
def create_pipeline(workspace):
# Retreive compute cluster
compute_target = workspace.compute_targets[args.compute_target]
# Setup batch scoring environment from conda dependencies
environment = Environment.from_conda_specification(
name=args.environment_name, file_path=args.environment_specification
)
# Add environment variables
environment.environment_variables = {
"APPLICATIONINSIGHTS_CONNECTION_STRING": args.ai_connection_string
}
# Enable docker run
environment.docker.enabled = True
# Create run config
run_config = RunConfiguration()
run_config.environment = environment
# Retreive input and output datastores
input_datastore = Datastore(workspace, args.input_datastore_name)
output_datastore = Datastore(workspace, args.output_datastore_name)
# Define build id parameter
build_id_param = PipelineParameter("build_id", default_value=args.build_id)
# Define input datapath parameter
input_datapath = DataPath(datastore=input_datastore, path_on_datastore="")
input_datapath_param = (
PipelineParameter(name="input_datapath", default_value=input_datapath),
DataPathComputeBinding(mode="mount"),
)
# Define output datapath parameter
output_datapath = DataPath(datastore=output_datastore, path_on_datastore="")
output_datapath_param = (
PipelineParameter(name="output_datapath", default_value=output_datapath),
DataPathComputeBinding(mode="mount"),
)
# Define score step for pipeline
score_step = PythonScriptStep(
name="score_data",
compute_target=compute_target,
source_directory="src/score",
script_name="score.py",
inputs=[input_datapath_param, output_datapath_param],
runconfig=run_config,
allow_reuse=False,
arguments=[
"--build_id",
build_id_param,
"--input_datapath",
input_datapath_param,
"--output_datapath",
output_datapath_param,
],
)
# Define pipeline for batch scoring
pipeline = Pipeline(workspace=workspace, steps=[score_step])
return pipeline
def main():
e = Env()
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group
)
print("get_workspace:")
print(aml_workspace)
aml_compute = get_compute(
aml_workspace,
e.compute_name,
e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
environment = get_environment(
aml_workspace, e.aml_env_name, create_new=e.rebuild_env)
run_config = RunConfiguration()
run_config.environment = environment
if (e.datastore_name):
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables["DATASTORE_NAME"] \
= datastore_name
dataset_name = e.dataset_name
file_name = e.file_name
datastore = Datastore.get(aml_workspace, datastore_name)
if (dataset_name not in aml_workspace.datasets):
raise Exception("Could not find dataset at \"%s\"." % dataset_name)
else:
dataset = Dataset.get_by_name(aml_workspace, name=dataset_name)
dataset.download(target_path='.', overwrite=True)
datastore.upload_files([file_name],
target_path=dataset_name,
overwrite=True)
raw_data_file = DataReference(datastore=datastore,
data_reference_name="Raw_Data_File",
path_on_datastore=dataset_name + '/'
+ file_name)
clean_data_file = PipelineParameter(name="clean_data_file",
default_value="/clean_data.csv")
clean_data_folder = PipelineData("clean_data_folder",
datastore=datastore)
prepDataStep = PythonScriptStep(name="Prepare Data",
source_directory=e.sources_directory_train,
script_name=e.data_prep_script_path,
arguments=["--raw_data_file",
raw_data_file,
"--clean_data_folder",
clean_data_folder,
"--clean_data_file",
clean_data_file],
inputs=[raw_data_file],
outputs=[clean_data_folder],
compute_target=aml_compute,
allow_reuse=False)
print("Step Prepare Data created")
new_model_file = PipelineParameter(name="new_model_file ",
default_value='/' + e.model_name
+ '.pkl')
new_model_folder = PipelineData("new_model_folder", datastore=datastore)
est = SKLearn(source_directory=e.sources_directory_train,
entry_script=e.train_script_path,
pip_packages=['azureml-sdk', 'scikit-learn==0.20.3',
'azureml-dataprep[pandas,fuse]>=1.1.14'],
compute_target=aml_compute)
trainingStep = EstimatorStep(
name="Model Training",
estimator=est,
estimator_entry_script_arguments=["--clean_data_folder",
clean_data_folder,
"--new_model_folder",
new_model_folder,
"--clean_data_file",
clean_data_file.default_value,
"--new_model_file",
new_model_file.default_value],
runconfig_pipeline_params=None,
inputs=[clean_data_folder],
outputs=[new_model_folder],
compute_target=aml_compute,
allow_reuse=False)
print("Step Train created")
model_name_param = PipelineParameter(name="model_name",
default_value=e.model_name)
evaluateStep = PythonScriptStep(
name="Evaluate Model",
source_directory=e.sources_directory_train,
script_name=e.evaluate_script_path,
arguments=["--model_name", model_name_param],
compute_target=aml_compute,
allow_reuse=False)
print("Step Evaluate created")
registerStep = PythonScriptStep(
name="Register Model",
source_directory=e.sources_directory_train,
script_name=e.register_script_path,
arguments=["--new_model_folder", new_model_folder,
"--new_model_file", new_model_file,
"--model_name", model_name_param],
inputs=[new_model_folder],
compute_target=aml_compute,
allow_reuse=False)
print("Step Register created")
if ((e.run_evaluation).lower() == 'true'):
print("Include evaluation step before register step.")
trainingStep.run_after(prepDataStep)
evaluateStep.run_after(trainingStep)
registerStep.run_after(evaluateStep)
else:
print("Exclude evaluation step and directly run register step.")
trainingStep.run_after(prepDataStep)
registerStep.run_after(trainingStep)
pipeline = Pipeline(workspace=aml_workspace, steps=[registerStep])
pipeline.validate()
print("Pipeline is built")
pipeline._set_experiment_name
published_pipeline = pipeline.publish(
name=e.pipeline_name,
description="Predict Employee Retention Model training pipeline",
version=e.build_id
)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
def main():
"""Build pipeline."""
# Environment variables
env = Env()
# Azure ML workspace
aml_workspace = Workspace.get(
name=env.workspace_name,
subscription_id=env.subscription_id,
resource_group=env.resource_group,
)
logger.info(f"Azure ML workspace: {aml_workspace}")
# Azure ML compute cluster
aml_compute = get_compute(aml_workspace, env.compute_name)
logger.info(f"Aazure ML compute cluster: {aml_compute}")
# Azure ML environment
environment = Environment(name=env.aml_env_name)
conda_dep = CondaDependencies(
conda_dependencies_file_path="./local_development/dev_dependencies.yml"
)
environment.python.conda_dependencies = conda_dep
run_config = RunConfiguration()
run_config.environment = environment
# Pipeline Data
preparation_pipelinedata = PipelineData("preparation_pipelinedata",
is_directory=True).as_dataset()
extraction_pipelinedata = PipelineData("extraction_pipelinedata",
is_directory=True)
training_pipelinedata = PipelineData("training_pipelinedata",
is_directory=True)
# List of pipeline steps
step_list = list()
preparation_step = PythonScriptStep(
name="preparation-step",
compute_target=aml_compute,
source_directory=env.sources_directory_train,
script_name=env.preparation_step_script_path,
outputs=[preparation_pipelinedata],
arguments=[
"--input_path", env.input_dir, "--output_path",
preparation_pipelinedata, "--datastore_name",
env.blob_datastore_name
],
runconfig=run_config)
step_list.append(preparation_step)
parallel_run_config = ParallelRunConfig(
source_directory=env.sources_directory_train,
entry_script=env.extraction_step_script_path,
mini_batch_size=env.mini_batch_size,
error_threshold=env.error_threshold,
output_action="append_row",
environment=environment,
compute_target=aml_compute,
node_count=env.node_count,
run_invocation_timeout=env.run_invocation_timeout,
process_count_per_node=env.process_count_per_node,
append_row_file_name="extraction_output.txt")
extraction_step = ParallelRunStep(
name="extraction-step",
inputs=[preparation_pipelinedata],
output=extraction_pipelinedata,
arguments=["--output_dir", extraction_pipelinedata],
parallel_run_config=parallel_run_config)
step_list.append(extraction_step)
training_step = PythonScriptStep(
name="traning-step",
compute_target=aml_compute,
source_directory=env.sources_directory_train,
script_name=env.training_step_script_path,
inputs=[extraction_pipelinedata],
outputs=[training_pipelinedata],
arguments=[
"--input_dir", extraction_pipelinedata, "--output_dir",
training_pipelinedata
],
runconfig=run_config)
step_list.append(training_step)
# Build pipeline
pipeline = Pipeline(workspace=aml_workspace, steps=step_list)
pipeline.validate()
logger.info(f"Built pipeline {pipeline}")
# Publish pipeline
published_pipeline = pipeline.publish(
env.pipeline_name,
description=env.pipeline_name,
version=datetime.utcnow().isoformat())
try:
pipeline_endpoint = PipelineEndpoint.get(
workspace=aml_workspace, name=env.pipeline_endpoint_name)
pipeline_endpoint.add_default(published_pipeline)
except ErrorResponseException:
pipeline_endpoint = PipelineEndpoint.publish(
workspace=aml_workspace,
name=env.pipeline_endpoint_name,
pipeline=published_pipeline,
description=env.pipeline_endpoint_name)
def main():
e = Env()
print(e.__dict__)
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable Azure ML environment
environment = get_environment(aml_workspace,
e.aml_env_name,
create_new=e.rebuild_env) #
run_config = RunConfiguration()
run_config.environment = environment
if e.datastore_name:
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables[
"DATASTORE_NAME"] = datastore_name # NOQA: E501
model_name_param = PipelineParameter(name="model_name",
default_value=e.model_name)
dataset_version_param = PipelineParameter(name="dataset_version",
default_value=e.dataset_version)
data_file_path_param = PipelineParameter(name="data_file_path",
default_value="none")
caller_run_id_param = PipelineParameter(name="caller_run_id",
default_value="none")
# Get dataset name
dataset_name = e.dataset_name
# Check to see if dataset exists
if dataset_name not in aml_workspace.datasets:
raise ValueError(
f"can't find dataset {dataset_name} in datastore {datastore_name}")
# Create PipelineData to pass data between steps
model_data = PipelineData("model_data",
datastore=aml_workspace.get_default_datastore())
train_ds = (PipelineData("train_ds",
datastore=aml_workspace.get_default_datastore()).
as_dataset().parse_delimited_files().register(
name="train", create_new_version=True))
test_ds = (PipelineData(
"test_ds", datastore=aml_workspace.get_default_datastore()).as_dataset(
).parse_delimited_files().register(name="test",
create_new_version=True))
prepare_step = PythonScriptStep(
name="Prepare Data",
script_name=e.prepare_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
outputs=[train_ds, test_ds],
arguments=[
"--dataset_version", dataset_version_param, "--data_file_path",
data_file_path_param, "--dataset_name", dataset_name,
"--caller_run_id", caller_run_id_param, "--train_ds", train_ds,
"--test_ds", test_ds
],
runconfig=run_config,
allow_reuse=True,
)
print("Step Prepare created")
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[
train_ds.as_named_input("training_data"),
test_ds.as_named_input("testing_data")
],
outputs=[model_data],
arguments=[
"--model_name", model_name_param, "--model_data", model_data
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--model_name",
model_name_param,
"--allow_run_cancel",
e.allow_run_cancel,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[model_data],
arguments=[
"--model_name", model_name_param, "--step_input", model_data
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
# Check run_evaluation flag to include or exclude evaluation step.
if (e.run_evaluation).lower() == "true":
print("Include evaluation step before register step.")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [prepare_step, train_step, evaluate_step, register_step]
else:
print("Exclude evaluation step and directly run register step.")
register_step.run_after(train_step)
steps = [prepare_step, train_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id)
print(f"Published pipeline: {published_pipeline.name}")
print(f"for build {published_pipeline.version}")
'azureml-sdk', 'PyYAML', 'azure-storage-blob', 'matplotlib',
'seaborn', 'tensorflow', 'Keras', 'tensorflow-hub', 'joblib',
'tqdm', 'Pillow', 'azureml-dataprep[pandas,fuse]>=1.1.14'
])
diagnoz_env = Environment("diagnoz-pipeline-env")
diagnoz_env.python.user_managed_dependencies = False # Let Azure ML manage dependencies
diagnoz_env.docker.enabled = True # Use a docker container
diagnoz_env.docker.base_image = DEFAULT_GPU_IMAGE
diagnoz_env.python.conda_dependencies = packages
diagnoz_env.register(workspace=ws)
# Runconfigs
pipeline_run_config = RunConfiguration()
pipeline_run_config.target = compute_target
pipeline_run_config.environment = diagnoz_env
print("Run configuration created.")
shutil.rmtree(script_folder, ignore_errors=True)
os.makedirs(script_folder, exist_ok=True)
#copy all necessary scripts
files = FilesProviders.get_path_files(
"../", [os.path.basename(__file__), "__init__.py"])
for f in files:
shutil.copy(f, script_folder)
#add generated config file to script folder
shutil.copy(generated_config_file, script_folder)
input_data = input_dataset.as_named_input('input_dataset').as_mount()
compute_target.wait_for_completion(show_output=True,
min_node_count=None,
timeout_in_minutes=20)
# For a more detailed view of current cluster status, use the 'status' property
print(compute_target.status.serialize())
aml_run_config = RunConfiguration()
# `compute_target` as defined in "Azure Machine Learning compute" section above
aml_run_config.target = compute_target
USE_CURATED_ENV = True
if USE_CURATED_ENV:
curated_environment = Environment.get(workspace=ws,
name="AzureML-Tutorial")
aml_run_config.environment = curated_environment
else:
aml_run_config.environment.python.user_managed_dependencies = False
# Add some packages relied on by data prep step
aml_run_config.environment.python.conda_dependencies = CondaDependencies.create(
conda_packages=['pandas', 'scikit-learn', 'seaborn', 'tqdm'],
pip_packages=[
'azureml-sdk', 'azureml-dataprep[fuse,pandas]', 'seaborn', 'tqdm'
],
pin_sdk_version=False)
web_path = 'https://dprepdata.blob.core.windows.net/demo/Titanic.csv'
my_dataset = Dataset.Tabular.from_delimited_files(
path=web_path, set_column_types={'Survived': DataType.to_bool()})
def main():
e = ENV()
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group
)
print(f'workspace:{aml_workspace}')
aml_compute = get_compute(workspace=aml_workspace,
compute_name=e.compute_name,
vm_size=e.vm_size)
if aml_compute is not None:
print(f'compute target: {aml_compute} is created')
environment = get_environment(workspace=aml_workspace,
env_name=e.aml_env_name,
conda_dependencies=e.aml_conda_train_dependent_files,
create_new=e.rebuild_env)
run_config = RunConfiguration()
run_config.environment = environment
if e.datastore_name:
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables['DATASTORE_NAME'] = datastore_name
model_name_param = PipelineParameter(name='model_name',default_value=e.model_name)
dataset_version_param = PipelineParameter(name='dataset_version',default_value=e.dataset_version)
dataset_file_path = PipelineParameter(name='dataset_file_path',default_value='none')
dataset_name = e.dataset_name
if dataset_name not in aml_workspace.datasets:
create_sample_data_csv()
file_name = 'diabetes.csv'
if not os.path.exists(file_name):
raise Exception(f'{file_name} does not exist!')
datastore = Datastore.get(aml_workspace,datastore_name)
target_path = 'training/'
datastore.upload_files(files=[file_name],
target_path=target_path,
overwrite=True,
show_progress=True)
path_on_datastore = os.path.join(target_path,file_name)
dataset = Dataset.Tabular.from_delimited_files(path=(datastore,path_on_datastore))
dataset.register(workspace=aml_workspace,
name=dataset_name,
description='registered dataset',
create_new_version=True,
tags={'format':'CSV'})
pipeline_data = PipelineData('train',datastore=aml_workspace.get_default_datastore())
train_step = PythonScriptStep(script_name=e.train_script_path,
name='train_step',
arguments=['--model-name',model_name_param,
'--dataset-name',dataset_name,
'--dataset-version',dataset_version_param,
'--dataset-file-path',dataset_file_path,
'--step-output',pipeline_data],
compute_target=aml_compute,
runconfig=run_config,
source_directory=e.source_train_directory,
outputs=[pipeline_data],
allow_reuse=False)
print('Train step created!')
eval_step = PythonScriptStep(name='eval_step',
script_name=e.eval_script_path,
compute_target=aml_compute,
source_directory=e.source_train_directory,
arguments=['--model-name',model_name_param,
'--allow-run-cancel',e.allow_run_cancel],
runconfig=run_config,
allow_reuse=False)
print('EVAL step created!')
register_step = PythonScriptStep(name='register_step',
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.source_train_directory,
inputs=[pipeline_data],
arguments=['--model-name',model_name_param,
'--step-input',pipeline_data],
runconfig=run_config,
allow_reuse=False)
print('Register step created!')
if e.run_evaluation:
print('evaluation step is included')
eval_step.run_after(train_step)
register_step.run_after(eval_step)
steps = [train_step,eval_step,register_step]
else:
print('evaluation step is excluded')
register_step.run_after(train_step)
steps = [train_step,register_step]
train_pipeline = Pipeline(workspace=aml_workspace,
steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(name=e.pipeline_name,
description='model training pipeline',
version=e.build_id)
print(f'{published_pipeline.name} is published1')
def build_prednet_pipeline(dataset, ws):
print("building pipeline for dataset %s in workspace %s" %
(dataset, ws.name))
base_dir = "."
def_blob_store = ws.get_default_datastore()
# folder for scripts that need to be uploaded to Aml compute target
script_folder = "./scripts"
os.makedirs(script_folder)
shutil.copytree(os.path.join(base_dir, "models"),
os.path.join(base_dir, script_folder, "models"))
shutil.copy(os.path.join(base_dir, "train.py"), script_folder)
shutil.copy(os.path.join(base_dir, "data_preparation.py"), script_folder)
shutil.copy(os.path.join(base_dir, "register_prednet.py"), script_folder)
shutil.copy(os.path.join(base_dir, "batch_scoring.py"), script_folder)
shutil.copy(os.path.join(base_dir, "train_clf.py"), script_folder)
shutil.copy(os.path.join(base_dir, "register_clf.py"), script_folder)
cpu_compute_name = args.cpu_compute_name
cpu_compute_target = AmlCompute(ws, cpu_compute_name)
print("found existing compute target: %s" % cpu_compute_name)
# use get_status() to get a detailed status for the current cluster.
print(cpu_compute_target.get_status().serialize())
# choose a name for your cluster
gpu_compute_name = args.gpu_compute_name
gpu_compute_target = AmlCompute(workspace=ws, name=gpu_compute_name)
print(gpu_compute_target.get_status().serialize())
env = Environment.get(ws, "prednet")
# Runconfigs
runconfig = RunConfiguration()
runconfig.environment = env
print("PipelineData object created")
# DataReference to where raw data is stored.
raw_data = DataReference(
datastore=def_blob_store,
data_reference_name="raw_data",
path_on_datastore=os.path.join("prednet", "data", "raw_data"),
)
print("DataReference object created")
# Naming the intermediate data as processed_data and assigning it to the
# variable processed_data.
preprocessed_data = PipelineData("preprocessed_data",
datastore=def_blob_store)
data_metrics = PipelineData("data_metrics", datastore=def_blob_store)
hd_child_cwd = PipelineData("prednet_model_path", datastore=def_blob_store)
# prednet_path = PipelineData("outputs", datastore=def_blob_store)
scored_data = PipelineData("scored_data", datastore=def_blob_store)
model_path = PipelineData("model_path", datastore=def_blob_store)
# prepare dataset for training/testing recurrent neural network
data_prep = PythonScriptStep(
name="prepare_data",
script_name="data_preparation.py",
arguments=[
"--raw_data",
raw_data,
"--preprocessed_data",
preprocessed_data,
"--dataset",
dataset,
],
inputs=[raw_data],
outputs=[preprocessed_data],
compute_target=cpu_compute_target,
source_directory=script_folder,
runconfig=runconfig,
allow_reuse=True,
)
# data_prep.run_after(video_decoding)
print("data_prep step created")
est = Estimator(
source_directory=script_folder,
compute_target=gpu_compute_target,
entry_script="train.py",
node_count=1,
environment_definition=env,
)
ps = BayesianParameterSampling({
"--batch_size":
choice(1, 2, 4, 10),
"--filter_sizes":
choice("3, 3, 3", "4, 4, 4", "5, 5, 5"),
"--stack_sizes":
choice("48, 96, 192", "36, 72, 144", "12, 24, 48"),
"--learning_rate":
uniform(1e-6, 1e-3),
"--lr_decay":
uniform(1e-9, 1e-2),
"--freeze_layers":
choice("0, 1, 2", "1, 2, 3", "0, 1", "1, 2", "2, 3", "0", "3"),
# "--fine_tuning": choice("True", "False"),
})
hdc = HyperDriveConfig(
estimator=est,
hyperparameter_sampling=ps,
primary_metric_name="val_loss",
primary_metric_goal=PrimaryMetricGoal.MINIMIZE,
max_total_runs=3,
max_concurrent_runs=3,
max_duration_minutes=60 * 6,
)
train_prednet = HyperDriveStep(
"train_w_hyperdrive",
hdc,
estimator_entry_script_arguments=[
"--preprocessed_data",
preprocessed_data,
"--remote_execution",
"--dataset",
dataset,
],
inputs=[preprocessed_data],
outputs=[hd_child_cwd],
metrics_output=data_metrics,
allow_reuse=True,
)
train_prednet.run_after(data_prep)
register_prednet = PythonScriptStep(
name="register_prednet",
script_name="register_prednet.py",
arguments=[
"--data_metrics",
data_metrics,
],
compute_target=cpu_compute_target,
inputs=[data_metrics, hd_child_cwd],
source_directory=script_folder,
allow_reuse=True,
)
register_prednet.run_after(train_prednet)
batch_scoring = PythonScriptStep(
name="batch_scoring",
script_name="batch_scoring.py",
arguments=[
"--preprocessed_data",
preprocessed_data,
"--scored_data",
scored_data,
"--dataset",
dataset,
# "--prednet_path",
# prednet_path
],
compute_target=gpu_compute_target,
inputs=[preprocessed_data],
outputs=[scored_data],
source_directory=script_folder,
runconfig=runconfig,
allow_reuse=True,
)
batch_scoring.run_after(register_prednet)
train_clf = PythonScriptStep(
name="train_clf",
script_name="train_clf.py",
arguments=[
"--preprocessed_data", preprocessed_data, "--scored_data",
scored_data, "--model_path", model_path
],
compute_target=cpu_compute_target,
inputs=[preprocessed_data, scored_data],
outputs=[model_path],
source_directory=script_folder,
runconfig=runconfig,
allow_reuse=True,
)
train_clf.run_after(batch_scoring)
register_clf = PythonScriptStep(
name="register_clf",
script_name="register_clf.py",
arguments=["--model_path", model_path],
inputs=[model_path],
compute_target=cpu_compute_target,
source_directory=script_folder,
allow_reuse=True,
runconfig=runconfig,
)
register_clf.run_after(train_clf)
pipeline = Pipeline(
workspace=ws,
steps=[
data_prep,
train_prednet,
register_prednet,
batch_scoring,
train_clf,
register_clf,
],
)
pipeline.validate()
pipeline_name = "prednet_" + dataset
published_pipeline = pipeline.publish(name=pipeline_name)
_ = Schedule.create(
workspace=ws,
name=pipeline_name + "_sch",
pipeline_id=published_pipeline.id,
experiment_name=pipeline_name,
datastore=def_blob_store,
wait_for_provisioning=True,
description="Datastore scheduler for Pipeline" + pipeline_name,
path_on_datastore=os.path.join("prednet/data/raw_data", dataset,
"Train"),
polling_interval=60 * 24,
)
published_pipeline.submit(ws, pipeline_name)
def main():
train_file = r"EdwardFry_Microsoft_issueDataset.csv"
ws = Workspace.from_config()
# Default datastore
def_data_store = ws.get_default_datastore() # Loads config.json
# Get the blob storage associated with the workspace
def_blob_store = Datastore(ws, "workspaceblobstore")
# Get file storage associated with the workspace
def_file_store = Datastore(ws, "workspacefilestore")
# Set data input and output
xyz_phishing_dataset = Dataset.File.from_files([(def_blob_store,
train_file)])
output_data1 = OutputFileDatasetConfig(
destination=(datastore, 'outputdataset/{run-id}'))
output_data_dataset = output_data1.register_on_complete(
name='prepared_output_data')
# Set compute
compute_name = "aml-compute"
vm_size = "STANDARD_NC6"
if compute_name in ws.compute_targets:
compute_target = ws.compute_targets[compute_name]
if compute_target and type(compute_target) is AmlCompute:
print('Found compute target: ' + compute_name)
else:
print('Creating a new compute target...')
provisioning_config = AmlCompute.provisioning_configuration(
vm_size=vm_size, # STANDARD_NC6 is GPU-enabled
min_nodes=0,
max_nodes=4)
# create the compute target
compute_target = ComputeTarget.create(ws, compute_name,
provisioning_config)
# Can poll for a minimum number of nodes and for a specific timeout.
# If no min node count is provided it will use the scale settings for the cluster
compute_target.wait_for_completion(show_output=True,
min_node_count=None,
timeout_in_minutes=20)
# For a more detailed view of current cluster status, use the 'status' property
print(compute_target.status.serialize())
aml_run_config = RunConfiguration()
# `compute_target` as defined in "Azure Machine Learning compute" section above
aml_run_config.target = compute_target
USE_CURATED_ENV = True
if USE_CURATED_ENV:
curated_environment = Environment.get(workspace=ws,
name="AzureML-Tutorial")
aml_run_config.environment = curated_environment
else:
aml_run_config.environment.python.user_managed_dependencies = False
# Add some packages relied on by data prep step
aml_run_config.environment.python.conda_dependencies = CondaDependencies.create(
conda_packages=['pandas', 'scikit-learn'],
pip_packages=['azureml-sdk', 'azureml-dataprep[fuse,pandas]'],
pin_sdk_version=False)
dataprep_source_dir = "./dataprep_src"
entry_point = "prepare.py"
# `my_dataset` as defined above
ds_input = xyz_phishing_dataset.as_named_input('input1')
# `output_data1`, `compute_target`, `aml_run_config` as defined above
data_prep_step = PythonScriptStep(script_name=entry_point,
source_directory=dataprep_source_dir,
arguments=[
"--input",
ds_input.as_download(), "--output",
output_data1
],
compute_target=compute_target,
runconfig=aml_run_config,
allow_reuse=True)
train_source_dir = "./train_src"
train_entry_point = "train.py"
training_results = OutputFileDatasetConfig(name="training_results",
destination=def_blob_store)
train_step = PythonScriptStep(script_name=train_entry_point,
source_directory=train_source_dir,
arguments=[
"--prepped_data",
output_data1.as_input(),
"--training_results", training_results
],
compute_target=compute_target,
runconfig=aml_run_config,
allow_reuse=True)
# list of steps to run (`compare_step` definition not shown)
compare_models = [data_prep_step, train_step, compare_step]
# Build the pipeline
pipeline1 = Pipeline(workspace=ws, steps=[compare_models])
#dataset_consuming_step = PythonScriptStep(
# script_name="iris_train.py",
# inputs=[iris_tabular_dataset.as_named_input("iris_data")],
# compute_target=compute_target,
# source_directory=project_folder
#)
#run_context = Run.get_context()
#iris_dataset = run_context.input_datasets['iris_data']
#dataframe = iris_dataset.to_pandas_dataframe()
## Within a PythonScriptStep
#ws = Run.get_context().experiment.workspace
#step = PythonScriptStep(name="Hello World",
# script_name="hello_world.py",
# compute_target=aml_compute,
# source_directory=source_directory,
# allow_reuse=False,
# hash_paths=['hello_world.ipynb'])
# Submit the pipeline to be run
pipeline_run1 = Experiment(ws, 'Compare_Models_Exp').submit(pipeline1)
pipeline_run1.wait_for_completion()
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group,
)
print(f"get_workspace:{aml_workspace}")
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print(f"aml_compute:{aml_compute}")
# Create a reusable Azure ML environment
environment = get_environment(
aml_workspace,
e.aml_env_name,
conda_dependencies_file=e.aml_env_train_conda_dep_file,
create_new=e.rebuild_env,
) #
run_config = RunConfiguration()
run_config.environment = environment
if e.datastore_name:
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables[
"DATASTORE_NAME"] = datastore_name # NOQA: E501
# datastore and dataset names are fixed for this pipeline, however
# data_file_path can be specified for registering new versions of dataset
# Note that AML pipeline parameters don't take empty string as default, "" won't work # NOQA: E501
model_name_param = PipelineParameter(
name="model_name", default_value=e.model_name) # NOQA: E501
data_file_path_param = PipelineParameter(
name="data_file_path", default_value="nopath") # NOQA: E501
ml_params = PipelineParameter(name="ml_params",
default_value="default") # NOQA: E501
# Create a PipelineData to pass data between steps
pipeline_data = PipelineData(
"pipeline_data", datastore=aml_workspace.get_default_datastore())
train_step = PythonScriptStep(
name="Train Model",
script_name="train/train_aml.py",
compute_target=aml_compute,
source_directory=e.sources_directory_train,
outputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_output",
pipeline_data,
"--data_file_path",
data_file_path_param,
"--dataset_name",
e.processed_dataset_name,
"--datastore_name",
datastore_name,
"--ml_params",
ml_params,
],
runconfig=run_config,
allow_reuse=True,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name="evaluate/evaluate_model.py",
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--model_name",
model_name_param,
"--ml_params",
ml_params,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name="register/register_model.py",
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_input",
pipeline_data,
"--ml_params",
ml_params,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.training_pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id,
)
print(f"Published pipeline: {published_pipeline.name}")
print(f"for build {published_pipeline.version}")
