def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable Azure ML environment
environment = get_environment(aml_workspace,
e.aml_env_name,
create_new=False) # NOQA: E501
run_config = RunConfiguration()
run_config.environment = environment
if (e.datastore_name):
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables[
"DATASTORE_NAME"] = datastore_name # NOQA: E501
def build_batchscore_pipeline():
"""
Main method that builds and publishes a scoring pipeline.
"""
try:
env = Env()
args = parse_args()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(
name=env.workspace_name,
subscription_id=env.subscription_id,
resource_group=env.resource_group,
)
# Get Azure machine learning cluster
aml_compute_score = get_compute(
aml_workspace,
env.compute_name_scoring,
env.vm_size_scoring,
for_batch_scoring=True,
)
input_dataset, output_location = get_inputds_outputloc(
aml_workspace, env) # NOQA: E501
scoring_runconfig, score_copy_runconfig = get_run_configs(
aml_workspace, aml_compute_score, env)
trained_model = get_model(aml_workspace, env, args.model_tag_name,
args.model_tag_value)
scoring_pipeline = get_scoring_pipeline(
trained_model,
input_dataset,
output_location,
scoring_runconfig,
score_copy_runconfig,
aml_compute_score,
aml_workspace,
env,
)
published_pipeline = scoring_pipeline.publish(
name=env.scoring_pipeline_name,
description="Diabetes Batch Scoring Pipeline",
)
pipeline_id_string = "##vso[task.setvariable variable=pipeline_id;isOutput=true]{}".format( # NOQA: E501
published_pipeline.id)
print(pipeline_id_string)
except Exception as e:
print(e)
exit(1)
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group,
)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable Azure ML environment
# Make sure to include `r-essentials'
# in h1c4driver/conda_dependencies.yml
environment = get_environment(
aml_workspace,
e.aml_env_name,
conda_dependencies_file=e.aml_env_train_conda_dep_file,
create_new=e.rebuild_env,
) # NOQA: E501
run_config = RunConfiguration()
run_config.environment = environment
train_step = PythonScriptStep(
name="Train Model",
script_name="train_with_r.py",
compute_target=aml_compute,
source_directory="h1c4driver/training/R",
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
steps = [train_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id,
)
print(f"Published pipeline: {published_pipeline.name}")
print(f"for build {published_pipeline.version}")
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable run configuration environment
# Read definition from diabetes_regression/azureml_environment.json
# Make sure to include `r-essentials'
# in diabetes_regression/conda_dependencies.yml
environment = Environment.load_from_directory(e.sources_directory_train)
if (e.collection_uri is not None and e.teamproject_name is not None):
builduri_base = e.collection_uri + e.teamproject_name
builduri_base = builduri_base + "/_build/results?buildId="
environment.environment_variables["BUILDURI_BASE"] = builduri_base
environment.register(aml_workspace)
run_config = RunConfiguration()
run_config.environment = environment
train_step = PythonScriptStep(
name="Train Model",
script_name="train_with_r.py",
compute_target=aml_compute,
source_directory="diabetes_regression/training/R",
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
steps = [train_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
run_config = RunConfiguration(conda_dependencies=CondaDependencies.create(
conda_packages=[
'numpy', 'pandas', 'scikit-learn', 'tensorflow', 'keras'
],
pip_packages=[
'azure', 'azureml-core', 'azure-storage', 'azure-storage-blob'
]))
run_config.environment.docker.enabled = True
run_config.environment.docker.base_image = "mcr.microsoft.com/mlops/python"
train_step = PythonScriptStep(
name="Train Model",
script_name="train_with_r.py",
compute_target=aml_compute,
source_directory="diabetes_regression/training/R",
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
steps = [train_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name + "_with_R",
description="Model training/retraining pipeline",
version=e.build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group
)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(
aml_workspace,
e.compute_name,
e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
train_step = DatabricksStep(
name="DBPythonInLocalMachine",
num_workers=1,
python_script_name="train_with_r_on_databricks.py",
source_directory="emp_retention/training/R",
run_name='DB_Python_R_demo',
existing_cluster_id=e.db_cluster_id,
compute_target=aml_compute,
allow_reuse=False
)
print("Step Train created")
steps = [train_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name + "_with_R_on_DB",
description="Model training/retraining pipeline",
version=e.build_id
)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a run configuration environment
conda_deps_file = "diabetes_regression/training_dependencies.yml"
conda_deps = CondaDependencies(conda_deps_file)
run_config = RunConfiguration(conda_dependencies=conda_deps)
run_config.environment.docker.enabled = True
train_step = PythonScriptStep(
name="Train Model",
script_name="train_with_r.py",
compute_target=aml_compute,
source_directory="diabetes_regression/training/R",
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
steps = [train_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print(f"get_workspace: {aml_workspace}")
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print(f"aml_compute: {aml_compute}")
# Prepare the dataset input
data_store = aml_workspace.get_default_datastore()
print("data_store: %s" % data_store.name)
# Parameters
sources_directory_train = e.sources_directory_train
build_id = e.build_id
pipeline_name = 'Prepare Data Pipeline'
train_ds_name = e.dataset_name
train_data_path = e.datafile_path
# Register the train dataset
if (train_ds_name not in aml_workspace.datasets):
train_path_on_datastore = train_data_path # +'/*.csv'
train_ds_data_path = [(data_store, train_path_on_datastore)]
train_ds = Dataset.File.from_files(path=train_ds_data_path,
validate=False)
train_ds = train_ds.register(workspace=aml_workspace,
name=train_ds_name,
description='train data',
tags={'format': 'CSV'},
create_new_version=True)
else:
train_ds = Dataset.get_by_name(aml_workspace, train_ds_name)
# Conda environment
environment = Environment.from_conda_specification(
"myenv", os.path.join(sources_directory_train,
"conda_dependencies.yml"))
run_config = RunConfiguration()
run_config.environment = environment
with open(os.path.join(sources_directory_train,
'pipeline_config.json')) as json_file:
pipe_param = json.load(json_file)
for param in pipe_param['pipeline_parameter']:
print(param)
# Prepare pipeline parameters
source_blob_url_param = PipelineParameter(name="source_blob_url",
default_value="url")
data_file_param = PipelineParameter(name="data_file",
default_value="data_file")
target_column_param = PipelineParameter(name="target_column",
default_value="target_column")
features_param = PipelineParameter(name="features", default_value="")
# train_storage_connection_string = "DefaultEndpointsProtocol=https;AccountName=forecastingml8724233808;AccountKey=9o2ZH/5cLtmYmNyoHpoeKEA7Xjw0zi1fHLjI0Z0CZeQL5i4Ky2FZ9Wa6VpSYgK6uwLaHC3eamwnfEAscNTcgYw==;EndpointSuffix=core.windows.net"
# Copy data step
copy_step = PythonScriptStep(
name="Copy Data",
script_name="copy_data.py",
arguments=[
"--source_blob_url", source_blob_url_param,
"--train_storage_connection_string",
e.train_storage_connection_string, "--train_storage_container",
e.train_storage_container, "--data_file", data_file_param,
"--data_file_path", train_data_path
],
runconfig=run_config,
compute_target=aml_compute,
source_directory=sources_directory_train)
print("Step Copy Data created")
# Prepare data step
prepare_step = PythonScriptStep(name="Prepare Data",
script_name="prepare.py",
arguments=[
"--data_file_path", train_data_path,
"--data_file", data_file_param,
"--target_column", target_column_param,
"--features", features_param
],
runconfig=run_config,
compute_target=aml_compute,
source_directory=sources_directory_train)
print("Step Prepare created")
# Publish the pipeline
prepare_step.run_after(copy_step)
pipeline_steps = [copy_step, prepare_step]
pipeline = Pipeline(workspace=aml_workspace, steps=pipeline_steps)
pipeline._set_experiment_name
pipeline.validate()
published_pipeline = pipeline.publish(name=pipeline_name,
description="Prepare Data pipeline",
version=build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
# Run the pipelines
runs = []
for param in pipe_param['pipeline_parameter']:
# pipeline_parameters = {"model_name": "nyc_energy_model", "build_id": build_id}
target_column = param['automl_settings']['label_column_name']
param.pop('automl_settings')
param.update({"target_column":
target_column}) # Special process target_column
print(param)
pipeline_run = published_pipeline.submit(aml_workspace,
e.experiment_name, param)
runs.append(pipeline_run)
print("Pipeline run initiated ", pipeline_run.id)
# Wait for all runs to finish
wait(lambda: are_all_runs_finished(runs),
timeout_seconds=3600,
sleep_seconds=5,
waiting_for="all runs are finished")
print("All prepare data pipeline runs done")
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group,
)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable Azure ML environment
environment = get_environment(
aml_workspace,
e.aml_env_name,
conda_dependencies_file=e.aml_env_train_conda_dep_file,
create_new=e.rebuild_env,
) #
run_config = RunConfiguration()
run_config.environment = environment
if e.datastore_name:
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables[
"DATASTORE_NAME"] = datastore_name # NOQA: E501
model_name_param = PipelineParameter(
name="model_name", default_value=e.model_name) # NOQA: E501
dataset_version_param = PipelineParameter(name="dataset_version",
default_value=e.dataset_version)
data_file_path_param = PipelineParameter(name="data_file_path",
default_value="none")
caller_run_id_param = PipelineParameter(name="caller_run_id",
default_value="none") # NOQA: E501
# Get dataset name
dataset_name = e.dataset_name
# Check to see if dataset exists
if dataset_name not in aml_workspace.datasets:
# This call creates an example CSV from sklearn sample data. If you
# have already bootstrapped your project, you can comment this line
# out and use your own CSV.
create_sample_data_csv()
# Use a CSV to read in the data set.
file_name = "safedriver.csv"
if not os.path.exists(file_name):
raise Exception(
'Could not find CSV dataset at "%s". If you have bootstrapped your project, you will need to provide a CSV.' # NOQA: E501
% file_name) # NOQA: E501
# Upload file to default datastore in workspace
datatstore = Datastore.get(aml_workspace, datastore_name)
target_path = "training-data/"
datatstore.upload_files(
files=[file_name],
target_path=target_path,
overwrite=True,
show_progress=False,
)
# Register dataset
path_on_datastore = os.path.join(target_path, file_name)
dataset = Dataset.Tabular.from_delimited_files(
path=(datatstore, path_on_datastore))
dataset = dataset.register(
workspace=aml_workspace,
name=dataset_name,
description="safedriver training data",
tags={"format": "CSV"},
create_new_version=True,
)
# Create a PipelineData to pass data between steps
pipeline_data = PipelineData(
"pipeline_data", datastore=aml_workspace.get_default_datastore())
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
outputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_output",
pipeline_data,
"--dataset_version",
dataset_version_param,
"--data_file_path",
data_file_path_param,
"--caller_run_id",
caller_run_id_param,
"--dataset_name",
dataset_name,
],
runconfig=run_config,
allow_reuse=True,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--model_name",
model_name_param,
"--allow_run_cancel",
e.allow_run_cancel,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_input",
pipeline_data,
], # NOQA: E501
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
# Check run_evaluation flag to include or exclude evaluation step.
if (e.run_evaluation).lower() == "true":
print("Include evaluation step before register step.")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
else:
print("Exclude evaluation step and directly run register step.")
register_step.run_after(train_step)
steps = [train_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id,
)
print(f"Published pipeline: {published_pipeline.name}")
print(f"for build {published_pipeline.version}")
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable Azure ML environment
environment = get_environment(aml_workspace,
e.aml_env_name,
create_new=False) # NOQA: E501
run_config = RunConfiguration()
run_config.environment = environment
if (e.datastore_name):
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables[
"DATASTORE_NAME"] = datastore_name # NOQA: E501
model_name_param = PipelineParameter(name="model_name",
default_value=e.model_name)
dataset_version_param = PipelineParameter(name="dataset_version",
default_value=e.dataset_version)
data_file_path_param = PipelineParameter(name="data_file_path",
default_value="none")
caller_run_id_param = PipelineParameter(name="caller_run_id",
default_value="none")
# Get dataset name
dataset_name = e.dataset_name
# # Check to see if dataset exists
# if (dataset_name not in aml_workspace.datasets):
# # Create dataset from lacemlops sample data
# sample_data = load_lacemlops()
# df = pd.DataFrame(
# data=sample_data.data,
# columns=sample_data.feature_names)
# df['Y'] = sample_data.target
# file_name = 'lacemlops.csv'
# df.to_csv(file_name, index=False)
# # Upload file to default datastore in workspace
# datatstore = Datastore.get(aml_workspace, datastore_name)
# target_path = 'training-data/'
# datatstore.upload_files(
# files=[file_name],
# target_path=target_path,
# overwrite=True,
# show_progress=False)
# # Register dataset
# path_on_datastore = os.path.join(target_path, file_name)
# dataset = Dataset.Tabular.from_delimited_files(
# path=(datatstore, path_on_datastore))
# dataset = dataset.register(
# workspace=aml_workspace,
# name=dataset_name,
# description='lacemlops training data',
# tags={'format': 'CSV'},
# create_new_version=True)
# Create a PipelineData to pass data between steps
pipeline_data = PipelineData(
'pipeline_data', datastore=aml_workspace.get_default_datastore())
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
outputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_output",
pipeline_data,
"--dataset_version",
dataset_version_param,
"--data_file_path",
data_file_path_param,
"--caller_run_id",
caller_run_id_param,
"--dataset_name",
dataset_name,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--model_name",
model_name_param,
"--allow_run_cancel",
e.allow_run_cancel,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_input",
pipeline_data,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
# Check run_evaluation flag to include or exclude evaluation step.
if ((e.run_evaluation).lower() == 'true'):
print("Include evaluation step before register step.")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
else:
print("Exclude evaluation step and directly run register step.")
register_step.run_after(train_step)
steps = [train_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group,
)
print(f"get_workspace:{aml_workspace}")
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print(f"aml_compute:{aml_compute}")
# Create a reusable Azure ML environment
environment = get_environment(
aml_workspace,
e.aml_env_name,
create_new=e.rebuild_env,
enable_docker=True,
dockerfile='ml_model/preprocess/Dockerfile'
) #
run_config = RunConfiguration()
run_config.environment = environment
if e.datastore_name:
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables["DATASTORE_NAME"] = datastore_name # NOQA: E501
datastore = Datastore(aml_workspace, name=datastore_name)
data_file_path_param = PipelineParameter(name="data_file_path", default_value=e.dataset_name) # NOQA: E501
# The version of the input/output dataset can't be determined at pipeline publish time, only run time. # NOQA: E501
# Options to store output data:
# Option 1: Use blob API to write output data. Otherwise, no way to dynamically change the output dataset based on PipelineParameter, # NOQA: E501
# The following will not work. It generate a path like "PipelineParameter_Name:data_file_path_Default:gear_images" # NOQA: E501
# output_ds = OutputFileDatasetConfig(destination=(datastore, data_file_path_param)) # NOQA: E501
# This option means writing a file locally and upload to the datastore. Fewer dataset, more code. # NOQA: E501
# Option 2: Use a dynamic path in OutputFileDatasetConfig, and register a new dataset at completion # NOQA: E501
# Output dataset can be mounted, so more dataset to maintain, less code. # NOQA: E501
# Using Option 2 below.
output_dataset = OutputFileDatasetConfig(
name=e.processed_dataset_name,
destination=(datastore, "/dataset/{output-name}/{run-id}")
).register_on_complete(
name=e.processed_dataset_name)
preprocess_step = PythonScriptStep(
name="Preprocess Data with OS cmd",
script_name='preprocess/preprocess_os_cmd_aml.py',
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--dataset_name", e.dataset_name,
"--datastore_name", datastore_name,
"--data_file_path", data_file_path_param,
"--output_dataset", output_dataset,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Preprocess OS cmd created")
steps = [preprocess_step]
preprocess_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
preprocess_pipeline._set_experiment_name
preprocess_pipeline.validate()
published_pipeline = preprocess_pipeline.publish(
name=e.preprocessing_pipeline_name,
description="Data preprocessing OS cmd pipeline",
version=e.build_id,
)
print(f"Published pipeline: {published_pipeline.name}")
print(f"for build {published_pipeline.version}")
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a run configuration environment
conda_deps_file = "diabetes_regression/training_dependencies.yml"
conda_deps = CondaDependencies(conda_deps_file)
run_config = RunConfiguration(conda_dependencies=conda_deps)
run_config.environment.docker.enabled = True
config_envvar = {}
if (e.collection_uri is not None and e.teamproject_name is not None):
builduri_base = e.collection_uri + e.teamproject_name
builduri_base = builduri_base + "/_build/results?buildId="
config_envvar["BUILDURI_BASE"] = builduri_base
run_config.environment.environment_variables = config_envvar
model_name_param = PipelineParameter(name="model_name",
default_value=e.model_name)
build_id_param = PipelineParameter(name="build_id",
default_value=e.build_id)
dataset_name = ""
if (e.datastore_name is not None and e.datafile_name is not None):
dataset_name = e.dataset_name
datastore = Datastore.get(aml_workspace, e.datastore_name)
data_path = [(datastore, e.datafile_name)]
dataset = Dataset.Tabular.from_delimited_files(path=data_path)
dataset.register(workspace=aml_workspace,
name=e.dataset_name,
description="dataset with training data",
create_new_version=True)
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
"--dataset_name",
dataset_name,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
"--allow_run_cancel",
e.allow_run_cancel,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
# Check run_evaluation flag to include or exclude evaluation step.
if ((e.run_evaluation).lower() == 'true'):
print("Include evaluation step before register step.")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
else:
print("Exclude evaluation step and directly run register step.")
register_step.run_after(train_step)
steps = [train_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
def main():
e = Env()
from azureml.core.authentication import InteractiveLoginAuthentication
myten=os.environ.get("AZURE_TENANT_ID")
interactive_auth = InteractiveLoginAuthentication(tenant_id=os.environ.get("AZURE_TENANT_ID"))
subscription=os.environ.get("CSUBSCRIPTION")
workspace_name=e.workspace_name
resource_group=e.resource_group
aml_workspace = Workspace.get(
name = workspace_name,
subscription_id = subscription,
resource_group=resource_group,
auth=interactive_auth
)
from ml_service.util.attach_compute import get_compute
# Get Azure machine learning cluster
# If not present then get_compute will create a compute based on environment variables
aml_compute = get_compute(
aml_workspace,
e.compute_name,
e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
print("SDK version: ", azureml.core.VERSION)
## Variable names that can be passed in as parameter values
from azureml.pipeline.core.graph import PipelineParameter
from azureml.core import Datastore
model_name_param = PipelineParameter(
name="model_name", default_value=e.model_name)
dataset_version_param = PipelineParameter(
name="dataset_version", default_value=e.dataset_version)
data_file_path_param = PipelineParameter(
name="data_file_path", default_value="none")
caller_run_id_param = PipelineParameter(
name="caller_run_id", default_value="none")
#model_path = PipelineParameter(
# name="model_path", default_value=e.model_path)
if (e.datastore_name):
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
# Get the datastore whether it is the default or named store
datastore = Datastore.get(aml_workspace, datastore_name)
dataset_name = e.dataset_name
# Create a reusable Azure ML environment
from ml_service.util.manage_environment import get_environment
from azureml.core import Environment
# RUN Configuration
## Must have this process to work with AzureML-SDK 1.0.85
from azureml.core.runconfig import RunConfiguration, DEFAULT_CPU_IMAGE
from azureml.core.conda_dependencies import CondaDependencies
try:
app_env=Environment(name="smartschedule_env")
app_env.register(workspace=aml_workspace)
except:
print("Environment not found")
# Create a new runconfig object
aml_run_config = RunConfiguration()
aml_run_config.environment.environment_variables["DATASTORE_NAME"] = e.datastore_name # NOQA: E501
# Use the aml_compute you created above.
aml_run_config.target = aml_compute
# Enable Docker
aml_run_config.environment.docker.enabled = True
# Set Docker base image to the default CPU-based image
aml_run_config.environment.docker.base_image = DEFAULT_CPU_IMAGE
#aml_run_config.environment.docker.base_image = "mcr.microsoft.com/azureml/base:0.2.1"
# Use conda_dependencies.yml to create a conda environment in the Docker image for execution
aml_run_config.environment.python.user_managed_dependencies = False
app_conda_deps=CondaDependencies.create(
conda_packages=['pandas','scikit-learn', 'libgcc','pyodbc', 'sqlalchemy', 'py-xgboost==0.90'],
pip_packages=['azureml-sdk[automl,explain,contrib,interpret]==1.4.0', 'xgboost==0.90', 'azureml-dataprep==1.4.6', 'pyarrow', 'azureml-defaults==1.4.0', 'azureml-train-automl-runtime==1.4.0'], pin_sdk_version=False)
# Specify CondaDependencies obj, add necessary packages
aml_run_config.environment.python.conda_dependencies = app_conda_deps
print ("Run configuration created.")
from azure.common.credentials import ServicePrincipalCredentials
#from azure.keyvault import KeyVaultClient, KeyVaultAuthentication
from azure.keyvault.secrets import SecretClient
from azure.identity import DefaultAzureCredential
import pandas as pd
#import sqlalchemy as sql
import pyodbc
def get_data(sql_string, columns):
credentials = None
credential = DefaultAzureCredential()
secret_client = SecretClient("https://smrtschd-aml-kv.vault.azure.net", credential=credential)
secret = secret_client.get_secret("database-connection")
#client = KeyVaultClient(KeyVaultAuthentication(auth_callback))
#secret_bundle = client.get_secret("https://smrtschd-aml-kv.vault.azure.net", "database-connection", "")
server = 'starlims-sql.database.windows.net'
database = 'QM12_DATA_AUTOMATION'
username = '******'
password = secret.value
driver= '{ODBC Driver 17 for SQL Server}'
conn = pyodbc.connect('Driver='+driver+';'+
'Server='+server+';'+
'Database='+database+';'+
'PORT=1433;'+
'UID='+username+';'+
'PWD='+password+'; MARS_Connection=Yes'
)
try:
SQL_Query = pd.read_sql_query(sql_string, conn)
df = pd.DataFrame(SQL_Query, columns=columns)
return df
except Exception as e:
print(e)
raise
sql_str = "SELECT " \
" Dept " \
", Method " \
", Servgrp " \
", Runno " \
", TestNo " \
", Testcode " \
", Total_Duration_Min " \
", Total_Duration_Hr " \
", Usrnam " \
", Eqid " \
", Eqtype " \
"FROM dbo.Draft " \
"order by TESTCODE, RUNNO, dept, method;"
columns = ["Dept", "Method", "Servgrp", "Runno", "TestNo", "Testcode", "Total_Duration_Min", "Total_Duration_Hr", "Usrnam", "Eqid","Eqtype"]
from azureml.core import Dataset
from sklearn.model_selection import train_test_split
if (e.train_dataset_name not in aml_workspace.datasets):
df = get_data(sql_str, columns)
train_df, test_df=train_test_split(df, test_size=0.2)
MY_DIR = "data"
CHECK_FOLDER = os.path.isdir(MY_DIR)
if not CHECK_FOLDER:
os.makedirs(MY_DIR)
else:
print("Folder ", MY_DIR, " is already created")
#files = ["data/analyst_tests.csv"]
files = ["data/train_data.csv","data/test_data.csv"]
def_file_store = Datastore(aml_workspace, "workspacefilestore")
dtfrm = df.to_csv(files[0], header=True, index=False)
train_dataframe=train_df.to_csv(files[0], header=True, index=False)
test_dataframe=test_df.to_csv(files[1], header=True, index=False)
datastore.upload_files(
files=files,
target_path='data/',
overwrite=True
)
from azureml.data.data_reference import DataReference
blob_input_data_test=DataReference(
datastore=datastore,
data_reference_name="smartschedulertest",
path_on_datastore="data/test_data.csv"
)
test_data=Dataset.Tabular.from_delimited_files(blob_input_data_test)
test_data.register(aml_workspace, e.test_dataset_name, create_new_version=True)
blob_input_data_train=DataReference(
datastore=datastore,
data_reference_name="smartschedulertrain",
path_on_datastore="data/train_data.csv"
)
train_data=Dataset.Tabular.from_delimited_files(blob_input_data_train)
train_data.register(aml_workspace, e.train_dataset_name, create_new_version=True)
else:
from azureml.data.data_reference import DataReference
print("getting from the datastore instead of uploading")
train_data=Dataset.get_by_name(aml_workspace, name=e.train_dataset_name)
test_data=Dataset.get_by_name(aml_workspace, name=e.test_dataset_name)
# check the training dataset to make sure it has at least 50 records.
tdf=train_data.to_pandas_dataframe().head(5)
print(tdf.shape)
print(tdf)
# display the first five rows of the data
# create a variable that can be used for other purposes
df=train_data.to_pandas_dataframe().head()
label_column="Total_Duration_Min"
import random
import string
def randomString(stringLength=15):
letters = string.ascii_lowercase
return ''.join(random.choice(letters) for i in range(stringLength))
from azureml.core import Experiment
experiment = Experiment(aml_workspace, "SmartScheduler_Pipeline")
import logging
aml_name = 'smart_scheduler_' + randomString(5)
print(aml_name)
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.ticker import StrMethodFormatter
print(df.head(5))
print(df.shape)
print(df.dtypes)
#df.hist(column='Dept')
list(df.columns.values)
# Remove Features that are not necessary.
#df.hist(column="Servgrp", bins=4)
train_data=train_data.drop_columns(["Runno","TestNo","Total_Duration_Hr"])
test_data=test_data.drop_columns(["Runno","TestNo","Total_Duration_Hr"])
print(train_data.to_pandas_dataframe())
print(test_data.to_pandas_dataframe())
from azureml.automl.core.featurization import FeaturizationConfig
# some of the columns could be change to one hot encoding especially if the categorical column
featurization_config=FeaturizationConfig()
featurization_config.blocked_transformers=['LabelEncoder']
featurization_config.add_column_purpose('Dept', 'CategoricalHash')
featurization_config.add_transformer_params('HashOneHotEncoder',['Method'], {"number_of_bits":3})
featurization_config.add_column_purpose('Servgrp', 'CategoricalHash')
featurization_config.add_column_purpose('Testcode', 'Numeric')
featurization_config.add_column_purpose('Usrnam', 'CategoricalHash')
featurization_config.add_column_purpose('Eqid', 'CategoricalHash')
featurization_config.add_column_purpose('Eqtype', 'CategoricalHash')
from azureml.pipeline.core import Pipeline, PipelineData
from azureml.pipeline.steps import PythonScriptStep
#train_model_folder = './scripts/trainmodel'
automl_settings = {
"iteration_timeout_minutes": 5,
"iterations": 5,
"enable_early_stopping": True,
"primary_metric": 'spearman_correlation',
"verbosity": logging.INFO,
"n_cross_validation":5
}
automl_config = AutoMLConfig(task="regression",
debug_log='automated_ml_errors.log',
#path = train_model_folder,
training_data=train_data,
featurization=featurization_config,
blacklist_models=['XGBoostRegressor'],
label_column_name=label_column,
compute_target=aml_compute,
**automl_settings)
from azureml.pipeline.steps import AutoMLStep
from azureml.pipeline.core import TrainingOutput
metrics_output_name = 'metrics_output'
best_model_output_name='best_model_output'
metrics_data = PipelineData(name = 'metrics_data',
datastore = datastore,
pipeline_output_name=metrics_output_name,
training_output=TrainingOutput(type='Metrics'))
model_data = PipelineData(name='model_data',
datastore=datastore,
pipeline_output_name=best_model_output_name,
training_output=TrainingOutput(type='Model'))
trainWithAutomlStep = AutoMLStep(
name=aml_name,
automl_config=automl_config,
passthru_automl_config=False,
outputs=[metrics_data, model_data],
allow_reuse=True
)
evaluate_step = PythonScriptStep(
name="Evaluate Model",
script_name='./evaluate/evaluate_model.py',
# e.evaluate_script_path,
compute_target=aml_compute,
source_directory='../app',
arguments=[
"--model_name", model_name_param,
"--allow_run_cancel", e.allow_run_cancel
]
)
register_step = PythonScriptStep(
name="Register Model ",
script_name='register/register_model2.py', #e.register_script_path,
compute_target=aml_compute,
source_directory='../app',
inputs=[model_data],
arguments=[
"--model_name", model_name_param,
"--model_path", model_data,
"--ds_name", e.train_dataset_name
],
runconfig=aml_run_config,
allow_reuse=False
)
if ((e.run_evaluation).lower() == 'true'):
print("Include evaluation step before register step.")
evaluate_step.run_after(trainWithAutomlStep)
register_step.run_after(evaluate_step)
pipeline_steps = [ trainWithAutomlStep, evaluate_step, register_step ]
else:
print("Exclude the evaluation step and run register step")
register_step.run_after(trainWithAutomlStep)
pipeline_steps = [ trainWithAutomlStep, register_step ]
print( "this is the value for execute pipeline: {}".format(e.execute_pipeline))
if( (e.execute_pipeline).lower() =='true' ):
# Execute the pipe normally during testing and debugging
print("Pipeline submitted for execution.")
pipeline = Pipeline(workspace = aml_workspace, steps=pipeline_steps)
pipeline_run = experiment.submit(pipeline)
pipeline_run.wait_for_completion()
print("Pipeline is built.")
else:
# Generates pipeline that will be called in ML Ops
train_pipeline = Pipeline(workspace=aml_workspace, steps=pipeline_steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id
)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
def main():
e = Env()
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group
)
print("get_workspace:")
print(aml_workspace)
aml_compute = get_compute(
aml_workspace,
e.compute_name,
e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
environment = get_environment(
aml_workspace, e.aml_env_name, create_new=e.rebuild_env)
run_config = RunConfiguration()
run_config.environment = environment
if (e.datastore_name):
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables["DATASTORE_NAME"] \
= datastore_name
dataset_name = e.dataset_name
file_name = e.file_name
datastore = Datastore.get(aml_workspace, datastore_name)
if (dataset_name not in aml_workspace.datasets):
raise Exception("Could not find dataset at \"%s\"." % dataset_name)
else:
dataset = Dataset.get_by_name(aml_workspace, name=dataset_name)
dataset.download(target_path='.', overwrite=True)
datastore.upload_files([file_name],
target_path=dataset_name,
overwrite=True)
raw_data_file = DataReference(datastore=datastore,
data_reference_name="Raw_Data_File",
path_on_datastore=dataset_name + '/'
+ file_name)
clean_data_file = PipelineParameter(name="clean_data_file",
default_value="/clean_data.csv")
clean_data_folder = PipelineData("clean_data_folder",
datastore=datastore)
prepDataStep = PythonScriptStep(name="Prepare Data",
source_directory=e.sources_directory_train,
script_name=e.data_prep_script_path,
arguments=["--raw_data_file",
raw_data_file,
"--clean_data_folder",
clean_data_folder,
"--clean_data_file",
clean_data_file],
inputs=[raw_data_file],
outputs=[clean_data_folder],
compute_target=aml_compute,
allow_reuse=False)
print("Step Prepare Data created")
new_model_file = PipelineParameter(name="new_model_file ",
default_value='/' + e.model_name
+ '.pkl')
new_model_folder = PipelineData("new_model_folder", datastore=datastore)
est = SKLearn(source_directory=e.sources_directory_train,
entry_script=e.train_script_path,
pip_packages=['azureml-sdk', 'scikit-learn==0.20.3',
'azureml-dataprep[pandas,fuse]>=1.1.14'],
compute_target=aml_compute)
trainingStep = EstimatorStep(
name="Model Training",
estimator=est,
estimator_entry_script_arguments=["--clean_data_folder",
clean_data_folder,
"--new_model_folder",
new_model_folder,
"--clean_data_file",
clean_data_file.default_value,
"--new_model_file",
new_model_file.default_value],
runconfig_pipeline_params=None,
inputs=[clean_data_folder],
outputs=[new_model_folder],
compute_target=aml_compute,
allow_reuse=False)
print("Step Train created")
model_name_param = PipelineParameter(name="model_name",
default_value=e.model_name)
evaluateStep = PythonScriptStep(
name="Evaluate Model",
source_directory=e.sources_directory_train,
script_name=e.evaluate_script_path,
arguments=["--model_name", model_name_param],
compute_target=aml_compute,
allow_reuse=False)
print("Step Evaluate created")
registerStep = PythonScriptStep(
name="Register Model",
source_directory=e.sources_directory_train,
script_name=e.register_script_path,
arguments=["--new_model_folder", new_model_folder,
"--new_model_file", new_model_file,
"--model_name", model_name_param],
inputs=[new_model_folder],
compute_target=aml_compute,
allow_reuse=False)
print("Step Register created")
if ((e.run_evaluation).lower() == 'true'):
print("Include evaluation step before register step.")
trainingStep.run_after(prepDataStep)
evaluateStep.run_after(trainingStep)
registerStep.run_after(evaluateStep)
else:
print("Exclude evaluation step and directly run register step.")
trainingStep.run_after(prepDataStep)
registerStep.run_after(trainingStep)
pipeline = Pipeline(workspace=aml_workspace, steps=[registerStep])
pipeline.validate()
print("Pipeline is built")
pipeline._set_experiment_name
published_pipeline = pipeline.publish(
name=e.pipeline_name,
description="Predict Employee Retention Model training pipeline",
version=e.build_id
)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
def main():
e = Env()
print(e.__dict__)
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable Azure ML environment
environment = get_environment(aml_workspace,
e.aml_env_name,
create_new=e.rebuild_env) #
run_config = RunConfiguration()
run_config.environment = environment
if e.datastore_name:
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables[
"DATASTORE_NAME"] = datastore_name # NOQA: E501
model_name_param = PipelineParameter(name="model_name",
default_value=e.model_name)
dataset_version_param = PipelineParameter(name="dataset_version",
default_value=e.dataset_version)
data_file_path_param = PipelineParameter(name="data_file_path",
default_value="none")
caller_run_id_param = PipelineParameter(name="caller_run_id",
default_value="none")
# Get dataset name
dataset_name = e.dataset_name
# Check to see if dataset exists
if dataset_name not in aml_workspace.datasets:
raise ValueError(
f"can't find dataset {dataset_name} in datastore {datastore_name}")
# Create PipelineData to pass data between steps
model_data = PipelineData("model_data",
datastore=aml_workspace.get_default_datastore())
train_ds = (PipelineData("train_ds",
datastore=aml_workspace.get_default_datastore()).
as_dataset().parse_delimited_files().register(
name="train", create_new_version=True))
test_ds = (PipelineData(
"test_ds", datastore=aml_workspace.get_default_datastore()).as_dataset(
).parse_delimited_files().register(name="test",
create_new_version=True))
prepare_step = PythonScriptStep(
name="Prepare Data",
script_name=e.prepare_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
outputs=[train_ds, test_ds],
arguments=[
"--dataset_version", dataset_version_param, "--data_file_path",
data_file_path_param, "--dataset_name", dataset_name,
"--caller_run_id", caller_run_id_param, "--train_ds", train_ds,
"--test_ds", test_ds
],
runconfig=run_config,
allow_reuse=True,
)
print("Step Prepare created")
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[
train_ds.as_named_input("training_data"),
test_ds.as_named_input("testing_data")
],
outputs=[model_data],
arguments=[
"--model_name", model_name_param, "--model_data", model_data
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--model_name",
model_name_param,
"--allow_run_cancel",
e.allow_run_cancel,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[model_data],
arguments=[
"--model_name", model_name_param, "--step_input", model_data
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
# Check run_evaluation flag to include or exclude evaluation step.
if (e.run_evaluation).lower() == "true":
print("Include evaluation step before register step.")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [prepare_step, train_step, evaluate_step, register_step]
else:
print("Exclude evaluation step and directly run register step.")
register_step.run_after(train_step)
steps = [prepare_step, train_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id)
print(f"Published pipeline: {published_pipeline.name}")
print(f"for build {published_pipeline.version}")
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print(f"get_workspace: {aml_workspace}")
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print(f"aml_compute: {aml_compute}")
# Prepare the dataset input
data_store = aml_workspace.get_default_datastore()
print("data_store: %s" % data_store.name)
train_ds_name = e.dataset_name
train_data_path = e.datafile_path
sources_directory_train = e.sources_directory_train
pipeline_name = e.pipeline_name
build_id = e.build_id
# Register the train dataset
if (train_ds_name not in aml_workspace.datasets):
train_path_on_datastore = train_data_path # +'/*.csv'
train_ds_data_path = [(data_store, train_path_on_datastore)]
train_ds = Dataset.File.from_files(path=train_ds_data_path,
validate=False)
train_ds = train_ds.register(workspace=aml_workspace,
name=train_ds_name,
description='train data',
tags={'format': 'CSV'},
create_new_version=True)
else:
train_ds = Dataset.get_by_name(aml_workspace, train_ds_name)
train_input = train_ds.as_named_input('train_input')
# Conda environment
environment = Environment.from_conda_specification(
"myenv", os.path.join(sources_directory_train,
"conda_dependencies.yml"))
# Logging into Azure Application Insights
env = {
"APPLICATIONINSIGHTS_CONNECTION_STRING":
e.applicationinsights_connection_string
}
env['AZUREML_FLUSH_INGEST_WAIT'] = ''
env['DISABLE_ENV_MISMATCH'] = True
environment.environment_variables = env
from ff.util.helper import build_parallel_run_config
# PLEASE MODIFY the following three settings based on your compute and
# experiment timeout.
process_count_per_node = 6
node_count = 3
# this timeout(in seconds) is inline with AutoML experiment timeout or (no
# of iterations * iteration timeout)
run_invocation_timeout = 3700
parallel_run_config = build_parallel_run_config(sources_directory_train,
environment, aml_compute,
node_count,
process_count_per_node,
run_invocation_timeout)
from azureml.pipeline.core import PipelineData
output_dir = PipelineData(name="training_output", datastore=data_store)
#from azureml.contrib.pipeline.steps import ParallelRunStep
from azureml.pipeline.steps import ParallelRunStep
parallel_run_step = ParallelRunStep(
name="many-models-training",
parallel_run_config=parallel_run_config,
allow_reuse=False,
inputs=[train_input],
output=output_dir
# models=[],
# arguments=[]
)
pipeline = Pipeline(workspace=aml_workspace, steps=parallel_run_step)
pipeline._set_experiment_name
pipeline.validate()
published_pipeline = pipeline.publish(name=pipeline_name,
description="FF AutomML pipeline",
version=build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group,
)
print(f"get_workspace:{aml_workspace}")
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print(f"aml_compute:{aml_compute}")
# Create a reusable Azure ML environment
environment = get_environment(
aml_workspace,
e.aml_env_name,
conda_dependencies_file=e.aml_env_train_conda_dep_file,
create_new=e.rebuild_env,
) #
run_config = RunConfiguration()
run_config.environment = environment
if e.datastore_name:
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables[
"DATASTORE_NAME"] = datastore_name # NOQA: E501
# datastore and dataset names are fixed for this pipeline, however
# data_file_path can be specified for registering new versions of dataset
# Note that AML pipeline parameters don't take empty string as default, "" won't work # NOQA: E501
model_name_param = PipelineParameter(
name="model_name", default_value=e.model_name) # NOQA: E501
data_file_path_param = PipelineParameter(
name="data_file_path", default_value="nopath") # NOQA: E501
ml_params = PipelineParameter(name="ml_params",
default_value="default") # NOQA: E501
# Create a PipelineData to pass data between steps
pipeline_data = PipelineData(
"pipeline_data", datastore=aml_workspace.get_default_datastore())
train_step = PythonScriptStep(
name="Train Model",
script_name="train/train_aml.py",
compute_target=aml_compute,
source_directory=e.sources_directory_train,
outputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_output",
pipeline_data,
"--data_file_path",
data_file_path_param,
"--dataset_name",
e.processed_dataset_name,
"--datastore_name",
datastore_name,
"--ml_params",
ml_params,
],
runconfig=run_config,
allow_reuse=True,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name="evaluate/evaluate_model.py",
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--model_name",
model_name_param,
"--ml_params",
ml_params,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name="register/register_model.py",
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_input",
pipeline_data,
"--ml_params",
ml_params,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.training_pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id,
)
print(f"Published pipeline: {published_pipeline.name}")
print(f"for build {published_pipeline.version}")
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable run configuration environment
# Read definition from diabetes_regression/azureml_environment.json
environment = Environment.load_from_directory(e.sources_directory_train)
if (e.collection_uri is not None and e.teamproject_name is not None):
builduri_base = e.collection_uri + e.teamproject_name
builduri_base = builduri_base + "/_build/results?buildId="
environment.environment_variables["BUILDURI_BASE"] = builduri_base
environment.register(aml_workspace)
run_config = RunConfiguration()
run_config.environment = environment
model_name_param = PipelineParameter(name="model_name",
default_value=e.model_name)
build_id_param = PipelineParameter(name="build_id",
default_value=e.build_id)
# Get dataset name
dataset_name = e.dataset_name
# Check to see if dataset exists
if (dataset_name not in aml_workspace.datasets):
# Create dataset from diabetes sample data
sample_data = load_diabetes()
df = pd.DataFrame(data=sample_data.data,
columns=sample_data.feature_names)
df['Y'] = sample_data.target
file_name = 'diabetes.csv'
df.to_csv(file_name, index=False)
# Upload file to default datastore in workspace
default_ds = aml_workspace.get_default_datastore()
target_path = 'training-data/'
default_ds.upload_files(files=[file_name],
target_path=target_path,
overwrite=True,
show_progress=False)
# Register dataset
path_on_datastore = os.path.join(target_path, file_name)
dataset = Dataset.Tabular.from_delimited_files(
path=(default_ds, path_on_datastore))
dataset = dataset.register(workspace=aml_workspace,
name=dataset_name,
description='diabetes training data',
tags={'format': 'CSV'},
create_new_version=True)
# Get the dataset
dataset = Dataset.get_by_name(aml_workspace, dataset_name)
# Create a PipelineData to pass data between steps
pipeline_data = PipelineData(
'pipeline_data', datastore=aml_workspace.get_default_datastore())
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[dataset.as_named_input('training_data')],
outputs=[pipeline_data],
arguments=[
"--build_id", build_id_param, "--model_name", model_name_param,
"--step_output", pipeline_data
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
"--allow_run_cancel",
e.allow_run_cancel,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[pipeline_data],
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
"--step_input",
pipeline_data,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
# Check run_evaluation flag to include or exclude evaluation step.
if ((e.run_evaluation).lower() == 'true'):
print("Include evaluation step before register step.")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
else:
print("Exclude evaluation step and directly run register step.")
register_step.run_after(train_step)
steps = [train_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')