def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group,
)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable Azure ML environment
environment = get_environment(
aml_workspace,
e.aml_env_name,
conda_dependencies_file=e.aml_env_train_conda_dep_file,
create_new=e.rebuild_env,
) #
run_config = RunConfiguration()
run_config.environment = environment
if e.datastore_name:
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables[
"DATASTORE_NAME"] = datastore_name # NOQA: E501
model_name_param = PipelineParameter(
name="model_name", default_value=e.model_name) # NOQA: E501
dataset_version_param = PipelineParameter(name="dataset_version",
default_value=e.dataset_version)
data_file_path_param = PipelineParameter(name="data_file_path",
default_value="none")
caller_run_id_param = PipelineParameter(name="caller_run_id",
default_value="none") # NOQA: E501
# Get dataset name
dataset_name = e.dataset_name
# Check to see if dataset exists
if dataset_name not in aml_workspace.datasets:
# Source dir for the data.
src_dir = "data"
if not os.path.exists(src_dir):
raise Exception('Could not find dataset at "%s".' # NOQA: E501
% src_dir) # NOQA: E501
# Upload file to default datastore in workspace
datatstore = Datastore.get(aml_workspace, datastore_name)
target_path = "training-data"
datatstore.upload(
src_dir=src_dir,
target_path=target_path,
overwrite=True,
show_progress=False,
)
# Register dataset
path_on_datastore = [(datatstore, target_path)]
dataset = Dataset.File.from_files(path=path_on_datastore)
dataset = dataset.register(
workspace=aml_workspace,
name=dataset_name,
description="img_class training data",
tags={"format": "gz"},
create_new_version=True,
)
# Create a PipelineData to pass data between steps
pipeline_data = PipelineData(
"pipeline_data", datastore=aml_workspace.get_default_datastore())
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
outputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_output",
pipeline_data,
"--dataset_version",
dataset_version_param,
"--data_file_path",
data_file_path_param,
"--caller_run_id",
caller_run_id_param,
"--dataset_name",
dataset_name,
],
runconfig=run_config,
allow_reuse=True,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--model_name",
model_name_param,
"--allow_run_cancel",
e.allow_run_cancel,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_input",
pipeline_data,
], # NOQA: E501
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
# Check run_evaluation flag to include or exclude evaluation step.
if (e.run_evaluation).lower() == "true":
print("Include evaluation step before register step.")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
else:
print("Exclude evaluation step and directly run register step.")
register_step.run_after(train_step)
steps = [train_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id,
)
print(f"Published pipeline: {published_pipeline.name}")
print(f"for build {published_pipeline.version}")
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print(f"get_workspace: {aml_workspace}")
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print(f"aml_compute: {aml_compute}")
# Prepare the dataset input
data_store = aml_workspace.get_default_datastore()
print("data_store: %s" % data_store.name)
# Parameters
sources_directory_train = e.sources_directory_train
build_id = e.build_id
pipeline_name = 'Prepare Data Pipeline'
train_ds_name = e.dataset_name
train_data_path = e.datafile_path
# Register the train dataset
if (train_ds_name not in aml_workspace.datasets):
train_path_on_datastore = train_data_path # +'/*.csv'
train_ds_data_path = [(data_store, train_path_on_datastore)]
train_ds = Dataset.File.from_files(path=train_ds_data_path,
validate=False)
train_ds = train_ds.register(workspace=aml_workspace,
name=train_ds_name,
description='train data',
tags={'format': 'CSV'},
create_new_version=True)
else:
train_ds = Dataset.get_by_name(aml_workspace, train_ds_name)
# Conda environment
environment = Environment.from_conda_specification(
"myenv", os.path.join(sources_directory_train,
"conda_dependencies.yml"))
run_config = RunConfiguration()
run_config.environment = environment
with open(os.path.join(sources_directory_train,
'pipeline_config.json')) as json_file:
pipe_param = json.load(json_file)
for param in pipe_param['pipeline_parameter']:
print(param)
# Prepare pipeline parameters
source_blob_url_param = PipelineParameter(name="source_blob_url",
default_value="url")
data_file_param = PipelineParameter(name="data_file",
default_value="data_file")
target_column_param = PipelineParameter(name="target_column",
default_value="target_column")
features_param = PipelineParameter(name="features", default_value="")
# train_storage_connection_string = "DefaultEndpointsProtocol=https;AccountName=forecastingml8724233808;AccountKey=9o2ZH/5cLtmYmNyoHpoeKEA7Xjw0zi1fHLjI0Z0CZeQL5i4Ky2FZ9Wa6VpSYgK6uwLaHC3eamwnfEAscNTcgYw==;EndpointSuffix=core.windows.net"
# Copy data step
copy_step = PythonScriptStep(
name="Copy Data",
script_name="copy_data.py",
arguments=[
"--source_blob_url", source_blob_url_param,
"--train_storage_connection_string",
e.train_storage_connection_string, "--train_storage_container",
e.train_storage_container, "--data_file", data_file_param,
"--data_file_path", train_data_path
],
runconfig=run_config,
compute_target=aml_compute,
source_directory=sources_directory_train)
print("Step Copy Data created")
# Prepare data step
prepare_step = PythonScriptStep(name="Prepare Data",
script_name="prepare.py",
arguments=[
"--data_file_path", train_data_path,
"--data_file", data_file_param,
"--target_column", target_column_param,
"--features", features_param
],
runconfig=run_config,
compute_target=aml_compute,
source_directory=sources_directory_train)
print("Step Prepare created")
# Publish the pipeline
prepare_step.run_after(copy_step)
pipeline_steps = [copy_step, prepare_step]
pipeline = Pipeline(workspace=aml_workspace, steps=pipeline_steps)
pipeline._set_experiment_name
pipeline.validate()
published_pipeline = pipeline.publish(name=pipeline_name,
description="Prepare Data pipeline",
version=build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
# Run the pipelines
runs = []
for param in pipe_param['pipeline_parameter']:
# pipeline_parameters = {"model_name": "nyc_energy_model", "build_id": build_id}
target_column = param['automl_settings']['label_column_name']
param.pop('automl_settings')
param.update({"target_column":
target_column}) # Special process target_column
print(param)
pipeline_run = published_pipeline.submit(aml_workspace,
e.experiment_name, param)
runs.append(pipeline_run)
print("Pipeline run initiated ", pipeline_run.id)
# Wait for all runs to finish
wait(lambda: are_all_runs_finished(runs),
timeout_seconds=3600,
sleep_seconds=5,
waiting_for="all runs are finished")
print("All prepare data pipeline runs done")
# runconfig=run_config,
# inputs=[jsonconfigs],
# # outputs=[jsonconfigs],
# allow_reuse=False,
# )
# print("Packed the model into a Scoring Image")
# Create Steps dependency such that they run in sequence
evaluate.run_after(train)
register_model.run_after(evaluate)
#package_model.run_after(register_model)
steps = [register_model]
# Build Pipeline
pipeline1 = Pipeline(workspace=ws, steps=steps)
print("Pipeline is built")
# Validate Pipeline
pipeline1.validate()
print("Pipeline validation complete")
# Submit unpublished pipeline with small data set for test
if args.pipeline_action == "pipeline-test":
pipeline_run1 = Experiment(ws,
experiment_name).submit(pipeline1,
regenerate_outputs=True)
print("Pipeline is submitted for execution")
pipeline_run1.wait_for_completion(show_output=True)
# RunDetails(pipeline_run1).show()
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group,
)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable Azure ML environment
environment = get_environment(
aml_workspace,
e.aml_env_name,
conda_dependencies_file=e.aml_env_train_conda_dep_file,
create_new=e.rebuild_env,
) #
run_config = RunConfiguration()
run_config.environment = environment
if e.datastore_name:
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables[
"DATASTORE_NAME"] = datastore_name # NOQA: E501
model_name_param = PipelineParameter(
name="model_name", default_value=e.model_name) # NOQA: E501
dataset_version_param = PipelineParameter(name="dataset_version",
default_value=e.dataset_version)
data_file_path_param = PipelineParameter(name="data_file_path",
default_value="none")
caller_run_id_param = PipelineParameter(name="caller_run_id",
default_value="none") # NOQA: E501
pipeline_parameters = {
"model_name": model_name_param,
"dataset_version": dataset_version_param,
"data_file_path": data_file_path_param,
"caller_run_id": caller_run_id_param,
}
# Get dataset name
dataset_name = e.dataset_name
# Check to see if dataset exists
if dataset_name not in aml_workspace.datasets:
# This call creates an example CSV from sklearn sample data. If you
# have already bootstrapped your project, you can comment this line
# out and use your own CSV.
create_sample_data_csv()
# Use a CSV to read in the data set.
file_name = "diabetes.csv"
if not os.path.exists(file_name):
raise Exception(
'Could not find CSV dataset at "%s". If you have bootstrapped your project, you will need to provide a CSV.' # NOQA: E501
% file_name) # NOQA: E501
# Upload file to default datastore in workspace
datatstore = Datastore.get(aml_workspace, datastore_name)
target_path = "training-data/"
datatstore.upload_files(
files=[file_name],
target_path=target_path,
overwrite=True,
show_progress=False,
)
# Register dataset
path_on_datastore = os.path.join(target_path, file_name)
dataset = Dataset.Tabular.from_delimited_files(
path=(datatstore, path_on_datastore))
dataset = dataset.register(
workspace=aml_workspace,
name=dataset_name,
description="diabetes training data",
tags={"format": "CSV"},
create_new_version=True,
)
# Create a PipelineData to pass data between steps
pipeline_data = PipelineData(
"pipeline_data", datastore=aml_workspace.get_default_datastore())
# List of pipeline steps
steps = list()
train_step = TrainStep(workspace=aml_workspace,
env=e,
compute=aml_compute,
config=run_config,
pipeline_parameters=pipeline_parameters,
output_pipelinedata=pipeline_data)
train_step.append_step(steps)
print("Step Train created")
if (e.run_evaluation).lower() == "true":
evaluate_step = EvaluateStep(workspace=aml_workspace,
env=e,
compute=aml_compute,
config=run_config,
pipeline_parameters=pipeline_parameters)
evaluate_step.append_step(steps)
print("Include evaluation step before register step.")
print("Step Evaluate created")
else:
print("Exclude evaluation step and directly run register step.")
register_step = RegisterStep(workspace=aml_workspace,
env=e,
compute=aml_compute,
config=run_config,
pipeline_parameters=pipeline_parameters,
input_pipelinedata=pipeline_data)
register_step.append_step(steps)
print("Step Register created")
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id,
)
print(f"Published pipeline: {published_pipeline.name}")
print(f"for build {published_pipeline.version}")
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
run_config = RunConfiguration(conda_dependencies=CondaDependencies.create(
conda_packages=[
'numpy', 'pandas', 'scikit-learn', 'tensorflow', 'keras'
],
pip_packages=[
'azure', 'azureml-core', 'azure-storage', 'azure-storage-blob',
'azureml-dataprep'
]))
run_config.environment.docker.enabled = True
config_envvar = {}
if (e.collection_uri is not None and e.teamproject_name is not None):
builduri_base = e.collection_uri + e.teamproject_name
builduri_base = builduri_base + "/_build/results?buildId="
config_envvar["BUILDURI_BASE"] = builduri_base
run_config.environment.environment_variables = config_envvar
model_name_param = PipelineParameter(name="model_name",
default_value=e.model_name)
build_id_param = PipelineParameter(name="build_id",
default_value=e.build_id)
hyperparameter_alpha_param = PipelineParameter(name="hyperparameter_alpha",
default_value=0.5)
dataset_name = ""
if (e.datastore_name is not None and e.datafile_name is not None):
dataset_name = e.dataset_name
datastore = Datastore.get(aml_workspace, e.datastore_name)
data_path = [(datastore, e.datafile_name)]
dataset = Dataset.Tabular.from_delimited_files(path=data_path)
dataset.register(workspace=aml_workspace,
name=e.dataset_name,
description="dataset with training data",
create_new_version=True)
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
"--alpha",
hyperparameter_alpha_param,
"--dataset_name",
dataset_name,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
"--allow_run_cancel",
e.allow_run_cancel,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--build_id",
build_id_param,
"--model_name",
model_name_param,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
# Check run_evaluation flag to include or exclude evaluation step.
if ((e.run_evaluation).lower() == 'true'):
print("Include evaluation step before register step.")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
else:
print("Exclude evaluation step and directly run register step.")
register_step.run_after(train_step)
steps = [train_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
args.storage_account, args.storage_key)
# get compute
compute = get_compute(ws, args.compute_target)
# prep step
pdata, pstep = process_step(datastore, compute, args.datastore_path)
# train step
tdata, tstep = train_step(datastore, pdata, compute)
# register step (tag model with version)
rdata, rstep = register_step(datastore, tdata, compute,
args.universal_package)
# create pipeline from steps
seer_pipeline = Pipeline(workspace=ws, steps=[pstep, tstep, rstep])
published_pipeline = seer_pipeline.publish(
name="Seer Pipeline",
description="Transfer learned image classifier. Uses folders as labels."
)
# add pipeline to endpoint
endpoint = add_endpoint(ws, published_pipeline, 'seer-endpoint')
# run pipeline
pipeline_run = endpoint.submit('seer')
pipeline_run.set_tags(
tags={'universalPackageVersion': args.universal_package})
print(f'Run created with ID: {pipeline_run.id}')
environment=batch_env,
compute_target=compute_target,
process_count_per_node=PipelineParameter(name="process_count_param",
default_value=2),
node_count=2)
from azureml.pipeline.steps import ParallelRunStep
from datetime import datetime
parallel_step_name = "batchscoring-" + datetime.now().strftime("%Y%m%d%H%M")
from azureml.pipeline.steps import ParallelRunStep
parallelrun_step = ParallelRunStep(name=parallel_step_name,
parallel_run_config=parallel_run_config,
inputs=[input_mnist_ds_consumption],
output=output_dir,
allow_reuse=True)
from azureml.pipeline.core import Pipeline
from azureml.core.experiment import Experiment
pipeline = Pipeline(workspace=ws, steps=[parallelrun_step])
experiment = Experiment(ws, 'batch_scoring')
pipeline_run = experiment.submit(pipeline)
#from azureml.widgets import RunDetails
#RunDetails(pipeline_run).show()
pipeline_run.wait_for_completion(show_output=True)
from azureml.pipeline.steps import DatabricksStep from azureml.core.databricks import PyPiLibrary notebook_path="/Users/[email protected]/ModelTraining" # Databricks notebook path dbNbStep = DatabricksStep( name="DBNotebookInWS", inputs=[datasetFilePath], outputs=[output], num_workers=1, notebook_path=notebook_path, run_name='DB_Notebook_demo', compute_target=databricks_compute, allow_reuse=True, spark_version="7.2.x-scala2.12", pypi_libraries=[PyPiLibrary(package = 'scikit-learn') ,PyPiLibrary(package = 'azureml-sdk') ,PyPiLibrary(package = 'lightgbm') ,PyPiLibrary(package = 'pandas')], node_type = "Standard_D13_v2" ) steps = [dbNbStep] pipeline = Pipeline(workspace=ws, steps=steps) pipeline_run = Experiment(ws, 'DB_Notebook_demo').submit(pipeline) pipeline_run.wait_for_completion() # COMMAND ---------- from azureml.widgets import RunDetails RunDetails(pipeline_run).show()
def main():
# Replace this with the Use of Service Principal for authenticating with the Workspace
ws = Workspace.from_config()
# Choose a name for your CPU cluster
cpu_cluster_name = "cpu-cluster"
# Verify that cluster does not exist already
try:
cpu_cluster = ComputeTarget(workspace=ws, name=cpu_cluster_name)
print('Found existing cluster, use it.')
except ComputeTargetException:
compute_config = AmlCompute.provisioning_configuration(
vm_size='STANDARD_D2_V2', max_nodes=4)
cpu_cluster = ComputeTarget.create(ws, cpu_cluster_name,
compute_config)
cpu_cluster.wait_for_completion(show_output=True)
# Run configuration for R
rc = RunConfiguration()
rc.framework = "R"
# Run configuration for python
py_rc = RunConfiguration()
py_rc.framework = "Python"
py_rc.environment.docker.enabled = True
# Combine GitHub and Cran packages for R env
rc.environment.r = RSection()
rc.environment.docker.enabled = True
# Upload iris data to the datastore
# target_path = "iris_data"
# upload_files_to_datastore(ds,
# list("./iris.csv"),
# target_path = target_path,
# overwrite = TRUE)
training_data = DataReference(
datastore=ws.get_default_datastore(),
data_reference_name="iris_data",
path_on_datastore="iris_data/iris.csv",
)
print('Succesfull')
print(training_data)
# PipelineData object for newly trained model
# trained_model_dir = PipelineData(
# name="trained_model", datastore=ws.get_default_datastore(), is_directory=True
# )
# Train and Register the model
train_step = RScriptStep(
script_name="train.R",
arguments=[training_data],
inputs=[training_data],
compute_target=cpu_cluster_name,
source_directory=".",
runconfig=rc,
allow_reuse=True,
)
# Deploy the trained model
print("Step Train created")
steps = [train_step]
train_pipeline = Pipeline(workspace=ws, steps=steps)
train_pipeline.validate()
pipeline_run = Experiment(ws, 'iris_training').submit(train_pipeline)
pipeline_run.wait_for_completion(show_output=True)
published_pipeline = train_pipeline.publish(
name="iris-train",
description="Model training/retraining pipeline",
)
print(f"Published pipeline: {published_pipeline.name}")
print(f"for build {published_pipeline.version}")
print('Error while retrieving compute', e)
sys.exit(-1)
################################
# If you want to use datastore
################################
# from azureml.core import Datastore
# from azureml.data.data_reference import DataReference
# from azureml.pipeline.core import PipelineData
# def_blob_store = Datastore(ws, "workspaceblobstore")
# input_data = DataReference(
# datastore=def_blob_store,
# data_reference_name="input_data",
# path_on_datastore="20newsgroups/20news.pkl")
# output = PipelineData("output", datastore=def_blob_store)
est_step = EstimatorStep(name="Estimator_Train",
estimator=est,
estimator_entry_script_arguments=["--datadir", input_data, "--output", output],
runconfig_pipeline_params=None,
inputs=[input_data],
outputs=[output],
compute_target=compute_target)
pipeline = Pipeline(workspace=ws, steps=[est_step])
pipeline_run = experiment.submit(pipeline)
def main():
load_dotenv()
workspace_name = os.environ.get("BASE_NAME") + "-AML-WS"
resource_group = "AML-RG-" + os.environ.get("BASE_NAME")
subscription_id = os.environ.get("SUBSCRIPTION_ID")
tenant_id = os.environ.get("TENANT_ID")
app_id = os.environ.get("SP_APP_ID")
app_secret = os.environ.get("SP_APP_SECRET")
sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN")
train_script_path = os.environ.get("TRAIN_SCRIPT_PATH")
evaluate_script_path = os.environ.get("EVALUATE_SCRIPT_PATH")
vm_size = os.environ.get("AML_COMPUTE_CLUSTER_SKU")
compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME")
aks_name = os.environ.get("AKS_CLUSTER_NAME")
model_name = os.environ.get("MODEL_NAME")
build_id = os.environ.get("BUILD_BUILDID")
pipeline_name = os.environ.get("TRAINING_PIPELINE_NAME")
experiment_name = os.environ.get("EXPERIMENT_NAME")
# Get Azure machine learning workspace
aml_workspace = get_workspace(workspace_name, resource_group,
subscription_id, tenant_id, app_id,
app_secret)
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, compute_name, vm_size)
if aml_compute is not None:
print(aml_compute)
run_config = RunConfiguration(conda_dependencies=CondaDependencies.create(
conda_packages=['numpy', 'pandas', 'scikit-learn', 'keras'],
pip_packages=[
'azure', 'azureml-sdk', 'azure-storage', 'azure-storage-blob',
'transformers>=2.1.1', 'tensorflow>=2.0.0', 'tensorflow-gpu>=2.0.0'
]))
run_config.environment.docker.enabled = True
datastore_name = 'tfworld'
container_name = 'azureml-blobstore-7c6bdd88-21fa-453a-9c80-16998f02935f'
account_name = 'tfworld6818510241'
sas_token = '?sv=2019-02-02&ss=bfqt&srt=sco&sp=rl&se=2019-11-08T05:12:15Z&st=2019-10-23T20:12:15Z&spr=https&sig=eDqnc51TkqiIklpQfloT5vcU70pgzDuKb5PAGTvCdx4%3D' # noqa: E501
try:
existing_datastore = Datastore.get(aml_workspace, datastore_name)
except: # noqa: E722
existing_datastore = Datastore \
.register_azure_blob_container(workspace=aml_workspace,
datastore_name=datastore_name,
container_name=container_name,
account_name=account_name,
sas_token=sas_token
)
azure_dataset = Dataset.File.from_files(
path=(existing_datastore, 'azure-service-classifier/data'))
azure_dataset = azure_dataset.register(
workspace=aml_workspace,
name='Azure Services Dataset',
description='Dataset containing azure related posts on Stackoverflow',
create_new_version=True)
azure_dataset.to_path()
input_data = azure_dataset.as_named_input('input_data1').as_mount(
'/tmp/data')
model_name = PipelineParameter(name="model_name", default_value=model_name)
max_seq_length = PipelineParameter(name="max_seq_length",
default_value=128)
learning_rate = PipelineParameter(name="learning_rate", default_value=3e-5)
num_epochs = PipelineParameter(name="num_epochs", default_value=3)
export_dir = PipelineParameter(name="export_dir",
default_value="./outputs/exports")
batch_size = PipelineParameter(name="batch_size", default_value=32)
steps_per_epoch = PipelineParameter(name="steps_per_epoch",
default_value=100)
# initialize the TensorFlow estimator
estimator = TensorFlow(source_directory=sources_directory_train,
entry_script=train_script_path,
compute_target=aml_compute,
framework_version='2.0',
use_gpu=True,
pip_packages=[
'transformers==2.0.0',
'azureml-dataprep[fuse,pandas]==1.1.22'
])
train_step = EstimatorStep(
name="Train Model",
estimator=estimator,
estimator_entry_script_arguments=[
"--data_dir", input_data, "--max_seq_length", max_seq_length,
"--learning_rate", learning_rate, "--num_epochs", num_epochs,
"--export_dir", export_dir, "--batch_size", batch_size,
"--steps_per_epoch", steps_per_epoch
],
compute_target=aml_compute,
inputs=[input_data],
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=evaluate_script_path,
compute_target=aml_compute,
source_directory=sources_directory_train,
arguments=[
"--model_name",
model_name,
"--build_id",
build_id,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
# Currently, the Evaluate step will automatically register
# the model if it performs better. This step is based on a
# previous version of the repo which utilized JSON files to
# track evaluation results.
evaluate_step.run_after(train_step)
steps = [evaluate_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=pipeline_name,
description="Model training/retraining pipeline",
version=build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
response = published_pipeline.submit( # noqa: F841
workspace=aml_workspace,
experiment_name=experiment_name)
# Get AKS cluster for deployment
aks_compute = get_aks(aml_workspace, aks_name)
if aks_compute is not None:
print(aks_compute)
step1 = PythonScriptStep(name = 'prepare data',
source_directory = 'scripts',
script_name = 'data_prep.py',
compute_target = 'aml-cluster')
# Step to train a model
step2 = PythonScriptStep(name = 'train model',
source_directory = 'scripts',
script_name = 'train_model.py',
compute_target = 'aml-cluster')
from azureml.pipeline.core import Pipeline
from azureml.core import Experiment
# Construct the pipeline
train_pipeline = Pipeline(workspace = ws, steps = [step1,step2])
# Create an experiment and run the pipeline
experiment = Experiment(workspace = ws, name = 'training-pipeline')
pipeline_run = experiment.submit(train_pipeline)
#Pass Data bn Pipeline steps w PipelineData obj
from azureml.pipeline.core import PipelineData
from azureml.pipeline.steps import PythonScriptStep, EstimatorStep
# Get a dataset for the initial data
raw_ds = Dataset.get_by_name(ws, 'raw_dataset')
# Define a PipelineData object to pass data between steps
data_store = ws.get_default_datastore()
"owner='Neelabh Kashyap'",
"--tag",
"team='Data Science'",
"--tag",
"comment=prod",
],
inputs=[model_outpath],
compute_target=compute_target,
source_directory=".",
runconfig=py_rc,
allow_reuse=False,
)
pipeline = Pipeline(
workspace=ws,
steps=[download_step, train_step, register_model],
description="Builds R model for iris dataset",
)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--publish", action="store_true")
args = parser.parse_args()
if args.publish:
p = pipeline.publish(
name="iris-classifier-train-r",
description="train a classifer on iris dataset and register model",
)
def build_prednet_pipeline(dataset, ws):
print("building pipeline for dataset %s in workspace %s" %
(dataset, ws.name))
base_dir = "."
def_blob_store = ws.get_default_datastore()
# folder for scripts that need to be uploaded to Aml compute target
script_folder = "./scripts"
os.makedirs(script_folder, exist_ok=True)
shutil.copytree(os.path.join(base_dir, "models"),
os.path.join(base_dir, script_folder, "models"))
shutil.copy(os.path.join(base_dir, "train.py"), script_folder)
shutil.copy(os.path.join(base_dir, "data_preparation.py"), script_folder)
shutil.copy(os.path.join(base_dir, "register_prednet.py"), script_folder)
shutil.copy(os.path.join(base_dir, "batch_scoring.py"), script_folder)
shutil.copy(os.path.join(base_dir, "train_clf.py"), script_folder)
shutil.copy(os.path.join(base_dir, "register_clf.py"), script_folder)
cpu_compute_name = args.cpu_compute_name
cpu_compute_target = AmlCompute(ws, cpu_compute_name)
print("found existing compute target: %s" % cpu_compute_name)
# use get_status() to get a detailed status for the current cluster.
print(cpu_compute_target.get_status().serialize())
# choose a name for your cluster
gpu_compute_name = args.gpu_compute_name
gpu_compute_target = AmlCompute(workspace=ws, name=gpu_compute_name)
print(gpu_compute_target.get_status().serialize())
env = Environment.get(ws, "prednet")
# Runconfigs
runconfig = RunConfiguration()
runconfig.environment = env
print("PipelineData object created")
# DataReference to where raw data is stored.
raw_data = DataReference(
datastore=def_blob_store,
data_reference_name="raw_data",
path_on_datastore=os.path.join("prednet", "data", "raw_data"),
)
print("DataReference object created")
# Naming the intermediate data as processed_data and assigning it to the
# variable processed_data.
preprocessed_data = PipelineData("preprocessed_data",
datastore=def_blob_store)
data_metrics = PipelineData("data_metrics", datastore=def_blob_store)
hd_child_cwd = PipelineData("prednet_model_path", datastore=def_blob_store)
# prednet_path = PipelineData("outputs", datastore=def_blob_store)
scored_data = PipelineData("scored_data", datastore=def_blob_store)
model_path = PipelineData("model_path", datastore=def_blob_store)
# prepare dataset for training/testing recurrent neural network
data_prep = PythonScriptStep(
name="prepare_data",
script_name="data_preparation.py",
arguments=[
"--raw_data",
raw_data,
"--preprocessed_data",
preprocessed_data,
"--dataset",
dataset,
],
inputs=[raw_data],
outputs=[preprocessed_data],
compute_target=cpu_compute_target,
source_directory=script_folder,
runconfig=runconfig,
allow_reuse=True,
)
# data_prep.run_after(video_decoding)
print("data_prep step created")
est = Estimator(
source_directory=script_folder,
compute_target=gpu_compute_target,
entry_script="train.py",
node_count=1,
environment_definition=env,
)
ps = BayesianParameterSampling({
"--batch_size":
choice(1, 2, 4, 10),
"--filter_sizes":
choice("3, 3, 3", "4, 4, 4", "5, 5, 5"),
"--stack_sizes":
choice("48, 96, 192", "36, 72, 144", "12, 24, 48"),
"--learning_rate":
uniform(1e-6, 1e-3),
"--lr_decay":
uniform(1e-9, 1e-2),
"--freeze_layers":
choice("0, 1, 2", "1, 2, 3", "0, 1", "1, 2", "2, 3", "0", "3"),
# "--fine_tuning": choice("True", "False"),
})
hdc = HyperDriveConfig(
estimator=est,
hyperparameter_sampling=ps,
primary_metric_name="val_loss",
primary_metric_goal=PrimaryMetricGoal.MINIMIZE,
max_total_runs=3,
max_concurrent_runs=3,
max_duration_minutes=60 * 6,
)
train_prednet = HyperDriveStep(
"train_w_hyperdrive",
hdc,
estimator_entry_script_arguments=[
"--preprocessed_data",
preprocessed_data,
"--remote_execution",
"--dataset",
dataset,
],
inputs=[preprocessed_data],
outputs=[hd_child_cwd],
metrics_output=data_metrics,
allow_reuse=True,
)
train_prednet.run_after(data_prep)
register_prednet = PythonScriptStep(
name="register_prednet",
script_name="register_prednet.py",
arguments=[
"--data_metrics",
data_metrics,
],
compute_target=cpu_compute_target,
inputs=[data_metrics, hd_child_cwd],
source_directory=script_folder,
allow_reuse=True,
)
register_prednet.run_after(train_prednet)
batch_scoring = PythonScriptStep(
name="batch_scoring",
script_name="batch_scoring.py",
arguments=[
"--preprocessed_data",
preprocessed_data,
"--scored_data",
scored_data,
"--dataset",
dataset,
# "--prednet_path",
# prednet_path
],
compute_target=gpu_compute_target,
inputs=[preprocessed_data],
outputs=[scored_data],
source_directory=script_folder,
runconfig=runconfig,
allow_reuse=True,
)
batch_scoring.run_after(register_prednet)
train_clf = PythonScriptStep(
name="train_clf",
script_name="train_clf.py",
arguments=[
"--preprocessed_data", preprocessed_data, "--scored_data",
scored_data, "--model_path", model_path
],
compute_target=cpu_compute_target,
inputs=[preprocessed_data, scored_data],
outputs=[model_path],
source_directory=script_folder,
runconfig=runconfig,
allow_reuse=True,
)
train_clf.run_after(batch_scoring)
register_clf = PythonScriptStep(
name="register_clf",
script_name="register_clf.py",
arguments=["--model_path", model_path],
inputs=[model_path],
compute_target=cpu_compute_target,
source_directory=script_folder,
allow_reuse=True,
runconfig=runconfig,
)
register_clf.run_after(train_clf)
pipeline = Pipeline(
workspace=ws,
steps=[
data_prep,
train_prednet,
register_prednet,
batch_scoring,
train_clf,
register_clf,
],
)
pipeline.validate()
pipeline_name = "prednet_" + dataset
published_pipeline = pipeline.publish(name=pipeline_name)
_ = Schedule.create(
workspace=ws,
name=pipeline_name + "_sch",
pipeline_id=published_pipeline.id,
experiment_name=pipeline_name,
datastore=def_blob_store,
wait_for_provisioning=True,
description="Datastore scheduler for Pipeline" + pipeline_name,
path_on_datastore=os.path.join("prednet/data/raw_data", dataset,
"Train"),
polling_interval=60 * 24,
)
published_pipeline.submit(ws, pipeline_name)
step3 = PythonScriptStep(name="register model",
compute_target=env.aml_compute_name,
source_directory='src/steps',
script_name='03_reg_model.py',
inputs=[model_dir],
outputs=[],
arguments=[
'--model_dir', model_dir, '--model_name',
env.aml_model_name
],
runconfig=run_config,
allow_reuse=False)
# Build pipeline
pipeline = Pipeline(workspace=ws, steps=[step3])
pipeline.validate()
# Publish pipeline & pipeline_endpoint
published_pipeline = pipeline.publish(name=env.aml_pipeline_name)
try:
pipeline_endpoint = PipelineEndpoint.get(
workspace=ws, name=env.aml_pipeline_endpoint_name)
pipeline_endpoint.add_default(published_pipeline)
except ErrorResponseException:
pipeline_endpoint = PipelineEndpoint.publish(
workspace=ws,
name=env.aml_pipeline_endpoint_name,
description=env.aml_pipeline_endpoint_name,
pipeline=published_pipeline)
def main():
load_dotenv()
workspace_name = os.environ.get("BASE_NAME") + "-AML-WS"
resource_group = os.environ.get("BASE_NAME") + "-AML-RG"
subscription_id = os.environ.get("SUBSCRIPTION_ID")
tenant_id = os.environ.get("TENANT_ID")
app_id = os.environ.get("SP_APP_ID")
app_secret = os.environ.get("SP_APP_SECRET")
sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN")
train_script_path = os.environ.get("TRAIN_SCRIPT_PATH")
vm_size = os.environ.get("AML_COMPUTE_CLUSTER_CPU_SKU")
compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME")
model_name = os.environ.get("MODEL_NAME")
build_id = os.environ.get("BUILD_BUILDID")
pipeline_name = os.environ.get("TRAINING_PIPELINE_NAME")
# Get Azure machine learning workspace
aml_workspace = get_workspace(workspace_name, resource_group,
subscription_id, tenant_id, app_id,
app_secret)
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, compute_name, vm_size)
if aml_compute is not None:
print(aml_compute)
run_config = RunConfiguration(conda_dependencies=CondaDependencies.create(
conda_packages=[
'numpy', 'pandas', 'scikit-learn', 'tensorflow', 'keras'
],
pip_packages=[
'azure', 'azureml-core', 'azure-storage', 'azure-storage-blob',
'azure-cli', 'azure-cli-telemetry', 'azure-cli-core',
'azure-cli-nspkg', 'azure-cli-command-modules-nspkg'
]))
run_config.environment.docker.enabled = True
model_name = PipelineParameter(name="model_name", default_value=model_name)
release_id = PipelineParameter(name="release_id", default_value="0")
storageacctname = PipelineParameter(
name="storageacctname",
default_value=os.environ.get("STORAGE_ACCT_NAME"))
storageacctkey = PipelineParameter(
name="storageacctkey",
default_value=os.environ.get("STORAGE_ACCT_KEY"))
containername = PipelineParameter(
name="containername",
default_value=os.environ.get("STORAGE_BLOB_NAME"))
train_step = PythonScriptStep(
name="Train Model",
script_name=train_script_path,
compute_target=aml_compute,
source_directory=sources_directory_train,
arguments=[
"--release_id", release_id, "--model_name", model_name,
"--storageacctname", storageacctname, "--storageacctkey",
storageacctkey, "--containername", containername
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
steps = [train_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=pipeline_name,
description="Model training/retraining pipeline",
version=build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
# environment.register(ws)
environment = Environment.get(ws, "sentiment-env")
estimator = TensorFlow(
source_directory="imdb",
entry_script="experiment.py",
framework_version="2.1",
conda_packages=["python=3.7.4", "tensorflow", "tensorflow-datasets"],
pip_packages=["azureml-sdk[notebooks,automl,explain]"],
compute_target="archi-trainer")
model_step = EstimatorStep(
name="training model",
estimator=estimator,
compute_target="archi-trainer",
estimator_entry_script_arguments=['--n-words', 5000, '--epochs', 2])
# register_step = PythonScriptStep(name="register pipeline", source_directory="sentiment_analysis", script_name="registration.py", compute_target="dummy", runconfig=run_config)
# register_step.run_after(model_step)
sentiment_pipe = Pipeline(workspace=ws, steps=[model_step])
sentiment_pipe.validate()
experiment = Experiment(workspace=ws, name="sentiment-analysis")
run = experiment.submit(config=sentiment_pipe)
run.wait_for_completion(show_output=True)
ds.upload('outputs/sentiment_model.h5',
'models',
overwrite=True,
show_progress=True)
def __init__(self, request_id, use_url, input_container_sas,
internal_datastore, model_name):
try:
self.request_id = request_id
aml_config = api_config.AML_CONFIG
self.ws = Workspace(subscription_id=aml_config['subscription_id'],
resource_group=aml_config['resource_group'],
workspace_name=aml_config['workspace_name'],
auth=svc_pr)
print('AMLCompute constructor, AML workspace obtained.')
internal_dir, output_dir = self._get_data_references(
request_id, internal_datastore)
compute_target = self.ws.compute_targets[
aml_config['aml_compute_name']]
dependencies = CondaDependencies.create(pip_packages=[
'tensorflow-gpu==1.12.0', 'pillow', 'numpy',
'azure-storage-blob==2.1.0', 'azureml-defaults==1.0.41'
])
amlcompute_run_config = RunConfiguration(
conda_dependencies=dependencies)
amlcompute_run_config.environment.docker.enabled = True
amlcompute_run_config.environment.docker.gpu_support = True
amlcompute_run_config.environment.docker.base_image = DEFAULT_GPU_IMAGE
amlcompute_run_config.environment.spark.precache_packages = False
# default values are required and need to be literal values or data references as JSON
param_job_id = PipelineParameter(name='param_job_id',
default_value='default_job_id')
param_begin_index = PipelineParameter(name='param_begin_index',
default_value=0)
param_end_index = PipelineParameter(name='param_end_index',
default_value=0)
param_detection_threshold = PipelineParameter(
name='param_detection_threshold', default_value=0.05)
batch_score_step = PythonScriptStep(
aml_config['script_name'],
source_directory=aml_config['source_dir'],
hash_paths=['.'], # include all contents of source_directory
name='batch_scoring',
arguments=[
'--job_id',
param_job_id,
'--request_id',
request_id,
'--model_name',
model_name,
'--input_container_sas',
input_container_sas, # can be None
'--use_url',
use_url,
'--internal_dir',
internal_dir,
'--begin_index',
param_begin_index, # inclusive
'--end_index',
param_end_index, # exclusive
'--output_dir',
output_dir,
'--detection_threshold',
param_detection_threshold
],
compute_target=compute_target,
inputs=[internal_dir],
outputs=[output_dir],
runconfig=amlcompute_run_config)
self.pipeline = Pipeline(workspace=self.ws,
steps=[batch_score_step])
self.aml_config = aml_config
print('AMLCompute constructor all good.')
except Exception as e:
raise RuntimeError(
'Error in setting up AML Compute resource: {}.'.format(str(e)))
"--input",
ds_input.as_download(), "--output",
prepared_data
],
inputs=[ds_input],
outputs=[prepared_data],
compute_target=compute_target,
runconfig=aml_run_config,
allow_reuse=True)
# Step 2: Train our model
training_results = PipelineData(name="training_results",
datastore=blob_store)
train_step = PythonScriptStep(script_name=train_entry_point,
source_directory=train_source_dir,
arguments=[
"--prepped_data",
prepared_data.as_input(),
"--training_results", training_results
],
compute_target=compute_target,
runconfig=aml_run_config,
allow_reuse=True)
# Step 3: Build the pipeline
pipeline_steps = [data_prep_step, train_step]
pipeline = Pipeline(workspace=workspace, steps=[pipeline_steps])
pipeline_run1 = Experiment(workspace, "mckee_bot").submit(pipeline)
pipeline_run1.wait_for_completion()
est = Estimator(source_directory='.',
compute_target=ws.compute_targets['cpu'],
entry_script='azureml-issues.py',
pip_packages=['azure-devops', 'pandas'])
data_processing = EstimatorStep(estimator=est,
estimator_entry_script_arguments=[
'--data_path', blob_output_data,
'--analyze', '--load_open', '--load_closed'
],
inputs=[blob_output_data],
compute_target=ws.compute_targets['cpu'],
allow_reuse=False)
pipeline = Pipeline(workspace=ws, steps=[data_processing])
print("Pipeline is built")
pipeline.validate()
print("Simple validation complete")
pipeline_run = Experiment(ws, 'issues_pipeline').submit(pipeline)
print("Pipeline is submitted for execution")
published_pipeline = pipeline.publish(
name="Issues_Stats",
description="Pull data from DevOps and aggregate for PowerBI")
print(published_pipeline.id)
from azureml.pipeline.core.schedule import ScheduleRecurrence, Schedule
def main():
load_dotenv()
workspace_name = os.environ.get("BASE_NAME") + "-AML-WS"
resource_group = os.environ.get("RESOURCE_GROUP")
subscription_id = os.environ.get("SUBSCRIPTION_ID")
tenant_id = os.environ.get("TENANT_ID")
app_id = os.environ.get("SP_APP_ID")
app_secret = os.environ.get("SP_APP_SECRET")
sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN")
train_script_path = os.environ.get("TRAIN_SCRIPT_PATH")
evaluate_script_path = os.environ.get("EVALUATE_SCRIPT_PATH")
register_script_path = os.environ.get("REGISTER_SCRIPT_PATH")
vm_size_cpu = os.environ.get("AML_COMPUTE_CLUSTER_CPU_SKU")
compute_name_cpu = os.environ.get("AML_COMPUTE_CLUSTER_NAME")
model_name = os.environ.get("MODEL_NAME")
# Get Azure machine learning workspace
aml_workspace = get_workspace(workspace_name, resource_group,
subscription_id, tenant_id, app_id,
app_secret)
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute_cpu = get_compute(aml_workspace, compute_name_cpu, vm_size_cpu)
if aml_compute_cpu is not None:
print(aml_compute_cpu)
run_config = RunConfiguration(conda_dependencies=CondaDependencies.create(
conda_packages=[
'numpy', 'pandas', 'scikit-learn', 'tensorflow', 'keras'
],
pip_packages=[
'azure', 'azureml-core', 'azure-storage', 'azure-storage-blob'
]))
run_config.environment.docker.enabled = True
model_name = PipelineParameter(name="model_name", default_value=model_name)
def_blob_store = Datastore(aml_workspace, "workspaceblobstore")
jsonconfigs = PipelineData("jsonconfigs", datastore=def_blob_store)
config_suffix = datetime.datetime.now().strftime("%Y%m%d%H")
train_step = PythonScriptStep(
name="Train Model",
script_name=train_script_path,
compute_target=aml_compute_cpu,
source_directory=sources_directory_train,
arguments=[
"--config_suffix",
config_suffix,
"--json_config",
jsonconfigs,
"--model_name",
model_name,
],
runconfig=run_config,
# inputs=[jsonconfigs],
outputs=[jsonconfigs],
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=evaluate_script_path,
compute_target=aml_compute_cpu,
source_directory=sources_directory_train,
arguments=[
"--config_suffix",
config_suffix,
"--json_config",
jsonconfigs,
],
runconfig=run_config,
inputs=[jsonconfigs],
# outputs=[jsonconfigs],
allow_reuse=False,
)
print("Step Evaluate created")
register_model_step = PythonScriptStep(
name="Register New Trained Model",
script_name=register_script_path,
compute_target=aml_compute_cpu,
source_directory=sources_directory_train,
arguments=[
"--config_suffix",
config_suffix,
"--json_config",
jsonconfigs,
"--model_name",
model_name,
],
runconfig=run_config,
inputs=[jsonconfigs],
# outputs=[jsonconfigs],
allow_reuse=False,
)
print("Step register model created")
evaluate_step.run_after(train_step)
register_model_step.run_after(evaluate_step)
steps = [register_model_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name="training-pipeline",
description="Model training/retraining pipeline")
train_pipeline_json = {}
train_pipeline_json["rest_endpoint"] = published_pipeline.endpoint
json_file_path = "ml_service/pipelines/train_pipeline.json"
with open(json_file_path, "w") as outfile:
json.dump(train_pipeline_json, outfile)
name='Train Estimator Step',
estimator=trainEstimator,
inputs=[training_data_location],
outputs=[model],
compute_target=computeCluster,
estimator_entry_script_arguments = script_params
)
# == Step 3 ==
model_name = "MargeOrHomer"
registerStep = PythonScriptStep(name="Register model for deployment",
script_name="register.py",
compute_target=computeCluster,
inputs=[model],
arguments=['--dataset_name', model_name,
'--model_assets_path', model
],
source_directory=script_folder)
# Create the pipeline
prep_train_register = [preProcessDataStep,trainOnGpuStep,registerStep]
pipeline = Pipeline(workspace=ws, steps=[prep_train_register])
pipeline.validate()
# Publish the pipeline
mlpipeline = pipeline.publish(name="Marge Or Homer - Training pipeline")
print("Pipeline Published ID:"+mlpipeline.id)
# Submit the pipeline to be run
mlpipeline.submit(ws,"Marge-or-Homer" ,pipeline_parameters={"source_dataset":DataPath(datastore=source_ds, path_on_datastore="trainingdata")})
arguments = ["--input_directory", anomaly_data],
inputs = [anomaly_data],
# outputs = [model],
compute_target = aml_compute,
source_directory = project_folder,
allow_reuse = True,
runconfig = amlcompute_run_config)
print("AutoML Training Step created.")
############################### set up, validate and run pipeline
steps = [anom_detect, automl_step]
print("Step lists created")
pipeline = Pipeline(workspace = ws, steps = steps)
print ("Pipeline is built")
pipeline.validate()
print("Pipeline validation complete")
pipeline_run = experiment.submit(pipeline) #, regenerate_outputs=True)
print("Pipeline is submitted for execution")
# Wait until the run finishes.
pipeline_run.wait_for_completion(show_output = False)
print("Pipeline run completed")
############################### upload artifacts to AML Workspace
# Download aml_config info and output of automl_step
def get_scoring_pipeline(
scoring_dataset: Dataset,
output_loc: PipelineData,
score_run_config: ParallelRunConfig,
copy_run_config: RunConfiguration,
computetarget: ComputeTarget,
ws: Workspace,
env: Env,
) -> Pipeline:
"""
Creates the scoring pipeline.
:param scoring_dataset: Data to score
:param output_loc: Location to save the scoring results
:param score_run_config: Parallel Run configuration to support
parallelized scoring
:param copy_run_config: Script Run configuration to support
score copying
:param computetarget: AML Compute target
:param ws: AML Workspace
:param env: Environment Variables
:returns: Scoring pipeline instance
"""
# To help filter the model make the model name, model version and a
# tag/value pair bindable parameters so that they can be passed to
# the pipeline when invoked either over REST or via the AML SDK.
model_name_param = PipelineParameter("model_name",
default_value=" ") # NOQA: E501
model_version_param = PipelineParameter("model_version",
default_value=" ") # NOQA: E501
model_tag_name_param = PipelineParameter("model_tag_name",
default_value=" ") # NOQA: E501
model_tag_value_param = PipelineParameter("model_tag_value",
default_value=" ") # NOQA: E501
scoring_step = ParallelRunStep(
name="scoringstep",
inputs=[scoring_dataset],
output=output_loc,
arguments=[
"--model_name",
model_name_param,
"--model_version",
model_version_param,
"--model_tag_name",
model_tag_name_param,
"--model_tag_value",
model_tag_value_param,
],
parallel_run_config=score_run_config,
allow_reuse=False,
)
copying_step = PythonScriptStep(
name="scorecopystep",
script_name=env.batchscore_copy_script_path,
source_directory=env.sources_directory_train,
arguments=[
"--output_path",
output_loc,
"--scoring_output_filename",
env.scoring_datastore_output_filename
if env.scoring_datastore_output_filename is not None else "",
"--scoring_datastore",
env.scoring_datastore_storage_name
if env.scoring_datastore_storage_name is not None else "",
"--score_container",
env.scoring_datastore_output_container
if env.scoring_datastore_output_container is not None else "",
"--scoring_datastore_key",
env.scoring_datastore_access_key
if env.scoring_datastore_access_key is not None else "",
],
inputs=[output_loc],
allow_reuse=False,
compute_target=computetarget,
runconfig=copy_run_config,
)
return Pipeline(workspace=ws, steps=[scoring_step, copying_step])
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable Azure ML environment
environment = get_environment(aml_workspace,
e.aml_env_name,
create_new=e.rebuild_env) #
run_config = RunConfiguration()
run_config.environment = environment
if (e.datastore_name):
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables[
"DATASTORE_NAME"] = datastore_name # NOQA: E501
model_name_param = PipelineParameter(name="model_name",
default_value=e.model_name)
dataset_version_param = PipelineParameter(name="dataset_version",
default_value=e.dataset_version)
data_file_path_param = PipelineParameter(name="data_file_path",
default_value="none")
caller_run_id_param = PipelineParameter(name="caller_run_id",
default_value="none")
# Get dataset name
dataset_name = e.dataset_name
# Check to see if dataset exists
if (dataset_name not in aml_workspace.datasets):
# Create dataset from diabetes sample data
sample_data = load_diabetes()
df = pd.DataFrame(data=sample_data.data,
columns=sample_data.feature_names)
df['Y'] = sample_data.target
file_name = 'diabetes.csv'
df.to_csv(file_name, index=False)
# Upload file to default datastore in workspace
datatstore = Datastore.get(aml_workspace, datastore_name)
target_path = 'training-data/'
datatstore.upload_files(files=[file_name],
target_path=target_path,
overwrite=True,
show_progress=False)
# Register dataset
path_on_datastore = os.path.join(target_path, file_name)
dataset = Dataset.Tabular.from_delimited_files(
path=(datatstore, path_on_datastore))
dataset = dataset.register(workspace=aml_workspace,
name=dataset_name,
description='diabetes training data',
tags={'format': 'CSV'},
create_new_version=True)
# Create a PipelineData to pass data between steps
pipeline_data = PipelineData(
'pipeline_data', datastore=aml_workspace.get_default_datastore())
train_step = PythonScriptStep(
name="Train Model",
script_name=e.train_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
outputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_output",
pipeline_data,
"--dataset_version",
dataset_version_param,
"--data_file_path",
data_file_path_param,
"--caller_run_id",
caller_run_id_param,
"--dataset_name",
dataset_name,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
evaluate_step = PythonScriptStep(
name="Evaluate Model ",
script_name=e.evaluate_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
arguments=[
"--model_name",
model_name_param,
"--allow_run_cancel",
e.allow_run_cancel,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Evaluate created")
register_step = PythonScriptStep(
name="Register Model ",
script_name=e.register_script_path,
compute_target=aml_compute,
source_directory=e.sources_directory_train,
inputs=[pipeline_data],
arguments=[
"--model_name",
model_name_param,
"--step_input",
pipeline_data,
],
runconfig=run_config,
allow_reuse=False,
)
print("Step Register created")
# Check run_evaluation flag to include or exclude evaluation step.
if ((e.run_evaluation).lower() == 'true'):
print("Include evaluation step before register step.")
evaluate_step.run_after(train_step)
register_step.run_after(evaluate_step)
steps = [train_step, evaluate_step, register_step]
else:
print("Exclude evaluation step and directly run register step.")
register_step.run_after(train_step)
steps = [train_step, register_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline._set_experiment_name
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
environment.python.conda_dependencies.add_pip_package("scipy")
environment.python.conda_dependencies.add_pip_package("joblib")
environment.python.conda_dependencies.add_pip_package("numpy")
environment.python.conda_dependencies.add_pip_package("pandas")
# Create a new runconfig object for the pipeline
pipeline_run_config = RunConfiguration()
# Use the compute you created above.
pipeline_run_config.target = pipeline_cluster
# Assign the environment to the run configuration
pipeline_run_config.environment = environment
print ("Run configuration created.")
pipeline_param = PipelineParameter(
name="data",
default_value=default)
register_step = PythonScriptStep(name = "Register Model",
script_name = "pipelineScript.py",
arguments = ['--data', pipeline_param],
compute_target = pipeline_cluster,
runconfig = pipeline_run_config,
allow_reuse = True)
reg_pipeline=Pipeline(workspace=ws,steps=[register_step])
# Submit the pipeline to be run
pipeline_run1 = Experiment(ws, 'RegExp3').submit(reg_pipeline)
pipeline_run1.wait_for_completion()
def main():
cluster_id = os.environ.get("DATABRICKS_CLUSTER_ID", None)
workspace_name = os.environ.get("AML_WORKSPACE_NAME", None)
resource_group = os.environ.get("RESOURCE_GROUP", None)
subscription_id = os.environ.get("SUBSCRIPTION_ID", None)
tenant_id = os.environ.get("TENANT_ID", None)
app_id = os.environ.get("SP_APP_ID", None)
app_secret = os.environ.get("SP_APP_SECRET", None)
experiment_subfolder = os.environ.get(
"EXPERIMENT_FOLDER",
'aml_service/experiment'
)
sources_directory = os.environ.get("SOURCES_DIR", None)
experiment_folder = os.path.join(sources_directory, experiment_subfolder)
train_script_path = os.environ.get("TRAIN_SCRIPT_PATH", None)
databricks_workspace_name = os.environ.get(
"DATABRICKS_WORKSPACE_NAME",
None
)
databricks_access_token = os.environ.get("DATABRICKS_ACCESS_TOKEN", None)
databricks_compute_name_aml = os.environ.get(
"DATABRICKS_COMPUTE_NAME_AML",
None
)
model_dir = os.environ.get("MODEL_DIR", '/dbfs/model')
model_name = os.environ.get("MODEL_NAME", 'local-model')
model_file_name = "%s.pth" % (model_name)
model_path = os.path.join(model_dir, model_file_name)
print("The model path will be %s" % (model_path))
aml_workspace = get_workspace(
workspace_name,
resource_group,
subscription_id,
tenant_id,
app_id,
app_secret)
print(aml_workspace)
databricks_compute = get_compute(
aml_workspace,
databricks_compute_name_aml,
resource_group,
databricks_workspace_name,
databricks_access_token)
print(databricks_compute)
step1 = DatabricksStep(
name="DBPythonInLocalMachine",
num_workers=1,
python_script_name=train_script_path,
source_directory=sources_directory,
run_name='DB_Python_Local_demo',
existing_cluster_id=cluster_id,
compute_target=databricks_compute,
allow_reuse=False,
python_script_params=['--MODEL_PATH', model_path]
)
step2 = DatabricksStep(
name="RegisterModel",
num_workers=1,
python_script_name="register_model.py",
source_directory=experiment_folder,
run_name='Register_model',
existing_cluster_id=cluster_id,
compute_target=databricks_compute,
allow_reuse=False,
python_script_params=[
'--MODEL_PATH', model_path,
'--TENANT_ID', tenant_id,
'--APP_ID', app_id,
'--APP_SECRET', app_secret,
'--MODEL_NAME', model_name]
)
step2.run_after(step1)
print("Step lists created")
pipeline = Pipeline(
workspace=aml_workspace,
# steps=[step1])
steps=[step1, step2])
print("Pipeline is built")
pipeline.validate()
print("Pipeline validation complete")
pipeline_run = pipeline.submit(experiment_name="pipetest")
print("Pipeline is submitted for execution")
pipeline_details = pipeline_run.get_details()
pipeline_run_id = pipeline_details['runId']
azure_run_url = get_experiment_run_url(
subscription_id,
resource_group,
workspace_name,
pipeline_run_id
)
print("To check details of the Pipeline run, go to " + azure_run_url)
pipeline_status = pipeline_run.get_status()
timer_mod = 0
while pipeline_status == 'Running' or pipeline_status == 'NotStarted':
timer_mod = timer_mod + 10
time.sleep(10)
if (timer_mod % 30) == 0:
print(
"Status: %s. %s seconds have passed." %
(pipeline_status, timer_mod)
)
pipeline_status = pipeline_run.get_status()
if pipeline_status == 'Failed':
print("AML Pipelne failed. Check %s for details." % (azure_run_url))
sys.exit(1)
else:
print(pipeline_status)
print("Pipeline completed")
environment.python.conda_dependencies.add_pip_package("numpy")
environment.python.conda_dependencies.add_pip_package("pandas")
# Create a new runconfig object for the pipeline
pipeline_run_config = RunConfiguration()
# Use the compute you created above.
pipeline_run_config.target = pipeline_cluster
# Assign the environment to the run configuration
pipeline_run_config.environment = environment
print("Run configuration created.")
pipeline_param = PipelineParameter(name="data", default_value=default)
register_step = PythonScriptStep(name="Register Model",
script_name="pipelineScript.py",
arguments=['--data', pipeline_param],
compute_target=pipeline_cluster,
runconfig=pipeline_run_config,
allow_reuse=False)
service_step = PythonScriptStep(
name="CreateService",
script_name="serviceCreation_PipelineScript.py",
compute_target=pipeline_cluster,
runconfig=pipeline_run_config,
allow_reuse=False)
succ_pipeline = Pipeline(workspace=ws, steps=[register_step, service_step])
# Submit the pipeline to be run
pipeline_run1 = Experiment(ws, 'RegExp3').submit(succ_pipeline)
pipeline_run1.wait_for_completion()
# step 2, submit pipelines
submit_pipelines = PythonScriptStep(
name='submit pipelines',
script_name="pipelines_submit.py",
# arguments=["--overwrite_published_pipelines", overwrite_published_pipelines],
compute_target=cpu_compute_target,
source_directory=script_folder,
runconfig=cpu_compute_run_config,
allow_reuse=False,
hash_paths=["."])
print("pipeline submit step created")
submit_pipelines.run_after(build_pipelines)
pipeline = Pipeline(workspace=ws, steps=[build_pipelines, submit_pipelines])
print("Pipeline created")
pipeline.validate()
print("Validation complete")
pipeline_name = 'prednet_master'
published_pipeline = pipeline.publish(name=pipeline_name)
print("pipeline id: ", published_pipeline.id)
datastore = ws.get_default_datastore()
schedule = Schedule.create(workspace=ws,
name=pipeline_name + "_sch",
pipeline_id=published_pipeline.id,
experiment_name='Schedule_Run',
def main(cluster_id, workspace_name, resource_group, subscription_id,
tenant_id, app_id, app_secret, experiment_folder, project_folder,
train_script_path, databricks_workspace, databricks_access_token,
databricks_compute_name, mdl_dir, mdl_name):
mdl_file_name = "%s.pth" % (mdl_name)
mdl_path = os.path.join(mdl_dir, mdl_file_name)
print("The model path will be %s" % (mdl_path))
ws = get_workspace(workspace_name, resource_group, subscription_id,
tenant_id, app_id, app_secret)
print(ws)
databricks_compute = get_compute(ws, databricks_compute_name,
resource_group, databricks_workspace,
databricks_access_token)
print(databricks_compute)
step1 = DatabricksStep(name="DBPythonInLocalMachine",
num_workers=1,
python_script_name=train_script_path,
source_directory=project_folder,
run_name='DB_Python_Local_demo',
existing_cluster_id=cluster_id,
compute_target=databricks_compute,
allow_reuse=False,
python_script_params=['--MODEL_PATH', mdl_path])
step2 = DatabricksStep(name="RegisterModel",
num_workers=1,
python_script_name="Register.py",
source_directory=experiment_folder,
run_name='Register_model',
existing_cluster_id=cluster_id,
compute_target=databricks_compute,
allow_reuse=False,
python_script_params=[
'--MODEL_PATH', mdl_path, '--TENANT_ID',
tenant_id, '--APP_ID', app_id, '--APP_SECRET',
app_secret, '--MODEL_NAME', mdl_name
])
step2.run_after(step1)
print("Step lists created")
pipeline = Pipeline(
workspace=ws,
# steps=[step1])
steps=[step1, step2])
print("Pipeline is built")
pipeline.validate()
print("Pipeline validation complete")
pipeline_run = pipeline.submit(experiment_name="pipetest")
print("Pipeline is submitted for execution")
pipeline_details = pipeline_run.get_details()
pipeline_run_id = pipeline_details['runId']
azure_run_url = get_experiment_run_url(subscription_id, resource_group,
workspace_name, pipeline_run_id)
print("To check details of the Pipeline run, go to " + azure_run_url)
pipeline_status = pipeline_run.get_status()
timer_mod = 0
while (pipeline_status == 'Running'):
timer_mod = timer_mod + 10
time.sleep(10)
if ((timer_mod % 30) == 0):
print("Still running. %s seconds have passed." % (timer_mod))
pipeline_status = pipeline_run.get_status()
if pipeline_status == 'Failed':
sys.exit("AML Pipeline failed")
else:
print(pipeline_status)
print("Pipeline completed")
