# convert to relative paths
prefix = Path(__file__).parent
source_dir = str(prefix.joinpath(source_dir))
environment_file = str(prefix.joinpath(environment_file))
# get workspace
ws = Workspace.from_config()
# create dataset
ds = Dataset.File.from_files(data_uri)
# create environment
env = Environment.from_pip_requirements(environment_name, environment_file)
# setup entry script arguments
args = ["--data-dir", ds.as_mount()]
# create a job configuration
src = ScriptRunConfig(
source_directory=source_dir,
script=entry_script,
arguments=args,
environment=env,
compute_target=compute_name,
)
# run the job
run = Experiment(ws, experiment_name).submit(src)
run.wait_for_completion(show_output=True)
# Build Pipeline
pipeline1 = Pipeline(workspace=ws, steps=steps)
print("Pipeline is built")
# Validate Pipeline
pipeline1.validate()
print("Pipeline validation complete")
# Submit unpublished pipeline with small data set for test
if args.pipeline_action == "pipeline-test":
pipeline_run1 = Experiment(ws,
experiment_name).submit(pipeline1,
regenerate_outputs=True)
print("Pipeline is submitted for execution")
pipeline_run1.wait_for_completion(show_output=True)
# RunDetails(pipeline_run1).show()
# Define pipeline parameters
# run_env = PipelineParameter(
# name="dev_flag",
# default_value=True)
# dbname = PipelineParameter(
# name="dbname",
# default_value='opex')
# Publish Pipeline
if args.pipeline_action == "publish":
published_pipeline1 = pipeline1.publish(
ws = Workspace.from_config()
print(ws.name, ws.resource_group, ws.location, ws.subscription_id, sep='\n')
datastore = ws.get_default_datastore()
input_ds = Dataset.get_by_name(ws, dataset_name)
batch_data = DatasetConsumptionConfig("batch_dataset", input_ds, mode='mount')
output_dir = PipelineData(name='batch_output', datastore=datastore)
parallel_run_config = ParallelRunConfig.load_yaml(workspace=ws,
path='convert_parallel.yml')
batch_step = ParallelRunStep(name="batch-conversion-step",
parallel_run_config=parallel_run_config,
arguments=['--data_output_path', output_dir],
inputs=[batch_data],
output=output_dir,
allow_reuse=False)
steps = [batch_step]
pipeline = Pipeline(workspace=ws, steps=steps)
pipeline.validate()
pipeline_run = Experiment(ws, 'convert-batch-pipeline').submit(pipeline)
pipeline_run.wait_for_completion()
#published_pipeline = pipeline.publish(pipeline_name)
#print("Published pipeline id: ", published_pipeline.id)
evaluateStep.run_after(trainStep)
steps = [evaluateStep]
pipeline = Pipeline(workspace=ws, steps=steps)
print ("Pipeline is built")
pipeline.validate()
print("Simple validation complete")
run = Run.get_context()
experiment_name = run.experiment.name
pipeline_run = Experiment(ws, experiment_name).submit(pipeline)
print("Pipeline is submitted for execution")
pipeline_run.wait_for_completion(show_output=True, timeout_seconds=43200)
print("Get StepRun for evaluate step...")
pipeline_run_id = pipeline_run.id
step_run_id = pipeline_run.find_step_run('evaluate')[0].id
node_id = pipeline_run.get_graph().node_name_dict['evaluate'][0].node_id
print('Pipeline Run ID: {} Step Run ID: {}, Step Run Node ID: {}'.format(pipeline_run_id, step_run_id, node_id))
step_run = StepRun(run.experiment, step_run_id, pipeline_run_id, node_id)
print(step_run)
print("Downloading evaluation results...")
# access the evaluate_output
#data = pipeline_run.find_step_run('evaluate')[0].get_output_data('evaluate_output')
data = step_run.get_output_data('evaluate_output')
# download the predictions to local path
data.download('.', show_progress=True)
modelEvalReg = PythonScriptStep(
name="Evaluate and Register Model",
script_name="evaluate_model.py",
arguments=["--release_id", 0, '--model_name', model_name],
compute_target=compute_target_cpu,
source_directory=source_directory,
runconfig=run_config,
allow_reuse=False)
modelEvalReg.run_after(trainingStep)
print("Model Evaluation and Registration Step is Created")
from azureml.pipeline.core import Pipeline
from azureml.core import Experiment
pipeline = Pipeline(workspace=ws,
steps=[extractDataStep, trainingStep, modelEvalReg])
pipeline_run = Experiment(ws, pipeline_experiment_name).submit(pipeline)
pipeline_run.wait_for_completion(show_output=True, raise_on_error=True)
published_pipeline = pipeline_run.publish_pipeline(
name=pipeline_name,
description=
"Steps are: data preparation, training, model validation and model registration",
version="0.1",
continue_on_step_failure=False)
## Free up compute resource(s) after computation is completed!
print(f'Deleting compute resource: [{cluster_name_cpu}]')
compute_target_cpu.delete()
# list of steps to run
compareModels = [trainStep]
from azureml.pipeline.core import Pipeline
# Build the pipeline
pipeline1 = Pipeline(workspace=ws, steps=[compareModels])
run = Run.get_context()
# Envoyer le pipeline
from azureml.core import Experiment
# Submit the pipeline to be run
pipeline_run1 = Experiment(ws, 'Compare_Models_Exp').submit(pipeline1)
pipeline_run1.wait_for_completion(False, 0, True)
#Récupération du run id de la step 1 sur 1 de la pipeline
step1_runid = [e for e in pipeline_run1.get_steps()][0].id
#création du path complet dans le blob storage
path_model_datastore = "azureml/" + step1_runid + "/outputs/"
#download en local du modele
def_data_store.download(target_path='datastore', prefix=path_model_datastore)
#chargement dans la workspace a partir du modele download en local
model = Model.register(workspace=ws,
model_path='datastore/' + path_model_datastore +
model_pklname + '.pkl',
model_name=model_basename)
#Create environment file
from azureml.core.conda_dependencies import CondaDependencies
def main():
train_file = r"EdwardFry_Microsoft_issueDataset.csv"
ws = Workspace.from_config()
# Default datastore
def_data_store = ws.get_default_datastore() # Loads config.json
# Get the blob storage associated with the workspace
def_blob_store = Datastore(ws, "workspaceblobstore")
# Get file storage associated with the workspace
def_file_store = Datastore(ws, "workspacefilestore")
# Set data input and output
xyz_phishing_dataset = Dataset.File.from_files([(def_blob_store,
train_file)])
output_data1 = OutputFileDatasetConfig(
destination=(datastore, 'outputdataset/{run-id}'))
output_data_dataset = output_data1.register_on_complete(
name='prepared_output_data')
# Set compute
compute_name = "aml-compute"
vm_size = "STANDARD_NC6"
if compute_name in ws.compute_targets:
compute_target = ws.compute_targets[compute_name]
if compute_target and type(compute_target) is AmlCompute:
print('Found compute target: ' + compute_name)
else:
print('Creating a new compute target...')
provisioning_config = AmlCompute.provisioning_configuration(
vm_size=vm_size, # STANDARD_NC6 is GPU-enabled
min_nodes=0,
max_nodes=4)
# create the compute target
compute_target = ComputeTarget.create(ws, compute_name,
provisioning_config)
# Can poll for a minimum number of nodes and for a specific timeout.
# If no min node count is provided it will use the scale settings for the cluster
compute_target.wait_for_completion(show_output=True,
min_node_count=None,
timeout_in_minutes=20)
# For a more detailed view of current cluster status, use the 'status' property
print(compute_target.status.serialize())
aml_run_config = RunConfiguration()
# `compute_target` as defined in "Azure Machine Learning compute" section above
aml_run_config.target = compute_target
USE_CURATED_ENV = True
if USE_CURATED_ENV:
curated_environment = Environment.get(workspace=ws,
name="AzureML-Tutorial")
aml_run_config.environment = curated_environment
else:
aml_run_config.environment.python.user_managed_dependencies = False
# Add some packages relied on by data prep step
aml_run_config.environment.python.conda_dependencies = CondaDependencies.create(
conda_packages=['pandas', 'scikit-learn'],
pip_packages=['azureml-sdk', 'azureml-dataprep[fuse,pandas]'],
pin_sdk_version=False)
dataprep_source_dir = "./dataprep_src"
entry_point = "prepare.py"
# `my_dataset` as defined above
ds_input = xyz_phishing_dataset.as_named_input('input1')
# `output_data1`, `compute_target`, `aml_run_config` as defined above
data_prep_step = PythonScriptStep(script_name=entry_point,
source_directory=dataprep_source_dir,
arguments=[
"--input",
ds_input.as_download(), "--output",
output_data1
],
compute_target=compute_target,
runconfig=aml_run_config,
allow_reuse=True)
train_source_dir = "./train_src"
train_entry_point = "train.py"
training_results = OutputFileDatasetConfig(name="training_results",
destination=def_blob_store)
train_step = PythonScriptStep(script_name=train_entry_point,
source_directory=train_source_dir,
arguments=[
"--prepped_data",
output_data1.as_input(),
"--training_results", training_results
],
compute_target=compute_target,
runconfig=aml_run_config,
allow_reuse=True)
# list of steps to run (`compare_step` definition not shown)
compare_models = [data_prep_step, train_step, compare_step]
# Build the pipeline
pipeline1 = Pipeline(workspace=ws, steps=[compare_models])
#dataset_consuming_step = PythonScriptStep(
# script_name="iris_train.py",
# inputs=[iris_tabular_dataset.as_named_input("iris_data")],
# compute_target=compute_target,
# source_directory=project_folder
#)
#run_context = Run.get_context()
#iris_dataset = run_context.input_datasets['iris_data']
#dataframe = iris_dataset.to_pandas_dataframe()
## Within a PythonScriptStep
#ws = Run.get_context().experiment.workspace
#step = PythonScriptStep(name="Hello World",
# script_name="hello_world.py",
# compute_target=aml_compute,
# source_directory=source_directory,
# allow_reuse=False,
# hash_paths=['hello_world.ipynb'])
# Submit the pipeline to be run
pipeline_run1 = Experiment(ws, 'Compare_Models_Exp').submit(pipeline1)
pipeline_run1.wait_for_completion()
min_node_count=None,
timeout_in_minutes=0)
script_params = {}
estimator = PyTorch(source_directory='./',
script_params=script_params,
compute_target=compute_target_gpu,
entry_script='train.py',
use_gpu=True,
pip_packages=[],
framework_version='1.2')
est_step = EstimatorStep(name="Train_Step",
estimator=estimator,
estimator_entry_script_arguments=auth_params,
runconfig_pipeline_params=None,
inputs=[],
outputs=[],
compute_target=compute_target_gpu)
est_step.run_after(process_step)
# step 3
pipeline = Pipeline(workspace=ws, steps=[process_step, est_step])
pipeline_run_first = Experiment(
ws, project_config['experiment_name']).submit(pipeline)
#https://docs.microsoft.com/en-us/python/api/azureml-pipeline-core/azureml.pipeline.core.pipelinerun?view=azure-ml-py
pipeline_run_first.wait_for_completion(show_output=True,
timeout_seconds=9223372036854775807,
raise_on_error=True)
print("..8. completed")
print('')
print('')
print("9. Create run object for the experiment...")
print('.............................................')
run = Run.get_context()
experimentName = run.experiment.name
print("..9. completed")
print('')
print('')
print("10. Submit build pipeline run, synchronously/blocking...")
print('.............................................')
pipelineRun = Experiment(amlWs, experimentName).submit(pipeline)
pipelineRun.wait_for_completion(show_output=True)
print("..10. completed")
print('')
print('')
print("11. Download pipeline output...")
print('.............................................')
# Get a handle to the output of containerize pipeline stage
pipelineStagesLog = pipelineRun.find_step_run(
'containerize')[0].get_output_data('containerize_output')
# Download locally
pipelineStagesLog.download('.', show_progress=True)
print("..11. completed")
print('')
print('')
def run_pipeline_steps(workspace, steps, experiment_name):
mlapp_pipeline = Pipeline(workspace=workspace, steps=steps)
pipeline_run = Experiment(workspace, experiment_name).submit(mlapp_pipeline)
pipeline_run.wait_for_completion()
def run_parallel(distributed_csv_iris_step,aml_interface):
ws = aml_interface.workspace
pipeline = Pipeline(workspace=ws, steps=[distributed_csv_iris_step])
pipeline_run = Experiment(ws, 'iris-prs').submit(pipeline)
pipeline_run.wait_for_completion(show_output=True)
def main():
# Ger our configs
with open("ptgnn/authentication.json") as jsonFile:
authData = json.load(jsonFile)[args.auth_cluster]
# Copy the convertCorpus script here. Done so we don't upload the corpus to Azure, or keep a copy of the script in here.
# (It's weird, I know. It works and has a purpose though)
convertCorpusLocation = Path("../convertCorpusForML.py")
convertCorpusAzureLocation = Path("./convertCorpusForML.py")
shutil.copy(convertCorpusLocation, convertCorpusAzureLocation)
# Grab the authentication data from the JSON file
subID = authData["subID"] # Get from Azure Portal; used for billing
resGroup = authData["resGroup"] # Name for the resource group
wsName = authData["wsName"] # Name for the workspace, which is the collection of compute clusters + experiments
computeName = authData["computeName"] # Name for computer cluster
datastoreName = authData["datastoreName"]
# Get the workspace, the compute target and the datastore
ws = Workspace.get(wsName, subscription_id=subID, resource_group=resGroup)
computeTarget = ComputeTarget(ws, computeName)
datastore = Datastore(ws, name=datastoreName)
# Download the entire corpus to the compute target. Save the DataReference obj here
# as_mount is also possible, but slows things down due to network opening of files
corpus_location = datastore.path(args.aml_location).as_download()
output_location = "./"
# The files that will be uploaded for usage by our script (everything in the azure folder)
source_directory = "."
# params for the script
params = {
"--corpus_location": corpus_location,
"--output_folder": output_location,
"--aml": "",
"--training_percent": args.training_percent,
"--validation_percent": args.validation_percent,
"-c": ""
}
if args.log_num is not None:
params["-l"] = args.log_num
tags = {
"logs": str(args.log_num)
}
else:
tags = {
"logs": "MAX"
}
if args.statement_generation:
params["-s"] = ""
tags["generationType"] = "Statement"
else:
tags["generationType"] = "Severity"
# Set up the estimator object. Note the inputs element, it tells azure that corpus_location in params
# will be a DataReference Object.
est = Estimator(source_directory=source_directory,
compute_target=computeTarget,
entry_script='convertCorpusForML.py',
script_params=params,
inputs=[corpus_location],
conda_packages=["pip"],
pip_packages=["azureml-core", "tqdm", "numpy", "protobuf"],
use_docker=True,
use_gpu=False)
# Start the experiment
run = Experiment(ws, args.exp_name).submit(config=est, tags=tags)
# remove the copy of convertCorpus (Remember, don't question this)
convertCorpusAzureLocation.unlink()
# print out the portral URL
# print("Portal URL: ", run.get_portal_url())
# this will stream everything that the compute target does.
print("Experiment Started. Remember you can exit out of this program but the experiment will still run on Azure!")
run.wait_for_completion(show_output=True)
def main():
# Replace this with the Use of Service Principal for authenticating with the Workspace
ws = Workspace.from_config()
# Choose a name for your CPU cluster
cpu_cluster_name = "cpu-cluster"
# Verify that cluster does not exist already
try:
cpu_cluster = ComputeTarget(workspace=ws, name=cpu_cluster_name)
print('Found existing cluster, use it.')
except ComputeTargetException:
compute_config = AmlCompute.provisioning_configuration(
vm_size='STANDARD_D2_V2', max_nodes=4)
cpu_cluster = ComputeTarget.create(ws, cpu_cluster_name,
compute_config)
cpu_cluster.wait_for_completion(show_output=True)
# Run configuration for R
rc = RunConfiguration()
rc.framework = "R"
# Run configuration for python
py_rc = RunConfiguration()
py_rc.framework = "Python"
py_rc.environment.docker.enabled = True
# Combine GitHub and Cran packages for R env
rc.environment.r = RSection()
rc.environment.docker.enabled = True
# Upload iris data to the datastore
# target_path = "iris_data"
# upload_files_to_datastore(ds,
# list("./iris.csv"),
# target_path = target_path,
# overwrite = TRUE)
training_data = DataReference(
datastore=ws.get_default_datastore(),
data_reference_name="iris_data",
path_on_datastore="iris_data/iris.csv",
)
print('Succesfull')
print(training_data)
# PipelineData object for newly trained model
# trained_model_dir = PipelineData(
# name="trained_model", datastore=ws.get_default_datastore(), is_directory=True
# )
# Train and Register the model
train_step = RScriptStep(
script_name="train.R",
arguments=[training_data],
inputs=[training_data],
compute_target=cpu_cluster_name,
source_directory=".",
runconfig=rc,
allow_reuse=True,
)
# Deploy the trained model
print("Step Train created")
steps = [train_step]
train_pipeline = Pipeline(workspace=ws, steps=steps)
train_pipeline.validate()
pipeline_run = Experiment(ws, 'iris_training').submit(train_pipeline)
pipeline_run.wait_for_completion(show_output=True)
published_pipeline = train_pipeline.publish(
name="iris-train",
description="Model training/retraining pipeline",
)
print(f"Published pipeline: {published_pipeline.name}")
print(f"for build {published_pipeline.version}")
ws = Workspace(sub_id, db_rg, az_ml_ws_name, auth=sp)
return ws
def get_db_compute(ws: Workspace) -> DatabricksCompute:
db_compute = None
try:
db_compute = ComputeTarget(ws, db_compute_name)
except ComputeTargetException:
attach_config = DatabricksCompute.attach_configuration(
resource_group=db_rg,
workspace_name=db_workspace_name,
access_token=db_access_token)
db_compute = ComputeTarget.attach(ws, db_compute_name, attach_config)
db_compute.wait_for_completion(True)
return db_compute
ws = get_workspace()
db_compute = get_db_compute(ws)
train_step = PythonScriptStep(script_name="train.py",
compute_target=db_compute,
source_directory="./notebooks",
num_workers=4)
steps = [train_step]
pipeline = Pipeline(workspace=ws, steps=steps)
run = Experiment(ws, "anomaly_detector").submit(pipeline)
run.wait_for_completion()
