def getpipelineSubmissionObject(ws,submission):
print("::debug::Loading module to receive experiment config")
root = os.environ.get("GITHUB_WORKSPACE", default="./")
print("::debug::Adding root to system path")
sys.path.insert(0, f"{root}")
pipeline_yaml_path = submission["pipeline_yamlFile"];
if pipeline_yaml_path != None and pipeline_yaml_path != "":
# Load pipeline yaml definition
print("::debug::Loading pipeline yaml definition")
try:
run_config = Pipeline.load_yaml(
workspace=ws,
filename=pipeline_yaml_path
)
return run_config; # this is a pipeline run config
except Exception as exception:
print(f"::error::Error when loading pipeline yaml definition your repository (Path: /{pipeline_yaml_path}): {exception}")
raise AMLExperimentConfigurationException(f"Error when loading pipeline yaml definition your repository (Path: /{pipeline_yaml_path}): {exception}")
# means yaml file doesn't exists.
else:
# means there are steps which user wants to execute.
# steps here
submissionSteps = []
for step in submission["steps"]:
pipelineStep = getRunSubmissionObject(ws, step["pipelineStep_run_config_file"], step["run_config_function"], step["inputs"])
submissionSteps.append(pipelineStep)
print(submissionSteps)
pipeline1 = Pipeline(workspace=ws, steps=submissionSteps)
print ("Pipeline is built")
print(pipeline1)
pipeline1.validate();
print("Pipeline validation complete")
return pipeline1
def load_pipeline_yaml(workspace, pipeline_yaml_file):
try:
run_config = Pipeline.load_yaml(workspace=workspace,
filename=pipeline_yaml_file)
except Exception as exception:
print(
f"::debug::Error when loading pipeline yaml definition your repository (Path: /{pipeline_yaml_file}): {exception}"
)
run_config = None
return run_config
def submit_pipeline(
workspace=None, # Auto populated args + object
pipeline_id=None,
experiment_name=None,
pipeline_yaml=None,
pipeline_params=None,
datapath_params=None,
output_file=None,
# We enforce a logger
logger=None):
"""
Submit a pipeline run based on a published pipeline ID
"""
if pipeline_id is None and pipeline_yaml is None:
raise UserErrorException("Please specify a pipeline ID or a pipeline YAML file")
published_pipeline = None
pipeline = None
if pipeline_id is not None:
from azureml.pipeline.core import PublishedPipeline
published_pipeline = PublishedPipeline.get(workspace, pipeline_id)
if experiment_name is None or experiment_name == '':
# Use the pipeline name as the experiment name
experiment_name = published_pipeline._sanitize_name()
else:
from azureml.pipeline.core import Pipeline
pipeline = Pipeline.load_yaml(workspace, pipeline_yaml)
if experiment_name is None:
raise UserErrorException("Please specify an experiment name")
assigned_params = _parse_key_values(pipeline_params, 'Parameter assignment')
datapaths = _parse_key_values(datapath_params, 'Datapath assignment')
for datapath_param_name in datapaths:
datastore_with_path = datapaths[datapath_param_name]
if '/' not in datastore_with_path:
raise UserErrorException("Datapath value %s should have format datastore/path" % datastore_with_path)
path_tokens = datastore_with_path.split('/', 1)
from azureml.core import Datastore
from azureml.data.datapath import DataPath
datastore = Datastore(workspace, path_tokens[0])
assigned_params[datapath_param_name] = DataPath(datastore=datastore, path_on_datastore=path_tokens[1])
dict_output = _pipeline_run_submit(experiment_name, assigned_params, published_pipeline, pipeline,
workspace, output_file, logger)
return dict_output
def load_pipeline_yaml(workspace, pipeline_yaml_file):
root = os.environ.get("GITHUB_WORKSPACE", default=None)
print("::debug::Adding root to system path")
sys.path.insert(1, f"{root}")
try:
run_config = Pipeline.load_yaml(workspace=workspace,
filename=pipeline_yaml_file)
except Exception as exception:
print(
f"::debug::Error when loading pipeline yaml definition your repository (Path: /{pipeline_yaml_file}): {exception}"
)
run_config = None
return run_config
def main():
load_dotenv()
ws = AzureMLUtils.get_workspace()
print("Workspace lookup successful")
# read command line parameters
pipeline_yaml_path, pipeline_name = getRuntimeArgs()
# Load the pipeline from YAML file
pipeline = Pipeline.load_yaml(ws, filename=pipeline_yaml_path)
# Publish the pipeline
published_pipeline = pipeline.publish(pipeline_name)
print("Published pipeline id: ", published_pipeline.id)
# This line is needed to for Azure Devops to set PIPELINE_ID as environment variable.
# This is needed in the next step (if there is need to run the pipeline)
print("##vso[task.setvariable variable=PIPELINE_ID;isOutput=true]",
published_pipeline.id)
from azureml.core import Workspace
from azureml.pipeline.core import Pipeline
from azureml.core.authentication import AzureCliAuthentication
# Parse Definition Arguments
parser = argparse.ArgumentParser()
parser.add_argument("-d",
"--definition",
dest='definition',
type=str,
help='The name of the ML pipeline YML definition file',
default='train_pipeline.yml')
args = parser.parse_args()
# Load Pipeline from YML
pipeline_definition_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)), '../../pipelines',
args.definition)
ws = Workspace.from_config(auth=AzureCliAuthentication())
pipeline = Pipeline.load_yaml(ws, pipeline_definition_path)
print("Parsed pipeline graph from YML {}".format(pipeline.graph))
# Run Pipeline
pipeline_run = pipeline.submit(experiment_name="test")
# Wait for Pipeline Run Completion
print(pipeline_run.wait_for_completion())
