def getpipelineSubmissionObject(ws,submission): print("::debug::Loading module to receive experiment config") root = os.environ.get("GITHUB_WORKSPACE", default="./") print("::debug::Adding root to system path") sys.path.insert(0, f"{root}") pipeline_yaml_path = submission["pipeline_yamlFile"]; if pipeline_yaml_path != None and pipeline_yaml_path != "": # Load pipeline yaml definition print("::debug::Loading pipeline yaml definition") try: run_config = Pipeline.load_yaml( workspace=ws, filename=pipeline_yaml_path ) return run_config; # this is a pipeline run config except Exception as exception: print(f"::error::Error when loading pipeline yaml definition your repository (Path: /{pipeline_yaml_path}): {exception}") raise AMLExperimentConfigurationException(f"Error when loading pipeline yaml definition your repository (Path: /{pipeline_yaml_path}): {exception}") # means yaml file doesn't exists. else: # means there are steps which user wants to execute. # steps here submissionSteps = [] for step in submission["steps"]: pipelineStep = getRunSubmissionObject(ws, step["pipelineStep_run_config_file"], step["run_config_function"], step["inputs"]) submissionSteps.append(pipelineStep) print(submissionSteps) pipeline1 = Pipeline(workspace=ws, steps=submissionSteps) print ("Pipeline is built") print(pipeline1) pipeline1.validate(); print("Pipeline validation complete") return pipeline1
def load_pipeline_yaml(workspace, pipeline_yaml_file): try: run_config = Pipeline.load_yaml(workspace=workspace, filename=pipeline_yaml_file) except Exception as exception: print( f"::debug::Error when loading pipeline yaml definition your repository (Path: /{pipeline_yaml_file}): {exception}" ) run_config = None return run_config
def submit_pipeline( workspace=None, # Auto populated args + object pipeline_id=None, experiment_name=None, pipeline_yaml=None, pipeline_params=None, datapath_params=None, output_file=None, # We enforce a logger logger=None): """ Submit a pipeline run based on a published pipeline ID """ if pipeline_id is None and pipeline_yaml is None: raise UserErrorException("Please specify a pipeline ID or a pipeline YAML file") published_pipeline = None pipeline = None if pipeline_id is not None: from azureml.pipeline.core import PublishedPipeline published_pipeline = PublishedPipeline.get(workspace, pipeline_id) if experiment_name is None or experiment_name == '': # Use the pipeline name as the experiment name experiment_name = published_pipeline._sanitize_name() else: from azureml.pipeline.core import Pipeline pipeline = Pipeline.load_yaml(workspace, pipeline_yaml) if experiment_name is None: raise UserErrorException("Please specify an experiment name") assigned_params = _parse_key_values(pipeline_params, 'Parameter assignment') datapaths = _parse_key_values(datapath_params, 'Datapath assignment') for datapath_param_name in datapaths: datastore_with_path = datapaths[datapath_param_name] if '/' not in datastore_with_path: raise UserErrorException("Datapath value %s should have format datastore/path" % datastore_with_path) path_tokens = datastore_with_path.split('/', 1) from azureml.core import Datastore from azureml.data.datapath import DataPath datastore = Datastore(workspace, path_tokens[0]) assigned_params[datapath_param_name] = DataPath(datastore=datastore, path_on_datastore=path_tokens[1]) dict_output = _pipeline_run_submit(experiment_name, assigned_params, published_pipeline, pipeline, workspace, output_file, logger) return dict_output
def load_pipeline_yaml(workspace, pipeline_yaml_file): root = os.environ.get("GITHUB_WORKSPACE", default=None) print("::debug::Adding root to system path") sys.path.insert(1, f"{root}") try: run_config = Pipeline.load_yaml(workspace=workspace, filename=pipeline_yaml_file) except Exception as exception: print( f"::debug::Error when loading pipeline yaml definition your repository (Path: /{pipeline_yaml_file}): {exception}" ) run_config = None return run_config
def main(): load_dotenv() ws = AzureMLUtils.get_workspace() print("Workspace lookup successful") # read command line parameters pipeline_yaml_path, pipeline_name = getRuntimeArgs() # Load the pipeline from YAML file pipeline = Pipeline.load_yaml(ws, filename=pipeline_yaml_path) # Publish the pipeline published_pipeline = pipeline.publish(pipeline_name) print("Published pipeline id: ", published_pipeline.id) # This line is needed to for Azure Devops to set PIPELINE_ID as environment variable. # This is needed in the next step (if there is need to run the pipeline) print("##vso[task.setvariable variable=PIPELINE_ID;isOutput=true]", published_pipeline.id)
from azureml.core import Workspace from azureml.pipeline.core import Pipeline from azureml.core.authentication import AzureCliAuthentication # Parse Definition Arguments parser = argparse.ArgumentParser() parser.add_argument("-d", "--definition", dest='definition', type=str, help='The name of the ML pipeline YML definition file', default='train_pipeline.yml') args = parser.parse_args() # Load Pipeline from YML pipeline_definition_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), '../../pipelines', args.definition) ws = Workspace.from_config(auth=AzureCliAuthentication()) pipeline = Pipeline.load_yaml(ws, pipeline_definition_path) print("Parsed pipeline graph from YML {}".format(pipeline.graph)) # Run Pipeline pipeline_run = pipeline.submit(experiment_name="test") # Wait for Pipeline Run Completion print(pipeline_run.wait_for_completion())