def submitPipelineRun(self, pipelineUrl, productName, deploymentName,
apiVersion, entryPoint, userInput,
predecessorOperationId, userId, subscriptionId):
operationId = str('a' + uuid4().hex[1:])
experimentName = subscriptionId
exp = Experiment(self._workspace, experimentName)
tags = {
'userId': userId,
'productName': productName,
'deploymentName': deploymentName,
'apiVersion': apiVersion,
'operationName': entryPoint,
'operationId': operationId,
'subscriptionId': subscriptionId,
'predecessorOperationId': predecessorOperationId
}
parameters = {
'userId': userId,
'userInput': userInput,
'productName': productName,
'deploymentName': deploymentName,
'apiVersion': apiVersion,
'operationName': entryPoint,
'operationId': operationId,
'subscriptionId': subscriptionId,
'predecessorOperationId': predecessorOperationId
}
pipeline = PublishedPipeline.get(
workspace=self._workspace,
id=self.get_pipeline_id_from_url(pipelineUrl))
exp.submit(pipeline, tags=tags, pipeline_parameters=parameters)
return operationId
def run_pipeline_endpoint(workspace: Workspace, pipeline_endpoint_id, experiment_name, config_str,
pipeline_version=None):
pipeline_endpoint_by_name = PublishedPipeline.get(workspace=workspace, id=pipeline_endpoint_id)
run_id = pipeline_endpoint_by_name.submit(
workspace=workspace,
experiment_name=experiment_name,
pipeline_parameters={
'config': config_str
})
print(run_id)
def submit_pipeline(
workspace=None, # Auto populated args + object
pipeline_id=None,
experiment_name=None,
pipeline_yaml=None,
pipeline_params=None,
datapath_params=None,
output_file=None,
# We enforce a logger
logger=None):
"""
Submit a pipeline run based on a published pipeline ID
"""
if pipeline_id is None and pipeline_yaml is None:
raise UserErrorException("Please specify a pipeline ID or a pipeline YAML file")
published_pipeline = None
pipeline = None
if pipeline_id is not None:
from azureml.pipeline.core import PublishedPipeline
published_pipeline = PublishedPipeline.get(workspace, pipeline_id)
if experiment_name is None or experiment_name == '':
# Use the pipeline name as the experiment name
experiment_name = published_pipeline._sanitize_name()
else:
from azureml.pipeline.core import Pipeline
pipeline = Pipeline.load_yaml(workspace, pipeline_yaml)
if experiment_name is None:
raise UserErrorException("Please specify an experiment name")
assigned_params = _parse_key_values(pipeline_params, 'Parameter assignment')
datapaths = _parse_key_values(datapath_params, 'Datapath assignment')
for datapath_param_name in datapaths:
datastore_with_path = datapaths[datapath_param_name]
if '/' not in datastore_with_path:
raise UserErrorException("Datapath value %s should have format datastore/path" % datastore_with_path)
path_tokens = datastore_with_path.split('/', 1)
from azureml.core import Datastore
from azureml.data.datapath import DataPath
datastore = Datastore(workspace, path_tokens[0])
assigned_params[datapath_param_name] = DataPath(datastore=datastore, path_on_datastore=path_tokens[1])
dict_output = _pipeline_run_submit(experiment_name, assigned_params, published_pipeline, pipeline,
workspace, output_file, logger)
return dict_output
def get_pipeline(workspace,
env,
pipeline_id=None):
if pipeline_id is not None:
scoring_pipeline = PublishedPipeline.get(workspace,id=pipeline_id)
else:
pipeline_list = PublishedPipeline.list(workspace)
scoring_pipeline = [pl for pl in pipeline_list if pl.name == env.scoring_pipeline_name]
if len(scoring_pipeline) == 0:
raise ValueError('no available pipeline to download!')
else:
scoring_pipeline = scoring_pipeline[0]
return scoring_pipeline
def get_pipeline(pipeline_id, ws: Workspace, env: Env):
if pipeline_id is not None:
scoringpipeline = PublishedPipeline.get(ws, pipeline_id)
else:
pipelines = PublishedPipeline.list(ws)
scoringpipelinelist = [
pl for pl in pipelines if pl.name == env.scoring_pipeline_name
] # noqa E501
if scoringpipelinelist.count == 0:
raise Exception("No pipeline found matching name:{}".format(
env.scoring_pipeline_name) # NOQA: E501
)
else:
# latest published
scoringpipeline = scoringpipelinelist[0]
return scoringpipeline
print("Azure ML SDK version:", azureml.core.VERSION)
parser = argparse.ArgumentParser("publish_to_pipeline_endpoint")
parser.add_argument("--pipeline_id", type=str, help="Id of the published pipeline that should get added to the Pipeline Endpoint", required=True)
parser.add_argument("--pipeline_endpoint_name", type=str, help="Name of the Pipeline Endpoint that the the pipeline should be added to", required=True)
parser.add_argument("--pipeline_endpoint_description", type=str, help="Description for the Pipeline Endpoint", default="Pipeline Endpoint", required=False)
args = parser.parse_args()
print(f'Arguments: {args}')
print('Connecting to workspace')
ws = Workspace.from_config()
print(f'WS name: {ws.name}\nRegion: {ws.location}\nSubscription id: {ws.subscription_id}\nResource group: {ws.resource_group}')
endpoint_name = args.pipeline_endpoint_name
pipeline_description = args.pipeline_endpoint_description
pipeline_id = args.pipeline_id
published_pipeline = PublishedPipeline.get(workspace=ws, id=pipeline_id)
# Add tested published pipeline to pipeline endpoint
try:
pl_endpoint = PipelineEndpoint.get(workspace=ws, name=endpoint_name)
pl_endpoint.add_default(published_pipeline)
print(f'Added pipeline {pipeline_id} to Pipeline Endpoint with name {endpoint_name}')
except Exception:
print(f'Will create new Pipeline Endpoint with name {endpoint_name} with pipeline {pipeline_id}')
pl_endpoint = PipelineEndpoint.publish(workspace=ws,
name=endpoint_name,
pipeline=published_pipeline,
description=pipeline_description)
cli_auth = AzureCliAuthentication()
##------------- Get Workspace
subscriptionId = "<your subscription id>" # make this a parameter
resourceGroup = "<your resource group>" # make this a parameter
workspaceName = "<your ml workspace name>" # make this a parameter
ws = Workspace(subscriptionId, resourceGroup, workspaceName, auth=cli_auth)
##------------- Run Published pipeline using REST endpoint
aad_token = cli_auth.get_authentication_header()
published_pipeline_id = "ab0691a9-438f-416b-a146-5c7660d1be11" # Replace this with the published pipeline id
published_pipeline = PublishedPipeline.get(ws, published_pipeline_id)
rest_endpoint = published_pipeline.endpoint
print("Rest endpoint: " + rest_endpoint)
response = requests.post(rest_endpoint,
headers=aad_token,
json={
"ExperimentName": "quality_prediction_gb",
"RunSource": "SDK",
"ParameterAssignments": {
"modelName": "quality_gbm_model.pkl",
"datasetName": "qualitydataset",
"datasetStorePath": "/inputdata/train.csv"
}
})
print(response)
def pipeline():
return PublishedPipeline.get(workspace=ws, id=pipeline_id)