def publish_automl_pipeline(ws, pipeline, build_id):
published_pipeline = pipeline.publish(name=PIPELINE_NAME,
description=build_id,
version=build_id)
try:
pipeline_endpoint = PipelineEndpoint.get(workspace=ws,
name=PIPELINE_ENDPOINT_NAME)
print("pipeline endpoint exists, add a version")
pipeline_endpoint.add_default(published_pipeline)
except Exception:
print("publish a new pipeline endpoint")
pipeline_endpoint = PipelineEndpoint.publish(
workspace=ws,
name=PIPELINE_ENDPOINT_NAME,
pipeline=published_pipeline,
description='NYCtaxi_automl_training_pipeline_endpoint')
print(f'Published pipeline: {published_pipeline.name}')
print(f' version: {published_pipeline.version}')
print(f'Pipeline endpoint: {pipeline_endpoint.name}')
print('##vso[task.setvariable variable=PIPELINE_ENDPOINT_NAME;]{}'.format(
pipeline_endpoint.name))
print('##vso[task.setvariable variable=PIPELINE_ENDPOINT_DEFAULT_VER;]{}'.
format(pipeline_endpoint.default_version))
print('##vso[task.setvariable variable=PUBLISHED_PIPELINE_VERSION;]{}'.
format(published_pipeline.version))
return pipeline_endpoint
def add_endpoint(ws: Workspace, pipeline: PublishedPipeline,
endpoint_name: str) -> PipelineEndpoint:
endpoint_list = [p.name for p in PipelineEndpoint.list(ws)]
endpoint = None
# endpoint does not exist so add
if endpoint_name in endpoint_list:
endpoint = PipelineEndpoint.get(workspace=ws, name=endpoint_name)
endpoint.add_default(published_pipeline)
else:
endpoint = PipelineEndpoint.publish(
workspace=ws,
name=endpoint_name,
pipeline=published_pipeline,
description="Seer Pipeline Endpoint")
return endpoint
def publish_pipeline(workspace: Workspace,
steps: List,
name: str,
description: str = "") -> Tuple[str, PipelineEndpoint]:
published_pipeline = Pipeline(workspace=workspace,
steps=steps).publish(name)
try:
pipeline_endpoint = PipelineEndpoint.get(workspace, name=name)
pipeline_endpoint.add_default(published_pipeline)
except ErrorResponseException:
pipeline_endpoint = PipelineEndpoint.publish(
workspace,
name=name,
pipeline=published_pipeline,
description=description)
return published_pipeline.id, pipeline_endpoint
def main(workspace):
ws = Workspace.from_config()
print(ws)
print("------------")
published_pipeline = PipelineEndpoint.get(workspace=ws, name="aml-run-val")
print(published_pipeline)
print("------------")
pipeline_run = published_pipeline.submit("aml-run-val")
print("run completed")
return pipeline_run
def start_sonntagsfrage_pipeline():
# --- read params
pipelineName = "Sonntagsfrage-Forecaster-Pipeline"
workspaceName = "Sonntagsfrage-predictor"
# --- load configuration
print("Loading configuration...")
for p in sys.path:
print(p)
env = Env()
env.read_env("Azure_ML/foundation.env")
azure_subscription_id = env("AZURE_SUBSCRIPTION_ID")
resource_group = env("RESOURCE_GROUP")
workspaceName = env("WORKSPACE_NAME")
# --- get creds for aservice principalv
with open('Azure_ML/service_principals/sonntagsfrage-ml-auth-file.json'
) as f:
svcpr_data = json.load(f)
# --- get service principal
svc_pr = ServicePrincipalAuthentication(
tenant_id=svcpr_data['tenantId'],
service_principal_id=svcpr_data['clientId'],
service_principal_password=svcpr_data['clientSecret'])
# --- get workspace, compute target, run config
print("Getting workspace and compute target...")
workspace = Workspace(subscription_id=azure_subscription_id,
resource_group=resource_group,
workspace_name=workspaceName,
auth=svc_pr)
print(f"Get pipeline endpoint '{pipelineName}' (as configured)...")
pipeline_endpoint = PipelineEndpoint.get(workspace=workspace,
name=pipelineName)
print(f"Triggering pipeline endpoint '{pipelineName}' (as configured)...")
pipeline_run = Experiment(workspace,
pipelineName).submit(pipeline_endpoint)
return f"Pipeline {pipelineName} has been successfully started!", pipeline_run
def clean_azml_workspace(ctx):
"""
[WARNING] Only use in test-only workspace. Remove or disable all compute clusters, published pipelines, published pipeline endpoints and schedules from Azure ML workspace.
"""
ws = Workspace.from_config()
# remove compute clusters
for _, compute in ws.compute_targets.items():
if not compute.provisioning_state == "Deleting":
compute.delete()
# deactivate schedules
for s in Schedule.list(ws):
s.disable()
# remove pipeline endpoints
for pe in PipelineEndpoint.list(ws):
pe.disable()
# remove pipelines
for p in PublishedPipeline.list(ws):
p.disable()
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
# Retrieve environment variables
subscriptionId = os.environ["SUBSCRIPTION_ID"]
rg_name = os.environ["RESOURCE_GROUP"]
ws_name = os.environ["WORKSPACE_NAME"]
pipeline_endpoint_name = os.environ["PIPELINE_ENDPOINT_NAME"]
experiment_name = os.environ["EXPERIMENT_NAME"]
# Managed identity authentication
msi_auth = MsiAuthentication()
# Azure ML workspace
aml_workspace = Workspace(subscription_id=subscriptionId,
resource_group=rg_name,
workspace_name=ws_name,
auth=msi_auth)
logging.info(f"Connected to workspace: {aml_workspace}")
try:
# Submit a job to a pipeline endpoint
pipeline_endpoint_by_name = PipelineEndpoint.get(
workspace=aml_workspace, name=pipeline_endpoint_name)
run_id = pipeline_endpoint_by_name.submit(experiment_name)
logging.info(f"Pipeline Endpoint: {run_id}")
logging.info(
"Successfully submit a job to the default version of a pipeline endpoint"
)
return func.HttpResponse(f"run_id: {run_id}")
except Exception as ex:
logging.exception(
f'main function stopped execution due to the following error: {ex}'
)
print("Azure ML SDK version:", azureml.core.VERSION)
parser = argparse.ArgumentParser("publish_to_pipeline_endpoint")
parser.add_argument("--pipeline_id", type=str, help="Id of the published pipeline that should get added to the Pipeline Endpoint", required=True)
parser.add_argument("--pipeline_endpoint_name", type=str, help="Name of the Pipeline Endpoint that the the pipeline should be added to", required=True)
parser.add_argument("--pipeline_endpoint_description", type=str, help="Description for the Pipeline Endpoint", default="Pipeline Endpoint", required=False)
args = parser.parse_args()
print(f'Arguments: {args}')
print('Connecting to workspace')
ws = Workspace.from_config()
print(f'WS name: {ws.name}\nRegion: {ws.location}\nSubscription id: {ws.subscription_id}\nResource group: {ws.resource_group}')
endpoint_name = args.pipeline_endpoint_name
pipeline_description = args.pipeline_endpoint_description
pipeline_id = args.pipeline_id
published_pipeline = PublishedPipeline.get(workspace=ws, id=pipeline_id)
# Add tested published pipeline to pipeline endpoint
try:
pl_endpoint = PipelineEndpoint.get(workspace=ws, name=endpoint_name)
pl_endpoint.add_default(published_pipeline)
print(f'Added pipeline {pipeline_id} to Pipeline Endpoint with name {endpoint_name}')
except Exception:
print(f'Will create new Pipeline Endpoint with name {endpoint_name} with pipeline {pipeline_id}')
pl_endpoint = PipelineEndpoint.publish(workspace=ws,
name=endpoint_name,
pipeline=published_pipeline,
description=pipeline_description)
from azureml.pipeline.core import PipelineEndpoint
from azureml.core import Workspace
from azureml.core.authentication import ServicePrincipalAuthentication
from azureml.core.authentication import AzureCliAuthentication
import os
from azureml.core import Run
run = Run.get_context()
ws = run.experiment.workspace
published_pipeline = PipelineEndpoint.get(workspace=ws,
name="aml-run-regression")
print(published_pipeline)
print("submitting pipeline aml-run-val")
pipeline_run = published_pipeline.submit("github-val-yueyue")
print("pipeline aml-run-val run completed")
def main():
"""Build pipeline."""
# Environment variables
env = Env()
# Azure ML workspace
aml_workspace = Workspace.get(
name=env.workspace_name,
subscription_id=env.subscription_id,
resource_group=env.resource_group,
)
logger.info(f"Azure ML workspace: {aml_workspace}")
# Azure ML compute cluster
aml_compute = get_compute(aml_workspace, env.compute_name)
logger.info(f"Aazure ML compute cluster: {aml_compute}")
# Azure ML environment
environment = Environment(name=env.aml_env_name)
conda_dep = CondaDependencies(
conda_dependencies_file_path="./local_development/dev_dependencies.yml"
)
environment.python.conda_dependencies = conda_dep
run_config = RunConfiguration()
run_config.environment = environment
# Pipeline Data
preparation_pipelinedata = PipelineData("preparation_pipelinedata",
is_directory=True).as_dataset()
extraction_pipelinedata = PipelineData("extraction_pipelinedata",
is_directory=True)
training_pipelinedata = PipelineData("training_pipelinedata",
is_directory=True)
# List of pipeline steps
step_list = list()
preparation_step = PythonScriptStep(
name="preparation-step",
compute_target=aml_compute,
source_directory=env.sources_directory_train,
script_name=env.preparation_step_script_path,
outputs=[preparation_pipelinedata],
arguments=[
"--input_path", env.input_dir, "--output_path",
preparation_pipelinedata, "--datastore_name",
env.blob_datastore_name
],
runconfig=run_config)
step_list.append(preparation_step)
parallel_run_config = ParallelRunConfig(
source_directory=env.sources_directory_train,
entry_script=env.extraction_step_script_path,
mini_batch_size=env.mini_batch_size,
error_threshold=env.error_threshold,
output_action="append_row",
environment=environment,
compute_target=aml_compute,
node_count=env.node_count,
run_invocation_timeout=env.run_invocation_timeout,
process_count_per_node=env.process_count_per_node,
append_row_file_name="extraction_output.txt")
extraction_step = ParallelRunStep(
name="extraction-step",
inputs=[preparation_pipelinedata],
output=extraction_pipelinedata,
arguments=["--output_dir", extraction_pipelinedata],
parallel_run_config=parallel_run_config)
step_list.append(extraction_step)
training_step = PythonScriptStep(
name="traning-step",
compute_target=aml_compute,
source_directory=env.sources_directory_train,
script_name=env.training_step_script_path,
inputs=[extraction_pipelinedata],
outputs=[training_pipelinedata],
arguments=[
"--input_dir", extraction_pipelinedata, "--output_dir",
training_pipelinedata
],
runconfig=run_config)
step_list.append(training_step)
# Build pipeline
pipeline = Pipeline(workspace=aml_workspace, steps=step_list)
pipeline.validate()
logger.info(f"Built pipeline {pipeline}")
# Publish pipeline
published_pipeline = pipeline.publish(
env.pipeline_name,
description=env.pipeline_name,
version=datetime.utcnow().isoformat())
try:
pipeline_endpoint = PipelineEndpoint.get(
workspace=aml_workspace, name=env.pipeline_endpoint_name)
pipeline_endpoint.add_default(published_pipeline)
except ErrorResponseException:
pipeline_endpoint = PipelineEndpoint.publish(
workspace=aml_workspace,
name=env.pipeline_endpoint_name,
pipeline=published_pipeline,
description=env.pipeline_endpoint_name)
pipeline = Pipeline(
workspace=workspace,
steps=pipeline_steps,
description=pipeline_description,
)
pipeline.validate()
published_pipeline = pipeline.publish(
# name=pipeline_name, description=pipeline_description, version={...some version...}
name=pipeline_name,
description=pipeline_description,
)
print(f"Newly published pipeline id: {published_pipeline.id}")
try:
pipeline_endpoint = PipelineEndpoint.get(workspace=workspace,
name=pipeline_name)
pipeline_endpoint.add(published_pipeline)
except:
pipeline_endpoint = PipelineEndpoint.publish(
workspace=workspace,
name=pipeline_name,
pipeline=published_pipeline,
description=f"Pipeline Endpoint for {pipeline_name}",
)
# TODO: cleanup older pipeline endpoints(?)
# --- add a schedule for the pipeline (if told to do so)
# note: this is a sample schedule which runs time-based.
# there is also the option to trigger the pipeline based on changes.
# details at https://github.com/Azure/MachineLearningNotebooks/blob/4e7b3784d50e81c313c62bcdf9a330194153d9cd/how-t
"""
$ python -m ml_service.pipelines.run_pipeline --test_size=0.2
"""
if __name__ == "__main__":
# Environment variables
env = Env()
# Parse args
parser = argparse.ArgumentParser()
parser.add_argument('--test_size', type=float, default=0.2)
args = parser.parse_args()
# get workspace
ws = Workspace.from_config()
exp = Experiment(workspace=ws, name=env.aml_experiment_name)
# customize parameters
custom_pipeline_parameters = {
"pipeparam_test_size": args.test_size
}
print('custom_pipeline_parameters=', custom_pipeline_parameters)
# run pipeline
pipeline_endpoint = PipelineEndpoint.get(workspace=ws, name=env.aml_pipeline_endpoint_name)
pipeline_run = exp.submit(pipeline_endpoint, pipeline_parameters=custom_pipeline_parameters)
# print url
aml_url = pipeline_run.get_portal_url()
print(aml_url)
from azureml.core import Workspace, Datastore, Dataset
from azureml.pipeline.core import Pipeline, PublishedPipeline, PipelineEndpoint
from azureml.data.datapath import DataPath
import requests
#multi tenant with my account
from azureml.core.authentication import InteractiveLoginAuthentication
int_auth = InteractiveLoginAuthentication(tenant_id='your_tenant_id')
ws = Workspace.from_config(auth=int_auth)
ws.name
pipe = PipelineEndpoint.get(workspace=ws,
name="Wine_demo_dry_run-batch inference")
#submit with default parameters and a new experiment name
run = pipe.submit("wine_exp_submitted")
run.wait_for_completion(show_output=True)
from azureml.pipeline.core import Pipeline, PublishedPipeline, PipelineEndpoint
print(f'Azure ML SDK version: {azureml.core.VERSION}')
endpoint_name = "training-pipeline-endpoint"
pipeline_id = os.getenv('PIPELINE_ID')
# Connect to the workspace
ws = Workspace.from_config()
print(f'WS name: {ws.name}')
print(f'Region: {ws.location}')
print(f'Subscription id: {ws.subscription_id}')
print(f'Resource group: {ws.resource_group}')
print(f'Pipeline ID: {pipeline_id}')
published_pipeline = PublishedPipeline.get(workspace=ws, id=pipeline_id)
print(f'Published Pipeline: {published_pipeline}')
# Check if PipelineEndpoint already exists
if any(pe.name == endpoint_name for pe in PipelineEndpoint.list(ws)):
print(f'Pipeline Endpoint with name {endpoint_name} already exists, will add pipeline to it')
pipeline_endpoint = PipelineEndpoint.get(workspace=ws, name=endpoint_name)
pipeline_endpoint.add(published_pipeline)
# Set it to default, as we already tested it beforehand!
pipeline_endpoint.set_default(published_pipeline)
else:
print(f'Will create Pipeline Endpoint with name {endpoint_name}')
pipeline_endpoint = PipelineEndpoint.publish(workspace=ws,
name=endpoint_name,
pipeline=published_pipeline,
description="New Training Pipeline Endpoint")
print(f'Azure ML SDK version: {azureml.core.VERSION}')
endpoint_name = "training-pipeline-endpoint"
pipeline_id = os.getenv('PIPELINE_ID')
# Connect to the workspace
ws = Workspace.from_config()
print(f'WS name: {ws.name}')
print(f'Region: {ws.location}')
print(f'Subscription id: {ws.subscription_id}')
print(f'Resource group: {ws.resource_group}')
print(f'Pipeline ID: {pipeline_id}')
published_pipeline = PublishedPipeline.get(workspace=ws, id=pipeline_id)
print(f'Published Pipeline: {published_pipeline}')
try:
print(
f'Pipeline Endpoint with name {endpoint_name} already exists, will add pipeline to it'
)
pipeline_endpoint = PipelineEndpoint.get(workspace=ws, name=endpoint_name)
pipeline_endpoint.add_default(published_pipeline)
except Exception:
print(f'Will create Pipeline Endpoint with name {endpoint_name}')
pipeline_endpoint = PipelineEndpoint.publish(
workspace=ws,
name=endpoint_name,
pipeline=published_pipeline,
description="New Training Pipeline Endpoint")
def main():
# Environment variables
env = Env()
# Azure ML workspace
aml_workspace = Workspace.get(
name=env.workspace_name,
subscription_id=env.subscription_id,
resource_group=env.resource_group,
)
logger.info(f"Azure ML workspace: {aml_workspace}")
# Azure ML compute cluster
aml_compute = get_compute(aml_workspace, env.compute_name)
logger.info(f"Aazure ML compute cluster: {aml_compute}")
# Azure ML environment
environment = Environment(name=env.aml_env_name)
conda_dep = CondaDependencies(
conda_dependencies_file_path="./local_development/dev_dependencies.yml"
)
environment.docker.enabled = True
environment.docker.base_image = env.acr_image
environment.docker.base_image_registry.address = env.acr_address
environment.docker.base_image_registry.username = env.acr_username
environment.docker.base_image_registry.password = env.acr_password
environment.python.conda_dependencies = conda_dep
run_config = RunConfiguration()
run_config.environment = environment
# List of pipeline steps
step_list = list()
first_step = PythonScriptStep(name="first_step",
compute_target=aml_compute,
source_directory=env.sources_directory_train,
script_name=env.first_step_script_path,
outputs=[],
arguments=[
"--input_dataset_name",
env.input_dataset_name,
"--waves_dataset_name",
env.waves_dataset_name
],
runconfig=run_config)
step_list.append(first_step)
# Build pipeline
pipeline = Pipeline(workspace=aml_workspace, steps=step_list)
pipeline.validate()
logger.info(f"Built pipeline {pipeline}")
# Publish pipeline
published_pipeline = pipeline.publish(
env.pipeline_name,
description=env.pipeline_name,
version=datetime.utcnow().isoformat())
try:
pipeline_endpoint = PipelineEndpoint.get(
workspace=aml_workspace, name=env.pipeline_endpoint_name)
pipeline_endpoint.add_default(published_pipeline)
except ErrorResponseException:
pipeline_endpoint = PipelineEndpoint.publish(
workspace=aml_workspace,
name=env.pipeline_endpoint_name,
pipeline=published_pipeline,
description=env.pipeline_endpoint_name)