def remove_schedules(workspace, published_pipeline_id):
schedules = Schedule.list(workspace, pipeline_id=published_pipeline_id)
print("Found these schedules for the pipeline id {}:".format(
published_pipeline_id))
for schedule in schedules:
print(schedule.id)
if schedule.recurrence is not None:
schedule_id = schedule.id
fetched_schedule = Schedule.get(workspace, schedule_id)
fetched_schedule.disable(wait_for_provisioning=True)
print("Disabled schedule {}. New status is: {}".format(
fetched_schedule.id, fetched_schedule.status))
def create_schedules(workspace, published_pipeline_id, schedule_name,
experiment_name, time):
recurrence = ScheduleRecurrence(
frequency="Week",
interval=1,
week_days=["Monday", "Tuesday", "Wednesday", "Thursday", "Friday"],
time_of_day=time)
schedule = Schedule.create(workspace,
name=schedule_name,
pipeline_id=published_pipeline_id,
experiment_name=experiment_name,
recurrence=recurrence,
wait_for_provisioning=True)
def create_or_update_schedule(
workspace: Workspace,
name: str,
pipeline_id: str,
experiment_name: str,
recurrence: ScheduleRecurrence,
pipeline_parameters: Dict[str, Any],
) -> Schedule:
matching_schedules = [
s for s in Schedule.list(workspace) if s.name == name
]
for s in matching_schedules:
s.disable()
return Schedule.create(
workspace,
name=name,
pipeline_id=pipeline_id,
experiment_name=experiment_name,
recurrence=recurrence,
pipeline_parameters=pipeline_parameters,
)
def clean_azml_workspace(ctx):
"""
[WARNING] Only use in test-only workspace. Remove or disable all compute clusters, published pipelines, published pipeline endpoints and schedules from Azure ML workspace.
"""
ws = Workspace.from_config()
# remove compute clusters
for _, compute in ws.compute_targets.items():
if not compute.provisioning_state == "Deleting":
compute.delete()
# deactivate schedules
for s in Schedule.list(ws):
s.disable()
# remove pipeline endpoints
for pe in PipelineEndpoint.list(ws):
pe.disable()
# remove pipelines
for p in PublishedPipeline.list(ws):
p.disable()
compute_target = 'aml-cluster',
arguments = ['--in_folder', prepped_data,
'--reg', reg_param],
inputs = [prepped_data]
)
#we can run the pipeline on ONdemand like using rest endpoint or we can make a schedule like once in a day it will run with the help of Schedule recurrence library
from azureml.pipeline.core import ScheduleRecurrence, Schedule
daily = ScheduleRecurrence(frequency = 'Day', interval = 1)
pipeline_schedule = Schedule.create(ws, name = 'Daily training',
description='trains model every day',
pipeline_id = published_pipeline.id)
experiment_name = 'Training_pipeline',
recurrence = daily)
# we can also create it like, whenevr data will change pipeline will be run
from azureml.core import Datastore
from azureml.pipeline.core import Schedule
training_ds = Datastore(workspace = ws, name = "blob_data")
pipeline_schedule = Schedule.create(ws, name = 'Reactive training',
description='trains model on data change',
pipeline_id = published_pipeline_id,
from azureml.pipeline.core import Schedule
from azureml.core.datastore import Datastore
datastore = Datastore(workspace="Playground", name="workspaceblobstore")
schedule = Schedule.create(workspace, name="TestSchedule", pipeline_id="3100e87c-3300-400b-a5a5-470e85a100b3"
experiment_name="working version", datastore=datastore,
polling_interval=25, path_on_datastore="file/path")
from azureml.core import Experiment, Workspace
from azureml.pipeline.core import PublishedPipeline, Schedule, ScheduleRecurrence
ws = Workspace.from_config()
exp = Experiment(ws,
"MaxFreezerTemperatureExceededPipeline",
_create_in_cloud=True)
pipeline_id = PublishedPipeline.list(ws)[0]
schedule = Schedule.create(
ws,
name="four_updates_per_day",
description="runs the pipeline every 6 hours",
pipeline_id=pipeline_id,
recurrence=ScheduleRecurrence(
frequency="Hour",
interval=6,
start_time=None, # run instantly
time_zone=None, # default UTC
),
experiment_name=exp.name,
)
# Schedule.list(ws)
# schedule = Schedule.list(ws)[0]
# schedule.get_last_pipeline_run()
ws = Workspace(
subscription_id="45b59352-da58-4e3a-beab-1a4518951e4e",
resource_group="kk6gpv-rg",
workspace_name="kk6gpv-aml",
auth=ServicePrincipalAuthentication(
tenant_id=os.environ["tenant_id"],
service_principal_id=os.environ["sp_id"],
service_principal_password=os.environ["sp_password"],
),
)
dstor = ws.get_default_datastore()
# cancel all pipeline schedules
print("Scheduled pipelines before:")
scheds = Schedule.list(ws)
print(scheds)
for sched in scheds:
sched.disable()
print(sched.id)
print("Scheduled pipelines after:")
scheds = Schedule.list(ws)
print(scheds)
# Choose a name for the run history container in the workspace.
experiment_name = "retrain-noaaweather"
experiment = Experiment(ws, experiment_name)
output = {}
output["Subscription ID"] = ws.subscription_id
#we can easily get the pipeline endpoint
endpoint = published_pipe.endpoint
#we can also schedule the pipelines for some periodic interval
from azureml.pipeline.core import Schedule, ScheduleRecurrence
#for daily
daily = ScheduleRecurrence(frequency = 'day', interval=1)
pipeline_schedule = Schedule.create(ws, name = "Daily training",
desciption = "",
pipeline_id= published_pipe.id,
experiment_name = 'training_pipe',
recurrence=daily)
#for triggering
from azureml.core import Datastore
from azureml.pipeline.core import Schdule
trining_datastore = Datastore(ws, "blob_data")
pipeline_schedule = Schedule.create(ws, name, description=
pipeline_id = published_pipeline_id,
experiment_name = "training_pipeline",
datastore=trining_datastore,
path_on_datastore="data/training")
#use endpoint to start job
import requests
response = requests.post(rest_endpoint,
headers=auth_header,
json={"ExperimentName": "Batch_Prediction"})
run_id = response.json()["Id"]
#have it run auto
from azureml.pipeline.core import ScheduleRecurrence, Schedule
weekly = ScheduleRecurrence(frequency='Week', interval=1)
pipeline_schedule = Schedule.create(ws, name='Weekly Predictions',
description='batch inferencing',
pipeline_id=published_pipeline.id,
experiment_name='Batch_Prediction',
recurrence=weekly)
##EXERCISE
#1 - connect to ws
import azureml.core
from azureml.core import Workspace
#load from config file
ws = workspace.from_config()
print('ready to use azure {} to work with {}'.format(azureml.core.VERSION, ws.name))
from azureml.train.estimator import Estimator
# Check core SDK version number
print("SDK version:", azureml.core.VERSION)
from azureml.core import Workspace
ws = Workspace.from_config()
print(ws.get_details())
from azureml.pipeline.core import PublishedPipeline, Schedule
old_pipes = PublishedPipeline.list(ws)
for old_pipe in old_pipes:
old_schedules = Schedule.list(ws, pipeline_id=old_pipe.id)
for schedule in old_schedules:
schedule.disable(wait_for_provisioning=True)
old_pipe.disable()
ds = ws.get_default_datastore()
params = {
'--data_path': ws.get_default_datastore().path('data'),
'--analyze': '',
'--load_open': '',
'--load_closed': '',
}
project_folder = '.'
def_blob_store = ws.get_default_datastore()
inputs=[prepped_data])
#run pipeline w parameter
response = requests.post(rest_endpoint,
headers=auth_header,
json={"ExperimentName": "run_training_pipeline",
"ParameterAssignments": {"reg_rate": 0.1}})
#set schedule
from azureml.pipeline.core import ScheduleRecurrence, Schedule
daily = ScheduleRecurrence(frequency='Day', interval=1)
pipeline_schedule = Schedule.create(ws, name='Daily Training',
description='trains model every day',
pipeline_id=published_pipeline.id,
experiment_name='Training_Pipeline',
recurrence=daily)
#schedule it to run when data changes
from azureml.core import Datastore
from azureml.pipeline.core import Schedule
training_datastore = Datastore(workspace=ws, name='blob_data')
pipeline_schedule = Schedule.create(ws, name='Reactive Training',
description='trains model on data change',
pipeline_id=published_pipeline_id,
experiment_name='Training_Pipeline',
datastore=training_datastore,
parallelrun_step = ParallelRunStep(name='skill-extractor-parallel',
parallel_run_config=parallel_run_config,
inputs=[named_emp_ds],
output=output_dir,
allow_reuse=True)
pipeline = Pipeline(workspace=ws, steps=[parallelrun_step])
experiment = Experiment(ws, 'skill-extractor-parallel')
pipeline_run = experiment.submit(pipeline)
pipeline_run.wait_for_completion(show_output=True)
# Publish pipeline as REST API
publised_pipeline = pipeline_run.publish_pipeline(
name='skill_extractor',
description='Extract skill from employees JD text',
version='1.0.0')
# Print REST end point
rest_endpoint = publised_pipeline.endpoint
print(rest_endpoint)
# Schedule pipeline weekly
daily = ScheduleRecurrence(frequency='week', interval=1)
pipeline_schedule = Schedule.create(
ws,
name='skill extractor weekly run',
description='skill extractor weekly scheduler',
pipeline_id=publised_pipeline.id,
experiment_name='skill-extractor-parallel',
recurrence=daily)