def ComputeExist():
subscription_id = request.json['subscription_id']
resource_group = request.json['resource_group']
workspace_name = request.json['workspace_name']
location = request.json['location']
Cluster_type = request.json['Cluster_type']
cluster_name = request.json['cluster_name']
ws = Workspace(subscription_id=subscription_id,
resource_group=resource_group,
workspace_name=workspace_name)
print("Found workspace {} at location {}".format(ws.name, ws.location))
print('Found existing Workspace.')
#aml_compute = AmlCompute(ws, cluster_name)
#cluster_name = 'cpu-cluster'
try:
if Cluster_type == 'Training':
aml_compute = AmlCompute(ws, cluster_name)
else:
aks_target = AksCompute(ws, cluster_name)
print('Found existing AML compute context.')
return "compute exist"
except:
print('need to create new Compute.')
return "compute not exist"
def get_aml_compute(workspace):
# TODO: Set desired name for compute target
aml_compute_target = "example_vm_name"
try:
aml_compute = AmlCompute(workspace, aml_compute_target)
print("found existing compute target.")
except ComputeTargetException:
print("creating new compute target")
# TODO: Configure desired VM, see: https://docs.microsoft.com/nl-nl/azure/virtual-machines/sizes-general
provisioning_config = AmlCompute.provisioning_configuration(
vm_size="STANDARD_D1_V2",
min_nodes=0,
max_nodes=1,
vnet_resourcegroup_name="",
vnet_name="-vn",
subnet_name="default",
idle_seconds_before_scaledown=1800,
vm_priority='lowpriority')
aml_compute = ComputeTarget.create(workspace, aml_compute_target,
provisioning_config)
aml_compute.wait_for_completion(show_output=True,
min_node_count=None,
timeout_in_minutes=20)
print("Azure Machine Learning Compute attached")
return aml_compute
def ComputeDelete():
subscription_id = request.json['subscription_id']
resource_group = request.json['resource_group']
workspace_name = request.json['workspace_name']
location = request.json['location']
Cluster_type = request.json['Cluster_type']
cluster_name = request.json['cluster_name']
ws = Workspace(subscription_id=subscription_id,
resource_group=resource_group,
workspace_name=workspace_name)
print("Found workspace {} at location {}".format(ws.name, ws.location))
try:
if Cluster_type == 'Training':
aml_compute = AmlCompute(ws, cluster_name)
print('Found existing AML compute context.')
aml_compute.delete()
else:
aks_target = AksCompute(ws, cluster_name)
print('Found existing AKS compute context.')
aks_target.delete()
print('compute deleted')
return "compute deleted"
except Exception as e:
error_statement = str(e)
print("Error statement: ", error_statement)
return error_statement
def _create_or_update_cluster(self, min_nodes, max_nodes,
idle_timeout_secs):
try:
self.cluster = AmlCompute(workspace=self.workspace,
name=self.cluster_name)
print('Updating existing cluster "{}"'.format(
colored(self.cluster_name, "green")))
self.cluster.update(
min_nodes=min_nodes,
max_nodes=max_nodes,
idle_seconds_before_scaledown=idle_timeout_secs,
)
except ComputeTargetException:
print('Creating new cluster "{}"'.format(
colored(self.cluster_name, "green")))
cluster_config = AmlCompute.provisioning_configuration(
vm_size=self.vm_type,
min_nodes=min_nodes,
max_nodes=max_nodes,
idle_seconds_before_scaledown=idle_timeout_secs,
admin_username=self.admin_username,
admin_user_ssh_key=self.ssh_key,
remote_login_port_public_access="Enabled",
)
self.cluster = AmlCompute.create(self.workspace, self.cluster_name,
cluster_config)
self.cluster.wait_for_completion()
if len(self.cluster_nodes) < min_nodes:
sleep(30)
if len(self.cluster_nodes) < min_nodes:
raise RuntimeError("Failed to provision sufficient nodes")
def ComputeCompute():
subscription_id = request.json['subscription_id']
resource_group = request.json['resource_group']
workspace_name = request.json['workspace_name']
location = request.json['location']
cluster_name = request.json['cluster_name']
vm_size = request.json['vm_size']
min_nodes = request.json['min_nodes']
max_nodes = request.json['max_nodes']
ws = Workspace(subscription_id=subscription_id,
resource_group=resource_group,
workspace_name=workspace_name)
print("Found workspace {} at location {}".format(ws.name, ws.location))
print('Found existing Workspace.')
#aml_compute = AmlCompute(ws, cluster_name)
#cluster_name = 'cpu-cluster'
try:
aml_compute = AmlCompute(ws, cluster_name)
print('Found existing AML compute context.')
return "Found existing AML compute context."
except:
print('need to create new Compute.')
print('Creating new AML compute context.')
aml_config = AmlCompute.provisioning_configuration(vm_size=vm_size,
min_nodes=min_nodes,
max_nodes=max_nodes)
aml_compute = AmlCompute.create(ws,
name=cluster_name,
provisioning_configuration=aml_config)
aml_compute.wait_for_completion(show_output=True)
return "Compute successfully created"
def main():
# workspace
ws = Workspace.from_config()
#compute
compute = AmlCompute(workspace=ws, name='gandalf')
# datasource
datastore = Datastore.get(ws, datastore_name='surfrider')
# experiment
script_params = {
"--datastore": datastore.as_mount()
}
# Create and run experiment
estimator = Estimator(source_directory='./',
script_params=script_params,
compute_target=compute,
entry_script='train.py',
use_gpu=True,
pip_packages=['opencv-python>=4.1',
'tensorpack==0.9.8',
'tensorflow-gpu>=1.3,<2.0',
'tqdm>=4.36.1',
'cython>=0.29.13',
'scipy>=1.3.1',
'ffmpeg-python',
'wget'])
exp = Experiment(ws, 'surfrider_rcnn')
run = exp.submit(estimator)
def choose_compute_target(workspace, name):
try:
aml_compute = AmlCompute(workspace, name)
print("Found existing compute target: {}".format(name))
except:
print("Creating new compute target: {}".format(name))
provisioning_config = AmlCompute.provisioning_configuration(vm_size="STANDARD_D2_V2",
min_nodes=1,
max_nodes=4)
aml_compute = ComputeTarget.create(workspace, name, provisioning_config)
aml_compute.wait_for_completion(show_output=True)
print(aml_compute)
return aml_compute
def setup(num):
workspace_name = '%s-%s-%02d' % (workspace_prefix, location, num)
try:
ws = Workspace.get(
name=workspace_name,
subscription_id=subscription_id,
resource_group=resource_group)
print('Found existing workspace %s' % workspace_name)
except WorkspaceException:
print('Creating new workspace %s...' % workspace_name)
ws = Workspace.create(
name=workspace_name,
subscription_id=subscription_id,
resource_group=resource_group,
location=location)
try:
compute_target = AmlCompute(ws, compute_name)
print('Found existing compute %s' % compute_name)
compute_target.update(min_nodes=min_nodes, max_nodes=max_nodes)
except ComputeTargetException:
print('Creating new compute target %s...' % compute_name)
compute_config = AmlCompute.provisioning_configuration(vm_size=vm_size, min_nodes=min_nodes, max_nodes=max_nodes)
compute_target = ComputeTarget.create(ws, compute_name, compute_config)
compute_target.wait_for_completion(show_output=True, timeout_in_minutes=20)
ds = ws.get_default_datastore()
ds.upload("testdata")
dataset_name = 'sample_dataset'
if dataset_name not in ws.datasets:
data = Dataset.File.from_files(path=[(ds, 'testdata.txt')])
data.register(
workspace = ws,
name = dataset_name,
description = 'Sample data for load test')
print('Dataset successfully registered')
else:
print('Dataset already exists')
def login(tenant, compute_target):
if tenant == 'vx':
tenant_id = d.VINX_TENANT_ID
target_name = d.VINX_COMPUTING
config = d.VINX_CONFIG_PATH
elif tenant == 'ds':
if compute_target:
target_name = compute_target
else:
target_name = d.DS_COMPUTING
tenant_id = d.DS_TENANT_ID
config = d.DS_CONFIG_PATH
else:
raise ValueError('tenant must be either on of vx or ds')
interactive_auth = InteractiveLoginAuthentication(tenant_id=tenant_id)
ws = Workspace.from_config(config)
compute = AmlCompute(ws, target_name)
return ws, compute
def main(args):
"""
Main
"""
workspace = package_utils.get_workspace()
if args.cluster_name is None:
cluster_name = package_utils.trim_cluster_name(workspace.name)
else:
cluster_name = args.cluster_name
try:
compute_target = AmlCompute(workspace=workspace, name=cluster_name)
logger.info(msg=f"Found existing cluster {cluster_name}")
except ComputeTargetException:
compute_config = AmlCompute.provisioning_configuration(
vm_size=args.cluster_sku,
vm_priority=args.cluster_vm_priority,
idle_seconds_before_scaledown=40 * 60,
min_nodes=args.cluster_min_nodes,
max_nodes=args.cluster_max_nodes,
)
compute_target_create = ComputeTarget.create(workspace, cluster_name,
compute_config)
compute_target_create.wait_for_completion(show_output=True,
min_node_count=None,
timeout_in_minutes=5)
logger.info(
msg="main",
extra={"status": compute_target_create.get_status().serialize()})
compute_target = workspace.compute_targets[cluster_name]
logger.info(msg="main",
extra={"compute_target": compute_target.serialize()})
return compute_target
# Creation of AmlCompute takes approximately 5 minutes. If the AmlCompute
# with that name is already in your workspace, this code will skip the
# creation process.
# As with other Azure services, there are limits on certain resources
# (e.g. AmlCompute) associated with the Azure Machine Learning service.
# Please read this article on the default limits and how to request more
# quota.
from azureml.core.compute import ComputeTarget, AmlCompute
from azureml.core.compute_target import ComputeTargetException
# choose a name for your cluster
cluster_name = "gpucluster"
try:
compute_target = AmlCompute(workspace=ws, name=cluster_name)
print('Found existing compute target.')
except ComputeTargetException:
print('Creating a new compute target...')
compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_NC6',
max_nodes=4,
idle_seconds_before_scaledown=1800)
# create the cluster
compute_target = ComputeTarget.create(ws, cluster_name, compute_config)
compute_target.wait_for_completion(show_output=True)
# use get_status() to get a detailed status for the current cluster.
print(compute_target.get_status().serialize())
def build_pipeline(dataset, ws, config):
print("building pipeline for dataset %s in workspace %s" % (dataset, ws.name))
base_dir = '.'
def_blob_store = ws.get_default_datastore()
# folder for scripts that need to be uploaded to Aml compute target
script_folder = './scripts'
os.makedirs(script_folder, exist_ok=True)
shutil.copy(os.path.join(base_dir, 'video_decoding.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'pipelines_submit.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'pipelines_create.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'train.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'data_utils.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'prednet.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'keras_utils.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'data_preparation.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'model_registration.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'config.json'), script_folder)
cpu_compute_name = config['cpu_compute']
try:
cpu_compute_target = AmlCompute(ws, cpu_compute_name)
print("found existing compute target: %s" % cpu_compute_name)
except:# ComputeTargetException:
print("creating new compute target")
provisioning_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_D2_V2',
max_nodes=4,
idle_seconds_before_scaledown=1800)
cpu_compute_target = ComputeTarget.create(ws, cpu_compute_name, provisioning_config)
cpu_compute_target.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20)
# use get_status() to get a detailed status for the current cluster.
print(cpu_compute_target.get_status().serialize())
# choose a name for your cluster
gpu_compute_name = config['gpu_compute']
try:
gpu_compute_target = AmlCompute(workspace=ws, name=gpu_compute_name)
print("found existing compute target: %s" % gpu_compute_name)
except:
print('Creating a new compute target...')
provisioning_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_NC6',
max_nodes=10,
idle_seconds_before_scaledown=1800)
# create the cluster
gpu_compute_target = ComputeTarget.create(ws, gpu_compute_name, provisioning_config)
# can poll for a minimum number of nodes and for a specific timeout.
# if no min node count is provided it uses the scale settings for the cluster
gpu_compute_target.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20)
# use get_status() to get a detailed status for the current cluster.
try:
print(gpu_compute_target.get_status().serialize())
except BaseException as e:
print("Could not get status of compute target.")
print(e)
# conda dependencies for compute targets
cpu_cd = CondaDependencies.create(conda_packages=["py-opencv=3.4.2"], pip_indexurl='https://azuremlsdktestpypi.azureedge.net/sdk-release/Candidate/604C89A437BA41BD942B4F46D9A3591D', pip_packages=["azure-storage-blob==1.5.0", "hickle==3.4.3", "requests==2.21.0", "sklearn", "pandas==0.24.2", "azureml-sdk", "numpy==1.16.2", "pillow==6.0.0"])
# Runconfigs
cpu_compute_run_config = RunConfiguration(conda_dependencies=cpu_cd)
cpu_compute_run_config.environment.docker.enabled = True
cpu_compute_run_config.environment.docker.gpu_support = False
cpu_compute_run_config.environment.docker.base_image = DEFAULT_CPU_IMAGE
cpu_compute_run_config.environment.spark.precache_packages = False
print("PipelineData object created")
# DataReference to where video data is stored.
video_data = DataReference(
datastore=def_blob_store,
data_reference_name="video_data",
path_on_datastore=os.path.join("prednet", "data", "video", dataset))
print("DataReference object created")
# Naming the intermediate data as processed_data1 and assigning it to the variable processed_data1.
raw_data = PipelineData("raw_video_fames", datastore=def_blob_store)
preprocessed_data = PipelineData("preprocessed_video_frames", datastore=def_blob_store)
data_metrics = PipelineData("data_metrics", datastore=def_blob_store)
data_output = PipelineData("output_data", datastore=def_blob_store)
# prepare dataset for training/testing prednet
video_decoding = PythonScriptStep(
name='decode_videos',
script_name="video_decoding.py",
arguments=["--input_data", video_data, "--output_data", raw_data],
inputs=[video_data],
outputs=[raw_data],
compute_target=cpu_compute_target,
source_directory=script_folder,
runconfig=cpu_compute_run_config,
allow_reuse=True,
hash_paths=['.']
)
print("video_decode step created")
# prepare dataset for training/testing recurrent neural network
data_prep = PythonScriptStep(
name='prepare_data',
script_name="data_preparation.py",
arguments=["--input_data", raw_data, "--output_data", preprocessed_data],
inputs=[raw_data],
outputs=[preprocessed_data],
compute_target=cpu_compute_target,
source_directory=script_folder,
runconfig=cpu_compute_run_config,
allow_reuse=True,
hash_paths=['.']
)
data_prep.run_after(video_decoding)
print("data_prep step created")
# configure access to ACR for pulling our custom docker image
acr = ContainerRegistry()
acr.address = config['acr_address']
acr.username = config['acr_username']
acr.password = config['acr_password']
est = Estimator(source_directory=script_folder,
compute_target=gpu_compute_target,
entry_script='train.py',
use_gpu=True,
node_count=1,
custom_docker_image = "wopauli_1.8-gpu:1",
image_registry_details=acr,
user_managed=True
)
ps = RandomParameterSampling(
{
'--batch_size': choice(1, 2, 4, 8),
'--filter_sizes': choice("3, 3, 3", "4, 4, 4", "5, 5, 5"),
'--stack_sizes': choice("48, 96, 192", "36, 72, 144", "12, 24, 48"), #, "48, 96"),
'--learning_rate': loguniform(-6, -1),
'--lr_decay': loguniform(-9, -1),
'--freeze_layers': choice("0, 1, 2", "1, 2, 3", "0, 1", "1, 2", "2, 3", "0", "3"),
'--transfer_learning': choice("True", "False")
}
)
policy = BanditPolicy(evaluation_interval=2, slack_factor=0.1, delay_evaluation=10)
hdc = HyperDriveConfig(estimator=est,
hyperparameter_sampling=ps,
policy=policy,
primary_metric_name='val_loss',
primary_metric_goal=PrimaryMetricGoal.MINIMIZE,
max_total_runs=10,
max_concurrent_runs=5,
max_duration_minutes=60*6
)
hd_step = HyperDriveStep(
name="train_w_hyperdrive",
hyperdrive_run_config=hdc,
estimator_entry_script_arguments=[
'--data-folder', preprocessed_data,
'--remote_execution',
'--dataset', dataset
],
inputs=[preprocessed_data],
metrics_output = data_metrics,
allow_reuse=True
)
hd_step.run_after(data_prep)
registration_step = PythonScriptStep(
name='register_model',
script_name='model_registration.py',
arguments=['--input_dir', data_metrics, '--output_dir', data_output],
compute_target=cpu_compute_target,
inputs=[data_metrics],
outputs=[data_output],
source_directory=script_folder,
allow_reuse=True,
hash_paths=['.']
)
registration_step.run_after(hd_step)
pipeline = Pipeline(workspace=ws, steps=[video_decoding, data_prep, hd_step, registration_step])
print ("Pipeline is built")
pipeline.validate()
print("Simple validation complete")
pipeline_name = 'prednet_' + dataset
published_pipeline = pipeline.publish(name=pipeline_name)
schedule = Schedule.create(workspace=ws, name=pipeline_name + "_sch",
pipeline_id=published_pipeline.id,
experiment_name=pipeline_name,
datastore=def_blob_store,
wait_for_provisioning=True,
description="Datastore scheduler for Pipeline" + pipeline_name,
path_on_datastore=os.path.join('prednet/data/video', dataset, 'Train'),
polling_interval=1
)
return pipeline_name
'tensorboard'
],
conda_packages=['python=3.6.11'])
myenv = Environment("yolov3")
myenv.python.conda_dependencies = cd
myenv.python.conda_dependencies.add_pip_package("azureml-sdk")
myenv.python.conda_dependencies.add_channel("conda-forge")
myenv.docker.enabled = True
myenv.docker.base_image = DEFAULT_GPU_IMAGE
# Choose a name for your CPU cluster
CLUSTER_NAME = "gpu-cluster"
# Verify that cluster does not exist already
try:
aml_cluster = AmlCompute(workspace=ws, name=CLUSTER_NAME)
print("Found existing cluster, use it.")
except ComputeTargetException:
print("provisioning new compute target")
compute_config = AmlCompute.provisioning_configuration(
vm_size="STANDARD_NC6", max_nodes=8, vm_priority="lowpriority")
aml_cluster = ComputeTarget.create(ws, CLUSTER_NAME, compute_config)
aml_cluster.wait_for_completion(show_output=True)
def_blob_store = ws.get_default_datastore()
dataset = Dataset.File.from_files(path=(def_blob_store, '/data/VOCdevkit'))
src = ScriptRunConfig(
source_directory=PROJECT_FOLDER,
output['SDK version'] = azureml.core.VERSION
output['Subscription ID'] = ws.subscription_id
output['Workspace Name'] = ws.name
output['Resource Group'] = ws.resource_group
output['Location'] = ws.location
output['Project Directory'] = project_folder
output['Experiment Name'] = experiment.name
pd.set_option('display.max_colwidth', -1)
outputDf = pd.DataFrame(data=output, index=[''])
outputDf.T
from azureml.core.compute import AmlCompute
aml_name = 'cpu-cluster'
try:
aml_compute = AmlCompute(ws, aml_name)
print('Found existing AML compute context.')
except:
print('Creating new AML compute context.')
aml_config = AmlCompute.provisioning_configuration(
vm_size="Standard_D2_v2", min_nodes=1, max_nodes=4)
aml_compute = AmlCompute.create(ws,
name=aml_name,
provisioning_configuration=aml_config)
aml_compute.wait_for_completion(show_output=True)
#writefile get_data.py
from sklearn import datasets
from sklearn.model_selection import train_test_split
from scipy import sparse
# Initialize Workspace
ws = Workspace.from_config()
print("Resource group: ", ws.resource_group)
print("Location: ", ws.location)
print("Workspace name: ", ws.name)
# Choose a name for the run history container in the workspace.
experiment_name = 'automl-remote-batchai'
experiment = Experiment(ws, experiment_name)
# Create Batch AI Cluster
compute_target_name = 'myazbai'
try:
batch_ai_compute = AmlCompute(workspace=ws, name=compute_target_name)
print('found existing Azure Batch AI cluster:', batch_ai_compute.name)
except ComputeTargetException:
print('creating new Azure Batch AI cluster...')
batch_ai_config = AmlCompute.provisioning_configuration(
vm_size="Standard_NC6",
vm_priority="dedicated",
min_nodes=0,
max_nodes=4,
idle_seconds_before_scaledown=300)
batch_ai_compute = AmlCompute.create(
ws,
name=compute_target_name,
provisioning_configuration=batch_ai_config)
batch_ai_compute.wait_for_completion(show_output=True)
# subscription_id = '7cb97533-0a52-4037-a51e-8b8d707367ad'
# resource_group = 'cd-mlops'
print("workspace_name: %s" % workspace_name)
print("compute_target_name: %s" % compute_target_name)
print("dataset_name: %s" % dataset_name)
print("subscription_id: %s" % subscription_id)
print("resource_group: %s" % resource_group)
# get the workspace
print("Getting a reference to workspace %s" % workspace_name)
ws = Workspace.get(name=workspace_name,
subscription_id=subscription_id,
resource_group=resource_group)
experiment = Experiment(workspace=ws, name='automl-diabetes')
aml_compute = AmlCompute(ws, compute_target_name)
# read in the data
print("Getting a reference to default datastore")
datastore = ws.get_default_datastore()
print("Preparing the 'prep data' step")
blob_diabetes_data = DataReference(
datastore=datastore,
data_reference_name="diabetes_data",
path_on_datastore="diabetesdata/diabetes_pima.csv")
# Create a new runconfig object
aml_run_config = RunConfiguration()
aml_run_config.target = aml_compute
aml_run_config.environment.docker.enabled = True
pass
ws = Workspace.from_config(path=config_json, auth=svc_pr)
# folder for scripts that need to be uploaded to Aml compute target
script_folder = './scripts/'
try:
os.makedirs(script_folder)
except BaseException as e:
print("Deleting:", script_folder)
shutil.rmtree(script_folder)
os.makedirs(script_folder)
cpu_compute_name = config['cpu_compute']
try:
cpu_compute_target = AmlCompute(ws, cpu_compute_name)
print("found existing compute target: %s" % cpu_compute_name)
except: # ComputeTargetException:
print("creating new compute target")
provisioning_config = AmlCompute.provisioning_configuration(
vm_size='STANDARD_D2_V2',
max_nodes=4,
idle_seconds_before_scaledown=1800)
cpu_compute_target = ComputeTarget.create(ws, cpu_compute_name,
provisioning_config)
cpu_compute_target.wait_for_completion(show_output=True,
min_node_count=None,
timeout_in_minutes=20)
# use get_status() to get a detailed status for the current cluster.
)
parser.add_argument("--path", type=str, help="path", dest="path", required=True)
args = parser.parse_args()
print("Argument 1: %s" % args.aml_compute_target)
print("Argument 2: %s" % args.path)
print("creating AzureCliAuthentication...")
cli_auth = AzureCliAuthentication()
print("done creating AzureCliAuthentication!")
print("get workspace...")
ws = Workspace.from_config(path=args.path, auth=cli_auth)
print("done getting workspace!")
try:
aml_compute = AmlCompute(ws, args.aml_compute_target)
print("found existing compute target.")
except ComputeTargetException:
print("creating new compute target")
provisioning_config = AmlCompute.provisioning_configuration(
vm_size="STANDARD_DS4_V2", min_nodes=1, max_nodes=8
)
aml_compute = ComputeTarget.create(ws, args.aml_compute_target, provisioning_config)
aml_compute.wait_for_completion(
show_output=True, min_node_count=None, timeout_in_minutes=20
)
print("Aml Compute attached")
image_arr = []
label_arr = []
for i in range(3):
image, label = sess.run([data_image, data_label])
image_arr.append(image)
label_arr.append(label)
# Predict
pred_fn = tf.contrib.predictor.from_saved_model('./distributed_model')
pred = pred_fn({'inputs': image_arr})
print('Predicted: ', pred['classes'].tolist())
print('Actual : ', label_arr)
#%% [markdown]
# ### Step 6 : Remove AML compute
#
# **You don't need to remove your AML compute** for saving money, because the nodes will be automatically terminated, when it's inactive.
# But if you want to clean up, please run the following.
#%%
# Delete cluster (nbodes) and remove from AML workspace
mycompute = AmlCompute(workspace=ws, name='mycluster01')
mycompute.delete()
#%%
run = experiment.submit(config=run_config) #%% run.wait_for_completion(show_output=True) #%% [markdown] # ## View logs #%% [markdown] # You can view logs using [Azure Portal](https://portal.azure.com/), but you can also view using AML run history widget in your notebook. #%% from azureml.widgets import RunDetails RunDetails(run_instance=run).show() #%% [markdown] # You can also explorer metrics with your python code. #%% allmetrics = run.get_metrics() print(allmetrics) #%% [markdown] # ## Remove AML compute #%% # Delete cluster (nbodes) and remove from AML workspace mycompute = AmlCompute(workspace=ws, name='hypertest01') mycompute.delete() #%%
def build_pipeline(dataset, ws, config):
print("building pipeline for dataset %s in workspace %s" %
(dataset, ws.name))
hostname = socket.gethostname()
if hostname == 'wopauliNC6':
base_dir = '.'
else:
base_dir = '.'
def_blob_store = ws.get_default_datastore()
# folder for scripts that need to be uploaded to Aml compute target
script_folder = './scripts'
os.makedirs(script_folder, exist_ok=True)
shutil.copy(os.path.join(base_dir, 'video_decoding.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'pipelines_submit.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'pipelines_build.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'train.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'data_utils.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'prednet.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'keras_utils.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'data_preparation.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'model_registration.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'config.json'), script_folder)
cpu_compute_name = config['cpu_compute']
try:
cpu_compute_target = AmlCompute(ws, cpu_compute_name)
print("found existing compute target: %s" % cpu_compute_name)
except ComputeTargetException:
print("creating new compute target")
provisioning_config = AmlCompute.provisioning_configuration(
vm_size='STANDARD_D2_V2',
max_nodes=4,
idle_seconds_before_scaledown=1800)
cpu_compute_target = ComputeTarget.create(ws, cpu_compute_name,
provisioning_config)
cpu_compute_target.wait_for_completion(show_output=True,
min_node_count=None,
timeout_in_minutes=20)
# use get_status() to get a detailed status for the current cluster.
print(cpu_compute_target.get_status().serialize())
# choose a name for your cluster
gpu_compute_name = config['gpu_compute']
try:
gpu_compute_target = AmlCompute(workspace=ws, name=gpu_compute_name)
print("found existing compute target: %s" % gpu_compute_name)
except ComputeTargetException:
print('Creating a new compute target...')
provisioning_config = AmlCompute.provisioning_configuration(
vm_size='STANDARD_NC6',
max_nodes=5,
idle_seconds_before_scaledown=1800)
# create the cluster
gpu_compute_target = ComputeTarget.create(ws, gpu_compute_name,
provisioning_config)
# can poll for a minimum number of nodes and for a specific timeout.
# if no min node count is provided it uses the scale settings for the cluster
gpu_compute_target.wait_for_completion(show_output=True,
min_node_count=None,
timeout_in_minutes=20)
# use get_status() to get a detailed status for the current cluster.
print(gpu_compute_target.get_status().serialize())
# conda dependencies for compute targets
cpu_cd = CondaDependencies.create(conda_packages=["py-opencv=3.4.2"],
pip_packages=[
"azure-storage-blob==1.5.0",
"hickle==3.4.3", "requests==2.21.0",
"sklearn", "pandas==0.24.2",
"azureml-sdk==1.0.21",
"numpy==1.16.2", "pillow==6.0.0"
])
gpu_cd = CondaDependencies.create(pip_packages=[
"keras==2.0.8", "theano==1.0.4", "tensorflow==1.8.0",
"tensorflow-gpu==1.8.0", "hickle==3.4.3", "matplotlib==3.0.3",
"seaborn==0.9.0", "requests==2.21.0", "bs4==0.0.1", "imageio==2.5.0",
"sklearn", "pandas==0.24.2", "azureml-sdk==1.0.21", "numpy==1.16.2"
])
# Runconfigs
cpu_compute_run_config = RunConfiguration(conda_dependencies=cpu_cd)
cpu_compute_run_config.environment.docker.enabled = True
cpu_compute_run_config.environment.docker.gpu_support = False
cpu_compute_run_config.environment.docker.base_image = DEFAULT_CPU_IMAGE
cpu_compute_run_config.environment.spark.precache_packages = False
gpu_compute_run_config = RunConfiguration(conda_dependencies=gpu_cd)
gpu_compute_run_config.environment.docker.enabled = True
gpu_compute_run_config.environment.docker.gpu_support = True
gpu_compute_run_config.environment.docker.base_image = DEFAULT_GPU_IMAGE
gpu_compute_run_config.environment.spark.precache_packages = False
print("PipelineData object created")
video_data = DataReference(datastore=def_blob_store,
data_reference_name="video_data",
path_on_datastore=os.path.join(
"prednet", "data", "video", dataset))
# Naming the intermediate data as processed_data1 and assigning it to the variable processed_data1.
raw_data = PipelineData("raw_video_fames", datastore=def_blob_store)
preprocessed_data = PipelineData("preprocessed_video_frames",
datastore=def_blob_store)
data_metrics = PipelineData("data_metrics", datastore=def_blob_store)
data_output = PipelineData("output_data", datastore=def_blob_store)
print("DataReference object created")
# prepare dataset for training/testing prednet
video_decoding = PythonScriptStep(
name='decode_videos',
script_name="video_decoding.py",
arguments=["--input_data", video_data, "--output_data", raw_data],
inputs=[video_data],
outputs=[raw_data],
compute_target=cpu_compute_target,
source_directory=script_folder,
runconfig=cpu_compute_run_config,
allow_reuse=True,
hash_paths=['.'])
print("video_decode created")
# prepare dataset for training/testing recurrent neural network
data_prep = PythonScriptStep(name='prepare_data',
script_name="data_preparation.py",
arguments=[
"--input_data", raw_data, "--output_data",
preprocessed_data
],
inputs=[raw_data],
outputs=[preprocessed_data],
compute_target=cpu_compute_target,
source_directory=script_folder,
runconfig=cpu_compute_run_config,
allow_reuse=True,
hash_paths=['.'])
data_prep.run_after(video_decoding)
print("data_prep created")
est = TensorFlow(source_directory=script_folder,
compute_target=gpu_compute_target,
pip_packages=[
'keras==2.0.8', 'theano', 'tensorflow==1.8.0',
'tensorflow-gpu==1.8.0', 'matplotlib', 'horovod',
'hickle'
],
entry_script='train.py',
use_gpu=True,
node_count=1)
ps = RandomParameterSampling({
'--batch_size':
choice(2, 4, 8, 16),
'--filter_sizes':
choice("3, 3, 3", "4, 4, 4", "5, 5, 5"),
'--stack_sizes':
choice("48, 96, 192", "36, 72, 144", "12, 24, 48"), #, "48, 96"),
'--learning_rate':
loguniform(-6, -1),
'--lr_decay':
loguniform(-9, -1),
'--freeze_layers':
choice("0, 1, 2", "1, 2, 3", "0, 1", "1, 2", "2, 3", "0", "1", "2",
"3"),
'--transfer_learning':
choice("True", "False")
})
policy = BanditPolicy(evaluation_interval=2,
slack_factor=0.1,
delay_evaluation=20)
hdc = HyperDriveRunConfig(
estimator=est,
hyperparameter_sampling=ps,
policy=policy,
primary_metric_name='val_loss',
primary_metric_goal=PrimaryMetricGoal.MINIMIZE,
max_total_runs=5, #100,
max_concurrent_runs=5, #10,
max_duration_minutes=60 * 6)
hd_step = HyperDriveStep(name="train_w_hyperdrive",
hyperdrive_run_config=hdc,
estimator_entry_script_arguments=[
'--data-folder', preprocessed_data,
'--remote_execution'
],
inputs=[preprocessed_data],
metrics_output=data_metrics,
allow_reuse=True)
hd_step.run_after(data_prep)
registration_step = PythonScriptStep(
name='register_model',
script_name='model_registration.py',
arguments=['--input_dir', data_metrics, '--output_dir', data_output],
compute_target=gpu_compute_target,
inputs=[data_metrics],
outputs=[data_output],
source_directory=script_folder,
allow_reuse=True,
hash_paths=['.'])
registration_step.run_after(hd_step)
pipeline = Pipeline(
workspace=ws,
steps=[video_decoding, data_prep, hd_step, registration_step])
print("Pipeline is built")
pipeline.validate()
print("Simple validation complete")
pipeline_name = 'prednet_' + dataset
pipeline.publish(name=pipeline_name)
return pipeline_name
def RunAutoMLForecast():
subscription_id = request.json['subscription_id']
resource_group = request.json['resource_group']
workspace_name = request.json['workspace_name']
file_name = request.json['file_name']
location = request.json['location']
target_var = request.json['target_var']
cluster_name = request.json['cluster_name']
best_model = request.json['best_model']
time_column_name = request.json['time_column_name']
max_horizon = request.json['max_horizon']
ws = Workspace(subscription_id=subscription_id,
resource_group=resource_group,
workspace_name=workspace_name)
print("Found workspace {} at location {}".format(ws.name, ws.location))
print('Found existing Workspace.')
compute_target = AmlCompute(ws, cluster_name)
print('Found existing AML compute context.')
dataset_name = file_name
time_column_name = time_column_name
# Get a dataset by name
dataset = Dataset.get_by_name(workspace=ws,
name=dataset_name).with_timestamp_columns(
fine_grain_timestamp=time_column_name)
print(dataset)
#df_ts = Dataset.Tabular.from_delimited_files(df_ts)
dataset.to_pandas_dataframe().describe()
dataset.take(3).to_pandas_dataframe()
print(dataset)
#y_df = df_ts[target_var].values
#x_df = df_ts.drop([target_var], axis=1)
print('file successfully recieved.')
#stock_dataset_df.head()
# create a new RunConfig object
conda_run_config = RunConfiguration(framework="python")
conda_run_config.environment.docker.enabled = True
conda_run_config.environment.docker.base_image = azureml.core.runconfig.DEFAULT_CPU_IMAGE
cd = CondaDependencies.create(pip_packages=['azureml-sdk[automl]'],
conda_packages=['numpy', 'py-xgboost<=0.80'])
conda_run_config.environment.python.conda_dependencies = cd
print('run config is ready')
ExperimentName = request.json['ExperimentName']
tasks = request.json['tasks']
iterations = request.json['iterations']
n_cross_validations = request.json['n_cross_validations']
iteration_timeout_minutes = request.json['iteration_timeout_minutes']
primary_metric = request.json['primary_metric']
#max_concurrent_iterations = request.json['max_concurrent_iterations']
automl_settings = {
'time_column_name': time_column_name,
'max_horizon': max_horizon,
"iterations": iterations,
}
automl_config = AutoMLConfig(
task=tasks,
primary_metric=primary_metric,
#blacklist_models = ['ExtremeRandomTrees', 'AutoArima', 'Prophet'],
experiment_timeout_minutes=iteration_timeout_minutes,
training_data=dataset,
label_column_name=target_var,
compute_target=compute_target,
enable_early_stopping=True,
n_cross_validations=n_cross_validations,
#verbosity=logging.INFO,
**automl_settings)
print("AutoML config created.")
experiment = Experiment(ws, ExperimentName)
remote_run = experiment.submit(automl_config, show_output=True)
children = list(remote_run.get_children())
metricslist = {}
for run in children:
properties = run.get_properties()
metrics = {
k: v
for k, v in run.get_metrics().items() if isinstance(v, float)
}
metricslist[int(properties['iteration'])] = metrics
rundata = pd.DataFrame(metricslist).sort_index(axis=1, by=primary_metric)
rundata.rename(columns={
0: "one",
1: "two",
2: "three",
3: "four",
4: "five",
5: "six",
6: "seven",
7: "eight",
8: "nine",
9: "ten",
},
inplace=True)
iterations_toJson = rundata.to_json(orient='columns')
print(iterations_toJson)
best_run, fitted_model = remote_run.get_output()
#best_run_toJson = best_run.get_metrics()
#dict = {}
#dict['iterations_toJson'] = iterations_toJson
#dict['best_run_toJson'] = best_run_toJson
#print(best_run.get_file_names())
#Register the model
#from datetime import date
model = remote_run.register_model(model_name=best_model,
description='AutoML Model')
print(model.name, model.id, model.version, sep='\t')
best_model = model.name
best_model
var1 = "@"
var2 = var1 + best_model
return '{} {}'.format(iterations_toJson, var2)
label_arr.append(label)
# Predict
pred_fn = tf.contrib.predictor.from_saved_model('./remote_model')
pred = pred_fn({'inputs': image_arr})
print('Predicted: ', pred['classes'].tolist())
print('Actual : ', label_arr)
#%% [markdown]
# ### Step 7 : Remove AML compute
#
# **You don't need to remove your AML compute** for saving money, because the nodes will be automatically terminated, when it's inactive.
# But if you want to clean up, please run the following.
#%%
# Delete cluster (nbodes) and remove from AML workspace
mycompute = AmlCompute(workspace=ws, name='mydsvm01')
mycompute.delete()
#%%
# get a status for the current cluster.
print(mycompute.status.serialize())
#%%
cliAuth = AzureCliAuthentication()
print('2. Authenticating...END')
print('')
print('')
print('3. Get workspace reference...START')
print('.............................................')
amlWs = Workspace.from_config(path=args.path, auth=cliAuth)
print('3. Get workspace reference...END')
print('')
print('')
print('4. Get compute reference or create new...START')
print('.............................................')
try:
amlCompute = AmlCompute(amlWs, args.aml_compute_target)
print("....found existing compute target.")
except ComputeTargetException:
print("....creating new compute target")
amlComputeProvisioningConfig = AmlCompute.provisioning_configuration(
vm_size="STANDARD_D12_V2", min_nodes=0, max_nodes=1)
amlCompute = ComputeTarget.create(amlWs, args.aml_compute_target,
amlComputeProvisioningConfig)
amlCompute.wait_for_completion(show_output=True,
min_node_count=None,
timeout_in_minutes=20)
print('4. Get compute reference or create new...END')
print('')
print('')
def_blob_store = ws.get_default_datastore()
# folder for scripts that need to be uploaded to Aml compute target
script_folder = './scripts'
os.makedirs(script_folder, exist_ok=True)
os.makedirs(os.path.join(script_folder, 'utils'), exist_ok=True)
# copy all relevant assets into the `script_folder` so they will be uploaded to the cloud and made available to the remote compute target
shutil.copy(os.path.join(base_dir, 'train_xception.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'xception.py'), script_folder)
shutil.copy(os.path.join(base_dir, 'config.json'), script_folder)
cpu_compute_name = config['cpu_compute']
try:
cpu_compute_target = AmlCompute(ws, cpu_compute_name)
print("found existing compute target: %s" % cpu_compute_name)
except: # ComputeTargetException:
print("creating new compute target")
provisioning_config = AmlCompute.provisioning_configuration(
vm_size='STANDARD_D2_V2',
max_nodes=4,
idle_seconds_before_scaledown=1800)
cpu_compute_target = ComputeTarget.create(ws, cpu_compute_name,
provisioning_config)
cpu_compute_target.wait_for_completion(show_output=True,
min_node_count=None,
timeout_in_minutes=20)
# use get_status() to get a detailed status for the current cluster.
print(datastore.datastore_type, datastore.account_name,
datastore.container_name)
file_path = request.json['file_path']
print(file_path)
file_name = request.json['file_name']
ds.upload(src_dir=file_path,
target_path=None,
overwrite=True,
show_progress=True)
stock_ds = Dataset.Tabular.from_delimited_files(path=datastore.path(file_name))
stock_ds = stock_ds.register(workspace=ws,
name=file_name,
description='Introact Owner Data')
compute_target = AmlCompute(ws, cluster_name)
print('Found existing AML compute context.')
dataset_name = file_name
# Get a dataset by name
df = Dataset.get_by_name(workspace=ws, name=dataset_name)
X = df.drop_columns(columns=[target_var])
y = df.keep_columns(columns=[target_var], validate=True)
print(y)
#y = diabetes.pop('Y')
#X_train, X_test, y_train, y_test = train_test_split(diabetes, y, test_size=0.2, random_state=0)
#data = {"train": {"X": X_train, "y": y_train}, "test": {"X": X_test, "y": y_test}}
conda_run_config = RunConfiguration(framework="python")
conda_run_config.environment.docker.enabled = True
conda_run_config.environment.docker.base_image = azureml.core.runconfig.DEFAULT_CPU_IMAGE
from azureml.core import Workspace
from azureml.core.compute import AmlCompute, ComputeTarget
from azureml.pipeline.wrapper import Module, Pipeline
# In[ ]:
workspace = Workspace.from_config()
print(workspace.name,
workspace.resource_group,
workspace.location,
workspace.subscription_id,
sep='\n')
aml_compute_target = "aml-compute"
try:
aml_compute = AmlCompute(workspace, aml_compute_target)
print("Found existing compute target: {}".format(aml_compute_target))
except:
print("Creating new compute target: {}".format(aml_compute_target))
provisioning_config = AmlCompute.provisioning_configuration(
vm_size="STANDARD_D2_V2", min_nodes=1, max_nodes=4)
aml_compute = ComputeTarget.create(workspace, aml_compute_target,
provisioning_config)
aml_compute.wait_for_completion(show_output=True,
min_node_count=None,
timeout_in_minutes=20)
# In[ ]:
try:
settings = json.load(f)
workspace_config_settings = settings["workspace"]["config"]
aml_settings = settings["compute_target"]["training"]["amlcompute"]
# Get workspace
print("Loading Workspace")
cli_auth = AzureCliAuthentication()
ws = Workspace.from_config(path=workspace_config_settings["path"],
auth=cli_auth,
_file_name=workspace_config_settings["file_name"])
print(ws.name, ws.resource_group, ws.location, ws.subscription_id, sep='\n')
try:
# Loading AMLCompute
print("Loading existing AML Compute")
cluster = AmlCompute(workspace=ws, name=aml_settings["name"])
# Check settings and redeploy if required settings have changed
print("Found existing cluster")
if cluster.vm_size.lower() != aml_settings["vm_size"].lower(
) or cluster.vm_priority.lower() != aml_settings["vm_priority"].lower():
cluster.delete()
cluster.wait_for_completion(show_output=True)
raise ComputeTargetException(
"Cluster is of incorrect size or has incorrect priority. Deleting cluster and provisioning a new one."
)
# Update AMLCompute
#if cluster.provisioning_configuration.min_nodes != aml_settings["min_nodes"] or cluster.provisioning_configuration.max_nodes != aml_settings["max_nodes"] or cluster.provisioning_configuration.idle_seconds_before_scaledown != aml_settings["idle_seconds_before_scaledown"]:
print("Updating settings of Cluster")
cluster.update(min_nodes=aml_settings["min_nodes"],
from azureml.core import Workspace
from azureml.core.compute import ComputeTarget, AmlCompute
from azureml.core.compute_target import ComputeTargetException
ws = Workspace.get(name='akws',
subscription_id='8b3748c0-bb0b-4913-ab5b-c462062118fe',
resource_group='akrg')
cpu_cluster_name = 'tdsp-cluster'
#verify that cluster does not exist
try:
cpu_cluster = AmlCompute(workspace=ws, name=cpu_cluster_name)
print('Cluster already exists.')
except ComputeTargetException:
compute_config = AmlCompute.provisioning_configuration(
vm_size='Standard_NC6', max_nodes=4)
cpu_cluster = AmlCompute.create(ws, cpu_cluster_name, compute_config)
cpu_cluster.wait_for_completion(show_output=True)
print('2. Authenticate...')
print('.............................................')
cliAuth = AzureCliAuthentication()
print('..2. completed')
print('')
print('')
print('3. Instantiate AML workspace object ref...')
print('.............................................')
amlWs = Workspace.from_config(path=args.path, auth=cliAuth)
amlWsStorageRef = amlWs.get_default_datastore()
print('..3. completed')
print('')
print('')
print('4. Instantiate AML managed compute ref...')
print('.............................................')
amlTrainingComputeRef = AmlCompute(amlWs, args.aml_compute_target)
print('..4. completed')
print('')
print('')
print("5. Instantiate and configure run object for the managed compute...")
print('.............................................')
# Create runconfig object
amlComputeRunConf = RunConfiguration()
# Use the compute provisioned
amlComputeRunConf.target = args.aml_compute_target
# Enable Docker
amlComputeRunConf.environment.docker.enabled = True
# Set Docker base image to the default CPU-based image
amlComputeRunConf.environment.docker.base_image = DEFAULT_CPU_IMAGE
# Use conda_dependencies.yml to create a conda environment in the Docker image for execution
