def load_module(workspace, namespace, name, yaml_file_path):
try:
module_func = Module.load(workspace=workspace, namespace=namespace, name=name)
print('found the module of {}'.format(name))
return module_func
except:
print('not found the module of {}, register it now...'.format(name))
module_func = Module.register(workspace=workspace, yaml_file=yaml_file_path)
return module_func
vm_size="STANDARD_D2_V2", min_nodes=1, max_nodes=4)
aml_compute = ComputeTarget.create(workspace, aml_compute_target,
provisioning_config)
aml_compute.wait_for_completion(show_output=True,
min_node_count=None,
timeout_in_minutes=20)
# In[ ]:
try:
mpi_train_module_func = Module.load(
workspace,
namespace="microsoft.com/azureml/samples",
name="Hello World MPI Job")
except:
mpi_train_module_func = Module.register(
workspace, os.path.join('modules', 'mpi_module', 'module_spec.yaml'))
from azureml.pipeline.wrapper._dataset import get_global_dataset_by_path
blob_input_data = get_global_dataset_by_path(
workspace, 'Automobile_price_data',
'GenericCSV/Automobile_price_data_(Raw)')
mpi_train = mpi_train_module_func(input_path=blob_input_data,
string_parameter="test1")
mpi_train.runsettings.configure(node_count=2, process_count_per_node=2)
print(mpi_train.runsettings.node_count)
mpi_train.runsettings.node_count = 1
# In[ ]:
provisioning_config = AmlCompute.provisioning_configuration(vm_size = "STANDARD_D2_V2",
min_nodes = 1,
max_nodes = 4)
aml_compute = ComputeTarget.create(ws, aml_compute_target, provisioning_config)
aml_compute.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20)
# In[ ]:
# modules
try:
ejoin_module_func = Module.load(ws, namespace='microsoft.com/bing', name='ejoin')
eselect_module_func = Module.load(ws, namespace='microsoft.com/bing', name='eselect')
except:
ejoin_module_func = Module.register(ws, os.path.join('modules', 'ejoin', 'amlmodule.yaml'))
eselect_module_func = Module.register(ws, os.path.join('modules', 'eselect', 'amlmodule.yaml'))
training_data_name = "Titanic.tsv"
if training_data_name not in ws.datasets:
print('Registering a training dataset for sample pipeline ...')
train_data = Dataset.File.from_files(path=['https://desginerdemo.blob.core.windows.net/demo/titanic.tsv'])
train_data.register(workspace = ws,
name = training_data_name,
description = 'Training data (just for illustrative purpose)')
print('Registerd')
else:
train_data = ws.datasets[training_data_name]
print('Training dataset found in workspace')
# datasets
provisioning_config = AmlCompute.provisioning_configuration(
vm_size="STANDARD_D2_V2", min_nodes=1, max_nodes=4)
aml_compute = ComputeTarget.create(ws, aml_compute_target,
provisioning_config)
aml_compute.wait_for_completion(show_output=True,
min_node_count=None,
timeout_in_minutes=20)
# In[ ]:
try:
train_module_func = Module.load(ws,
namespace='microsoft.com/aml/samples',
name='Train')
except:
train_module_func = Module.register(
ws, os.path.join('modules', 'train-score-eval', 'train.yaml'))
try:
score_module_func = Module.load(ws,
namespace='microsoft.com/aml/samples',
name='Score')
except:
score_module_func = Module.register(
ws, os.path.join('modules', 'train-score-eval', 'score.yaml'))
try:
eval_module_func = Module.load(ws,
namespace='microsoft.com/aml/samples',
name='Evaluate')
except:
eval_module_func = Module.register(