def _create_default_run_configs(project_directory, compute_target_dict):
"""
Creates a local.runconfig and docker.runconfig for a project.
:return: None
"""
from azureml.core.runconfig import RunConfiguration
# Mocking a project object, as RunConfiguration requires a Project object, but only requires
# project_directory field.
project_object = empty_function
project_object.project_directory = project_directory
# Creating a local runconfig.
local_run_config = RunConfiguration()
local_run_config.save(name="local", path=project_directory)
# Creating a docker runconfig.
docker_run_config = RunConfiguration()
docker_run_config.environment.docker.enabled = True
docker_run_config.save(name="docker", path=project_directory)
for compute_target_name, compute_target in compute_target_dict.items():
# Creating a compute runconfig.
compute_config = RunConfiguration()
if compute_target.type == 'HDInsight':
compute_config.framework = "PySpark"
else:
compute_config.framework = "Python"
compute_config.environment.docker.enabled = True
compute_config.target = compute_target_name
compute_config.save(name=compute_target_name, path=project_directory)
def create_run_config(azure_config: AzureConfig,
source_config: SourceConfig,
all_azure_dataset_ids: List[str],
all_dataset_mountpoints: List[str],
environment_name: str = "") -> ScriptRunConfig:
"""
Creates a configuration to run the InnerEye training script in AzureML.
:param azure_config: azure related configurations to use for model scale-out behaviour
:param source_config: configurations for model execution, such as name and execution mode
:param all_azure_dataset_ids: The name of all datasets on blob storage that will be used for this run.
:param all_dataset_mountpoints: When using the datasets in AzureML, these are the per-dataset mount points.
:param environment_name: If specified, try to retrieve the existing Python environment with this name. If that
is not found, create one from the Conda files provided in `source_config`. This parameter is meant to be used
when running inference for an existing model.
:return: The configured script run.
"""
dataset_consumptions = create_dataset_consumptions(
azure_config, all_azure_dataset_ids, all_dataset_mountpoints)
# AzureML seems to sometimes expect the entry script path in Linux format, hence convert to posix path
entry_script_relative_path = source_config.entry_script.relative_to(
source_config.root_folder).as_posix()
logging.info(
f"Entry script {entry_script_relative_path} ({source_config.entry_script} relative to "
f"source directory {source_config.root_folder})")
max_run_duration = None
if azure_config.max_run_duration:
max_run_duration = run_duration_string_to_seconds(
azure_config.max_run_duration)
workspace = azure_config.get_workspace()
run_config = RunConfiguration(
script=entry_script_relative_path,
arguments=source_config.script_params,
)
run_config.environment = get_or_create_python_environment(
azure_config, source_config, environment_name=environment_name)
run_config.target = azure_config.cluster
run_config.max_run_duration_seconds = max_run_duration
if azure_config.num_nodes > 1:
distributed_job_config = MpiConfiguration(
node_count=azure_config.num_nodes)
run_config.mpi = distributed_job_config
run_config.framework = "Python"
run_config.communicator = "IntelMpi"
run_config.node_count = distributed_job_config.node_count
if len(dataset_consumptions) > 0:
run_config.data = {
dataset.name: dataset
for dataset in dataset_consumptions
}
# Use blob storage for storing the source, rather than the FileShares section of the storage account.
run_config.source_directory_data_store = workspace.datastores.get(
WORKSPACE_DEFAULT_BLOB_STORE_NAME).name
script_run_config = ScriptRunConfig(
source_directory=str(source_config.root_folder),
run_config=run_config,
)
if azure_config.hyperdrive:
script_run_config = source_config.hyperdrive_config_func(
script_run_config) # type: ignore
return script_run_config
def _write_compute_run_config(source_directory, compute_target_object, compute_yaml):
"""
:param source_directory:
:type source_directory: str
:param compute_target_object:
:type compute_target_object: azureml.core.compute_target.AbstractComputeTarget
:param compute_yaml:
:type compute_yaml: dict
:return:
"""
from azureml.core.compute_target import _BatchAITarget
# Writing the target.compute file.
run_config_dir_name = get_run_config_dir_name(source_directory)
file_path = os.path.join(source_directory, run_config_dir_name,
compute_target_object.name + COMPUTECONTEXT_EXTENSION)
with open(file_path, 'w') as outfile:
ruamel.yaml.dump(compute_yaml, outfile, default_flow_style=False)
# This creates a run config and writes it in the aml_config/<compute_target_name>.runconfig file
run_config_object = RunConfiguration()
run_config_object.target = compute_target_object
if compute_target_object.type == _BatchAITarget._BATCH_AI_TYPE:
run_config_object.environment.docker.enabled = True
run_config_object.framework = compute_target_object._default_framework
run_config_object.save(name=compute_target_object.name, path=source_directory)
def create_run_config(azure_config: AzureConfig,
source_config: SourceConfig,
azure_dataset_id: str = "",
environment_name: str = "") -> ScriptRunConfig:
"""
Creates a configuration to run the InnerEye training script in AzureML.
:param azure_config: azure related configurations to use for model scale-out behaviour
:param source_config: configurations for model execution, such as name and execution mode
:param azure_dataset_id: The name of the dataset in blob storage to be used for this run. This can be an empty
string to not use any datasets.
:param environment_name: If specified, try to retrieve the existing Python environment with this name. If that
is not found, create one from the Conda files provided in `source_config`. This parameter is meant to be used
when running inference for an existing model.
:return: The configured script run.
"""
if azure_dataset_id:
azureml_dataset = get_or_create_dataset(azure_config, azure_dataset_id=azure_dataset_id)
if not azureml_dataset:
raise ValueError(f"AzureML dataset {azure_dataset_id} could not be found or created.")
named_input = azureml_dataset.as_named_input(INPUT_DATA_KEY)
dataset_consumption = named_input.as_mount() if azure_config.use_dataset_mount else named_input.as_download()
else:
dataset_consumption = None
# AzureML seems to sometimes expect the entry script path in Linux format, hence convert to posix path
entry_script_relative_path = source_config.entry_script.relative_to(source_config.root_folder).as_posix()
logging.info(f"Entry script {entry_script_relative_path} ({source_config.entry_script} relative to "
f"source directory {source_config.root_folder})")
max_run_duration = None
if azure_config.max_run_duration:
max_run_duration = run_duration_string_to_seconds(azure_config.max_run_duration)
workspace = azure_config.get_workspace()
run_config = RunConfiguration(
script=entry_script_relative_path,
arguments=source_config.script_params,
)
run_config.environment = get_or_create_python_environment(azure_config, source_config,
environment_name=environment_name)
run_config.target = azure_config.cluster
run_config.max_run_duration_seconds = max_run_duration
if azure_config.num_nodes > 1:
distributed_job_config = MpiConfiguration(node_count=azure_config.num_nodes)
run_config.mpi = distributed_job_config
run_config.framework = "Python"
run_config.communicator = "IntelMpi"
run_config.node_count = distributed_job_config.node_count
if dataset_consumption:
run_config.data = {dataset_consumption.name: dataset_consumption}
# Use blob storage for storing the source, rather than the FileShares section of the storage account.
run_config.source_directory_data_store = workspace.datastores.get(WORKSPACE_DEFAULT_BLOB_STORE_NAME).name
script_run_config = ScriptRunConfig(
source_directory=str(source_config.root_folder),
run_config=run_config,
)
if azure_config.hyperdrive:
script_run_config = source_config.hyperdrive_config_func(script_run_config) # type: ignore
return script_run_config
try:
# Attaches a remote docker on a remote vm as a compute target.
project.attach_legacy_compute_target(RemoteTarget(name = "cpu-dsvm",
address = "hai2.eastus2.cloudapp.azure.com:5022",
username = "******",
password = '******'))
except UserErrorException as e:
print("Caught = {}".format(e.message))
print("Compute config already attached.")
print('Load the cpu-dsvm config and make it run Spark:')
# Load the "cpu-dsvm.runconfig" file (created by the above attach operation) in memory
run_config = RunConfiguration.load(project_object = project, run_config_name = "cpu-dsvm")
# set framework to PySpark
run_config.framework = "PySpark"
# Use Docker in the remote VM
run_config.environment.docker.enabled = True
# Use the MMLSpark CPU based image.
# https://hub.docker.com/r/microsoft/mmlspark/
run_config.environment.docker.base_image = azureml.core.runconfig.DEFAULT_MMLSPARK_CPU_IMAGE
print('base image is:', run_config.environment.docker.base_image)
# signal use the user-managed environment
# do NOT provision a new one based on the conda.yml file
run_config.environment.python.user_managed_dependencies = False
# Prepare the Docker and conda environment automatically when execute for the first time.
run_config.prepare_environment = True
from azureml.core.compute import ComputeTarget, AmlCompute
from azureml.core.compute_target import ComputeTargetException
from azureml.core.conda_dependencies import CondaDependencies
from azureml.core.runconfig import MpiConfiguration, RunConfiguration, DEFAULT_CPU_IMAGE, DEFAULT_GPU_IMAGE
from azureml.train.estimator import Estimator
# Load tenant ID from config.json; the tenant ID must be manually obtained
# from the Azure portal:
config = json.load(open('config.json', 'rt'))
tenant_id = config['tenant_id']
interactive_auth = InteractiveLoginAuthentication(tenant_id=tenant_id)
# This will open a web page to enable one to authenticate:
ws = Workspace.from_config(auth=interactive_auth)
run_conf = RunConfiguration()
run_conf.framework = 'Python'
# Number of folds for cross validation; if set to None, no cross validation is
# performed (and hence dask is not used):
cv = 3
# Use local development environment:
compute_name = config['compute_name']
if compute_name == 'local':
run_conf.environment.python.user_managed_dependencies = True
compute_target = compute_name
# Use AzureML compute target:
else:
# Create compute target if it doesn't already exist:
## Environment: docker section
myenv.docker.enabled = True
myenv.docker.base_image = "mcr.microsoft.com/azureml/base-gpu:openmpi3.1.2-cuda10.1-cudnn7-ubuntu18.04"
# comment out this environment variable if you don't have it set!
myenv.environment_variables = {'WANDB_API_KEY': os.environ['WANDB_API_KEY']}
## Environment: python section
conda_dep = CondaDependencies(conda_dependencies_file_path='environment.yml')
myenv.python.conda_dependencies = conda_dep
# create configuration for Run
# Use RunConfiguration to specify compute target / env deps part of run
run_config = RunConfiguration()
# Attach compute target to run config
run_config.framework = 'python'
run_config.target = "gpu-cluster"
# This doesn't actuallly do anything since my target is a persistent compute instead of amlcompute
run_config.amlcompute.vm_size = "Standard_NC24"
run_config.node_count = 1
run_config.environment = myenv
# ScriptRunConfig packaages together environment configuration of
# RunConfiguration with a script for training to create a **script** run.
""" RunConfiguration(script=None, arguments=None, framework=None, communicator=None,
conda_dependencies=None, _history_enabled=None, _path=None, _name=None)
"""
src = ScriptRunConfig(source_directory='.',
script='rl_credit/examples/distractor_delay_expt.py',
run_config=run_config)
def main():
# Replace this with the Use of Service Principal for authenticating with the Workspace
ws = Workspace.from_config()
# Choose a name for your CPU cluster
cpu_cluster_name = "cpu-cluster"
# Verify that cluster does not exist already
try:
cpu_cluster = ComputeTarget(workspace=ws, name=cpu_cluster_name)
print('Found existing cluster, use it.')
except ComputeTargetException:
compute_config = AmlCompute.provisioning_configuration(
vm_size='STANDARD_D2_V2', max_nodes=4)
cpu_cluster = ComputeTarget.create(ws, cpu_cluster_name,
compute_config)
cpu_cluster.wait_for_completion(show_output=True)
# Run configuration for R
rc = RunConfiguration()
rc.framework = "R"
# Run configuration for python
py_rc = RunConfiguration()
py_rc.framework = "Python"
py_rc.environment.docker.enabled = True
# Combine GitHub and Cran packages for R env
rc.environment.r = RSection()
rc.environment.docker.enabled = True
# Upload iris data to the datastore
# target_path = "iris_data"
# upload_files_to_datastore(ds,
# list("./iris.csv"),
# target_path = target_path,
# overwrite = TRUE)
training_data = DataReference(
datastore=ws.get_default_datastore(),
data_reference_name="iris_data",
path_on_datastore="iris_data/iris.csv",
)
print('Succesfull')
print(training_data)
# PipelineData object for newly trained model
# trained_model_dir = PipelineData(
# name="trained_model", datastore=ws.get_default_datastore(), is_directory=True
# )
# Train and Register the model
train_step = RScriptStep(
script_name="train.R",
arguments=[training_data],
inputs=[training_data],
compute_target=cpu_cluster_name,
source_directory=".",
runconfig=rc,
allow_reuse=True,
)
# Deploy the trained model
print("Step Train created")
steps = [train_step]
train_pipeline = Pipeline(workspace=ws, steps=steps)
train_pipeline.validate()
pipeline_run = Experiment(ws, 'iris_training').submit(train_pipeline)
pipeline_run.wait_for_completion(show_output=True)
published_pipeline = train_pipeline.publish(
name="iris-train",
description="Model training/retraining pipeline",
)
print(f"Published pipeline: {published_pipeline.name}")
print(f"for build {published_pipeline.version}")
