def __get_run_config(self,
compute_target,
channels=None,
conda_packages=None,
pip_packages=None):
# Load the "cpu-dsvm.runconfig" file (created by the above attach operation) in memory
run_config = RunConfiguration(framework="python")
# Set compute target to the Linux DSVM
run_config.target = compute_target.name
# Use Docker in the remote VM
run_config.environment.docker.enabled = False
# Ask system to provision a new one based on the conda_dependencies.yml file
run_config.environment.python.user_managed_dependencies = False
# Prepare the Docker and conda environment automatically when used the first time.
run_config.auto_prepare_environment = True
# specify dependencies obj
conda_dependencies = CondaDependencies.create(
conda_packages=conda_packages, pip_packages=pip_packages)
if (channels):
for channel in channels:
conda_dependencies.add_channel(channel)
run_config.environment.python.conda_dependencies = conda_dependencies
return run_config
path_on_datastore=dataPathRemote,
path_on_compute='/tmp',
mode='download',
overwrite=False)
# Create a new RunConfiguration and attach data
runConfig = RunConfiguration()
runConfig.data_references = {
ds.name: dataRef
} # This syntax is not documented!
if not os.environ.get('AZML_RUN_LOCAL', 'false') == "true":
# Set it up for running in Azure ML compute
runConfig.target = computeTarget
runConfig.environment.docker.enabled = True
runConfig.auto_prepare_environment = True
runConfig.environment.python.conda_dependencies = CondaDependencies.create(
conda_packages=['scikit-learn==0.20.3', 'pandas', 'matplotlib'])
print(
f"### Will execute script {trainingScriptDir}/{trainingScript} on REMOTE compute"
)
else:
# OR set up RunConfig to run local, needs a pre-set up Python 3 virtual env
runConfig.environment.python.user_managed_dependencies = True
runConfig.environment.python.interpreter_path = os.environ[
'VIRTUAL_ENV'] + "/bin/python"
print(
f"### Will execute script {trainingScriptDir}/{trainingScript} on LOCAL compute"
)
# Pass two args to the training script
run_amlcompute = RunConfiguration()
# Use the cpu_cluster you created above.
run_amlcompute.target = args.aml_compute_target
# Enable Docker
run_amlcompute.environment.docker.enabled = True
# Set Docker base image to the default CPU-based image
run_amlcompute.environment.docker.base_image = DEFAULT_CPU_IMAGE
# Use conda_dependencies.yml to create a conda environment in the Docker image for execution
run_amlcompute.environment.python.user_managed_dependencies = False
# Auto-prepare the Docker image when used for execution (if it is not already prepared)
run_amlcompute.auto_prepare_environment = True
# Specify CondaDependencies obj, add necessary packages
run_amlcompute.environment.python.conda_dependencies = CondaDependencies.create(
pip_packages=[
'numpy', 'pandas', 'scikit-learn==0.20.3', 'sklearn_pandas',
'azureml-sdk'
])
scripts_folder = 'scripts'
def_blob_store = ws.get_default_datastore()
train_output = PipelineData('train_output', datastore=def_blob_store)
print("train_output PipelineData object created")
trainStep = PythonScriptStep(name="train",
print('')
print("5. Instantiate and configure run object for the managed compute...")
print('.............................................')
# Create runconfig object
amlComputeRunConf = RunConfiguration()
# Use the compute provisioned
amlComputeRunConf.target = args.aml_compute_target
# Enable Docker
amlComputeRunConf.environment.docker.enabled = True
# Set Docker base image to the default CPU-based image
amlComputeRunConf.environment.docker.base_image = DEFAULT_CPU_IMAGE
# Use conda_dependencies.yml to create a conda environment in the Docker image for execution
amlComputeRunConf.environment.python.user_managed_dependencies = False
# Auto-prepare the Docker image when used for execution (if it is not already prepared)
amlComputeRunConf.auto_prepare_environment = True
# Specify CondaDependencies obj, add necessary packages
amlComputeRunConf.environment.python.conda_dependencies = CondaDependencies.create(
pip_packages=['numpy', 'pandas', 'scikit-learn', 'azureml-sdk'])
print("..5. completed")
print('')
print('')
print("6. Define pipeline stage - training...")
print('.............................................')
training_output = PipelineData('train_output', datastore=amlWsStorageRef)
trainPipelineStep = PythonScriptStep(name="train",
script_name="train.py",
arguments=[
"--model_name", args.model_name,
"--build_number", args.build_number
