def __get_run_config(self,
compute_target,
channels=None,
conda_packages=None,
pip_packages=None):
# Load the "cpu-dsvm.runconfig" file (created by the above attach operation) in memory
run_config = RunConfiguration(framework="python")
# Set compute target to the Linux DSVM
run_config.target = compute_target.name
# Use Docker in the remote VM
run_config.environment.docker.enabled = False
# Ask system to provision a new one based on the conda_dependencies.yml file
run_config.environment.python.user_managed_dependencies = False
# Prepare the Docker and conda environment automatically when used the first time.
run_config.auto_prepare_environment = True
# specify dependencies obj
conda_dependencies = CondaDependencies.create(
conda_packages=conda_packages, pip_packages=pip_packages)
if (channels):
for channel in channels:
conda_dependencies.add_channel(channel)
run_config.environment.python.conda_dependencies = conda_dependencies
return run_config
def create_run_config(cpu_cluster, docker_proc_type, conda_env_file):
"""
AzureML requires the run environment to be setup prior to submission.
This configures a docker persistent compute. Even though
it is called Persistent compute, AzureML handles startup/shutdown
of the compute environment.
Args:
cpu_cluster (str) : Names the cluster for the test
In the case of unit tests, any of
the following:
- Reco_cpu_test
- Reco_gpu_test
docker_proc_type (str) : processor type, cpu or gpu
conda_env_file (str) : filename which contains info to
set up conda env
Return:
run_amlcompute : AzureML run config
"""
# runconfig with max_run_duration_seconds did not work, check why:
# run_amlcompute = RunConfiguration(max_run_duration_seconds=60*30)
run_amlcompute = RunConfiguration()
run_amlcompute.target = cpu_cluster
run_amlcompute.environment.docker.enabled = True
run_amlcompute.environment.docker.base_image = docker_proc_type
# Use conda_dependencies.yml to create a conda environment in
# the Docker image for execution
# False means the user will provide a conda file for setup
# True means the user will manually configure the environment
run_amlcompute.environment.python.user_managed_dependencies = False
run_amlcompute.environment.python.conda_dependencies = CondaDependencies(
conda_dependencies_file_path=conda_env_file)
return run_amlcompute
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable Azure ML environment
environment = get_environment(aml_workspace,
e.aml_env_name,
create_new=False) # NOQA: E501
run_config = RunConfiguration()
run_config.environment = environment
if (e.datastore_name):
datastore_name = e.datastore_name
else:
datastore_name = aml_workspace.get_default_datastore().name
run_config.environment.environment_variables[
"DATASTORE_NAME"] = datastore_name # NOQA: E501
def fetch_run_config(compute_target, base_image, sp_username, sp_tenant,
sp_password):
""" Generates a Run Configuration based on the pipeline parameters,
specifying such things as the Compute Target and Conda Dependencies.
"""
# Inits configuration for Python
run_config = RunConfiguration(framework="python")
# Specifies compute target
run_config.target = compute_target
# Configures Docker/Image/Environment Variable parameters
run_config.environment.docker.enabled = True
run_config.environment.docker.base_image = base_image
run_config.environment.environment_variables = {
"SP_USERNAME": sp_username,
"SP_TENANT": sp_tenant,
"SP_PASSWORD": sp_password
}
# Specifies Conda file location (Auto-injected from preparing staging)
run_config.environment.python.conda_dependencies = CondaDependencies(
os.path.join("snapshot", "inputs", "environment.yml"))
# Returns configuration
return run_config
def create_runconfig(aml_compute, env=None):
# Create a new runconfig object
aml_run_config = RunConfiguration()
# Use the aml_compute you created above.
aml_run_config.target = aml_compute
if env:
aml_run_config.environment = env
else:
aml_run_config.environment = create_env_from_requirements()
return aml_run_config
def get_run_config(self, config):
environment_config = config.get("environment")
environment = self.get_environment(environment_config)
cluster_name = config.get("cluster")
cluster = ComputeTarget(workspace=self.workspace, name=cluster_name)
pipeline_run_config = RunConfiguration()
pipeline_run_config.target = cluster
pipeline_run_config.environment = environment
return pipeline_run_config
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group,
)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable Azure ML environment
# Make sure to include `r-essentials'
# in h1c4driver/conda_dependencies.yml
environment = get_environment(
aml_workspace,
e.aml_env_name,
conda_dependencies_file=e.aml_env_train_conda_dep_file,
create_new=e.rebuild_env,
) # NOQA: E501
run_config = RunConfiguration()
run_config.environment = environment
train_step = PythonScriptStep(
name="Train Model",
script_name="train_with_r.py",
compute_target=aml_compute,
source_directory="h1c4driver/training/R",
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
steps = [train_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id,
)
print(f"Published pipeline: {published_pipeline.name}")
print(f"for build {published_pipeline.version}")
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a reusable run configuration environment
# Read definition from diabetes_regression/azureml_environment.json
# Make sure to include `r-essentials'
# in diabetes_regression/conda_dependencies.yml
environment = Environment.load_from_directory(e.sources_directory_train)
if (e.collection_uri is not None and e.teamproject_name is not None):
builduri_base = e.collection_uri + e.teamproject_name
builduri_base = builduri_base + "/_build/results?buildId="
environment.environment_variables["BUILDURI_BASE"] = builduri_base
environment.register(aml_workspace)
run_config = RunConfiguration()
run_config.environment = environment
train_step = PythonScriptStep(
name="Train Model",
script_name="train_with_r.py",
compute_target=aml_compute,
source_directory="diabetes_regression/training/R",
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
steps = [train_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
def main():
print("Beginning ML runs")
#### This doc should contain meta-data for ML experiment runs ####
#This sets the compute target to your local machine.
run_user_managed = RunConfiguration()
run_user_managed.environment.python.user_managed_dependencies = False
#Can also adjust for a specific python interpreter if desired. Otherwise, it uses the specific one declared by IDE.
#This determines what workspace to run experiments in. Should not need any edits.
try:
#This provides information to access ML Service workspace. Do not change.
ws = Workspace(subscription_id="8402fd02-6a15-499c-bb04-1dee338962d6",
resource_group="NDSUCapstone2019",
workspace_name="Capstone2019MLServerWorksapce")
# write the details of the workspace to a configuration file.
ws.write_config()
print(
"Workspace configuration succeeded. Skip the workspace creation steps below"
)
except:
print(
"Workspace not accessible. Change your parameters or create a new workspace below"
)
ws = Workspace.from_config()
#The remainder of this doc should run the models defined in other documents.
#This experiment runs the REGRESSION model.
experiment = Experiment(ws, "Regression_Attempt_1")
automated_ml_config = model(preppedData[0], preppedData[2],
"FOLDER HARD CODED", 10, 20,
'spearman_correlation', 5, 'regression')
local_run = experiment.submit(automated_ml_config, show_output=True)
def get_run_cfg(ws, pip_packages, conda_packages, ext_wheels, gpu=True):
'''
get_run_cfg - Retrieves the AMLS run configuration.
:returns: AMLS run configuration
:rtype: RunConfiguration object
'''
conda_dep = CondaDependencies()
for pip_package in pip_packages:
conda_dep.add_pip_package(pip_package)
for conda_package in conda_packages:
conda_dep.add_conda_package(conda_package)
for whl_path in ext_wheels:
whl_url = Environment.add_private_pip_wheel(workspace=ws,
file_path=whl_path,
exist_ok=True)
conda_dep.add_pip_package(whl_url)
run_cfg = RunConfiguration(conda_dependencies=conda_dep)
run_cfg.environment.docker.enabled = True
run_cfg.environment.docker.gpu_support = gpu
if gpu:
run_cfg.environment.docker.base_image = DEFAULT_GPU_IMAGE
else:
run_cfg.environment.docker.base_image = DEFAULT_CPU_IMAGE
run_cfg.environment.spark.precache_packages = False
return run_cfg
def is_compute_target_prepared(experiment, source_directory, run_config):
"""Check compute target is prepared.
Checks whether the compute target, specified in run_config, is already prepared or not for the specified run
configuration.
:param experiment:
:type experiment: azureml.core.experiment.Experiment
:param source_directory:
:type source_directory: str
:param run_config: The run configuration. This can be a run configuration name, as string, or a
azureml.core.runconfig.RunConfiguration object.
:type run_config: str or azureml.core.runconfig.RunConfiguration
:return: True, if the compute target is prepared.
:rtype: bool
"""
from azureml._execution import _commands
from azureml.core.runconfig import RunConfiguration
from azureml._project.project import Project
run_config_object = RunConfiguration._get_run_config_object(
path=source_directory, run_config=run_config)
project_object = Project(experiment=experiment, directory=source_directory)
return _commands.prepare_compute_target(project_object,
run_config_object,
check=True)
def data_ingestion_step(datastore_reference, compute_target):
run_config = RunConfiguration()
run_config.environment.docker.enabled = True
raw_data_dir = PipelineData(
name='raw_data_dir',
pipeline_output_name='raw_data_dir',
datastore=datastore_reference.datastore,
output_mode='mount',
is_directory=True)
outputs = [raw_data_dir]
outputs_map = { 'raw_data_dir': raw_data_dir }
step = PythonScriptStep(
script_name='data_ingestion.py',
arguments=['--output_dir', raw_data_dir, ],
inputs=[datastore_reference],
outputs=outputs,
compute_target=compute_target,
source_directory=os.path.dirname(os.path.abspath(__file__)),
runconfig=run_config,
allow_reuse=True
)
return step, outputs_map
def main(workspace,inputs):
print("Loading compute target")
compute_target = ComputeTarget(
workspace=workspace,
name=inputs["compute"]
)
# create a new runconfig object
run_config = RunConfiguration()
# enable Docker
run_config.environment.docker.enabled = True
# set Docker base image to the default CPU-based image
run_config.environment.docker.base_image = DEFAULT_CPU_IMAGE
# use conda_dependencies.yml to create a conda environment in the Docker image for execution
run_config.environment.python.user_managed_dependencies = False
# specify CondaDependencies obj
run_config.environment.python.conda_dependencies = CondaDependencies.create(conda_packages=['scikit-learn'])
# For this step, we use yet another source_directory
step = PythonScriptStep(name=inputs["step_name"],
script_name=inputs["train_script"],
compute_target=compute_target,
source_directory=inputs["source_directory"],
runconfig=run_config,
allow_reuse=True)
return step
def tb():
import os
ws = Workspace.from_config()
print(ws.name)
proj = Project.attach(ws, 'tbhistory', '/tmp/tb-test')
shutil.copy('tftb.py', os.path.join(proj.project_directory, 'tftb.py'))
from azureml.core.compute_target import RemoteTarget
rt = RemoteTarget(name='dsvm', address='hai2.eastus2.cloudapp.azure.com:5022', username='******', password='******')
proj.attach_legacy_compute_target(rt)
rc = RunConfiguration.load(proj, "dsvm")
rc.environment.python.user_managed_dependencies = True
rc.environment.python.interpreter_path = '/anaconda/envs/tf/bin/python'
print(rc.target)
run = Run.submit(proj, rc, 'tftb.py')
print(run.id)
#run.wait_for_completion(show_output=True)
from azureml.contrib.tensorboard import Tensorboard
tb = Tensorboard([run])
print('starting tensorboard...')
print(tb.start())
print('tensorboard started.')
run.wait_for_completion(show_output=True)
tb.stop()
def submit_job():
ws = Workspace.from_config()
proj = Project.attach(ws, 'util', '/tmp/random_proj')
rc = RunConfiguration(proj, "local")
shutil.copy('./train-sklearn-one-model.py', '/tmp/random_proj/train-sklearn-one-model.py')
#run = Run.submit(proj, rc, "train-sklearn-one-model.py", "--alpha 0.9")
run = Run.submit(proj, rc, "train-sklearn-one-model.py", arguments_list=["--alpha", "0.9"])
run.wait_for_completion(show_output=True)
def run_local_compute_experiment(ws, experiment_name, entry_script, source_directory=os.getcwd()):
# Edit a run configuration property on the fly.
run_local = RunConfiguration()
run_local.environment.python.user_managed_dependencies = True
exp = Experiment(workspace=ws, name=experiment_name)
src = ScriptRunConfig(source_directory=source_directory, script=entry_script, run_config=run_local)
run = exp.submit(src)
run.wait_for_completion(show_output=True)
def test():
ws = Workspace.from_config()
proj = Project.attach(ws, 'test_rh', '/tmp/randomproj1')
rc = RunConfiguration(proj, 'local')
rc.environment.python.interpreter_path = '/Users/haining/miniconda3/envs/comet/bin/python'
with open('/tmp/randomproj1/test.py', 'w') as file:
file.write('import sys; print(sys.version);import os;os.makedirs("./outputs", exist_ok=True);fs=open("./outputs/f.txt","w");fs.write("hello!");')
r = Run.submit(proj, rc, 'test.py')
print(helpers.get_run_history_url(r))
r.wait_for_completion(show_output=True)
def create_runconfig(aml_compute, env=None):
# Create a new runconfig object
aml_run_config = RunConfiguration()
# Use the aml_compute you created above.
aml_run_config.target = aml_compute
if env is not None:
aml_run_config.environment = env
else:
# Enable Docker
aml_run_config.environment.docker.enabled = True
# Set Docker base image to the default CPU-based image
aml_run_config.environment.docker.base_image = "mcr.microsoft.com/azureml/base:0.2.1"
# Use conda_dependencies.yml to create a conda environment in the Docker image for execution
aml_run_config.environment.python.user_managed_dependencies = False
return aml_run_config
def main():
load_dotenv()
workspace_name = os.environ.get("BASE_NAME") + "-AML-WS"
resource_group = os.environ.get("BASE_NAME") + "-AML-RG"
subscription_id = os.environ.get("SUBSCRIPTION_ID")
tenant_id = os.environ.get("TENANT_ID")
app_id = os.environ.get("SP_APP_ID")
app_secret = os.environ.get("SP_APP_SECRET")
vm_size = os.environ.get("AML_COMPUTE_CLUSTER_CPU_SKU")
compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME")
build_id = os.environ.get("BUILD_BUILDID")
pipeline_name = os.environ.get("TRAINING_PIPELINE_NAME")
# Get Azure machine learning workspace
aml_workspace = get_workspace(workspace_name, resource_group,
subscription_id, tenant_id, app_id,
app_secret)
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, compute_name, vm_size)
if aml_compute is not None:
print(aml_compute)
run_config = RunConfiguration(conda_dependencies=CondaDependencies.create(
conda_packages=[
'numpy', 'pandas', 'scikit-learn', 'tensorflow', 'keras'
],
pip_packages=[
'azure', 'azureml-core', 'azure-storage', 'azure-storage-blob'
]))
run_config.environment.docker.enabled = True
run_config.environment.docker.base_image = "mcr.microsoft.com/mlops/python"
train_step = PythonScriptStep(
name="Train Model",
script_name="train_with_r.py",
compute_target=aml_compute,
source_directory="code/training/R",
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
steps = [train_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=pipeline_name + "_with_R",
description="Model training/retraining pipeline",
version=build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
def get_local_run_configuration() -> RunConfiguration:
"""
Get Local Run Config
:return:
"""
# Editing a run configuration property on-fly.
run_config_user_managed = RunConfiguration()
run_config_user_managed.environment.python.user_managed_dependencies = True
# Choose the specific Python environment of this tutorial by pointing to the Python path
run_config_user_managed.environment.python.interpreter_path = sys.executable
return run_config_user_managed
def _write_compute_run_config(source_directory, compute_target_object, compute_yaml):
"""
:param source_directory:
:type source_directory: str
:param compute_target_object:
:type compute_target_object: azureml.core.compute_target.AbstractComputeTarget
:param compute_yaml:
:type compute_yaml: dict
:return:
"""
from azureml.core.compute_target import _BatchAITarget
# Writing the target.compute file.
run_config_dir_name = get_run_config_dir_name(source_directory)
file_path = os.path.join(source_directory, run_config_dir_name,
compute_target_object.name + COMPUTECONTEXT_EXTENSION)
with open(file_path, 'w') as outfile:
ruamel.yaml.dump(compute_yaml, outfile, default_flow_style=False)
# This creates a run config and writes it in the aml_config/<compute_target_name>.runconfig file
run_config_object = RunConfiguration()
run_config_object.target = compute_target_object
if compute_target_object.type == _BatchAITarget._BATCH_AI_TYPE:
run_config_object.environment.docker.enabled = True
run_config_object.framework = compute_target_object._default_framework
run_config_object.save(name=compute_target_object.name, path=source_directory)
def _get_run_config_object(self, run_config):
if isinstance(run_config, str):
# If it is a string then we don't need to create a copy.
return RunConfiguration.load(self.project_directory, run_config)
elif isinstance(run_config, RunConfiguration):
# TODO: Deep copy of project and auth object too.
import copy
return copy.deepcopy(run_config)
else:
raise UserErrorException(
"Unsupported runconfig type {}. run_config can be of str or "
"azureml.core.runconfig.RunConfiguration type.".format(
type(run_config)))
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(
name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group
)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(
aml_workspace,
e.compute_name,
e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
run_config = RunConfiguration(conda_dependencies=CondaDependencies.create(
conda_packages=['numpy', 'pandas',
'scikit-learn', 'tensorflow', 'keras'],
pip_packages=['azure', 'azureml-core',
'azure-storage',
'azure-storage-blob'])
)
run_config.environment.docker.enabled = True
run_config.environment.docker.base_image = "mcr.microsoft.com/mlops/python"
train_step = PythonScriptStep(
name="Train Model",
script_name="train_with_r.py",
compute_target=aml_compute,
source_directory="code/training/R",
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
steps = [train_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name + "_with_R",
description="Model training/retraining pipeline",
version=e.build_id
)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
def mi_run_config(ws, compute):
whl_url = Environment.add_private_pip_wheel(workspace=ws,
file_path=d.WHL_VINX_AZURE_ML,
exist_ok=True)
run_config = RunConfiguration()
run_config.target = compute
run_config.environment.docker.enabled = True
run_config.environment.docker.base_image = None
run_config.environment.docker.base_dockerfile = 'FROM mcr.microsoft.com/azureml/base:latest\nRUN apt-get update && apt-get -y install freetds-dev freetds-bin vim gcc'
run_config.environment.python.user_managed_dependencies = False
run_config.environment.python.conda_dependencies = CondaDependencies.create(
conda_packages=[
'tqdm', 'cython', 'matplotlib', 'scikit-learn', 'fbprophet'
],
pip_packages=[
'azureml-sdk', 'pandas', 'lightgbm', 'scipy==1.4.1', 'statsmodels',
'mlxtend', 'optuna', 'xgboost', 'CatBoost', 'tensorflow', 'keras',
'jpholiday', 'joblib', 'pymssql==2.1.1'
],
pin_sdk_version=False)
run_config.environment.python.conda_dependencies.add_pip_package(whl_url)
return run_config
def get_run_config(aml_compute, conda_dep):
run_amlcompute = RunConfiguration()
run_amlcompute.target = aml_compute
dockerfile = r"""
FROM mcr.microsoft.com/azureml/base:intelmpi2018.3-ubuntu16.04
RUN apt-get update && \
apt-get install -y sudo curl apt-transport-https && \
apt-get update && \
sudo su && \
curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \
curl https://packages.microsoft.com/config/ubuntu/16.04/prod.list > /etc/apt/sources.list.d/mssql-release.list && \
sudo apt-get update && \
sudo ACCEPT_EULA=Y apt-get install -y msodbcsql17 && \
sudo apt-get install -y unixodbc-dev
"""
run_amlcompute.environment.docker.enabled = True
run_amlcompute.environment.docker.base_image = None
run_amlcompute.environment.docker.base_dockerfile = dockerfile
run_amlcompute.environment.python.conda_dependencies = conda_dep
return run_amlcompute
def data_ingestion_step(datastore, compute_target):
'''
This step will leverage Azure Cognitive Services to search the web for images
to create a dataset. This replicates the real-world scenario of data being
ingested from a constantly changing source. The same 10 classes in the CIFAR-10 dataset
will be used (airplane, automobile, bird, cat, deer, dog, frog, horse, ship, truck).
:param datastore: The datastore that will be used
:type datastore: Datastore
:param compute_target: The compute target to run the step on
:type compute_target: ComputeTarget
:return: The ingestion step, step outputs dictionary (keys: raw_data_dir)
:rtype: PythonScriptStep, dict
'''
run_config = RunConfiguration()
run_config.environment.environment_variables = {
'BING_SEARCH_V7_SUBSCRIPTION_KEY': os.environ['BING_SEARCH_V7_SUBSCRIPTION_KEY'],
'BING_SEARCH_V7_ENDPOINT': os.environ['BING_SEARCH_V7_ENDPOINT'],
'AZURE_REGION': datastore._workspace.location
}
run_config.environment.docker.enabled = True
num_images = PipelineParameter(name='num_images', default_value=25)
raw_data_dir = PipelineData(
name='raw_data_dir',
pipeline_output_name='raw_data_dir',
datastore=datastore,
output_mode='mount',
is_directory=True)
outputs = [raw_data_dir]
outputs_map = { 'raw_data_dir': raw_data_dir }
step = PythonScriptStep(
name="Data Ingestion",
script_name='data_ingestion.py',
arguments=['--output_dir', raw_data_dir, '--num_images', num_images],
outputs=outputs,
compute_target=compute_target,
source_directory=os.path.dirname(os.path.abspath(__file__)),
runconfig=run_config,
allow_reuse=True
)
return step, outputs_map
def data_preprocess_step(raw_data_dir, compute_target):
run_config = RunConfiguration()
run_config.environment.python.conda_dependencies = CondaDependencies.create(
pip_packages=['pandas'])
run_config.environment.docker.enabled = True
train_dir = PipelineData(name='train_dir',
pipeline_output_name='train_dir',
datastore=raw_data_dir.datastore,
output_mode='mount',
is_directory=True)
test_dir = PipelineData(name='test_dir',
pipeline_output_name='test_dir',
datastore=raw_data_dir.datastore,
output_mode='mount',
is_directory=True)
outputs = [train_dir, test_dir]
outputs_map = {
'train_dir': train_dir,
'test_dir': test_dir,
}
step = PythonScriptStep(script_name='data_preprocess.py',
arguments=[
'--raw_data_dir',
raw_data_dir,
'--train_dir',
train_dir,
'--test_dir',
test_dir,
],
inputs=[raw_data_dir],
outputs=outputs,
compute_target=compute_target,
runconfig=run_config,
source_directory=os.path.dirname(
os.path.abspath(__file__)),
allow_reuse=True)
return step, outputs_map
def main():
e = Env()
# Get Azure machine learning workspace
aml_workspace = Workspace.get(name=e.workspace_name,
subscription_id=e.subscription_id,
resource_group=e.resource_group)
print("get_workspace:")
print(aml_workspace)
# Get Azure machine learning cluster
aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size)
if aml_compute is not None:
print("aml_compute:")
print(aml_compute)
# Create a run configuration environment
conda_deps_file = "diabetes_regression/training_dependencies.yml"
conda_deps = CondaDependencies(conda_deps_file)
run_config = RunConfiguration(conda_dependencies=conda_deps)
run_config.environment.docker.enabled = True
train_step = PythonScriptStep(
name="Train Model",
script_name="train_with_r.py",
compute_target=aml_compute,
source_directory="diabetes_regression/training/R",
runconfig=run_config,
allow_reuse=False,
)
print("Step Train created")
steps = [train_step]
train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
train_pipeline.validate()
published_pipeline = train_pipeline.publish(
name=e.pipeline_name,
description="Model training/retraining pipeline",
version=e.build_id)
print(f'Published pipeline: {published_pipeline.name}')
print(f'for build {published_pipeline.version}')
def publish(azureml_workspace, entry_point, name, description, parameters={}):
luna_config = utils.Init()
if azureml_workspace:
run_config = RunConfiguration.load(
luna_config['azureml']['run_config'])
arguments = utils.GetPipelineArguments(luna_config['MLproject'],
entry_point, parameters)
trainStep = PythonScriptStep(
script_name=luna_config['code'][entry_point],
arguments=arguments,
inputs=[],
outputs=[],
source_directory=os.getcwd(),
runconfig=run_config)
pipeline = Pipeline(workspace=azureml_workspace, steps=[trainStep])
published_pipeline = pipeline.publish(name=name,
description=description)
return published_pipeline.endpoint
def main():
print("Beginning ML runs")
#### This doc should contain meta-data for ML experiment runs ####
#This sets the compute target to your local machine.
run_user_managed = RunConfiguration()
run_user_managed.environment.python.user_managed_dependencies = False
#Can also adjust for a specific python interpreter if desired. Otherwise, it uses the specific one declared by IDE.
#This determines what workspace to run experiments in. Should not need any edits.
ws = Workspace.from_config()
#The remainder of this doc should run the models defined in other documents.
#This experiment runs the REGRESSION model.
experiment = Experiment(ws, "RunFinal1")
automated_ml_config = model(X_toTrain, Y_toTrain, "FOLDER HARD CODED", 10,
30, 'normalized_root_mean_squared_error', 5,
'regression')
local_run = experiment.submit(automated_ml_config, show_output=True)
####Code to explore runs####
def retrieveRunData():
children = list(local_run.get_children())
metricslist = {}
for run in children:
properties = run.get_properties()
metrics = {
k: v
for k, v in run.get_metrics().items() if isinstance(v, float)
}
metricslist[int(properties['iteration'])] = metrics
rundata = pd.DataFrame(metricslist).sort_index(1)
print(rundata)
retrieveRunData()
#The following method retrieves the best test run's model, and makes a prediction using the previously defined .csv file.
for toPredict in to_predict_array:
retrieveAndPredict('normalized_root_mean_squared_error', 'runFinal1',
toPredict, predicted_values_folder)
