def main():
# workspace
ws = Workspace.from_config()
#compute
compute = AmlCompute(workspace=ws, name='gandalf')
# datasource
datastore = Datastore.get(ws, datastore_name='surfrider')
# experiment
script_params = {
"--datastore": datastore.as_mount()
}
# Create and run experiment
estimator = Estimator(source_directory='./',
script_params=script_params,
compute_target=compute,
entry_script='train.py',
use_gpu=True,
pip_packages=['opencv-python>=4.1',
'tensorpack==0.9.8',
'tensorflow-gpu>=1.3,<2.0',
'tqdm>=4.36.1',
'cython>=0.29.13',
'scipy>=1.3.1',
'ffmpeg-python',
'wget'])
exp = Experiment(ws, 'surfrider_rcnn')
run = exp.submit(estimator)
def run_inference(test_experiment, compute_target, script_folder, train_run,
test_dataset, target_column_name, model_name):
train_run.download_file('outputs/conda_env_v_1_0_0.yml',
'inference/condafile.yml')
inference_env = Environment("myenv")
inference_env.docker.enabled = True
inference_env.python.conda_dependencies = CondaDependencies(
conda_dependencies_file_path='inference/condafile.yml')
est = Estimator(source_directory=script_folder,
entry_script='infer.py',
script_params={
'--target_column_name': target_column_name,
'--model_name': model_name
},
inputs=[test_dataset.as_named_input('test_data')],
compute_target=compute_target,
environment_definition=inference_env)
run = test_experiment.submit(est,
tags={
'training_run_id':
train_run.id,
'run_algorithm':
train_run.properties['run_algorithm'],
'valid_score':
train_run.properties['score'],
'primary_metric':
train_run.properties['primary_metric']
})
run.log("run_algorithm", run.tags['run_algorithm'])
return run
def main(workspace):
# Loading compute target
print("Loading compute target")
compute_target = ComputeTarget(
workspace=workspace,
name="aml-intTest"
)
# Loading script parameters
print("Loading script parameters")
script_params = {
"--kernel": "linear",
"--penalty": 0.9
}
# Creating experiment config
print("Creating experiment config")
estimator = Estimator(
source_directory="./tests/train/train_with_python_config",
entry_script="train.py",
script_params=script_params,
compute_target=compute_target,
conda_dependencies_file="environment.yml"
)
return estimator
def run_rolling_forecast(test_experiment,
compute_target,
train_run,
test_dataset,
target_column_name,
inference_folder='./forecast'):
train_run.download_file('outputs/model.pkl',
inference_folder + '/model.pkl')
inference_env = train_run.get_environment()
est = Estimator(source_directory=inference_folder,
entry_script='forecasting_script.py',
script_params={'--target_column_name': target_column_name},
inputs=[test_dataset.as_named_input('test_data')],
compute_target=compute_target,
environment_definition=inference_env)
run = test_experiment.submit(est,
tags={
'training_run_id':
train_run.id,
'run_algorithm':
train_run.properties['run_algorithm'],
'valid_score':
train_run.properties['score'],
'primary_metric':
train_run.properties['primary_metric']
})
run.log("run_algorithm", run.tags['run_algorithm'])
return run
def main():
with open("authentication.json") as jsonFile:
authData = json.load(jsonFile)[args.auth_cluster]
# AzureML Subscription Details (get details from the Azure Portal)
subID = authData["subID"] # Get from Azure Portal; used for billing
resGroup = authData["resGroup"] # Name for the resource group
wsName = authData[
"wsName"] # Name for the workspace, which is the collection of compute clusters + experiments
computeName = authData["computeName"] # Name for computer cluster
### Get workspace and compute target
ws = Workspace.get(wsName, subscription_id=subID, resource_group=resGroup)
compute_target = ComputeTarget(ws, computeName)
# The path to the dataset. If using RichPath then this should be prefixed with azure://
# otherwise this is the location where the AzureML Datastore will be mounted
# datapath_prefix = "azure://example1234/data/"
# Set up by using the URL like above as well as a generated SAS key, placed into azureinfo.json
datapath_prefix = authData["dataPath"]
script_folder = "."
script_params = OrderedDict([(datapath_prefix + args.train_file_name, ""),
(datapath_prefix + args.validate_file_name,
""),
(datapath_prefix + args.test_file_name, ""),
("./model.pkl.gz", ""),
("--max-num-epochs", args.max_epochs),
("--aml", ""),
("--azure-info", "azureinfo.json"),
("--quiet", "")])
# we are trying to predict statements
if args.predicting_statement:
script_params["--predicting-statement"] = ""
with open("Dockerfile") as f:
docker = DockerSection()
docker.base_image = None
docker.base_dockerfile = f.read()
docker.enabled = True
environment = Environment(name="pytorchenv")
environment.docker = docker
environment.python.user_managed_dependencies = True
est = Estimator(
source_directory=script_folder,
script_params=script_params,
compute_target=compute_target,
entry_script="ptgnn/implementations/graph2seq/trainandtest.py",
environment_definition=environment,
use_docker=True,
)
### Submit the experiment
exp = Experiment(workspace=ws, name=args.exp_name)
run = exp.submit(config=est, tags=args.tags)
print(
"Experiment Started. Remember you can exit out of this program but the experiment will still run on Azure!"
)
# print("Portal URL: ", run.get_portal_url())
run.wait_for_completion(show_output=True)
def main(workspace):
# Load compute target
print("Loading compute target")
compute_target = ComputeTarget(
workspace=workspace,
name="mycluster"
)
# Load script parameters
print("Loading script parameters")
script_params = {
"--kernel": "linear",
"--penalty": 1.0
}
# Create experiment config
print("Creating experiment config")
estimator = Estimator(
source_directory="code/train",
entry_script="train.py",
script_params=script_params,
compute_target=compute_target,
pip_packages=["azureml-dataprep[pandas,fuse]", "scikit-learn", "pandas", "matplotlib"]
)
return estimator
def run_inference(test_experiment, compute_target, script_folder, train_run,
test_dataset, target_column_name, model_name):
inference_env = train_run.get_environment()
est = Estimator(source_directory=script_folder,
entry_script='infer.py',
script_params={
'--target_column_name': target_column_name,
'--model_name': model_name
},
inputs=[test_dataset.as_named_input('test_data')],
compute_target=compute_target,
environment_definition=inference_env)
run = test_experiment.submit(est,
tags={
'training_run_id':
train_run.id,
'run_algorithm':
train_run.properties['run_algorithm'],
'valid_score':
train_run.properties['score'],
'primary_metric':
train_run.properties['primary_metric']
})
run.log("run_algorithm", run.tags['run_algorithm'])
return run
def trigger_training_job(compute_name, script_folder):
# Define Vars < Change the vars>
tenant_id = "<Enter Your Tenant Id>"
app_id = "<Application Id of the SPN you Create>"
app_key = "<Key for the SPN>"
workspace = "<Name of your workspace>"
subscription_id = "<Subscription id>"
resource_grp = "<Name of your resource group where aml service is created>"
experiment_name = '<Name of your experiment you defined in dataprep.py>'
print("Starting trigger engine")
# Start creating
# Point file to conf directory containing details for the aml service
spn = ServicePrincipalAuthentication(tenant_id, app_id, app_key)
ws = Workspace(auth=spn,
workspace_name=workspace,
subscription_id=subscription_id,
resource_group=resource_grp)
#ws = Workspace.from_config(path="../conf/config.json")
ds = ws.get_default_datastore()
print(ds.datastore_type, ds.account_name, ds.container_name)
exp = Experiment(workspace=ws, name=experiment_name)
compute_target = ws.compute_targets[compute_name]
script_params = {'--data-folder': ds.as_mount(), '--regularization': 0.8}
est = Estimator(source_directory=script_folder,
script_params=script_params,
compute_target=compute_target,
entry_script='train.py',
conda_packages=['scikit-learn'])
print("Submitting Runs to AML compute " + compute_name)
run = exp.submit(config=est)
run.wait_for_completion(show_output=True) # specify True for a verbose log
def process_step(datastore: Datastore, compute: ComputeTarget,
path_on_datastore: str) -> (PipelineData, EstimatorStep):
datapath = DataPath(datastore=datastore,
path_on_datastore=path_on_datastore)
data_path_pipeline_param = (PipelineParameter(name="data",
default_value=datapath),
DataPathComputeBinding(mode='mount'))
seer_tfrecords = PipelineData("tfrecords_set",
datastore=datastore,
is_directory=True)
prep = Estimator(source_directory='.',
compute_target=compute,
entry_script='prep.py',
pip_requirements_file='requirements.txt')
prepStep = EstimatorStep(name='Data Preparation',
estimator=prep,
estimator_entry_script_arguments=[
"--source_path", data_path_pipeline_param,
"--target_path", seer_tfrecords
],
inputs=[data_path_pipeline_param],
outputs=[seer_tfrecords],
compute_target=compute)
return seer_tfrecords, prepStep
def submit(script_params):
global SUBMIT_ALL
input_data = None
# the files in source_directory will be uploaded to the cluster
est = Estimator(source_directory='src',
script_params=script_params,
compute_target=compute_target,
entry_script='train_cv.py',
environment_definition=rc.environment)
print('script_params', script_params)
while not SUBMIT_ALL and True:
input_data = input("Submit? [Y/y/n/s]")
if input_data in ['Y', 'y', 'n', 's']:
break
if input_data == 'Y':
SUBMIT_ALL = True
if SUBMIT_ALL or input_data == 'y':
run = exp.submit(est)
print('=' * 40)
print(run)
print('Monitor the training progress on the portal: URL=',
run.get_portal_url())
elif input_data == 'n':
print("aborting!")
sys.exit(0)
else:
print("skip!")
def createEstimator(ws, args, folders):
data_folder = args.remoteDataFolder
if (args.mountDataStore == True):
ds = ws.get_default_datastore()
data_folder = ds.path(args.remoteDataFolder).as_mount()
if (args.verbose):
print("Remote data folder : {0}".format(data_folder))
script_params = {
'--data-folder': data_folder,
'--modelFilePath': args.modelFilePath
}
compute_target = createCompute(ws, args)
conda_packages = []
## hack until conda_dependencies_file_path works
with open(args.conda_dependencies_file_path, 'r') as stream:
yml = yaml.safe_load(stream)
conda_packages = yml['dependencies']
##
est = Estimator(source_directory=folders.script_folder,
script_params=script_params,
compute_target=compute_target,
entry_script=args.entryScript,
use_gpu=args.useGPU,
user_managed=args.userManaged,
use_docker=args.useDocker,
conda_packages=conda_packages,
pip_requirements_file_path=args.pip_requirements_file_path)
return est
def setup_etcd_cluster(self):
# Define the project folder
project_folder = '.' # This is to allow the libraries stored under pytorch/ to be loaded
## Using a public image published on Docker Hub.
image_name = 'rskolli/pet-etcd:latest'
# Define the Pytorch estimator
etcd_estimator = Estimator(
source_directory=project_folder,
# Compute configuration
compute_target=self.pet_etcd_target,
node_count=1,
#Docker image
use_docker=True,
custom_docker_image=image_name,
user_managed=True,
# Training script parameters
#script_params = { },
entry_script='etcd-setup.py')
self.pet_etcd_run = self.pet_experiment.submit(etcd_estimator)
RunDetails(self.pet_etcd_run).show()
async def __create_cluster(self):
self.__print_message("Setting up cluster")
exp = Experiment(self.workspace, self.experiment_name)
estimator = Estimator(
os.path.join(self.abs_path, "setup"),
compute_target=self.compute_target,
entry_script="start_jupyter.py",
environment_definition=self.environment_definition,
script_params=self.scheduler_params,
node_count=1, ### start only scheduler
distributed_training=MpiConfiguration(),
use_docker=True,
inputs=self.datastores,
)
run = exp.submit(estimator, tags=self.tags)
self.__print_message("Waiting for compute cluster's IP")
while (
run.get_status() != "Canceled"
and run.get_status() != "Failed"
and "jupyter" not in run.get_metrics() #and "scheduler" not in run.get_metrics()
):
print(".", end="")
logger.info("Compute Cluster not ready")
time.sleep(5)
if run.get_status() == "Canceled" or run.get_status() == "Failed":
logger.exception("Failed to start the AzureML Compute Cluster")
raise Exception("Failed to start the AzureML Compute Cluster.")
print("\n\n")
### SET FLAGS
####---self.scheduler_ip_port = run.get_metrics()["scheduler"]
self.scheduler_ip_port = run.get_metrics()["jupyter"]
self.worker_params["--scheduler_ip_port"] = self.scheduler_ip_port
self.__print_message(f'Scheduler: {run.get_metrics()["scheduler"]}')
self.run = run
logger.info(f'Scheduler: {run.get_metrics()["scheduler"]}')
### CHECK IF ON THE SAME VNET
while self.same_vnet is None:
self.__check_if_scheduler_ip_reachable()
time.sleep(1)
### REQUIRED BY dask.distributed.deploy.cluster.Cluster
####---_scheduler = self.__prepare_rpc_connection_to_headnode()
####---self.scheduler_comm = rpc(_scheduler)
####---await self.sync(self.__setup_port_forwarding)
####---await self.sync(super()._start)
_scheduler = self.__prepare_rpc_connection_to_headnode()
self.__setup_port_forwarding()
self.__update_links()
self.__print_message("Connections established")
def run_inference(
test_experiment,
compute_target,
script_folder,
train_run,
test_dataset,
lookback_dataset,
max_horizon,
target_column_name,
time_column_name,
freq,
):
model_base_name = "model.pkl"
if "model_data_location" in train_run.properties:
model_location = train_run.properties["model_data_location"]
_, model_base_name = model_location.rsplit("/", 1)
train_run.download_file(
"outputs/{}".format(model_base_name), "inference/{}".format(model_base_name)
)
train_run.download_file("outputs/conda_env_v_1_0_0.yml", "inference/condafile.yml")
inference_env = Environment("myenv")
inference_env.docker.enabled = True
inference_env.python.conda_dependencies = CondaDependencies(
conda_dependencies_file_path="inference/condafile.yml"
)
est = Estimator(
source_directory=script_folder,
entry_script="infer.py",
script_params={
"--max_horizon": max_horizon,
"--target_column_name": target_column_name,
"--time_column_name": time_column_name,
"--frequency": freq,
"--model_path": model_base_name,
},
inputs=[
test_dataset.as_named_input("test_data"),
lookback_dataset.as_named_input("lookback_data"),
],
compute_target=compute_target,
environment_definition=inference_env,
)
run = test_experiment.submit(
est,
tags={
"training_run_id": train_run.id,
"run_algorithm": train_run.properties["run_algorithm"],
"valid_score": train_run.properties["score"],
"primary_metric": train_run.properties["primary_metric"],
},
)
run.log("run_algorithm", run.tags["run_algorithm"])
return run
def test_get_hyperdrive_config(
number_of_cross_validation_splits: int,
number_of_cross_validation_splits_per_fold: int,
test_output_dirs: TestOutputDirectories) -> None:
"""
Test to make sure the number of dataset reader workers are set correctly
"""
if number_of_cross_validation_splits_per_fold > 0:
config = HyperDriveTestModelScalar()
config.number_of_cross_validation_splits_per_fold = number_of_cross_validation_splits_per_fold
else:
config = HyperDriveTestModelSegmentation()
config.number_of_cross_validation_splits = number_of_cross_validation_splits
# create HyperDrive config with dummy estimator for testing
source_config = SourceConfig(root_folder=test_output_dirs.root_dir,
entry_script="something.py",
conda_dependencies_files=[])
estimator = Estimator(source_directory=source_config.root_folder,
entry_script=source_config.entry_script,
compute_target="Local")
hd_config = config.get_hyperdrive_config(estimator=estimator)
assert hd_config.estimator.source_directory == source_config.root_folder
assert hd_config.estimator.run_config.script == source_config.entry_script
assert hd_config.estimator._script_params == source_config.script_params
if number_of_cross_validation_splits > 0 and number_of_cross_validation_splits_per_fold > 0:
assert hd_config._max_total_runs == number_of_cross_validation_splits * \
number_of_cross_validation_splits_per_fold
elif number_of_cross_validation_splits > 0:
assert hd_config._max_total_runs == number_of_cross_validation_splits
else:
assert hd_config._max_total_runs == HYPERDRIVE_TOTAL_RUNS
if config.perform_cross_validation:
# check sampler is as expected
sampler = config.get_cross_validation_hyperdrive_sampler()
expected_sampler_dict = {
CROSS_VALIDATION_SPLIT_INDEX_TAG_KEY:
choice(list(range(number_of_cross_validation_splits)))
}
if number_of_cross_validation_splits_per_fold > 0:
expected_sampler_dict[
CROSS_VALIDATION_SUB_FOLD_SPLIT_INDEX_TAG_KEY] = choice(
list(range(number_of_cross_validation_splits_per_fold)))
assert sampler._parameter_space == expected_sampler_dict
else:
assert vars(config.get_hyperdrive_config(estimator)) \
== vars(_create_dummy_hyperdrive_param_search_config(estimator))
def create_estimator(ws, ds, compute_target, run_env, input_folder,
output_folder):
script_params = {
'--raw_folder': ds.path(input_folder).as_mount(),
'--processed_folder': ds.path(output_folder).as_mount()
}
est = Estimator(source_directory=CODE_PATH,
entry_script='data_prep.py',
script_params=script_params,
compute_target=compute_target,
environment_definition=run_env)
return est
def launch_experiment(self, exp_name: str, script_params: dict,
entry_script: str):
experiment = Experiment(self.ws, name=exp_name)
est = Estimator(source_directory=self.project_folder,
script_params=script_params,
compute_target=self.compute_target,
entry_script=entry_script,
custom_docker_image=self.conf_docker['image_name'],
image_registry_details=self.image_registry_details,
user_managed=self.user_managed_dependencies,
source_directory_data_store=self.input_ds)
run = experiment.submit(est)
def create_estimator(ws, ds, compute_target, input_folder):
training_env_file = os.path.join(CODE_PATH, 'train_env.yml')
training_env = amlutils.create_azureml_env(
ws, consts.train_environment_name, training_env_file)
script_params = {
'--data_folder': ds.path(input_folder).as_mount()
}
est = Estimator(
source_directory=CODE_PATH,
entry_script='train.py',
script_params=script_params,
compute_target=compute_target,
environment_definition=training_env)
return est
def get_script_config(self,
script_folder,
entry_script,
script_params,
compute_target,
channels=None,
conda_packages=None,
pip_packages=None):
estimator = Estimator(source_directory=script_folder,
entry_script=entry_script,
script_params=script_params,
conda_packages=conda_packages,
pip_packages=pip_packages,
compute_target=compute_target)
return estimator
def run_inference(test_experiment, compute_target, script_folder, train_run,
test_dataset, lookback_dataset, max_horizon,
target_column_name, time_column_name, freq):
model_base_name = 'model.pkl'
if 'model_data_location' in train_run.properties:
model_location = train_run.properties['model_data_location']
_, model_base_name = model_location.rsplit('/', 1)
train_run.download_file('outputs/{}'.format(model_base_name),
'inference/{}'.format(model_base_name))
train_run.download_file('outputs/conda_env_v_1_0_0.yml',
'inference/condafile.yml')
inference_env = Environment("myenv")
inference_env.docker.enabled = True
inference_env.python.conda_dependencies = CondaDependencies(
conda_dependencies_file_path='inference/condafile.yml')
est = Estimator(source_directory=script_folder,
entry_script='infer.py',
script_params={
'--max_horizon': max_horizon,
'--target_column_name': target_column_name,
'--time_column_name': time_column_name,
'--frequency': freq,
'--model_path': model_base_name
},
inputs=[
test_dataset.as_named_input('test_data'),
lookback_dataset.as_named_input('lookback_data')
],
compute_target=compute_target,
environment_definition=inference_env)
run = test_experiment.submit(est,
tags={
'training_run_id':
train_run.id,
'run_algorithm':
train_run.properties['run_algorithm'],
'valid_score':
train_run.properties['score'],
'primary_metric':
train_run.properties['primary_metric']
})
run.log("run_algorithm", run.tags['run_algorithm'])
return run
def main(workspace, inputs):
# Loading compute target
print("Loading compute target")
compute_target = ComputeTarget(workspace=workspace, name=inputs["compute"])
# Loading script parameters
print("Loading script parameters")
script_params = {"--kernel": "linear", "--penalty": 0.9}
# Creating experiment config
print("Creating experiment config")
estimator = Estimator(source_directory=inputs["source_directory"],
entry_script=inputs["train_script"],
script_params=script_params,
compute_target=compute_target,
conda_dependencies_file="environment.yml")
return estimator
def run_rolling_forecast(test_experiment,
compute_target,
train_run,
test_dataset,
max_horizon,
target_column_name,
time_column_name,
freq='D',
inference_folder='./forecast'):
condafile = inference_folder + '/condafile.yml'
train_run.download_file('outputs/model.pkl',
inference_folder + '/model.pkl')
train_run.download_file('outputs/conda_env_v_1_0_0.yml', condafile)
inference_env = Environment("myenv")
inference_env.docker.enabled = True
inference_env.python.conda_dependencies = CondaDependencies(
conda_dependencies_file_path=condafile)
est = Estimator(source_directory=inference_folder,
entry_script='forecasting_script.py',
script_params={
'--max_horizon': max_horizon,
'--target_column_name': target_column_name,
'--time_column_name': time_column_name,
'--frequency': freq
},
inputs=[test_dataset.as_named_input('test_data')],
compute_target=compute_target,
environment_definition=inference_env)
run = test_experiment.submit(est,
tags={
'training_run_id':
train_run.id,
'run_algorithm':
train_run.properties['run_algorithm'],
'valid_score':
train_run.properties['score'],
'primary_metric':
train_run.properties['primary_metric']
})
run.log("run_algorithm", run.tags['run_algorithm'])
return run
def estimator(data, store, compute):
estimator = Estimator(source_directory=os.path.dirname(
os.path.abspath(__file__)),
compute_target=compute,
entry_script='train.py',
pip_packages=['azureml-dataprep', 'lightgbm'])
output = PipelineData("output", datastore=store)
step = EstimatorStep(name=os.path.basename(__file__),
estimator=estimator,
estimator_entry_script_arguments=[
'--input_dir', data, '--output_dir', output
],
inputs=[data],
outputs=[output],
compute_target=estimator._compute_target,
allow_reuse=True)
return step, output
def register_step(datastore: Datastore, input_data: PipelineData,
compute: ComputeTarget,
build: str) -> (PipelineData, EstimatorStep):
seer_model = PipelineData("model", datastore=datastore, is_directory=True)
register = Estimator(source_directory='.',
compute_target=compute,
entry_script='register.py')
registerStep = EstimatorStep(name='Model Registration',
estimator=register,
estimator_entry_script_arguments=[
"--source_path", input_data,
"--target_path", seer_model, "--build",
build
],
inputs=[input_data],
outputs=[seer_model],
compute_target=compute)
return seer_model, registerStep
def run_azure_experiment_with_storage(
subscription_id: str,
resource_group: str,
workspace_name: str,
datastore_name: str,
container_name: str,
storage_account_name: str,
storage_account_key: str,
compute_name: str,
experiment_name: Optional[str] = None,
source_directory: Optional[str] = None,
image_name: Optional[str] = None,
use_gpu=True,
) -> Run:
workspace = Workspace(subscription_id, resource_group, workspace_name,)
data_store = Datastore.register_azure_blob_container(
workspace=workspace,
datastore_name=datastore_name,
container_name=container_name,
account_name=storage_account_name,
account_key=storage_account_key,
)
source_directory = source_directory or dirname(__file__)
assert (
compute_name in workspace.compute_targets
), f"compute {compute_name} is not created in {workspace_name} workspace"
estimator = Estimator(
source_directory=source_directory,
script_params={"--data-folder": data_store.as_mount()},
compute_target=workspace.compute_targets[compute_name],
pip_packages=pip_packages(),
entry_script=os.path.join(source_directory, "azure_train.py"),
use_gpu=use_gpu,
custom_docker_image=image_name,
)
experiment_name = experiment_name or __file__.split(os.sep)[-1].split(".py")[0]
experiment = Experiment(workspace=workspace, name=experiment_name)
run = experiment.submit(estimator)
return run
def __create_cluster(self):
print("\n")
self.__print_message("Setting up cluster")
exp = Experiment(self.workspace, self.experiment_name)
estimator = Estimator(
os.path.join(self.abs_path, "setup"),
compute_target=self.compute_target,
entry_script="start_jupyter.py",
environment_definition=self.environment_definition,
script_params=self.script_params,
node_count=1, ### start only scheduler
use_docker=True,
)
run = exp.submit(estimator, tags=self.tags)
self.run = run
self.status = "running"
self.__print_message("Waiting for compute cluster's IP")
while (run.get_status() != "Canceled" and run.get_status() != "Failed"
and "jupyter" not in
run.get_metrics() # and "scheduler" not in run.get_metrics()
):
print(".", end="")
# logger.info("Compute Cluster not ready")
time.sleep(5)
if run.get_status() == "Canceled" or run.get_status() == "Failed":
logger.exception("Failed to start the AzureML Compute Cluster")
raise Exception("Failed to start the AzureML Compute Cluster.")
print()
self.__print_message("Jupyter session is running...")
print("\n\n")
self.__setup_port_forwarding()
self.__update_links()
self.__print_message("Connections established")
def run_inference(
test_experiment,
compute_target,
script_folder,
train_run,
test_dataset,
target_column_name,
model_name,
):
inference_env = train_run.get_environment()
est = Estimator(
source_directory=script_folder,
entry_script="infer.py",
script_params={
"--target_column_name": target_column_name,
"--model_name": model_name,
},
inputs=[test_dataset.as_named_input("test_data")],
compute_target=compute_target,
environment_definition=inference_env,
)
run = test_experiment.submit(
est,
tags={
"training_run_id": train_run.id,
"run_algorithm": train_run.properties["run_algorithm"],
"valid_score": train_run.properties["score"],
"primary_metric": train_run.properties["primary_metric"],
},
)
run.log("run_algorithm", run.tags["run_algorithm"])
return run
def train_step(datastore: Datastore, input_data: PipelineData,
compute: ComputeTarget) -> (PipelineData, EstimatorStep):
seer_training = PipelineData("train",
datastore=datastore,
is_directory=True)
train = Estimator(source_directory='.',
compute_target=compute,
entry_script='train.py',
use_gpu=True,
pip_requirements_file='requirements.txt')
trainStep = EstimatorStep(name='Model Training',
estimator=train,
estimator_entry_script_arguments=[
"--source_path", input_data, "--target_path",
seer_training, "--epochs", 15, "--batch", 10,
"--lr", 0.001
],
inputs=[input_data],
outputs=[seer_training],
compute_target=compute)
return seer_training, trainStep
# load Azure ML workspace
azureml_workspace = Workspace.from_config(auth=AzureCliAuthentication())
# Retrieve a pointer to the dataset versions
redditcomments_gaming = Dataset.get_by_name(azureml_workspace,
name='redditcomments',
version='latest')
redditcomments = Dataset.get_by_name(azureml_workspace,
name='redditcomments_gaming',
version='latest')
# Configure the training run
est = Estimator(entry_script='train.py',
script_params={'--alpha': 1.0},
source_directory=os.path.dirname(os.path.realpath(__file__)),
compute_target='ml-e2e',
inputs=[redditcomments_gaming.as_named_input('comments')],
pip_packages=[
"azureml-sdk", "azureml-mlflow", "matplotlib", "scipy",
"sklearn", "azure-cli", "pandas", "numpy"
])
# Submit an experiment run
experiment = Experiment(azureml_workspace, "redditcomments_gaming_experiment")
run = experiment.submit(est)
run.wait_for_completion(show_output=True)
# - See ./training/train.py
################################
#%%
# Step 14 - Create estimator
#############################
from azureml.train.estimator import Estimator
script_params = {
'--data-folder': ds.as_mount(),
'--training-set-percentage': 0.3
}
est_config = Estimator(source_directory='./training',
script_params=script_params,
compute_target=compute_target,
entry_script='train.py',
conda_packages=['scikit-learn', 'pandas'])
#%%
# Step 15 - Execute the estimator job
#####################################
run = exp.submit(config=est_config)
run
# Poll for job status
run.wait_for_completion(
show_output=True) # value of True will display a verbose, streaming log
# Examine the recorded metrics from the run
print(run.get_metrics())
