def mirrored(train_fn,
name='no-name',
local_logdir=False,
description=None,
evaluator=False,
metric_key=None):
"""
*Distributed Training*
Example usage:
>>> from hops import experiment
>>> def mirrored_training():
>>> # Do all imports in the function
>>> import tensorflow
>>> # Put all code inside the train_fn function
>>> from hops import tensorboard
>>> from hops import devices
>>> logdir = tensorboard.logdir()
>>> ...MirroredStrategy()...
>>> experiment.mirrored(mirrored_training, local_logdir=True)
Args:
:train_fn: contains the code where you are using MirroredStrategy.
:name: name of the experiment
:local_logdir: True if *tensorboard.logdir()* should be in the local filesystem, otherwise it is in HDFS
:description: a longer description for the experiment
:evaluator: whether to run one of the workers as an evaluator
:metric_key: If returning a dict with multiple return values, this key should match the name of the key in the dict for the metric you want to associate with the experiment
Returns:
HDFS path in your project where the experiment is stored and return value from the process running as chief
"""
num_ps = util.num_param_servers()
assert num_ps == 0, "number of parameter servers should be 0"
global running
if running:
raise RuntimeError("An experiment is currently running.")
num_workers = util.num_executors()
if evaluator:
assert num_workers > 2, "number of workers must be atleast 3 if evaluator is set to True"
start = time.time()
sc = util._find_spark().sparkContext
try:
global app_id
global experiment_json
global run_id
app_id = str(sc.applicationId)
_start_run()
experiment_utils._create_experiment_dir(app_id, run_id)
experiment_json = experiment_utils._populate_experiment(
name, 'mirrored', 'DISTRIBUTED_TRAINING', None, description,
app_id, None, None)
experiment_json = experiment_utils._attach_experiment_xattr(
app_id, run_id, experiment_json, 'CREATE')
logdir, return_dict = mirrored_impl._run(sc,
train_fn,
run_id,
local_logdir=local_logdir,
name=name,
evaluator=evaluator)
duration = experiment_utils._seconds_to_milliseconds(time.time() -
start)
metric = experiment_utils._get_metric(return_dict, metric_key)
experiment_utils._finalize_experiment(experiment_json, metric, app_id,
run_id, 'FINISHED', duration,
logdir, None, None)
return logdir, return_dict
except:
_exception_handler(
experiment_utils._seconds_to_milliseconds(time.time() - start))
raise
finally:
_end_run(sc)
def launch(train_fn,
args_dict=None,
name='no-name',
local_logdir=False,
description=None,
metric_key=None):
"""
*Experiment* or *Parallel Experiment*
Run an Experiment contained in *train_fn* one time with no arguments or multiple times with different arguments if
*args_dict* is specified.
Example usage:
>>> from hops import experiment
>>> def train_nn():
>>> # Do all imports in the function
>>> import tensorflow
>>> # Put all code inside the train_fn function
>>> accuracy, loss = network.evaluate(learning_rate, layers, dropout)
>>> experiment.launch(train_nn)
Returning multiple outputs, including images and logs:
>>> from hops import experiment
>>> def train_nn():
>>> # Do all imports in the function
>>> import tensorflow
>>> # Put all code inside the train_fn function
>>> from PIL import Image
>>> f = open('logfile.txt', 'w')
>>> f.write('Starting training...')
>>> accuracy, loss = network.evaluate(learning_rate, layers, dropout)
>>> img = Image.new(.....)
>>> img.save('diagram.png')
>>> return {'accuracy': accuracy, 'loss': loss, 'logfile': 'logfile.txt', 'diagram': 'diagram.png'}
>>> experiment.launch(train_nn)
Args:
:train_fn: The function to run
:args_dict: If specified will run the same function multiple times with different arguments, {'a':[1,2], 'b':[5,3]} would run the function two times with arguments (1,5) and (2,3) provided that the function signature contains two arguments like *def func(a,b):*
:name: name of the experiment
:local_logdir: True if *tensorboard.logdir()* should be in the local filesystem, otherwise it is in HDFS
:description: A longer description for the experiment
:metric_key: If returning a dict with multiple return values, this key should match the name of the key in the dict for the metric you want to associate with the experiment
Returns:
HDFS path in your project where the experiment is stored
"""
num_ps = util.num_param_servers()
assert num_ps == 0, "number of parameter servers should be 0"
global running
if running:
raise RuntimeError(
"An experiment is currently running. Please call experiment.end() to stop it."
)
start = time.time()
sc = util._find_spark().sparkContext
try:
global app_id
global experiment_json
global run_id
app_id = str(sc.applicationId)
_start_run()
experiment_utils._create_experiment_dir(app_id, run_id)
experiment_json = None
if args_dict:
experiment_json = experiment_utils._populate_experiment(
name, 'launch', 'EXPERIMENT', json.dumps(args_dict),
description, app_id, None, None)
else:
experiment_json = experiment_utils._populate_experiment(
name, 'launch', 'EXPERIMENT', None, description, app_id, None,
None)
experiment_json = experiment_utils._attach_experiment_xattr(
app_id, run_id, experiment_json, 'CREATE')
logdir, return_dict = launcher._run(sc, train_fn, run_id, args_dict,
local_logdir)
duration = experiment_utils._seconds_to_milliseconds(time.time() -
start)
metric = experiment_utils._get_metric(return_dict, metric_key)
experiment_utils._finalize_experiment(experiment_json, metric, app_id,
run_id, 'FINISHED', duration,
logdir, None, None)
return logdir, return_dict
except:
_exception_handler(
experiment_utils._seconds_to_milliseconds(time.time() - start))
raise
finally:
_end_run(sc)