def _wrapper_fun(iter):
"""
Args:
iter:
Returns:
"""
for i in iter:
executor_num = i
experiment_utils._set_ml_id(app_id, run_id)
tb_hdfs_path = ''
hdfs_exec_logdir = ''
t = threading.Thread(target=devices._print_periodic_gpu_utilization)
if devices.get_num_gpus() > 0:
t.start()
try:
#Arguments
if args_dict:
param_string, params, args = experiment_utils.build_parameters(
map_fun, executor_num, args_dict)
hdfs_exec_logdir, hdfs_appid_logdir = experiment_utils._create_experiment_subdirectories(
app_id,
run_id,
param_string,
'random_search',
params=params)
logfile = experiment_utils._init_logger(hdfs_exec_logdir)
tb_hdfs_path, tb_pid = tensorboard._register(
hdfs_exec_logdir,
hdfs_appid_logdir,
executor_num,
local_logdir=local_logdir)
print(devices._get_gpu_info())
print(
'-------------------------------------------------------')
print('Started running task ' + param_string)
task_start = time.time()
retval = map_fun(*args)
task_end = time.time()
experiment_utils._handle_return(retval, hdfs_exec_logdir,
optimization_key, logfile)
time_str = 'Finished task ' + param_string + ' - took ' + experiment_utils._time_diff(
task_start, task_end)
print(time_str)
print('Returning metric ' + str(retval))
print(
'-------------------------------------------------------')
except:
raise
finally:
experiment_utils._cleanup(tensorboard, t)
def _wrapper_fun(iter):
"""
Args:
:iter:
Returns:
"""
for i in iter:
executor_num = i
experiment_utils._set_ml_id(app_id, run_id)
tb_hdfs_path = ''
hdfs_exec_logdir = ''
t = threading.Thread(target=devices._print_periodic_gpu_utilization)
if devices.get_num_gpus() > 0:
t.start()
global local_logdir_bool
try:
#Arguments
if args_dict:
param_string, params, args = experiment_utils.build_parameters(map_fun, executor_num, args_dict)
val = _get_return_file(param_string, app_id, generation_id, run_id)
hdfs_exec_logdir, hdfs_appid_logdir = experiment_utils._create_experiment_subdirectories(app_id, run_id, param_string, 'differential_evolution', sub_type='generation.' + str(generation_id), params=params)
logfile = experiment_utils._init_logger(hdfs_exec_logdir)
tb_hdfs_path, tb_pid = tensorboard._register(hdfs_exec_logdir, hdfs_appid_logdir, executor_num, local_logdir=local_logdir_bool)
print(devices._get_gpu_info())
print('-------------------------------------------------------')
print('Started running task ' + param_string)
if val is not None:
val = json.loads(val)
task_start = time.time()
if val is None:
val = map_fun(*args)
task_end = time.time()
time_str = 'Finished task ' + param_string + ' - took ' + experiment_utils._time_diff(task_start, task_end)
print(time_str)
experiment_utils._handle_return(val, hdfs_exec_logdir, opt_key, logfile)
print('Returning metric ' + str(val))
print('-------------------------------------------------------')
except:
raise
finally:
experiment_utils._cleanup(tensorboard, t)