def test_nan_mean(self):
self.assertEqual(wrapping_util.nan_mean(np.array([1, 5])), 3)
self.assertEqual(wrapping_util.nan_mean((1, 5)), 3)
# self.assertEqual(wrapping_util.nan_mean("abc"), 3)
self.assertRaises(TypeError, wrapping_util.nan_mean, ("abc"))
self.assertEqual(wrapping_util.nan_mean(np.array([1, 5, np.nan])), 3)
self.assertEqual(wrapping_util.nan_mean(np.array([1, 5, np.inf])), 3)
self.assertTrue(np.isnan(wrapping_util.nan_mean(np.array([np.inf]))))
self.assertEqual(wrapping_util.nan_mean(np.array([-1, 1])), 0)
self.assertTrue(np.isnan(wrapping_util.nan_mean(np.array([]))))
def output_experiment_pickle(console_output_delay, printed_start_configuration,
printed_end_configuration,
optimizer_dir_in_experiment, optimizer,
experiment_directory_prefix, lock, Experiment, np,
exit):
current_best = -1
while True:
try:
trials = Experiment.Experiment(
optimizer_dir_in_experiment,
experiment_directory_prefix + optimizer)
except Exception as e:
logger.error(e)
time.sleep(console_output_delay)
continue
with lock:
for i in range(len(printed_end_configuration),
len(trials.instance_order)):
configuration = trials.instance_order[i][0]
fold = trials.instance_order[i][1]
if i + 1 > len(printed_start_configuration):
logger.info("Starting configuration %5d, fold %2d",
configuration, fold)
printed_start_configuration.append(i)
if np.isfinite(trials.trials[configuration]["instance_results"]
[fold]):
last_result = trials.trials[configuration] \
["instance_results"][fold]
tmp_current_best = trials.get_arg_best()
if tmp_current_best <= i:
current_best = tmp_current_best
# Calculate current best
# Check if last result is finite, if not calc nanmean over all instances
dct_helper = trials.trials[current_best]
res = dct_helper["result"] if \
np.isfinite(dct_helper["result"]) \
else wrapping_util.nan_mean(dct_helper["instance_results"])
#np.nanmean(trials.trials[current_best]["instance_results"])
# nanmean does not work for all numpy version
logger.info("Result %10f, current best %10f", last_result,
res)
printed_end_configuration.append(i)
del trials
time.sleep(console_output_delay)
if exit:
break
def output_experiment_pickle(console_output_delay,
printed_start_configuration,
printed_end_configuration,
optimizer_dir_in_experiment,
optimizer, experiment_directory_prefix, lock,
Experiment, np, exit):
current_best = -1
while True:
try:
trials = Experiment.Experiment(optimizer_dir_in_experiment,
experiment_directory_prefix + optimizer)
except Exception as e:
logger.error(e)
time.sleep(console_output_delay)
continue
with lock:
for i in range(len(printed_end_configuration), len(trials.instance_order)):
configuration = trials.instance_order[i][0]
fold = trials.instance_order[i][1]
if i + 1 > len(printed_start_configuration):
logger.info("Starting configuration %5d, fold %2d",
configuration, fold)
printed_start_configuration.append(i)
if np.isfinite(trials.trials[configuration]
["instance_results"][fold]):
last_result = trials.trials[configuration] \
["instance_results"][fold]
tmp_current_best = trials.get_arg_best()
if tmp_current_best <= i:
current_best = tmp_current_best
# Calculate current best
# Check if last result is finite, if not calc nanmean over all instances
dct_helper = trials.trials[current_best]
res = dct_helper["result"] if \
np.isfinite(dct_helper["result"]) \
else wrapping_util.nan_mean(dct_helper["instance_results"])
#np.nanmean(trials.trials[current_best]["instance_results"])
# nanmean does not work for all numpy version
logger.info("Result %10f, current best %10f",
last_result, res)
printed_end_configuration.append(i)
del trials
time.sleep(console_output_delay)
if exit:
break
def get_arg_best(self):
best_idx = -1
best_value = sys.maxint
for i, trial in enumerate(self.trials):
tmp_res = np.NaN
if np.isfinite(trial['result']):
tmp_res = trial['result']
elif np.isfinite(trial['instance_results']).any():
tmp_res = wrapping_util.nan_mean(trial['instance_results'])
# np.nanmean is not available in older numpy versions
# tmp_res = scipy.nanmean(trial['instance_results'])
else:
continue
if tmp_res < best_value:
best_idx = i
best_value = tmp_res
if best_idx == -1:
raise ValueError("No best value found.")
return best_idx
def get_arg_best(self, consider_incomplete=False):
"""Get the job id of the best result.
If there is no result for a configuration, the behaviour depends on
the argument ``consider_incomplete``. If it is set to True, the mean
of all non-NaN values is considered.
Parameters
----------
consider_incomplete : bool, default=False
Consider the nanmean of incomplete trials if True.
Returns
-------
int
ID of the best hyperparameter configuration found so far.
Raises
------
ValueError
If no non-NaN value is found.
"""
best_idx = -1
best_value = sys.maxint
for i, trial in enumerate(self.trials):
tmp_res = np.NaN
if np.isfinite(trial['result']):
tmp_res = trial['result']
elif consider_incomplete and np.isfinite(trial[\
'instance_results']).any():
tmp_res = wrapping_util.nan_mean(trial['instance_results'])
else:
continue
if tmp_res < best_value:
best_idx = i
best_value = tmp_res
if best_idx == -1:
raise ValueError("No best value found.")
return best_idx
def get_arg_best(self, consider_incomplete=False):
"""Get the job id of the best result.
If there is no result for a configuration, the behaviour depends on
the argument ``consider_incomplete``. If it is set to True, the mean
of all non-NaN values is considered.
Parameters
----------
consider_incomplete : bool, default=False
Consider the nanmean of incomplete trials if True.
Returns
-------
int
ID of the best hyperparameter configuration found so far.
Raises
------
ValueError
If no non-NaN value is found.
"""
best_idx = -1
best_value = sys.maxint
for i, trial in enumerate(self.trials):
tmp_res = np.NaN
if np.isfinite(trial['result']):
tmp_res = trial['result']
elif consider_incomplete and np.isfinite(trial[\
'instance_results']).any():
tmp_res = wrapping_util.nan_mean(trial['instance_results'])
else:
continue
if tmp_res < best_value:
best_idx = i
best_value = tmp_res
if best_idx == -1:
raise ValueError("No best value found.")
return best_idx
