def test_nan_mean(self): self.assertEqual(wrapping_util.nan_mean(np.array([1, 5])), 3) self.assertEqual(wrapping_util.nan_mean((1, 5)), 3) # self.assertEqual(wrapping_util.nan_mean("abc"), 3) self.assertRaises(TypeError, wrapping_util.nan_mean, ("abc")) self.assertEqual(wrapping_util.nan_mean(np.array([1, 5, np.nan])), 3) self.assertEqual(wrapping_util.nan_mean(np.array([1, 5, np.inf])), 3) self.assertTrue(np.isnan(wrapping_util.nan_mean(np.array([np.inf])))) self.assertEqual(wrapping_util.nan_mean(np.array([-1, 1])), 0) self.assertTrue(np.isnan(wrapping_util.nan_mean(np.array([]))))
def output_experiment_pickle(console_output_delay, printed_start_configuration, printed_end_configuration, optimizer_dir_in_experiment, optimizer, experiment_directory_prefix, lock, Experiment, np, exit): current_best = -1 while True: try: trials = Experiment.Experiment( optimizer_dir_in_experiment, experiment_directory_prefix + optimizer) except Exception as e: logger.error(e) time.sleep(console_output_delay) continue with lock: for i in range(len(printed_end_configuration), len(trials.instance_order)): configuration = trials.instance_order[i][0] fold = trials.instance_order[i][1] if i + 1 > len(printed_start_configuration): logger.info("Starting configuration %5d, fold %2d", configuration, fold) printed_start_configuration.append(i) if np.isfinite(trials.trials[configuration]["instance_results"] [fold]): last_result = trials.trials[configuration] \ ["instance_results"][fold] tmp_current_best = trials.get_arg_best() if tmp_current_best <= i: current_best = tmp_current_best # Calculate current best # Check if last result is finite, if not calc nanmean over all instances dct_helper = trials.trials[current_best] res = dct_helper["result"] if \ np.isfinite(dct_helper["result"]) \ else wrapping_util.nan_mean(dct_helper["instance_results"]) #np.nanmean(trials.trials[current_best]["instance_results"]) # nanmean does not work for all numpy version logger.info("Result %10f, current best %10f", last_result, res) printed_end_configuration.append(i) del trials time.sleep(console_output_delay) if exit: break
def output_experiment_pickle(console_output_delay, printed_start_configuration, printed_end_configuration, optimizer_dir_in_experiment, optimizer, experiment_directory_prefix, lock, Experiment, np, exit): current_best = -1 while True: try: trials = Experiment.Experiment(optimizer_dir_in_experiment, experiment_directory_prefix + optimizer) except Exception as e: logger.error(e) time.sleep(console_output_delay) continue with lock: for i in range(len(printed_end_configuration), len(trials.instance_order)): configuration = trials.instance_order[i][0] fold = trials.instance_order[i][1] if i + 1 > len(printed_start_configuration): logger.info("Starting configuration %5d, fold %2d", configuration, fold) printed_start_configuration.append(i) if np.isfinite(trials.trials[configuration] ["instance_results"][fold]): last_result = trials.trials[configuration] \ ["instance_results"][fold] tmp_current_best = trials.get_arg_best() if tmp_current_best <= i: current_best = tmp_current_best # Calculate current best # Check if last result is finite, if not calc nanmean over all instances dct_helper = trials.trials[current_best] res = dct_helper["result"] if \ np.isfinite(dct_helper["result"]) \ else wrapping_util.nan_mean(dct_helper["instance_results"]) #np.nanmean(trials.trials[current_best]["instance_results"]) # nanmean does not work for all numpy version logger.info("Result %10f, current best %10f", last_result, res) printed_end_configuration.append(i) del trials time.sleep(console_output_delay) if exit: break
def get_arg_best(self): best_idx = -1 best_value = sys.maxint for i, trial in enumerate(self.trials): tmp_res = np.NaN if np.isfinite(trial['result']): tmp_res = trial['result'] elif np.isfinite(trial['instance_results']).any(): tmp_res = wrapping_util.nan_mean(trial['instance_results']) # np.nanmean is not available in older numpy versions # tmp_res = scipy.nanmean(trial['instance_results']) else: continue if tmp_res < best_value: best_idx = i best_value = tmp_res if best_idx == -1: raise ValueError("No best value found.") return best_idx
def get_arg_best(self, consider_incomplete=False): """Get the job id of the best result. If there is no result for a configuration, the behaviour depends on the argument ``consider_incomplete``. If it is set to True, the mean of all non-NaN values is considered. Parameters ---------- consider_incomplete : bool, default=False Consider the nanmean of incomplete trials if True. Returns ------- int ID of the best hyperparameter configuration found so far. Raises ------ ValueError If no non-NaN value is found. """ best_idx = -1 best_value = sys.maxint for i, trial in enumerate(self.trials): tmp_res = np.NaN if np.isfinite(trial['result']): tmp_res = trial['result'] elif consider_incomplete and np.isfinite(trial[\ 'instance_results']).any(): tmp_res = wrapping_util.nan_mean(trial['instance_results']) else: continue if tmp_res < best_value: best_idx = i best_value = tmp_res if best_idx == -1: raise ValueError("No best value found.") return best_idx