def __init__(self, working_dir, config_space_file):
self.working_dir = working_dir
self.config_space_file = config_space_file
self.result_logger = ResultLogger(working_dir=working_dir)
self.config_space = ExperimentArguments.read_configuration_space(config_space_file)
self.results = {}
experiment_start_time = np.inf
self.alpha = 0.4
self.size = 40
colors = ["windows blue", "amber", "green", "red", ]
self.pal = sns.xkcd_palette(colors)
self.lines = {}
# First find starting time
for configuration, result in self.result_logger.get_results(self.config_space):
budget, result_timestamps, result, exception = result
t_started = int(result_timestamps['started'])
experiment_start_time = min(t_started, experiment_start_time)
for configuration, result, info in self.result_logger.get_results(self.config_space, with_info=True):
budget, result_timestamps, result, exception = result
if result is None:
result = np.inf
else:
result = 100 + float(result['loss']) * 100
#result = np.sqrt(float(result['loss']) * stdv**2)
t_finished = int(result_timestamps['finished'])
point = ((t_finished - experiment_start_time) / 3600, result, info['model_based_pick'])
if point[0] > 170:
continue
try:
self.results[budget].append(point)
except KeyError:
self.results[budget] = [point]
try:
self.lines[configuration].append(point)
except KeyError:
self.lines[configuration] = [point]
self.best_points = {}
for key, values in self.results.items():
self.best_points[key] = []
values = sorted(values, key=lambda c: c[0])
best = np.inf
for time, result, mb in values:
if result < best:
best = result
self.best_points[key].append((time, result))
def __init__(self, working_dir, config_space_file):
self.working_dir = working_dir
self.config_space_file = config_space_file
self.result_logger = ResultLogger(working_dir=working_dir)
self.config_space = ExperimentArguments.read_configuration_space(
config_space_file)
self.results = {}
experiment_start_time = np.inf
self.alpha = 0.8
self.size = 50
# First find starting time
for configuration, result in self.result_logger.get_results(
self.config_space):
budget, result_timestamps, result, exception = result
t_started = int(result_timestamps['started'])
experiment_start_time = min(t_started, experiment_start_time)
for configuration, result, info in self.result_logger.get_results(
self.config_space, with_info=True):
budget, result_timestamps, result, exception = result
if result is None:
result = np.inf
else:
result = float(result['loss'])
t_finished = int(result_timestamps['finished'])
point = ((t_finished - experiment_start_time) / 3600, result,
info['model_based_pick'])
try:
self.results[budget].append(point)
except KeyError:
self.results[budget] = [point]
self.best_points = {}
for key, values in self.results.items():
self.best_points[key] = []
values = sorted(values, key=lambda c: c[0])
best = np.inf
for time, result, mb in values:
if result < best:
best = result
self.best_points[key].append((time, result))
def main():
# Experiment will read all arguments from the .ini file and command line interface (CLI).
experiment = Experiment()
train_manager = experiment.train_manager
experiment_arguments = experiment.experiment_arguments
# Sample random configuration
config_space = ExperimentArguments.read_configuration_space(
experiment_arguments.config_space_file)
random_config = config_space.sample_configuration()
# Update experiment arguments using random configuration
experiment_arguments = experiment_arguments.updated_with_configuration(
random_config)
# Run for different budgets
full_budget_decoder = FullBudgetDecoder()
adjusted_arguments_list = full_budget_decoder.adjusted_arguments(
experiment_arguments, budget=None)
for experiment_args in adjusted_arguments_list:
# Initialize a new directory for each training run
experiment_args.run_log_folder = train_manager.get_unique_dir()
train_metrics = train_manager.train(experiment_args)
valid_metrics = train_manager.validate(
experiment_args,
data_type=experiment_arguments.validation_data_type)
# Print for the user
logger.info('Train Metrics:')
logger.info(
json.dumps(train_metrics.get_summarized_results(),
indent=2,
sort_keys=True))
logger.info('%s Metrics:' %
experiment_arguments.validation_data_type.title())
logger.info(
json.dumps(valid_metrics.get_summarized_results(),
indent=2,
sort_keys=True))
def __init__(self, working_dir, config_space_file, n_iterations, run_id,
eta, min_budget, max_budget, ping_interval, nic_name,
**kwargs):
# Class that is used by the HpBandSter Master to store and manage job configurations and results
# At the beginning will load results from the previous HpBandSter runs. Those can be used to initialize
# config_generator
self.results_logger = ResultLogger(working_dir)
# Config space that holds all hyperparameters, default values and possible ranges
self.config_space = ExperimentArguments.read_configuration_space(
config_space_file)
# Config generator that builds a model and samples promising configurations.
# Initialized from previous configurations if those are present
self.config_generator = ConfigGenerator(self.config_space,
working_dir=working_dir,
**kwargs)
self.config_generator.load_from_results_logger(self.results_logger)
self.pyro_conf_file = os.path.join(working_dir, 'pyro.conf')
ns_host, ns_port = start_local_nameserver(nic_name=nic_name)
logger.info('Started nameserver with %s %s' % (ns_host, ns_port))
if os.path.exists(self.pyro_conf_file):
raise RuntimeError('Pyro conf file already exists')
with open(self.pyro_conf_file, 'w') as f:
logger.info('Creating new Pyro conf file.')
json.dump({
'ns_host': ns_host,
'ns_port': ns_port
},
f,
sort_keys=True,
indent=2)
super().__init__(run_id=run_id,
config_generator=self.config_generator,
working_directory=working_dir,
ping_interval=ping_interval,
nameserver=ns_host,
nameserver_port=ns_port,
host=ns_host,
logger=master_logger,
result_logger=self.results_logger)
# Ugly, but no other way to set it
self.dispatcher.logger = dispatcher_logger
# Hyperband related stuff
self.eta = eta
self.min_budget = min_budget
self.max_budget = max_budget
# Precompute some HB stuff
self.max_SH_iter = -int(
np.log(min_budget / max_budget) / np.log(eta)) + 1
self.budgets = max_budget * np.power(
eta, -np.linspace(self.max_SH_iter - 1, 0, self.max_SH_iter))
self.config.update({
'eta': eta,
'min_budget': min_budget,
'max_budget': max_budget,
'budgets': self.budgets,
'max_SH_iter': self.max_SH_iter,
'min_points_in_model': self.config_generator.min_points_in_model,
'top_n_percent': self.config_generator.top_n_percent,
'num_samples': self.config_generator.num_samples,
'random_fraction': self.config_generator.random_fraction,
'bandwidth_factor': self.config_generator.bw_factor,
'min_bandwidth': self.config_generator.min_bandwidth
})
self.n_iterations = n_iterations
self.min_budget = min_budget
self.max_budget = max_budget
self.eta = eta
self.ping_interval = ping_interval
self.run_id = run_id
self.nic_name = nic_name
# Update models for this budget
self.kde_models[budget] = dict(good=good_kde, bad=bad_kde)
logger.debug('Build new model for budget %f based on %i/%i split.' % (budget, n_good, n_bad))
logger.debug('Best loss for this budget:%f' % (np.min(train_losses)))
return True
if __name__ == '__main__':
from src.experiment_arguments import ExperimentArguments
logger.setLevel(logging.DEBUG)
working_dir = '/mhome/chrabasp/EEG_Results/BO_Anomaly_6'
config_space_file = '/mhome/chrabasp/Workspace/EEG/config/anomaly_simple.pcs'
results_logger = ResultLogger(working_dir=working_dir)
config_space = ExperimentArguments.read_configuration_space(config_space_file)
config_generator = ConfigGenerator(config_space=config_space,
working_dir=working_dir,
min_points_in_model=0,
top_n_percent=20,
num_samples=100,
random_fraction=0.3,
bandwidth_factor=3.0,
min_bandwidth=0.001,
bw_estimation_method='normal_reference')
config_generator.load_from_results_logger(results_logger)
model = config_generator.kde_models[27.0]['good']
model_bad = config_generator.kde_models[27.0]['bad']
# print(model)