def get_config_updates(updates):
"""
Parse the UPDATES given on the commandline.
Parameters
----------
updates (list[str]):
list of update-strings of the form NAME=LITERAL or just NAME.
Returns
-------
(dict, list):
Config updates and named configs to use
"""
config_updates = {}
named_configs = []
if not updates:
return config_updates, named_configs
for upd in updates:
if upd == '':
continue
path, sep, value = upd.partition('=')
if sep == '=':
path = path.strip() # get rid of surrounding whitespace
value = value.strip() # get rid of surrounding whitespace
set_by_dotted_path(config_updates, path, _convert_value(value))
else:
named_configs.append(path)
return config_updates, named_configs
def get_configuration(scaffolding):
config = {}
for sc_path, scaffold in reversed(list(scaffolding.items())):
if sc_path:
set_by_dotted_path(config, sc_path, scaffold.config)
else:
config.update(scaffold.config)
return config
def get_configuration(scaffolding):
config = {}
for sc_path, scaffold in reversed(list(scaffolding.items())):
if not scaffold.config:
continue
if sc_path:
set_by_dotted_path(config, sc_path, scaffold.config)
else:
config.update(scaffold.config)
return config
def distribute_config_updates(scaffolding, config_updates):
if config_updates is None:
return
nested_config_updates = convert_to_nested_dict(config_updates)
for path, value in iterate_flattened(nested_config_updates):
for prefix, suffix in reversed(list(iter_path_splits(path))):
if prefix in scaffolding:
set_by_dotted_path(scaffolding[prefix].config_updates, suffix,
value)
break
def get_fixture_recursive(runner):
for sr_path, subrunner in runner.subrunners.items():
# I am not sure if it is necessary to trigger all
subrunner.get_fixture()
get_fixture_recursive(subrunner)
sub_fix = copy(subrunner.config)
sub_path = sr_path
if is_prefix(self.path, sub_path):
sub_path = sr_path[len(self.path):].strip('.')
# Note: This might fail if we allow non-dict fixtures
set_by_dotted_path(self.fixture, sub_path, sub_fix)
def get_fixture_recursive(runner):
for sr_path, subrunner in runner.subrunners.items():
# I am not sure if it is necessary to trigger all
subrunner.get_fixture()
get_fixture_recursive(subrunner)
sub_fix = copy(subrunner.config)
sub_path = sr_path
if is_prefix(self.path, sub_path):
sub_path = sr_path[len(self.path):].strip('.')
# Note: This might fail if we allow non-dict fixtures
set_by_dotted_path(self.fixture, sub_path, sub_fix)
def gather_fallbacks(self):
fallback = {}
for sr_path, subrunner in self.subrunners.items():
if self.path and is_prefix(self.path, sr_path):
path = sr_path[len(self.path):].strip('.')
set_by_dotted_path(fallback, path, subrunner.config)
else:
set_by_dotted_path(fallback, sr_path, subrunner.config)
# dogmatize to make the subrunner configurations read-only
self.fallback = dogmatize(fallback)
self.fallback.revelation()
def pick_relevant_config_updates(self, config_updates, past_paths):
if config_updates is None:
return
for path, value in iterate_flattened(config_updates):
for prefix, suffix in reversed(list(iter_path_splits(path))):
if prefix in past_paths:
# don't use config_updates for prior ingredients
break
elif prefix == self.path:
set_by_dotted_path(self.config_updates, suffix, value)
break
def gather_fallbacks(self):
fallback = {}
for sr_path, subrunner in self.subrunners.items():
if self.path and is_prefix(self.path, sr_path):
path = sr_path[len(self.path):].strip('.')
set_by_dotted_path(fallback, path, subrunner.config)
else:
set_by_dotted_path(fallback, sr_path, subrunner.config)
# dogmatize to make the subrunner configurations read-only
self.fallback = dogmatize(fallback)
self.fallback.revelation()
def pick_relevant_config_updates(self, config_updates, past_paths):
if config_updates is None:
return
for path, value in iterate_flattened(config_updates):
for prefix, suffix in reversed(list(iter_path_splits(path))):
if prefix in past_paths:
# don't use config_updates for prior ingredients
break
elif prefix == self.path:
set_by_dotted_path(self.config_updates, suffix, value)
break
def get_fixture(self):
if self.fixture is not None:
return self.fixture
self.fixture = copy(self.config)
for sr_path, subrunner in self.subrunners.items():
sub_fix = subrunner.get_fixture()
sub_path = sr_path
if is_prefix(self.path, sub_path):
sub_path = sr_path[len(self.path):].strip('.')
# Note: This might fail if we allow non-dict fixtures
set_by_dotted_path(self.fixture, sub_path, sub_fix)
return self.fixture
def get_fixture(self):
if self.fixture is not None:
return self.fixture
self.fixture = copy(self.config)
for sr_path, subrunner in self.subrunners.items():
sub_fix = subrunner.get_fixture()
sub_path = sr_path
if is_prefix(self.path, sub_path):
sub_path = sr_path[len(self.path):].strip('.')
# Note: This might fail if we allow non-dict fixtures
set_by_dotted_path(self.fixture, sub_path, sub_fix)
return self.fixture
def get_config_updates(updates):
config_updates = {}
named_configs = []
if not updates:
return config_updates, named_configs
for upd in updates:
if upd == '':
continue
path, sep, value = upd.partition('=')
if sep == '=':
path = path.strip() # get rid of surrounding whitespace
value = value.strip() # get rid of surrounding whitespace
set_by_dotted_path(config_updates, path, _convert_value(value))
else:
named_configs.append(path)
return config_updates, named_configs
def get_config_updates(updates):
config_updates = {}
named_configs = []
if not updates:
return config_updates, named_configs
for upd in updates:
if upd == '':
continue
path, sep, value = upd.partition('=')
if sep == '=':
path = path.strip() # get rid of surrounding whitespace
value = value.strip() # get rid of surrounding whitespace
set_by_dotted_path(config_updates, path, _convert_value(value))
else:
named_configs.append(path)
return config_updates, named_configs
def set_up_config(self):
if self.config is not None:
return self.config
# gather presets
fallback = {}
for sr_path, subrunner in self.subrunners.items():
if self.path and is_prefix(self.path, sr_path):
path = sr_path[len(self.path):].strip('.')
set_by_dotted_path(fallback, path, subrunner.config)
else:
set_by_dotted_path(fallback, sr_path, subrunner.config)
# dogmatize to make the subrunner configurations read-only
const_fallback = dogmatize(fallback)
const_fallback.revelation()
self.config = {}
# named configs first
cfg_list = []
for ncfg in self.named_configs_to_use:
if os.path.exists(ncfg):
cfg_list.append(ConfigDict(load_config_file(ncfg)))
else:
cfg_list.append(self.named_configs[ncfg])
self.config_updates, _ = chain_evaluate_config_scopes(
cfg_list,
fixed=self.config_updates,
preset=self.config,
fallback=const_fallback)
# unnamed (default) configs second
self.config, self.summaries = chain_evaluate_config_scopes(
self.config_scopes,
fixed=self.config_updates,
preset=self.config,
fallback=const_fallback)
self.get_config_modifications()
def distribute_config_updates(prefixes, scaffolding, config_updates):
for path, value in iterate_flattened(config_updates):
scaffold_name, suffix = find_best_match(path, prefixes)
scaff = scaffolding[scaffold_name]
set_by_dotted_path(scaff.config_updates, suffix, value)
def test_set_by_dotted_path():
d = {'foo': {'bar': 7}}
set_by_dotted_path(d, 'foo.bar', 10)
assert d == {'foo': {'bar': 10}}
def test_set_by_dotted_path():
d = {"foo": {"bar": 7}}
set_by_dotted_path(d, "foo.bar", 10)
assert d == {"foo": {"bar": 10}}
def test_set_by_dotted_path():
d = {'foo': {'bar': 7}}
set_by_dotted_path(d, 'foo.bar', 10)
assert d == {'foo': {'bar': 10}}
def create_run(experiment, command_name, config_updates=None,
named_configs=(), force=False):
sorted_ingredients = gather_ingredients_topological(experiment)
scaffolding = create_scaffolding(experiment, sorted_ingredients)
# get all split non-empty prefixes sorted from deepest to shallowest
prefixes = sorted([s.split('.') for s in scaffolding if s != ''],
reverse=True, key=lambda p: len(p))
# --------- configuration process -------------------
# Phase 1: Config updates
config_updates = config_updates or {}
config_updates = convert_to_nested_dict(config_updates)
root_logger, run_logger = initialize_logging(experiment, scaffolding)
distribute_config_updates(prefixes, scaffolding, config_updates)
# Phase 2: Named Configs
for ncfg in named_configs:
scaff, cfg_name = get_scaffolding_and_config_name(ncfg, scaffolding)
scaff.gather_fallbacks()
ncfg_updates = scaff.run_named_config(cfg_name)
distribute_presets(prefixes, scaffolding, ncfg_updates)
for ncfg_key, value in iterate_flattened(ncfg_updates):
set_by_dotted_path(config_updates,
join_paths(scaff.path, ncfg_key),
value)
distribute_config_updates(prefixes, scaffolding, config_updates)
# Phase 3: Normal config scopes
for scaffold in scaffolding.values():
scaffold.gather_fallbacks()
scaffold.set_up_config()
# update global config
config = get_configuration(scaffolding)
# run config hooks
config_updates = scaffold.run_config_hooks(config, config_updates,
command_name, run_logger)
# Phase 4: finalize seeding
for scaffold in reversed(list(scaffolding.values())):
scaffold.set_up_seed() # partially recursive
config = get_configuration(scaffolding)
config_modifications = get_config_modifications(scaffolding)
# ----------------------------------------------------
experiment_info = experiment.get_experiment_info()
host_info = get_host_info()
main_function = get_command(scaffolding, command_name)
pre_runs = [pr for ing in sorted_ingredients for pr in ing.pre_run_hooks]
post_runs = [pr for ing in sorted_ingredients for pr in ing.post_run_hooks]
run = Run(config, config_modifications, main_function,
copy(experiment.observers), root_logger, run_logger,
experiment_info, host_info, pre_runs, post_runs,
experiment.captured_out_filter)
if hasattr(main_function, 'unobserved'):
run.unobserved = main_function.unobserved
run.force = force
for scaffold in scaffolding.values():
scaffold.finalize_initialization(run=run)
return run
def test_set_by_dotted_path_creates_missing_dicts():
d = {'foo': {'bar': 7}}
set_by_dotted_path(d, 'foo.d.baz', 3)
assert d == {'foo': {'bar': 7, 'd': {'baz': 3}}}
def distribute_config_updates(prefixes, scaffolding, config_updates):
for path, value in iterate_flattened(config_updates):
scaffold_name, suffix = find_best_match(path, prefixes)
scaff = scaffolding[scaffold_name]
set_by_dotted_path(scaff.config_updates, suffix, value)
def test_set_by_dotted_path_creates_missing_dicts():
d = {"foo": {"bar": 7}}
set_by_dotted_path(d, "foo.d.baz", 3)
assert d == {"foo": {"bar": 7, "d": {"baz": 3}}}
def create_run(experiment,
command_name,
config_updates=None,
named_configs=(),
force=False,
log_level=None):
sorted_ingredients = gather_ingredients_topological(experiment)
scaffolding = create_scaffolding(experiment, sorted_ingredients)
# get all split non-empty prefixes sorted from deepest to shallowest
prefixes = sorted([s.split('.') for s in scaffolding if s != ''],
reverse=True,
key=lambda p: len(p))
# --------- configuration process -------------------
# Phase 1: Config updates
config_updates = config_updates or {}
config_updates = convert_to_nested_dict(config_updates)
root_logger, run_logger = initialize_logging(experiment, scaffolding,
log_level)
distribute_config_updates(prefixes, scaffolding, config_updates)
# Phase 2: Named Configs
for ncfg in named_configs:
scaff, cfg_name = get_scaffolding_and_config_name(ncfg, scaffolding)
scaff.gather_fallbacks()
ncfg_updates = scaff.run_named_config(cfg_name)
distribute_presets(prefixes, scaffolding, ncfg_updates)
for ncfg_key, value in iterate_flattened(ncfg_updates):
set_by_dotted_path(config_updates,
join_paths(scaff.path, ncfg_key), value)
distribute_config_updates(prefixes, scaffolding, config_updates)
# Phase 3: Normal config scopes
for scaffold in scaffolding.values():
scaffold.gather_fallbacks()
scaffold.set_up_config()
# update global config
config = get_configuration(scaffolding)
# run config hooks
config_hook_updates = scaffold.run_config_hooks(
config, command_name, run_logger)
recursive_update(scaffold.config, config_hook_updates)
# Phase 4: finalize seeding
for scaffold in reversed(list(scaffolding.values())):
scaffold.set_up_seed() # partially recursive
config = get_configuration(scaffolding)
config_modifications = get_config_modifications(scaffolding)
# ----------------------------------------------------
experiment_info = experiment.get_experiment_info()
host_info = get_host_info()
main_function = get_command(scaffolding, command_name)
pre_runs = [pr for ing in sorted_ingredients for pr in ing.pre_run_hooks]
post_runs = [pr for ing in sorted_ingredients for pr in ing.post_run_hooks]
run = Run(config, config_modifications, main_function,
copy(experiment.observers), root_logger, run_logger,
experiment_info, host_info, pre_runs, post_runs,
experiment.captured_out_filter)
if hasattr(main_function, 'unobserved'):
run.unobserved = main_function.unobserved
run.force = force
for scaffold in scaffolding.values():
scaffold.finalize_initialization(run=run)
return run
def test_set_by_dotted_path_creates_missing_dicts():
d = {'foo': {'bar': 7}}
set_by_dotted_path(d, 'foo.d.baz', 3)
assert d == {'foo': {'bar': 7, 'd': {'baz': 3}}}
def objective(**params):
for key in params.keys():
if isinstance(params[key], np.int64):
# Strangeley, it seems like we dont get 'real' ints here,
# but a numpy datatypes. So lets cast them.
params[key] = int(params[key])
# Need to do this here in order to get rid
# of leftovers from previous evaluations
plt.close('all')
# Update the parameters we go with constant overrides
params.update(overrides)
# Transform the search space and overrides into structure of nested
# dicts
# This workaround as sacred does not allow '.' in dict keys
params = {
key.replace('__', '.'): value
for key, value in params.items()
}
# Convert to nested dict
transformed_params = {}
for key, value in params.items():
# This function is from sacred and used to convert x.y=z notation
# into nested dicts: {'x': {'y': z}}
set_by_dotted_path(transformed_params, key, value)
_log.debug(f'Running training with parameters: {transformed_params}')
try:
# Run the experiment and update config according to overrides
# to overrides and sampled parameters
run = train_experiment.run(config_updates=transformed_params)
result = run.result
results.append(result)
# gp optimize does not handle nan values, thus we need
# to return something fake if we diverge before the end
# of the first epoch
metric_values = [
result[eval_metric] for eval_metric in evaluation_metrics
]
all_finite = all((np.isfinite(val) for val in metric_values))
if all_finite:
return_value = np.mean(metric_values)
else:
return_value = nan_replacement
result['hyperparam_optimization_objective'] = return_value
except Exception as e:
_log.error('An exception occured during fitting: {}'.format(e))
results.append({})
return_value = nan_replacement
result = {}
# Store the results into sacred infrastructure
# Ensures they can be used even if the experiment is terminated
params.update(result)
evaluated_parameters.append(params)
_run.result['parameter_evaluations'] = evaluated_parameters
with NamedTemporaryFile(suffix='.csv') as f:
df = pd.DataFrame(evaluated_parameters)
df.to_csv(f.name)
_run.add_artifact(f.name, 'parameter_evaluations.csv')
return return_value
