def distribute_presets(sc_path, prefixes, scaffolding, config_updates):
for path, value in iterate_flattened(config_updates):
if sc_path:
path = sc_path + "." + path
scaffold_name, suffix = find_best_match(sc_path + "." + path, prefixes)
scaff = scaffolding[scaffold_name]
set_by_dotted_path(scaff.presets, suffix, value)
def get_config_modifications(self):
self.config_mods = ConfigSummary(added={
key
for key, value in iterate_flattened(self.config_updates)
})
for cfg_summary in self.summaries:
self.config_mods.update_from(cfg_summary)
def distribute_config_updates(scaffolding, config_updates):
if config_updates is None:
return
nested_config_updates = convert_to_nested_dict(config_updates)
for path, value in iterate_flattened(nested_config_updates):
for prefix, suffix in reversed(list(iter_path_splits(path))):
if prefix in scaffolding:
set_by_dotted_path(scaffolding[prefix].config_updates, suffix,
value)
break
def pick_relevant_config_updates(self, config_updates, past_paths):
if config_updates is None:
return
for path, value in iterate_flattened(config_updates):
for prefix, suffix in reversed(list(iter_path_splits(path))):
if prefix in past_paths:
# don't use config_updates for prior ingredients
break
elif prefix == self.path:
set_by_dotted_path(self.config_updates, suffix, value)
break
def pick_relevant_config_updates(self, config_updates, past_paths):
if config_updates is None:
return
for path, value in iterate_flattened(config_updates):
for prefix, suffix in reversed(list(iter_path_splits(path))):
if prefix in past_paths:
# don't use config_updates for prior ingredients
break
elif prefix == self.path:
set_by_dotted_path(self.config_updates, suffix, value)
break
def test_iterate_flattened():
d = {'a': {'aa': 1, 'ab': {'aba': 8}}, 'b': 3}
assert list(iterate_flattened(d)) == \
[('a.aa', 1), ('a.ab.aba', 8), ('b', 3)]
def test_iterate_flattened():
d = {"a": {"aa": 1, "ab": {"aba": 8}}, "b": 3}
assert list(iterate_flattened(d)) == [("a.aa", 1), ("a.ab.aba", 8), ("b", 3)]
def test_iterate_flattened():
d = {'a': {'aa': 1, 'ab': {'aba': 8}}, 'b': 3}
assert list(iterate_flattened(d)) == \
[('a.aa', 1), ('a.ab.aba', 8), ('b', 3)]
def create_run(experiment,
command_name,
config_updates=None,
named_configs=(),
force=False,
log_level=None):
sorted_ingredients = gather_ingredients_topological(experiment)
scaffolding = create_scaffolding(experiment, sorted_ingredients)
# get all split non-empty prefixes sorted from deepest to shallowest
prefixes = sorted([s.split('.') for s in scaffolding if s != ''],
reverse=True,
key=lambda p: len(p))
# --------- configuration process -------------------
# Phase 1: Config updates
config_updates = config_updates or {}
config_updates = convert_to_nested_dict(config_updates)
root_logger, run_logger = initialize_logging(experiment, scaffolding,
log_level)
distribute_config_updates(prefixes, scaffolding, config_updates)
# Phase 2: Named Configs
for ncfg in named_configs:
scaff, cfg_name = get_scaffolding_and_config_name(ncfg, scaffolding)
scaff.gather_fallbacks()
ncfg_updates = scaff.run_named_config(cfg_name)
distribute_presets(prefixes, scaffolding, ncfg_updates)
for ncfg_key, value in iterate_flattened(ncfg_updates):
set_by_dotted_path(config_updates,
join_paths(scaff.path, ncfg_key), value)
distribute_config_updates(prefixes, scaffolding, config_updates)
# Phase 3: Normal config scopes
for scaffold in scaffolding.values():
scaffold.gather_fallbacks()
scaffold.set_up_config()
# update global config
config = get_configuration(scaffolding)
# run config hooks
config_hook_updates = scaffold.run_config_hooks(
config, command_name, run_logger)
recursive_update(scaffold.config, config_hook_updates)
# Phase 4: finalize seeding
for scaffold in reversed(list(scaffolding.values())):
scaffold.set_up_seed() # partially recursive
config = get_configuration(scaffolding)
config_modifications = get_config_modifications(scaffolding)
# ----------------------------------------------------
experiment_info = experiment.get_experiment_info()
host_info = get_host_info()
main_function = get_command(scaffolding, command_name)
pre_runs = [pr for ing in sorted_ingredients for pr in ing.pre_run_hooks]
post_runs = [pr for ing in sorted_ingredients for pr in ing.post_run_hooks]
run = Run(config, config_modifications, main_function,
copy(experiment.observers), root_logger, run_logger,
experiment_info, host_info, pre_runs, post_runs,
experiment.captured_out_filter)
if hasattr(main_function, 'unobserved'):
run.unobserved = main_function.unobserved
run.force = force
for scaffold in scaffolding.values():
scaffold.finalize_initialization(run=run)
return run
def distribute_config_updates(prefixes, scaffolding, config_updates):
for path, value in iterate_flattened(config_updates):
scaffold_name, suffix = find_best_match(path, prefixes)
scaff = scaffolding[scaffold_name]
set_by_dotted_path(scaff.config_updates, suffix, value)
def get_config_modifications(self):
self.config_mods = ConfigSummary(
added={key
for key, value in iterate_flattened(self.config_updates)})
for cfg_summary in self.summaries:
self.config_mods.update_from(cfg_summary)
def create_run(experiment, command_name, config_updates=None,
named_configs=(), force=False):
sorted_ingredients = gather_ingredients_topological(experiment)
scaffolding = create_scaffolding(experiment, sorted_ingredients)
# get all split non-empty prefixes sorted from deepest to shallowest
prefixes = sorted([s.split('.') for s in scaffolding if s != ''],
reverse=True, key=lambda p: len(p))
# --------- configuration process -------------------
# Phase 1: Config updates
config_updates = config_updates or {}
config_updates = convert_to_nested_dict(config_updates)
root_logger, run_logger = initialize_logging(experiment, scaffolding)
distribute_config_updates(prefixes, scaffolding, config_updates)
# Phase 2: Named Configs
for ncfg in named_configs:
scaff, cfg_name = get_scaffolding_and_config_name(ncfg, scaffolding)
scaff.gather_fallbacks()
ncfg_updates = scaff.run_named_config(cfg_name)
distribute_presets(prefixes, scaffolding, ncfg_updates)
for ncfg_key, value in iterate_flattened(ncfg_updates):
set_by_dotted_path(config_updates,
join_paths(scaff.path, ncfg_key),
value)
distribute_config_updates(prefixes, scaffolding, config_updates)
# Phase 3: Normal config scopes
for scaffold in scaffolding.values():
scaffold.gather_fallbacks()
scaffold.set_up_config()
# update global config
config = get_configuration(scaffolding)
# run config hooks
config_updates = scaffold.run_config_hooks(config, config_updates,
command_name, run_logger)
# Phase 4: finalize seeding
for scaffold in reversed(list(scaffolding.values())):
scaffold.set_up_seed() # partially recursive
config = get_configuration(scaffolding)
config_modifications = get_config_modifications(scaffolding)
# ----------------------------------------------------
experiment_info = experiment.get_experiment_info()
host_info = get_host_info()
main_function = get_command(scaffolding, command_name)
pre_runs = [pr for ing in sorted_ingredients for pr in ing.pre_run_hooks]
post_runs = [pr for ing in sorted_ingredients for pr in ing.post_run_hooks]
run = Run(config, config_modifications, main_function,
copy(experiment.observers), root_logger, run_logger,
experiment_info, host_info, pre_runs, post_runs,
experiment.captured_out_filter)
if hasattr(main_function, 'unobserved'):
run.unobserved = main_function.unobserved
run.force = force
for scaffold in scaffolding.values():
scaffold.finalize_initialization(run=run)
return run
def distribute_config_updates(prefixes, scaffolding, config_updates):
for path, value in iterate_flattened(config_updates):
scaffold_name, suffix = find_best_match(path, prefixes)
scaff = scaffolding[scaffold_name]
set_by_dotted_path(scaff.config_updates, suffix, value)
