def distribute_presets(sc_path, prefixes, scaffolding, config_updates): for path, value in iterate_flattened(config_updates): if sc_path: path = sc_path + "." + path scaffold_name, suffix = find_best_match(sc_path + "." + path, prefixes) scaff = scaffolding[scaffold_name] set_by_dotted_path(scaff.presets, suffix, value)
def get_config_modifications(self): self.config_mods = ConfigSummary(added={ key for key, value in iterate_flattened(self.config_updates) }) for cfg_summary in self.summaries: self.config_mods.update_from(cfg_summary)
def distribute_config_updates(scaffolding, config_updates): if config_updates is None: return nested_config_updates = convert_to_nested_dict(config_updates) for path, value in iterate_flattened(nested_config_updates): for prefix, suffix in reversed(list(iter_path_splits(path))): if prefix in scaffolding: set_by_dotted_path(scaffolding[prefix].config_updates, suffix, value) break
def pick_relevant_config_updates(self, config_updates, past_paths): if config_updates is None: return for path, value in iterate_flattened(config_updates): for prefix, suffix in reversed(list(iter_path_splits(path))): if prefix in past_paths: # don't use config_updates for prior ingredients break elif prefix == self.path: set_by_dotted_path(self.config_updates, suffix, value) break
def test_iterate_flattened(): d = {'a': {'aa': 1, 'ab': {'aba': 8}}, 'b': 3} assert list(iterate_flattened(d)) == \ [('a.aa', 1), ('a.ab.aba', 8), ('b', 3)]
def test_iterate_flattened(): d = {"a": {"aa": 1, "ab": {"aba": 8}}, "b": 3} assert list(iterate_flattened(d)) == [("a.aa", 1), ("a.ab.aba", 8), ("b", 3)]
def create_run(experiment, command_name, config_updates=None, named_configs=(), force=False, log_level=None): sorted_ingredients = gather_ingredients_topological(experiment) scaffolding = create_scaffolding(experiment, sorted_ingredients) # get all split non-empty prefixes sorted from deepest to shallowest prefixes = sorted([s.split('.') for s in scaffolding if s != ''], reverse=True, key=lambda p: len(p)) # --------- configuration process ------------------- # Phase 1: Config updates config_updates = config_updates or {} config_updates = convert_to_nested_dict(config_updates) root_logger, run_logger = initialize_logging(experiment, scaffolding, log_level) distribute_config_updates(prefixes, scaffolding, config_updates) # Phase 2: Named Configs for ncfg in named_configs: scaff, cfg_name = get_scaffolding_and_config_name(ncfg, scaffolding) scaff.gather_fallbacks() ncfg_updates = scaff.run_named_config(cfg_name) distribute_presets(prefixes, scaffolding, ncfg_updates) for ncfg_key, value in iterate_flattened(ncfg_updates): set_by_dotted_path(config_updates, join_paths(scaff.path, ncfg_key), value) distribute_config_updates(prefixes, scaffolding, config_updates) # Phase 3: Normal config scopes for scaffold in scaffolding.values(): scaffold.gather_fallbacks() scaffold.set_up_config() # update global config config = get_configuration(scaffolding) # run config hooks config_hook_updates = scaffold.run_config_hooks( config, command_name, run_logger) recursive_update(scaffold.config, config_hook_updates) # Phase 4: finalize seeding for scaffold in reversed(list(scaffolding.values())): scaffold.set_up_seed() # partially recursive config = get_configuration(scaffolding) config_modifications = get_config_modifications(scaffolding) # ---------------------------------------------------- experiment_info = experiment.get_experiment_info() host_info = get_host_info() main_function = get_command(scaffolding, command_name) pre_runs = [pr for ing in sorted_ingredients for pr in ing.pre_run_hooks] post_runs = [pr for ing in sorted_ingredients for pr in ing.post_run_hooks] run = Run(config, config_modifications, main_function, copy(experiment.observers), root_logger, run_logger, experiment_info, host_info, pre_runs, post_runs, experiment.captured_out_filter) if hasattr(main_function, 'unobserved'): run.unobserved = main_function.unobserved run.force = force for scaffold in scaffolding.values(): scaffold.finalize_initialization(run=run) return run
def distribute_config_updates(prefixes, scaffolding, config_updates): for path, value in iterate_flattened(config_updates): scaffold_name, suffix = find_best_match(path, prefixes) scaff = scaffolding[scaffold_name] set_by_dotted_path(scaff.config_updates, suffix, value)
def get_config_modifications(self): self.config_mods = ConfigSummary( added={key for key, value in iterate_flattened(self.config_updates)}) for cfg_summary in self.summaries: self.config_mods.update_from(cfg_summary)
def create_run(experiment, command_name, config_updates=None, named_configs=(), force=False): sorted_ingredients = gather_ingredients_topological(experiment) scaffolding = create_scaffolding(experiment, sorted_ingredients) # get all split non-empty prefixes sorted from deepest to shallowest prefixes = sorted([s.split('.') for s in scaffolding if s != ''], reverse=True, key=lambda p: len(p)) # --------- configuration process ------------------- # Phase 1: Config updates config_updates = config_updates or {} config_updates = convert_to_nested_dict(config_updates) root_logger, run_logger = initialize_logging(experiment, scaffolding) distribute_config_updates(prefixes, scaffolding, config_updates) # Phase 2: Named Configs for ncfg in named_configs: scaff, cfg_name = get_scaffolding_and_config_name(ncfg, scaffolding) scaff.gather_fallbacks() ncfg_updates = scaff.run_named_config(cfg_name) distribute_presets(prefixes, scaffolding, ncfg_updates) for ncfg_key, value in iterate_flattened(ncfg_updates): set_by_dotted_path(config_updates, join_paths(scaff.path, ncfg_key), value) distribute_config_updates(prefixes, scaffolding, config_updates) # Phase 3: Normal config scopes for scaffold in scaffolding.values(): scaffold.gather_fallbacks() scaffold.set_up_config() # update global config config = get_configuration(scaffolding) # run config hooks config_updates = scaffold.run_config_hooks(config, config_updates, command_name, run_logger) # Phase 4: finalize seeding for scaffold in reversed(list(scaffolding.values())): scaffold.set_up_seed() # partially recursive config = get_configuration(scaffolding) config_modifications = get_config_modifications(scaffolding) # ---------------------------------------------------- experiment_info = experiment.get_experiment_info() host_info = get_host_info() main_function = get_command(scaffolding, command_name) pre_runs = [pr for ing in sorted_ingredients for pr in ing.pre_run_hooks] post_runs = [pr for ing in sorted_ingredients for pr in ing.post_run_hooks] run = Run(config, config_modifications, main_function, copy(experiment.observers), root_logger, run_logger, experiment_info, host_info, pre_runs, post_runs, experiment.captured_out_filter) if hasattr(main_function, 'unobserved'): run.unobserved = main_function.unobserved run.force = force for scaffold in scaffolding.values(): scaffold.finalize_initialization(run=run) return run