def get_config_updates(updates): """ Parse the UPDATES given on the commandline. Parameters ---------- updates (list[str]): list of update-strings of the form NAME=LITERAL or just NAME. Returns ------- (dict, list): Config updates and named configs to use """ config_updates = {} named_configs = [] if not updates: return config_updates, named_configs for upd in updates: if upd == '': continue path, sep, value = upd.partition('=') if sep == '=': path = path.strip() # get rid of surrounding whitespace value = value.strip() # get rid of surrounding whitespace set_by_dotted_path(config_updates, path, _convert_value(value)) else: named_configs.append(path) return config_updates, named_configs
def get_configuration(scaffolding): config = {} for sc_path, scaffold in reversed(list(scaffolding.items())): if sc_path: set_by_dotted_path(config, sc_path, scaffold.config) else: config.update(scaffold.config) return config
def get_configuration(scaffolding): config = {} for sc_path, scaffold in reversed(list(scaffolding.items())): if not scaffold.config: continue if sc_path: set_by_dotted_path(config, sc_path, scaffold.config) else: config.update(scaffold.config) return config
def distribute_config_updates(scaffolding, config_updates): if config_updates is None: return nested_config_updates = convert_to_nested_dict(config_updates) for path, value in iterate_flattened(nested_config_updates): for prefix, suffix in reversed(list(iter_path_splits(path))): if prefix in scaffolding: set_by_dotted_path(scaffolding[prefix].config_updates, suffix, value) break
def get_fixture_recursive(runner): for sr_path, subrunner in runner.subrunners.items(): # I am not sure if it is necessary to trigger all subrunner.get_fixture() get_fixture_recursive(subrunner) sub_fix = copy(subrunner.config) sub_path = sr_path if is_prefix(self.path, sub_path): sub_path = sr_path[len(self.path):].strip('.') # Note: This might fail if we allow non-dict fixtures set_by_dotted_path(self.fixture, sub_path, sub_fix)
def gather_fallbacks(self): fallback = {} for sr_path, subrunner in self.subrunners.items(): if self.path and is_prefix(self.path, sr_path): path = sr_path[len(self.path):].strip('.') set_by_dotted_path(fallback, path, subrunner.config) else: set_by_dotted_path(fallback, sr_path, subrunner.config) # dogmatize to make the subrunner configurations read-only self.fallback = dogmatize(fallback) self.fallback.revelation()
def pick_relevant_config_updates(self, config_updates, past_paths): if config_updates is None: return for path, value in iterate_flattened(config_updates): for prefix, suffix in reversed(list(iter_path_splits(path))): if prefix in past_paths: # don't use config_updates for prior ingredients break elif prefix == self.path: set_by_dotted_path(self.config_updates, suffix, value) break
def get_fixture(self): if self.fixture is not None: return self.fixture self.fixture = copy(self.config) for sr_path, subrunner in self.subrunners.items(): sub_fix = subrunner.get_fixture() sub_path = sr_path if is_prefix(self.path, sub_path): sub_path = sr_path[len(self.path):].strip('.') # Note: This might fail if we allow non-dict fixtures set_by_dotted_path(self.fixture, sub_path, sub_fix) return self.fixture
def get_config_updates(updates): config_updates = {} named_configs = [] if not updates: return config_updates, named_configs for upd in updates: if upd == '': continue path, sep, value = upd.partition('=') if sep == '=': path = path.strip() # get rid of surrounding whitespace value = value.strip() # get rid of surrounding whitespace set_by_dotted_path(config_updates, path, _convert_value(value)) else: named_configs.append(path) return config_updates, named_configs
def set_up_config(self): if self.config is not None: return self.config # gather presets fallback = {} for sr_path, subrunner in self.subrunners.items(): if self.path and is_prefix(self.path, sr_path): path = sr_path[len(self.path):].strip('.') set_by_dotted_path(fallback, path, subrunner.config) else: set_by_dotted_path(fallback, sr_path, subrunner.config) # dogmatize to make the subrunner configurations read-only const_fallback = dogmatize(fallback) const_fallback.revelation() self.config = {} # named configs first cfg_list = [] for ncfg in self.named_configs_to_use: if os.path.exists(ncfg): cfg_list.append(ConfigDict(load_config_file(ncfg))) else: cfg_list.append(self.named_configs[ncfg]) self.config_updates, _ = chain_evaluate_config_scopes( cfg_list, fixed=self.config_updates, preset=self.config, fallback=const_fallback) # unnamed (default) configs second self.config, self.summaries = chain_evaluate_config_scopes( self.config_scopes, fixed=self.config_updates, preset=self.config, fallback=const_fallback) self.get_config_modifications()
def distribute_config_updates(prefixes, scaffolding, config_updates): for path, value in iterate_flattened(config_updates): scaffold_name, suffix = find_best_match(path, prefixes) scaff = scaffolding[scaffold_name] set_by_dotted_path(scaff.config_updates, suffix, value)
def test_set_by_dotted_path(): d = {'foo': {'bar': 7}} set_by_dotted_path(d, 'foo.bar', 10) assert d == {'foo': {'bar': 10}}
def test_set_by_dotted_path(): d = {"foo": {"bar": 7}} set_by_dotted_path(d, "foo.bar", 10) assert d == {"foo": {"bar": 10}}
def create_run(experiment, command_name, config_updates=None, named_configs=(), force=False): sorted_ingredients = gather_ingredients_topological(experiment) scaffolding = create_scaffolding(experiment, sorted_ingredients) # get all split non-empty prefixes sorted from deepest to shallowest prefixes = sorted([s.split('.') for s in scaffolding if s != ''], reverse=True, key=lambda p: len(p)) # --------- configuration process ------------------- # Phase 1: Config updates config_updates = config_updates or {} config_updates = convert_to_nested_dict(config_updates) root_logger, run_logger = initialize_logging(experiment, scaffolding) distribute_config_updates(prefixes, scaffolding, config_updates) # Phase 2: Named Configs for ncfg in named_configs: scaff, cfg_name = get_scaffolding_and_config_name(ncfg, scaffolding) scaff.gather_fallbacks() ncfg_updates = scaff.run_named_config(cfg_name) distribute_presets(prefixes, scaffolding, ncfg_updates) for ncfg_key, value in iterate_flattened(ncfg_updates): set_by_dotted_path(config_updates, join_paths(scaff.path, ncfg_key), value) distribute_config_updates(prefixes, scaffolding, config_updates) # Phase 3: Normal config scopes for scaffold in scaffolding.values(): scaffold.gather_fallbacks() scaffold.set_up_config() # update global config config = get_configuration(scaffolding) # run config hooks config_updates = scaffold.run_config_hooks(config, config_updates, command_name, run_logger) # Phase 4: finalize seeding for scaffold in reversed(list(scaffolding.values())): scaffold.set_up_seed() # partially recursive config = get_configuration(scaffolding) config_modifications = get_config_modifications(scaffolding) # ---------------------------------------------------- experiment_info = experiment.get_experiment_info() host_info = get_host_info() main_function = get_command(scaffolding, command_name) pre_runs = [pr for ing in sorted_ingredients for pr in ing.pre_run_hooks] post_runs = [pr for ing in sorted_ingredients for pr in ing.post_run_hooks] run = Run(config, config_modifications, main_function, copy(experiment.observers), root_logger, run_logger, experiment_info, host_info, pre_runs, post_runs, experiment.captured_out_filter) if hasattr(main_function, 'unobserved'): run.unobserved = main_function.unobserved run.force = force for scaffold in scaffolding.values(): scaffold.finalize_initialization(run=run) return run
def test_set_by_dotted_path_creates_missing_dicts(): d = {'foo': {'bar': 7}} set_by_dotted_path(d, 'foo.d.baz', 3) assert d == {'foo': {'bar': 7, 'd': {'baz': 3}}}
def test_set_by_dotted_path_creates_missing_dicts(): d = {"foo": {"bar": 7}} set_by_dotted_path(d, "foo.d.baz", 3) assert d == {"foo": {"bar": 7, "d": {"baz": 3}}}
def create_run(experiment, command_name, config_updates=None, named_configs=(), force=False, log_level=None): sorted_ingredients = gather_ingredients_topological(experiment) scaffolding = create_scaffolding(experiment, sorted_ingredients) # get all split non-empty prefixes sorted from deepest to shallowest prefixes = sorted([s.split('.') for s in scaffolding if s != ''], reverse=True, key=lambda p: len(p)) # --------- configuration process ------------------- # Phase 1: Config updates config_updates = config_updates or {} config_updates = convert_to_nested_dict(config_updates) root_logger, run_logger = initialize_logging(experiment, scaffolding, log_level) distribute_config_updates(prefixes, scaffolding, config_updates) # Phase 2: Named Configs for ncfg in named_configs: scaff, cfg_name = get_scaffolding_and_config_name(ncfg, scaffolding) scaff.gather_fallbacks() ncfg_updates = scaff.run_named_config(cfg_name) distribute_presets(prefixes, scaffolding, ncfg_updates) for ncfg_key, value in iterate_flattened(ncfg_updates): set_by_dotted_path(config_updates, join_paths(scaff.path, ncfg_key), value) distribute_config_updates(prefixes, scaffolding, config_updates) # Phase 3: Normal config scopes for scaffold in scaffolding.values(): scaffold.gather_fallbacks() scaffold.set_up_config() # update global config config = get_configuration(scaffolding) # run config hooks config_hook_updates = scaffold.run_config_hooks( config, command_name, run_logger) recursive_update(scaffold.config, config_hook_updates) # Phase 4: finalize seeding for scaffold in reversed(list(scaffolding.values())): scaffold.set_up_seed() # partially recursive config = get_configuration(scaffolding) config_modifications = get_config_modifications(scaffolding) # ---------------------------------------------------- experiment_info = experiment.get_experiment_info() host_info = get_host_info() main_function = get_command(scaffolding, command_name) pre_runs = [pr for ing in sorted_ingredients for pr in ing.pre_run_hooks] post_runs = [pr for ing in sorted_ingredients for pr in ing.post_run_hooks] run = Run(config, config_modifications, main_function, copy(experiment.observers), root_logger, run_logger, experiment_info, host_info, pre_runs, post_runs, experiment.captured_out_filter) if hasattr(main_function, 'unobserved'): run.unobserved = main_function.unobserved run.force = force for scaffold in scaffolding.values(): scaffold.finalize_initialization(run=run) return run
def objective(**params): for key in params.keys(): if isinstance(params[key], np.int64): # Strangeley, it seems like we dont get 'real' ints here, # but a numpy datatypes. So lets cast them. params[key] = int(params[key]) # Need to do this here in order to get rid # of leftovers from previous evaluations plt.close('all') # Update the parameters we go with constant overrides params.update(overrides) # Transform the search space and overrides into structure of nested # dicts # This workaround as sacred does not allow '.' in dict keys params = { key.replace('__', '.'): value for key, value in params.items() } # Convert to nested dict transformed_params = {} for key, value in params.items(): # This function is from sacred and used to convert x.y=z notation # into nested dicts: {'x': {'y': z}} set_by_dotted_path(transformed_params, key, value) _log.debug(f'Running training with parameters: {transformed_params}') try: # Run the experiment and update config according to overrides # to overrides and sampled parameters run = train_experiment.run(config_updates=transformed_params) result = run.result results.append(result) # gp optimize does not handle nan values, thus we need # to return something fake if we diverge before the end # of the first epoch metric_values = [ result[eval_metric] for eval_metric in evaluation_metrics ] all_finite = all((np.isfinite(val) for val in metric_values)) if all_finite: return_value = np.mean(metric_values) else: return_value = nan_replacement result['hyperparam_optimization_objective'] = return_value except Exception as e: _log.error('An exception occured during fitting: {}'.format(e)) results.append({}) return_value = nan_replacement result = {} # Store the results into sacred infrastructure # Ensures they can be used even if the experiment is terminated params.update(result) evaluated_parameters.append(params) _run.result['parameter_evaluations'] = evaluated_parameters with NamedTemporaryFile(suffix='.csv') as f: df = pd.DataFrame(evaluated_parameters) df.to_csv(f.name) _run.add_artifact(f.name, 'parameter_evaluations.csv') return return_value