def update_from(self, config_mod, path=''): added = config_mod.added updated = config_mod.modified typechanged = config_mod.typechanged self.added |= {join_paths(path, a) for a in added} self.modified |= {join_paths(path, u) for u in updated} self.typechanged.update({join_paths(path, k): v for k, v in typechanged.items()}) self.ensure_coherence()
def update_from(self, config_mod, path=""): added = config_mod.added updated = config_mod.modified typechanged = config_mod.typechanged self.added &= {join_paths(path, a) for a in added} self.modified |= {join_paths(path, u) for u in updated} self.typechanged.update( {join_paths(path, k): v for k, v in typechanged.items()}) self.ensure_coherence() for k, v in config_mod.docs.items(): if not self.docs.get(k, ""): self.docs[k] = v
def _log_blocked_setitem(self, key, value, fixed_value): if type_changed(value, fixed_value): self.typechanges[key] = (type(value), type(fixed_value)) if value != fixed_value: self.modified.add(key) # if both are dicts recursively collect modified and typechanges if isinstance(fixed_value, DogmaticDict) and isinstance(value, dict): for k, val in fixed_value.typechanges.items(): self.typechanges[join_paths(key, k)] = val self.modified |= {join_paths(key, m) for m in fixed_value.modified}
def update_add(self, config_mod, path=""): added = config_mod.added updated = config_mod.modified typechanged = config_mod.typechanged self.added |= {join_paths(path, a) for a in added} self.modified |= {join_paths(path, u) for u in updated} self.typechanged.update( {join_paths(path, k): v for k, v in typechanged.items()}) self.docs.update({ join_paths(path, k): v for k, v in config_mod.docs.items() if path == "" or k != "seed" }) self.ensure_coherence()
def __init__(self, config_scopes, subrunners, path, captured_functions, commands, named_configs, config_hooks, generate_seed): self.config_scopes = config_scopes self.named_configs = named_configs self.subrunners = subrunners self.path = path self.generate_seed = generate_seed self.config_hooks = config_hooks self.config_updates = {} self.named_configs_to_use = [] self.config = {} self.fallback = None self.presets = {} self.fixture = None # TODO: rename self.logger = None self.seed = None self.rnd = None self._captured_functions = captured_functions self.commands = commands self.config_mods = None self.summaries = [] self.captured_args = {join_paths(cf.prefix, n) for cf in self._captured_functions for n in cf.signature.arguments} self.captured_args.add('__doc__') # allow setting the config docstring
def __init__(self, config_scopes, subrunners, path, captured_functions, commands, named_configs, config_hooks, generate_seed): self.config_scopes = config_scopes self.named_configs = named_configs self.subrunners = subrunners self.path = path self.generate_seed = generate_seed self.config_hooks = config_hooks self.config_updates = {} self.named_configs_to_use = [] self.config = {} self.fallback = None self.presets = {} self.fixture = None # TODO: rename self.logger = None self.seed = None self.rnd = None self._captured_functions = captured_functions self.commands = commands self.config_mods = None self.summaries = [] self.captured_args = { join_paths(cf.prefix, n) for cf in self._captured_functions for n in cf.signature.arguments } self.captured_args.add('__doc__') # allow setting the config docstring
def _emit_queued(self): self.status = 'QUEUED' queue_time = datetime.datetime.utcnow() self.meta_info['queue_time'] = queue_time command = join_paths(self.main_function.prefix, self.main_function.signature.name) self.run_logger.info("Queuing-up command '%s'", command) for observer in self.observers: if hasattr(observer, 'queued_event'): _id = observer.queued_event( ex_info=self.experiment_info, command=command, host_info=self.host_info, queue_time=queue_time, config=self.config, meta_info=self.meta_info, _id=self._id ) if self._id is None: self._id = _id # do not catch any exceptions on startup: # the experiment SHOULD fail if any of the observers fails if self._id is None: self.run_logger.info('Queued') else: self.run_logger.info('Queued-up run with ID "{}"'.format(self._id))
def _emit_started(self): self.status = 'RUNNING' self.start_time = datetime.datetime.utcnow() command = join_paths(self.main_function.prefix, self.main_function.signature.name) self.run_logger.info("Running command '%s'", command) for observer in self.observers: if hasattr(observer, 'started_event'): _id = observer.started_event( ex_info=self.experiment_info, command=command, host_info=self.host_info, start_time=self.start_time, config=self.config, meta_info=self.meta_info, _id=self._id ) if self._id is None: self._id = _id # do not catch any exceptions on startup: # the experiment SHOULD fail if any of the observers fails if self._id is None: self.run_logger.info('Started') else: self.run_logger.info('Started run with ID "{}"'.format(self._id))
def _emit_queued(self): self.status = 'QUEUED' queue_time = datetime.datetime.utcnow() self.meta_info['queue_time'] = queue_time command = join_paths(self.main_function.prefix, self.main_function.signature.name) self.run_logger.info("Queuing-up command '%s'", command) for observer in self.observers: if hasattr(observer, 'queued_event'): _id = observer.queued_event( ex_info=self.experiment_info, command=command, queue_time=queue_time, config=self.config, meta_info=self.meta_info, _id=self._id ) if self._id is None: self._id = _id # do not catch any exceptions on startup: # the experiment SHOULD fail if any of the observers fails if self._id is None: self.run_logger.info('Queued') else: self.run_logger.info('Queued-up run with ID "{}"'.format(self._id))
def test_join_paths(): assert join_paths() == '' assert join_paths('foo') == 'foo' assert join_paths('foo', 'bar') == 'foo.bar' assert join_paths('a', 'b', 'c', 'd') == 'a.b.c.d' assert join_paths('', 'b', '', 'd') == 'b.d' assert join_paths('a.b', 'c.d.e') == 'a.b.c.d.e' assert join_paths('a.b.', 'c.d.e') == 'a.b.c.d.e'
def test_join_paths(): assert join_paths() == "" assert join_paths("foo") == "foo" assert join_paths("foo", "bar") == "foo.bar" assert join_paths("a", "b", "c", "d") == "a.b.c.d" assert join_paths("", "b", "", "d") == "b.d" assert join_paths("a.b", "c.d.e") == "a.b.c.d.e" assert join_paths("a.b.", "c.d.e") == "a.b.c.d.e"
def gather_commands(self, ingredient): """Collect all commands from this ingredient and its sub-ingredients. Yields ------ cmd_name: str The full (dotted) name of the command. cmd: function The corresponding captured function. """ for command_name, command in ingredient.commands.items(): yield join_paths(ingredient.path, command_name), command
def gather_named_configs(self, ingredient): """Collect all named configs from this ingredient and its sub-ingredients. Yields ------ config_name: str The full (dotted) name of the named config. config: ConfigScope or ConfigDict or basestring The corresponding named config. """ for config_name, config in ingredient.named_configs.items(): yield join_paths(ingredient.path, config_name), config
def gather_commands(self): """Collect all commands from this ingredient and its sub-ingredients. Yields ------ cmd_name: str The full (dotted) name of the command. cmd: function The corresponding captured function. """ for ingredient, _ in self.traverse_ingredients(): for command_name, command in ingredient.commands.items(): cmd_name = join_paths(ingredient.path, command_name) cmd_name = self.post_process_name(cmd_name, ingredient) yield cmd_name, command
def gather_named_configs(self): """Collect all named configs from this ingredient and its sub-ingredients. Yields ------ config_name: str The full (dotted) name of the named config. config: ConfigScope or ConfigDict or basestring The corresponding named config. """ for ingredient, _ in self.traverse_ingredients(): for config_name, config in ingredient.named_configs.items(): config_name = join_paths(ingredient.path, config_name) config_name = self.post_process_name(config_name, ingredient) yield config_name, config
def gather_named_configs( self, ) -> Generator[Tuple[str, Union[ConfigScope, ConfigDict, str]], None, None]: """Collect all named configs from this ingredient and its sub-ingredients. Yields ------ config_name The full (dotted) name of the named config. config The corresponding named config. """ for ingredient, _ in self.traverse_ingredients(): for config_name, config in ingredient.named_configs.items(): config_name = join_paths(ingredient.path, config_name) config_name = self.post_process_name(config_name, ingredient) yield config_name, config
def collect_hyperparameters(search_space, path=''): """ Recursively collect all the hyperparameters from a search space. Parameters ---------- search_space : dict A JSON-like structure that describes the search space. path : str The path to the current entry. Used to determine the name of the detected hyperparameters. Optional: Only used for the recursion. Returns ------- parameters : dict A dictionary that to all the collected hyperparameters from their uids. """ # First try to decode to hyperparameter if isinstance(search_space, dict): try: hparam = decode_param_or_op(search_space) set_name(hparam, path) return {hparam['uid']: hparam} except ValueError: pass parameters = {} # if the space is a dict (but not a hyperparameter) we parse it recursively if isinstance(search_space, dict): for k, v in search_space.items(): # add the current key and a '.' as prefix when recursing sub_params = collect_hyperparameters(v, join_paths(path, k)) parameters = merge_parameters(parameters, sub_params) return parameters # if the space is a list we iterate it recursively elif isinstance(search_space, (tuple, list)): for i, v in enumerate(search_space): # add '[N]' to the name when recursing sub_params = collect_hyperparameters(v, path + '[{}]'.format(i)) parameters = merge_parameters(parameters, sub_params) return parameters else: # if the space is anything else do nothing return parameters
def _warn_about_suspicious_changes(self): for add in sorted(self.config_mods.added): if not set(iter_prefixes(add)).intersection(self.captured_args): if self.path: add = join_paths(self.path, add) raise ConfigAddedError(add, config=self.config) else: self.logger.warning('Added new config entry: "%s"' % add) for key, (type_old, type_new) in self.config_mods.typechanged.items(): if type_old in (int, float) and type_new in (int, float): continue self.logger.warning( 'Changed type of config entry "%s" from %s to %s' % (key, type_old.__name__, type_new.__name__)) for cfg_summary in self.summaries: for key in cfg_summary.ignored_fallbacks: self.logger.warning( 'Ignored attempt to set value of "%s", because it is an ' 'ingredient.' % key)
def _warn_about_suspicious_changes(self): for add in sorted(self.config_mods.added): if not set(iter_prefixes(add)).intersection(self.captured_args): if self.path: add = join_paths(self.path, add) raise ConfigAddedError(add, config=self.config) else: self.logger.warning('Added new config entry: "%s"' % add) for key, (type_old, type_new) in self.config_mods.typechanged.items(): if type_old in (int, float) and type_new in (int, float): continue self.logger.warning( 'Changed type of config entry "%s" from %s to %s' % (key, type_old.__name__, type_new.__name__)) for cfg_summary in self.summaries: for key in cfg_summary.ignored_fallbacks: self.logger.warning( 'Ignored attempt to set value of "%s", because it is an ' 'ingredient.' % key )
def create_run(experiment, command_name, config_updates=None, named_configs=(), force=False): sorted_ingredients = gather_ingredients_topological(experiment) scaffolding = create_scaffolding(experiment, sorted_ingredients) # get all split non-empty prefixes sorted from deepest to shallowest prefixes = sorted([s.split('.') for s in scaffolding if s != ''], reverse=True, key=lambda p: len(p)) # --------- configuration process ------------------- # Phase 1: Config updates config_updates = config_updates or {} config_updates = convert_to_nested_dict(config_updates) root_logger, run_logger = initialize_logging(experiment, scaffolding) distribute_config_updates(prefixes, scaffolding, config_updates) # Phase 2: Named Configs for ncfg in named_configs: scaff, cfg_name = get_scaffolding_and_config_name(ncfg, scaffolding) scaff.gather_fallbacks() ncfg_updates = scaff.run_named_config(cfg_name) distribute_presets(prefixes, scaffolding, ncfg_updates) for ncfg_key, value in iterate_flattened(ncfg_updates): set_by_dotted_path(config_updates, join_paths(scaff.path, ncfg_key), value) distribute_config_updates(prefixes, scaffolding, config_updates) # Phase 3: Normal config scopes for scaffold in scaffolding.values(): scaffold.gather_fallbacks() scaffold.set_up_config() # update global config config = get_configuration(scaffolding) # run config hooks config_updates = scaffold.run_config_hooks(config, config_updates, command_name, run_logger) # Phase 4: finalize seeding for scaffold in reversed(list(scaffolding.values())): scaffold.set_up_seed() # partially recursive config = get_configuration(scaffolding) config_modifications = get_config_modifications(scaffolding) # ---------------------------------------------------- experiment_info = experiment.get_experiment_info() host_info = get_host_info() main_function = get_command(scaffolding, command_name) pre_runs = [pr for ing in sorted_ingredients for pr in ing.pre_run_hooks] post_runs = [pr for ing in sorted_ingredients for pr in ing.post_run_hooks] run = Run(config, config_modifications, main_function, copy(experiment.observers), root_logger, run_logger, experiment_info, host_info, pre_runs, post_runs, experiment.captured_out_filter) if hasattr(main_function, 'unobserved'): run.unobserved = main_function.unobserved run.force = force for scaffold in scaffolding.values(): scaffold.finalize_initialization(run=run) return run
def create_run(experiment, command_name, config_updates=None, named_configs=(), force=False, log_level=None): sorted_ingredients = gather_ingredients_topological(experiment) scaffolding = create_scaffolding(experiment, sorted_ingredients) # get all split non-empty prefixes sorted from deepest to shallowest prefixes = sorted([s.split('.') for s in scaffolding if s != ''], reverse=True, key=lambda p: len(p)) # --------- configuration process ------------------- # Phase 1: Config updates config_updates = config_updates or {} config_updates = convert_to_nested_dict(config_updates) root_logger, run_logger = initialize_logging(experiment, scaffolding, log_level) distribute_config_updates(prefixes, scaffolding, config_updates) # Phase 2: Named Configs for ncfg in named_configs: scaff, cfg_name = get_scaffolding_and_config_name(ncfg, scaffolding) scaff.gather_fallbacks() ncfg_updates = scaff.run_named_config(cfg_name) distribute_presets(prefixes, scaffolding, ncfg_updates) for ncfg_key, value in iterate_flattened(ncfg_updates): set_by_dotted_path(config_updates, join_paths(scaff.path, ncfg_key), value) distribute_config_updates(prefixes, scaffolding, config_updates) # Phase 3: Normal config scopes for scaffold in scaffolding.values(): scaffold.gather_fallbacks() scaffold.set_up_config() # update global config config = get_configuration(scaffolding) # run config hooks config_hook_updates = scaffold.run_config_hooks( config, command_name, run_logger) recursive_update(scaffold.config, config_hook_updates) # Phase 4: finalize seeding for scaffold in reversed(list(scaffolding.values())): scaffold.set_up_seed() # partially recursive config = get_configuration(scaffolding) config_modifications = get_config_modifications(scaffolding) # ---------------------------------------------------- experiment_info = experiment.get_experiment_info() host_info = get_host_info() main_function = get_command(scaffolding, command_name) pre_runs = [pr for ing in sorted_ingredients for pr in ing.pre_run_hooks] post_runs = [pr for ing in sorted_ingredients for pr in ing.post_run_hooks] run = Run(config, config_modifications, main_function, copy(experiment.observers), root_logger, run_logger, experiment_info, host_info, pre_runs, post_runs, experiment.captured_out_filter) if hasattr(main_function, 'unobserved'): run.unobserved = main_function.unobserved run.force = force for scaffold in scaffolding.values(): scaffold.finalize_initialization(run=run) return run