def update_from(self, config_mod, path=''):
added = config_mod.added
updated = config_mod.modified
typechanged = config_mod.typechanged
self.added |= {join_paths(path, a) for a in added}
self.modified |= {join_paths(path, u) for u in updated}
self.typechanged.update({join_paths(path, k): v
for k, v in typechanged.items()})
self.ensure_coherence()
def update_from(self, config_mod, path=""):
added = config_mod.added
updated = config_mod.modified
typechanged = config_mod.typechanged
self.added &= {join_paths(path, a) for a in added}
self.modified |= {join_paths(path, u) for u in updated}
self.typechanged.update(
{join_paths(path, k): v
for k, v in typechanged.items()})
self.ensure_coherence()
for k, v in config_mod.docs.items():
if not self.docs.get(k, ""):
self.docs[k] = v
def _log_blocked_setitem(self, key, value, fixed_value):
if type_changed(value, fixed_value):
self.typechanges[key] = (type(value), type(fixed_value))
if value != fixed_value:
self.modified.add(key)
# if both are dicts recursively collect modified and typechanges
if isinstance(fixed_value, DogmaticDict) and isinstance(value, dict):
for k, val in fixed_value.typechanges.items():
self.typechanges[join_paths(key, k)] = val
self.modified |= {join_paths(key, m) for m in fixed_value.modified}
def _log_blocked_setitem(self, key, value, fixed_value):
if type_changed(value, fixed_value):
self.typechanges[key] = (type(value), type(fixed_value))
if value != fixed_value:
self.modified.add(key)
# if both are dicts recursively collect modified and typechanges
if isinstance(fixed_value, DogmaticDict) and isinstance(value, dict):
for k, val in fixed_value.typechanges.items():
self.typechanges[join_paths(key, k)] = val
self.modified |= {join_paths(key, m) for m in fixed_value.modified}
def update_add(self, config_mod, path=""):
added = config_mod.added
updated = config_mod.modified
typechanged = config_mod.typechanged
self.added |= {join_paths(path, a) for a in added}
self.modified |= {join_paths(path, u) for u in updated}
self.typechanged.update(
{join_paths(path, k): v
for k, v in typechanged.items()})
self.docs.update({
join_paths(path, k): v
for k, v in config_mod.docs.items() if path == "" or k != "seed"
})
self.ensure_coherence()
def __init__(self, config_scopes, subrunners, path, captured_functions,
commands, named_configs, config_hooks, generate_seed):
self.config_scopes = config_scopes
self.named_configs = named_configs
self.subrunners = subrunners
self.path = path
self.generate_seed = generate_seed
self.config_hooks = config_hooks
self.config_updates = {}
self.named_configs_to_use = []
self.config = {}
self.fallback = None
self.presets = {}
self.fixture = None # TODO: rename
self.logger = None
self.seed = None
self.rnd = None
self._captured_functions = captured_functions
self.commands = commands
self.config_mods = None
self.summaries = []
self.captured_args = {join_paths(cf.prefix, n)
for cf in self._captured_functions
for n in cf.signature.arguments}
self.captured_args.add('__doc__') # allow setting the config docstring
def __init__(self, config_scopes, subrunners, path, captured_functions,
commands, named_configs, config_hooks, generate_seed):
self.config_scopes = config_scopes
self.named_configs = named_configs
self.subrunners = subrunners
self.path = path
self.generate_seed = generate_seed
self.config_hooks = config_hooks
self.config_updates = {}
self.named_configs_to_use = []
self.config = {}
self.fallback = None
self.presets = {}
self.fixture = None # TODO: rename
self.logger = None
self.seed = None
self.rnd = None
self._captured_functions = captured_functions
self.commands = commands
self.config_mods = None
self.summaries = []
self.captured_args = {
join_paths(cf.prefix, n)
for cf in self._captured_functions for n in cf.signature.arguments
}
self.captured_args.add('__doc__') # allow setting the config docstring
def _emit_queued(self):
self.status = 'QUEUED'
queue_time = datetime.datetime.utcnow()
self.meta_info['queue_time'] = queue_time
command = join_paths(self.main_function.prefix,
self.main_function.signature.name)
self.run_logger.info("Queuing-up command '%s'", command)
for observer in self.observers:
if hasattr(observer, 'queued_event'):
_id = observer.queued_event(
ex_info=self.experiment_info,
command=command,
host_info=self.host_info,
queue_time=queue_time,
config=self.config,
meta_info=self.meta_info,
_id=self._id
)
if self._id is None:
self._id = _id
# do not catch any exceptions on startup:
# the experiment SHOULD fail if any of the observers fails
if self._id is None:
self.run_logger.info('Queued')
else:
self.run_logger.info('Queued-up run with ID "{}"'.format(self._id))
def _emit_started(self):
self.status = 'RUNNING'
self.start_time = datetime.datetime.utcnow()
command = join_paths(self.main_function.prefix,
self.main_function.signature.name)
self.run_logger.info("Running command '%s'", command)
for observer in self.observers:
if hasattr(observer, 'started_event'):
_id = observer.started_event(
ex_info=self.experiment_info,
command=command,
host_info=self.host_info,
start_time=self.start_time,
config=self.config,
meta_info=self.meta_info,
_id=self._id
)
if self._id is None:
self._id = _id
# do not catch any exceptions on startup:
# the experiment SHOULD fail if any of the observers fails
if self._id is None:
self.run_logger.info('Started')
else:
self.run_logger.info('Started run with ID "{}"'.format(self._id))
def _emit_started(self):
self.status = 'RUNNING'
self.start_time = datetime.datetime.utcnow()
command = join_paths(self.main_function.prefix,
self.main_function.signature.name)
self.run_logger.info("Running command '%s'", command)
for observer in self.observers:
if hasattr(observer, 'started_event'):
_id = observer.started_event(
ex_info=self.experiment_info,
command=command,
host_info=self.host_info,
start_time=self.start_time,
config=self.config,
meta_info=self.meta_info,
_id=self._id
)
if self._id is None:
self._id = _id
# do not catch any exceptions on startup:
# the experiment SHOULD fail if any of the observers fails
if self._id is None:
self.run_logger.info('Started')
else:
self.run_logger.info('Started run with ID "{}"'.format(self._id))
def _emit_queued(self):
self.status = 'QUEUED'
queue_time = datetime.datetime.utcnow()
self.meta_info['queue_time'] = queue_time
command = join_paths(self.main_function.prefix,
self.main_function.signature.name)
self.run_logger.info("Queuing-up command '%s'", command)
for observer in self.observers:
if hasattr(observer, 'queued_event'):
_id = observer.queued_event(
ex_info=self.experiment_info,
command=command,
queue_time=queue_time,
config=self.config,
meta_info=self.meta_info,
_id=self._id
)
if self._id is None:
self._id = _id
# do not catch any exceptions on startup:
# the experiment SHOULD fail if any of the observers fails
if self._id is None:
self.run_logger.info('Queued')
else:
self.run_logger.info('Queued-up run with ID "{}"'.format(self._id))
def test_join_paths():
assert join_paths() == ''
assert join_paths('foo') == 'foo'
assert join_paths('foo', 'bar') == 'foo.bar'
assert join_paths('a', 'b', 'c', 'd') == 'a.b.c.d'
assert join_paths('', 'b', '', 'd') == 'b.d'
assert join_paths('a.b', 'c.d.e') == 'a.b.c.d.e'
assert join_paths('a.b.', 'c.d.e') == 'a.b.c.d.e'
def test_join_paths():
assert join_paths() == ''
assert join_paths('foo') == 'foo'
assert join_paths('foo', 'bar') == 'foo.bar'
assert join_paths('a', 'b', 'c', 'd') == 'a.b.c.d'
assert join_paths('', 'b', '', 'd') == 'b.d'
assert join_paths('a.b', 'c.d.e') == 'a.b.c.d.e'
assert join_paths('a.b.', 'c.d.e') == 'a.b.c.d.e'
def test_join_paths():
assert join_paths() == ""
assert join_paths("foo") == "foo"
assert join_paths("foo", "bar") == "foo.bar"
assert join_paths("a", "b", "c", "d") == "a.b.c.d"
assert join_paths("", "b", "", "d") == "b.d"
assert join_paths("a.b", "c.d.e") == "a.b.c.d.e"
assert join_paths("a.b.", "c.d.e") == "a.b.c.d.e"
def gather_commands(self, ingredient):
"""Collect all commands from this ingredient and its sub-ingredients.
Yields
------
cmd_name: str
The full (dotted) name of the command.
cmd: function
The corresponding captured function.
"""
for command_name, command in ingredient.commands.items():
yield join_paths(ingredient.path, command_name), command
def gather_commands(self, ingredient):
"""Collect all commands from this ingredient and its sub-ingredients.
Yields
------
cmd_name: str
The full (dotted) name of the command.
cmd: function
The corresponding captured function.
"""
for command_name, command in ingredient.commands.items():
yield join_paths(ingredient.path, command_name), command
def gather_named_configs(self, ingredient):
"""Collect all named configs from this ingredient and its
sub-ingredients.
Yields
------
config_name: str
The full (dotted) name of the named config.
config: ConfigScope or ConfigDict or basestring
The corresponding named config.
"""
for config_name, config in ingredient.named_configs.items():
yield join_paths(ingredient.path, config_name), config
def gather_named_configs(self, ingredient):
"""Collect all named configs from this ingredient and its
sub-ingredients.
Yields
------
config_name: str
The full (dotted) name of the named config.
config: ConfigScope or ConfigDict or basestring
The corresponding named config.
"""
for config_name, config in ingredient.named_configs.items():
yield join_paths(ingredient.path, config_name), config
def gather_commands(self):
"""Collect all commands from this ingredient and its sub-ingredients.
Yields
------
cmd_name: str
The full (dotted) name of the command.
cmd: function
The corresponding captured function.
"""
for ingredient, _ in self.traverse_ingredients():
for command_name, command in ingredient.commands.items():
cmd_name = join_paths(ingredient.path, command_name)
cmd_name = self.post_process_name(cmd_name, ingredient)
yield cmd_name, command
def gather_named_configs(self):
"""Collect all named configs from this ingredient and its
sub-ingredients.
Yields
------
config_name: str
The full (dotted) name of the named config.
config: ConfigScope or ConfigDict or basestring
The corresponding named config.
"""
for ingredient, _ in self.traverse_ingredients():
for config_name, config in ingredient.named_configs.items():
config_name = join_paths(ingredient.path, config_name)
config_name = self.post_process_name(config_name, ingredient)
yield config_name, config
def gather_named_configs(
self,
) -> Generator[Tuple[str, Union[ConfigScope, ConfigDict, str]], None, None]:
"""Collect all named configs from this ingredient and its sub-ingredients.
Yields
------
config_name
The full (dotted) name of the named config.
config
The corresponding named config.
"""
for ingredient, _ in self.traverse_ingredients():
for config_name, config in ingredient.named_configs.items():
config_name = join_paths(ingredient.path, config_name)
config_name = self.post_process_name(config_name, ingredient)
yield config_name, config
def collect_hyperparameters(search_space, path=''):
"""
Recursively collect all the hyperparameters from a search space.
Parameters
----------
search_space : dict
A JSON-like structure that describes the search space.
path : str
The path to the current entry. Used to determine the name of the
detected hyperparameters. Optional: Only used for the recursion.
Returns
-------
parameters : dict
A dictionary that to all the collected hyperparameters from their uids.
"""
# First try to decode to hyperparameter
if isinstance(search_space, dict):
try:
hparam = decode_param_or_op(search_space)
set_name(hparam, path)
return {hparam['uid']: hparam}
except ValueError:
pass
parameters = {}
# if the space is a dict (but not a hyperparameter) we parse it recursively
if isinstance(search_space, dict):
for k, v in search_space.items():
# add the current key and a '.' as prefix when recursing
sub_params = collect_hyperparameters(v, join_paths(path, k))
parameters = merge_parameters(parameters, sub_params)
return parameters
# if the space is a list we iterate it recursively
elif isinstance(search_space, (tuple, list)):
for i, v in enumerate(search_space):
# add '[N]' to the name when recursing
sub_params = collect_hyperparameters(v, path + '[{}]'.format(i))
parameters = merge_parameters(parameters, sub_params)
return parameters
else:
# if the space is anything else do nothing
return parameters
def _warn_about_suspicious_changes(self):
for add in sorted(self.config_mods.added):
if not set(iter_prefixes(add)).intersection(self.captured_args):
if self.path:
add = join_paths(self.path, add)
raise ConfigAddedError(add, config=self.config)
else:
self.logger.warning('Added new config entry: "%s"' % add)
for key, (type_old, type_new) in self.config_mods.typechanged.items():
if type_old in (int, float) and type_new in (int, float):
continue
self.logger.warning(
'Changed type of config entry "%s" from %s to %s' %
(key, type_old.__name__, type_new.__name__))
for cfg_summary in self.summaries:
for key in cfg_summary.ignored_fallbacks:
self.logger.warning(
'Ignored attempt to set value of "%s", because it is an '
'ingredient.' % key)
def _warn_about_suspicious_changes(self):
for add in sorted(self.config_mods.added):
if not set(iter_prefixes(add)).intersection(self.captured_args):
if self.path:
add = join_paths(self.path, add)
raise ConfigAddedError(add, config=self.config)
else:
self.logger.warning('Added new config entry: "%s"' % add)
for key, (type_old, type_new) in self.config_mods.typechanged.items():
if type_old in (int, float) and type_new in (int, float):
continue
self.logger.warning(
'Changed type of config entry "%s" from %s to %s' %
(key, type_old.__name__, type_new.__name__))
for cfg_summary in self.summaries:
for key in cfg_summary.ignored_fallbacks:
self.logger.warning(
'Ignored attempt to set value of "%s", because it is an '
'ingredient.' % key
)
def create_run(experiment, command_name, config_updates=None,
named_configs=(), force=False):
sorted_ingredients = gather_ingredients_topological(experiment)
scaffolding = create_scaffolding(experiment, sorted_ingredients)
# get all split non-empty prefixes sorted from deepest to shallowest
prefixes = sorted([s.split('.') for s in scaffolding if s != ''],
reverse=True, key=lambda p: len(p))
# --------- configuration process -------------------
# Phase 1: Config updates
config_updates = config_updates or {}
config_updates = convert_to_nested_dict(config_updates)
root_logger, run_logger = initialize_logging(experiment, scaffolding)
distribute_config_updates(prefixes, scaffolding, config_updates)
# Phase 2: Named Configs
for ncfg in named_configs:
scaff, cfg_name = get_scaffolding_and_config_name(ncfg, scaffolding)
scaff.gather_fallbacks()
ncfg_updates = scaff.run_named_config(cfg_name)
distribute_presets(prefixes, scaffolding, ncfg_updates)
for ncfg_key, value in iterate_flattened(ncfg_updates):
set_by_dotted_path(config_updates,
join_paths(scaff.path, ncfg_key),
value)
distribute_config_updates(prefixes, scaffolding, config_updates)
# Phase 3: Normal config scopes
for scaffold in scaffolding.values():
scaffold.gather_fallbacks()
scaffold.set_up_config()
# update global config
config = get_configuration(scaffolding)
# run config hooks
config_updates = scaffold.run_config_hooks(config, config_updates,
command_name, run_logger)
# Phase 4: finalize seeding
for scaffold in reversed(list(scaffolding.values())):
scaffold.set_up_seed() # partially recursive
config = get_configuration(scaffolding)
config_modifications = get_config_modifications(scaffolding)
# ----------------------------------------------------
experiment_info = experiment.get_experiment_info()
host_info = get_host_info()
main_function = get_command(scaffolding, command_name)
pre_runs = [pr for ing in sorted_ingredients for pr in ing.pre_run_hooks]
post_runs = [pr for ing in sorted_ingredients for pr in ing.post_run_hooks]
run = Run(config, config_modifications, main_function,
copy(experiment.observers), root_logger, run_logger,
experiment_info, host_info, pre_runs, post_runs,
experiment.captured_out_filter)
if hasattr(main_function, 'unobserved'):
run.unobserved = main_function.unobserved
run.force = force
for scaffold in scaffolding.values():
scaffold.finalize_initialization(run=run)
return run
def create_run(experiment,
command_name,
config_updates=None,
named_configs=(),
force=False,
log_level=None):
sorted_ingredients = gather_ingredients_topological(experiment)
scaffolding = create_scaffolding(experiment, sorted_ingredients)
# get all split non-empty prefixes sorted from deepest to shallowest
prefixes = sorted([s.split('.') for s in scaffolding if s != ''],
reverse=True,
key=lambda p: len(p))
# --------- configuration process -------------------
# Phase 1: Config updates
config_updates = config_updates or {}
config_updates = convert_to_nested_dict(config_updates)
root_logger, run_logger = initialize_logging(experiment, scaffolding,
log_level)
distribute_config_updates(prefixes, scaffolding, config_updates)
# Phase 2: Named Configs
for ncfg in named_configs:
scaff, cfg_name = get_scaffolding_and_config_name(ncfg, scaffolding)
scaff.gather_fallbacks()
ncfg_updates = scaff.run_named_config(cfg_name)
distribute_presets(prefixes, scaffolding, ncfg_updates)
for ncfg_key, value in iterate_flattened(ncfg_updates):
set_by_dotted_path(config_updates,
join_paths(scaff.path, ncfg_key), value)
distribute_config_updates(prefixes, scaffolding, config_updates)
# Phase 3: Normal config scopes
for scaffold in scaffolding.values():
scaffold.gather_fallbacks()
scaffold.set_up_config()
# update global config
config = get_configuration(scaffolding)
# run config hooks
config_hook_updates = scaffold.run_config_hooks(
config, command_name, run_logger)
recursive_update(scaffold.config, config_hook_updates)
# Phase 4: finalize seeding
for scaffold in reversed(list(scaffolding.values())):
scaffold.set_up_seed() # partially recursive
config = get_configuration(scaffolding)
config_modifications = get_config_modifications(scaffolding)
# ----------------------------------------------------
experiment_info = experiment.get_experiment_info()
host_info = get_host_info()
main_function = get_command(scaffolding, command_name)
pre_runs = [pr for ing in sorted_ingredients for pr in ing.pre_run_hooks]
post_runs = [pr for ing in sorted_ingredients for pr in ing.post_run_hooks]
run = Run(config, config_modifications, main_function,
copy(experiment.observers), root_logger, run_logger,
experiment_info, host_info, pre_runs, post_runs,
experiment.captured_out_filter)
if hasattr(main_function, 'unobserved'):
run.unobserved = main_function.unobserved
run.force = force
for scaffold in scaffolding.values():
scaffold.finalize_initialization(run=run)
return run