Exemplo n.º 1
0
 def update_from(self, config_mod, path=''):
     added = config_mod.added
     updated = config_mod.modified
     typechanged = config_mod.typechanged
     self.added |= {join_paths(path, a) for a in added}
     self.modified |= {join_paths(path, u) for u in updated}
     self.typechanged.update({join_paths(path, k): v
                              for k, v in typechanged.items()})
     self.ensure_coherence()
Exemplo n.º 2
0
 def update_from(self, config_mod, path=""):
     added = config_mod.added
     updated = config_mod.modified
     typechanged = config_mod.typechanged
     self.added &= {join_paths(path, a) for a in added}
     self.modified |= {join_paths(path, u) for u in updated}
     self.typechanged.update(
         {join_paths(path, k): v
          for k, v in typechanged.items()})
     self.ensure_coherence()
     for k, v in config_mod.docs.items():
         if not self.docs.get(k, ""):
             self.docs[k] = v
Exemplo n.º 3
0
    def _log_blocked_setitem(self, key, value, fixed_value):
        if type_changed(value, fixed_value):
            self.typechanges[key] = (type(value), type(fixed_value))

        if value != fixed_value:
            self.modified.add(key)

        # if both are dicts recursively collect modified and typechanges
        if isinstance(fixed_value, DogmaticDict) and isinstance(value, dict):
            for k, val in fixed_value.typechanges.items():
                self.typechanges[join_paths(key, k)] = val

            self.modified |= {join_paths(key, m) for m in fixed_value.modified}
Exemplo n.º 4
0
    def _log_blocked_setitem(self, key, value, fixed_value):
        if type_changed(value, fixed_value):
            self.typechanges[key] = (type(value), type(fixed_value))

        if value != fixed_value:
            self.modified.add(key)

        # if both are dicts recursively collect modified and typechanges
        if isinstance(fixed_value, DogmaticDict) and isinstance(value, dict):
            for k, val in fixed_value.typechanges.items():
                self.typechanges[join_paths(key, k)] = val

            self.modified |= {join_paths(key, m) for m in fixed_value.modified}
Exemplo n.º 5
0
 def update_add(self, config_mod, path=""):
     added = config_mod.added
     updated = config_mod.modified
     typechanged = config_mod.typechanged
     self.added |= {join_paths(path, a) for a in added}
     self.modified |= {join_paths(path, u) for u in updated}
     self.typechanged.update(
         {join_paths(path, k): v
          for k, v in typechanged.items()})
     self.docs.update({
         join_paths(path, k): v
         for k, v in config_mod.docs.items() if path == "" or k != "seed"
     })
     self.ensure_coherence()
Exemplo n.º 6
0
 def __init__(self, config_scopes, subrunners, path, captured_functions,
              commands, named_configs, config_hooks, generate_seed):
     self.config_scopes = config_scopes
     self.named_configs = named_configs
     self.subrunners = subrunners
     self.path = path
     self.generate_seed = generate_seed
     self.config_hooks = config_hooks
     self.config_updates = {}
     self.named_configs_to_use = []
     self.config = {}
     self.fallback = None
     self.presets = {}
     self.fixture = None  # TODO: rename
     self.logger = None
     self.seed = None
     self.rnd = None
     self._captured_functions = captured_functions
     self.commands = commands
     self.config_mods = None
     self.summaries = []
     self.captured_args = {join_paths(cf.prefix, n)
                           for cf in self._captured_functions
                           for n in cf.signature.arguments}
     self.captured_args.add('__doc__')  # allow setting the config docstring
Exemplo n.º 7
0
 def __init__(self, config_scopes, subrunners, path, captured_functions,
              commands, named_configs, config_hooks, generate_seed):
     self.config_scopes = config_scopes
     self.named_configs = named_configs
     self.subrunners = subrunners
     self.path = path
     self.generate_seed = generate_seed
     self.config_hooks = config_hooks
     self.config_updates = {}
     self.named_configs_to_use = []
     self.config = {}
     self.fallback = None
     self.presets = {}
     self.fixture = None  # TODO: rename
     self.logger = None
     self.seed = None
     self.rnd = None
     self._captured_functions = captured_functions
     self.commands = commands
     self.config_mods = None
     self.summaries = []
     self.captured_args = {
         join_paths(cf.prefix, n)
         for cf in self._captured_functions for n in cf.signature.arguments
     }
     self.captured_args.add('__doc__')  # allow setting the config docstring
Exemplo n.º 8
0
    def _emit_queued(self):
        self.status = 'QUEUED'
        queue_time = datetime.datetime.utcnow()
        self.meta_info['queue_time'] = queue_time
        command = join_paths(self.main_function.prefix,
                             self.main_function.signature.name)
        self.run_logger.info("Queuing-up command '%s'", command)
        for observer in self.observers:
            if hasattr(observer, 'queued_event'):
                _id = observer.queued_event(
                    ex_info=self.experiment_info,
                    command=command,
                    host_info=self.host_info,
                    queue_time=queue_time,
                    config=self.config,
                    meta_info=self.meta_info,
                    _id=self._id
                )
                if self._id is None:
                    self._id = _id
                # do not catch any exceptions on startup:
                # the experiment SHOULD fail if any of the observers fails

        if self._id is None:
            self.run_logger.info('Queued')
        else:
            self.run_logger.info('Queued-up run with ID "{}"'.format(self._id))
Exemplo n.º 9
0
 def _emit_started(self):
     self.status = 'RUNNING'
     self.start_time = datetime.datetime.utcnow()
     command = join_paths(self.main_function.prefix,
                          self.main_function.signature.name)
     self.run_logger.info("Running command '%s'", command)
     for observer in self.observers:
         if hasattr(observer, 'started_event'):
             _id = observer.started_event(
                 ex_info=self.experiment_info,
                 command=command,
                 host_info=self.host_info,
                 start_time=self.start_time,
                 config=self.config,
                 meta_info=self.meta_info,
                 _id=self._id
             )
             if self._id is None:
                 self._id = _id
             # do not catch any exceptions on startup:
             # the experiment SHOULD fail if any of the observers fails
     if self._id is None:
         self.run_logger.info('Started')
     else:
         self.run_logger.info('Started run with ID "{}"'.format(self._id))
Exemplo n.º 10
0
 def _emit_started(self):
     self.status = 'RUNNING'
     self.start_time = datetime.datetime.utcnow()
     command = join_paths(self.main_function.prefix,
                          self.main_function.signature.name)
     self.run_logger.info("Running command '%s'", command)
     for observer in self.observers:
         if hasattr(observer, 'started_event'):
             _id = observer.started_event(
                 ex_info=self.experiment_info,
                 command=command,
                 host_info=self.host_info,
                 start_time=self.start_time,
                 config=self.config,
                 meta_info=self.meta_info,
                 _id=self._id
             )
             if self._id is None:
                 self._id = _id
             # do not catch any exceptions on startup:
             # the experiment SHOULD fail if any of the observers fails
     if self._id is None:
         self.run_logger.info('Started')
     else:
         self.run_logger.info('Started run with ID "{}"'.format(self._id))
Exemplo n.º 11
0
    def _emit_queued(self):
        self.status = 'QUEUED'
        queue_time = datetime.datetime.utcnow()
        self.meta_info['queue_time'] = queue_time
        command = join_paths(self.main_function.prefix,
                             self.main_function.signature.name)
        self.run_logger.info("Queuing-up command '%s'", command)
        for observer in self.observers:
            if hasattr(observer, 'queued_event'):
                _id = observer.queued_event(
                    ex_info=self.experiment_info,
                    command=command,
                    queue_time=queue_time,
                    config=self.config,
                    meta_info=self.meta_info,
                    _id=self._id
                )
                if self._id is None:
                    self._id = _id
                # do not catch any exceptions on startup:
                # the experiment SHOULD fail if any of the observers fails

        if self._id is None:
            self.run_logger.info('Queued')
        else:
            self.run_logger.info('Queued-up run with ID "{}"'.format(self._id))
Exemplo n.º 12
0
def test_join_paths():
    assert join_paths() == ''
    assert join_paths('foo') == 'foo'
    assert join_paths('foo', 'bar') == 'foo.bar'
    assert join_paths('a', 'b', 'c', 'd') == 'a.b.c.d'
    assert join_paths('', 'b', '', 'd') == 'b.d'
    assert join_paths('a.b', 'c.d.e') == 'a.b.c.d.e'
    assert join_paths('a.b.', 'c.d.e') == 'a.b.c.d.e'
Exemplo n.º 13
0
def test_join_paths():
    assert join_paths() == ''
    assert join_paths('foo') == 'foo'
    assert join_paths('foo', 'bar') == 'foo.bar'
    assert join_paths('a', 'b', 'c', 'd') == 'a.b.c.d'
    assert join_paths('', 'b', '', 'd') == 'b.d'
    assert join_paths('a.b', 'c.d.e') == 'a.b.c.d.e'
    assert join_paths('a.b.', 'c.d.e') == 'a.b.c.d.e'
Exemplo n.º 14
0
def test_join_paths():
    assert join_paths() == ""
    assert join_paths("foo") == "foo"
    assert join_paths("foo", "bar") == "foo.bar"
    assert join_paths("a", "b", "c", "d") == "a.b.c.d"
    assert join_paths("", "b", "", "d") == "b.d"
    assert join_paths("a.b", "c.d.e") == "a.b.c.d.e"
    assert join_paths("a.b.", "c.d.e") == "a.b.c.d.e"
Exemplo n.º 15
0
    def gather_commands(self, ingredient):
        """Collect all commands from this ingredient and its sub-ingredients.

        Yields
        ------
        cmd_name: str
            The full (dotted) name of the command.
        cmd: function
            The corresponding captured function.
        """
        for command_name, command in ingredient.commands.items():
            yield join_paths(ingredient.path, command_name), command
Exemplo n.º 16
0
    def gather_commands(self, ingredient):
        """Collect all commands from this ingredient and its sub-ingredients.

        Yields
        ------
        cmd_name: str
            The full (dotted) name of the command.
        cmd: function
            The corresponding captured function.
        """
        for command_name, command in ingredient.commands.items():
            yield join_paths(ingredient.path, command_name), command
Exemplo n.º 17
0
    def gather_named_configs(self, ingredient):
        """Collect all named configs from this ingredient and its
        sub-ingredients.

        Yields
        ------
        config_name: str
            The full (dotted) name of the named config.
        config: ConfigScope or ConfigDict or basestring
            The corresponding named config.
        """
        for config_name, config in ingredient.named_configs.items():
            yield join_paths(ingredient.path, config_name), config
Exemplo n.º 18
0
    def gather_named_configs(self, ingredient):
        """Collect all named configs from this ingredient and its
        sub-ingredients.

        Yields
        ------
        config_name: str
            The full (dotted) name of the named config.
        config: ConfigScope or ConfigDict or basestring
            The corresponding named config.
        """
        for config_name, config in ingredient.named_configs.items():
            yield join_paths(ingredient.path, config_name), config
Exemplo n.º 19
0
    def gather_commands(self):
        """Collect all commands from this ingredient and its sub-ingredients.

        Yields
        ------
        cmd_name: str
            The full (dotted) name of the command.
        cmd: function
            The corresponding captured function.
        """
        for ingredient, _ in self.traverse_ingredients():
            for command_name, command in ingredient.commands.items():
                cmd_name = join_paths(ingredient.path, command_name)
                cmd_name = self.post_process_name(cmd_name, ingredient)
                yield cmd_name, command
Exemplo n.º 20
0
    def gather_named_configs(self):
        """Collect all named configs from this ingredient and its
        sub-ingredients.

        Yields
        ------
        config_name: str
            The full (dotted) name of the named config.
        config: ConfigScope or ConfigDict or basestring
            The corresponding named config.
        """
        for ingredient, _ in self.traverse_ingredients():
            for config_name, config in ingredient.named_configs.items():
                config_name = join_paths(ingredient.path, config_name)
                config_name = self.post_process_name(config_name, ingredient)
                yield config_name, config
Exemplo n.º 21
0
    def gather_named_configs(
        self,
    ) -> Generator[Tuple[str, Union[ConfigScope, ConfigDict, str]], None, None]:
        """Collect all named configs from this ingredient and its sub-ingredients.

        Yields
        ------
        config_name
            The full (dotted) name of the named config.
        config
            The corresponding named config.
        """
        for ingredient, _ in self.traverse_ingredients():
            for config_name, config in ingredient.named_configs.items():
                config_name = join_paths(ingredient.path, config_name)
                config_name = self.post_process_name(config_name, ingredient)
                yield config_name, config
Exemplo n.º 22
0
def collect_hyperparameters(search_space, path=''):
    """
    Recursively collect all the hyperparameters from a search space.

    Parameters
    ----------
    search_space : dict
        A JSON-like structure that describes the search space.
    path : str
        The path to the current entry. Used to determine the name of the
        detected hyperparameters. Optional: Only used for the recursion.

    Returns
    -------
    parameters : dict
        A dictionary that to all the collected hyperparameters from their uids.
    """
    # First try to decode to hyperparameter
    if isinstance(search_space, dict):
        try:
            hparam = decode_param_or_op(search_space)
            set_name(hparam, path)
            return {hparam['uid']: hparam}
        except ValueError:
            pass

    parameters = {}
    # if the space is a dict (but not a hyperparameter) we parse it recursively
    if isinstance(search_space, dict):
        for k, v in search_space.items():
            # add the current key and a '.' as prefix when recursing
            sub_params = collect_hyperparameters(v, join_paths(path, k))
            parameters = merge_parameters(parameters, sub_params)
        return parameters

    # if the space is a list we iterate it recursively
    elif isinstance(search_space, (tuple, list)):
        for i, v in enumerate(search_space):
            # add '[N]' to the name when recursing
            sub_params = collect_hyperparameters(v, path + '[{}]'.format(i))
            parameters = merge_parameters(parameters, sub_params)
        return parameters
    else:
        # if the space is anything else do nothing
        return parameters
Exemplo n.º 23
0
    def _warn_about_suspicious_changes(self):
        for add in sorted(self.config_mods.added):
            if not set(iter_prefixes(add)).intersection(self.captured_args):
                if self.path:
                    add = join_paths(self.path, add)
                raise ConfigAddedError(add, config=self.config)
            else:
                self.logger.warning('Added new config entry: "%s"' % add)

        for key, (type_old, type_new) in self.config_mods.typechanged.items():
            if type_old in (int, float) and type_new in (int, float):
                continue
            self.logger.warning(
                'Changed type of config entry "%s" from %s to %s' %
                (key, type_old.__name__, type_new.__name__))

        for cfg_summary in self.summaries:
            for key in cfg_summary.ignored_fallbacks:
                self.logger.warning(
                    'Ignored attempt to set value of "%s", because it is an '
                    'ingredient.' % key)
Exemplo n.º 24
0
    def _warn_about_suspicious_changes(self):
        for add in sorted(self.config_mods.added):
            if not set(iter_prefixes(add)).intersection(self.captured_args):
                if self.path:
                    add = join_paths(self.path, add)
                raise ConfigAddedError(add, config=self.config)
            else:
                self.logger.warning('Added new config entry: "%s"' % add)

        for key, (type_old, type_new) in self.config_mods.typechanged.items():
            if type_old in (int, float) and type_new in (int, float):
                continue
            self.logger.warning(
                'Changed type of config entry "%s" from %s to %s' %
                (key, type_old.__name__, type_new.__name__))

        for cfg_summary in self.summaries:
            for key in cfg_summary.ignored_fallbacks:
                self.logger.warning(
                    'Ignored attempt to set value of "%s", because it is an '
                    'ingredient.' % key
                )
Exemplo n.º 25
0
def create_run(experiment, command_name, config_updates=None,
               named_configs=(), force=False):

    sorted_ingredients = gather_ingredients_topological(experiment)
    scaffolding = create_scaffolding(experiment, sorted_ingredients)
    # get all split non-empty prefixes sorted from deepest to shallowest
    prefixes = sorted([s.split('.') for s in scaffolding if s != ''],
                      reverse=True, key=lambda p: len(p))

    # --------- configuration process -------------------

    # Phase 1: Config updates
    config_updates = config_updates or {}
    config_updates = convert_to_nested_dict(config_updates)
    root_logger, run_logger = initialize_logging(experiment, scaffolding)
    distribute_config_updates(prefixes, scaffolding, config_updates)

    # Phase 2: Named Configs
    for ncfg in named_configs:
        scaff, cfg_name = get_scaffolding_and_config_name(ncfg, scaffolding)
        scaff.gather_fallbacks()
        ncfg_updates = scaff.run_named_config(cfg_name)
        distribute_presets(prefixes, scaffolding, ncfg_updates)
        for ncfg_key, value in iterate_flattened(ncfg_updates):
            set_by_dotted_path(config_updates,
                               join_paths(scaff.path, ncfg_key),
                               value)

    distribute_config_updates(prefixes, scaffolding, config_updates)

    # Phase 3: Normal config scopes
    for scaffold in scaffolding.values():
        scaffold.gather_fallbacks()
        scaffold.set_up_config()

        # update global config
        config = get_configuration(scaffolding)
        # run config hooks
        config_updates = scaffold.run_config_hooks(config, config_updates,
                                                   command_name, run_logger)

    # Phase 4: finalize seeding
    for scaffold in reversed(list(scaffolding.values())):
        scaffold.set_up_seed()  # partially recursive

    config = get_configuration(scaffolding)
    config_modifications = get_config_modifications(scaffolding)

    # ----------------------------------------------------

    experiment_info = experiment.get_experiment_info()
    host_info = get_host_info()
    main_function = get_command(scaffolding, command_name)
    pre_runs = [pr for ing in sorted_ingredients for pr in ing.pre_run_hooks]
    post_runs = [pr for ing in sorted_ingredients for pr in ing.post_run_hooks]

    run = Run(config, config_modifications, main_function,
              copy(experiment.observers), root_logger, run_logger,
              experiment_info, host_info, pre_runs, post_runs,
              experiment.captured_out_filter)

    if hasattr(main_function, 'unobserved'):
        run.unobserved = main_function.unobserved

    run.force = force

    for scaffold in scaffolding.values():
        scaffold.finalize_initialization(run=run)

    return run
Exemplo n.º 26
0
def create_run(experiment,
               command_name,
               config_updates=None,
               named_configs=(),
               force=False,
               log_level=None):

    sorted_ingredients = gather_ingredients_topological(experiment)
    scaffolding = create_scaffolding(experiment, sorted_ingredients)
    # get all split non-empty prefixes sorted from deepest to shallowest
    prefixes = sorted([s.split('.') for s in scaffolding if s != ''],
                      reverse=True,
                      key=lambda p: len(p))

    # --------- configuration process -------------------

    # Phase 1: Config updates
    config_updates = config_updates or {}
    config_updates = convert_to_nested_dict(config_updates)
    root_logger, run_logger = initialize_logging(experiment, scaffolding,
                                                 log_level)
    distribute_config_updates(prefixes, scaffolding, config_updates)

    # Phase 2: Named Configs
    for ncfg in named_configs:
        scaff, cfg_name = get_scaffolding_and_config_name(ncfg, scaffolding)
        scaff.gather_fallbacks()
        ncfg_updates = scaff.run_named_config(cfg_name)
        distribute_presets(prefixes, scaffolding, ncfg_updates)
        for ncfg_key, value in iterate_flattened(ncfg_updates):
            set_by_dotted_path(config_updates,
                               join_paths(scaff.path, ncfg_key), value)

    distribute_config_updates(prefixes, scaffolding, config_updates)

    # Phase 3: Normal config scopes
    for scaffold in scaffolding.values():
        scaffold.gather_fallbacks()
        scaffold.set_up_config()

        # update global config
        config = get_configuration(scaffolding)
        # run config hooks
        config_hook_updates = scaffold.run_config_hooks(
            config, command_name, run_logger)
        recursive_update(scaffold.config, config_hook_updates)

    # Phase 4: finalize seeding
    for scaffold in reversed(list(scaffolding.values())):
        scaffold.set_up_seed()  # partially recursive

    config = get_configuration(scaffolding)
    config_modifications = get_config_modifications(scaffolding)

    # ----------------------------------------------------

    experiment_info = experiment.get_experiment_info()
    host_info = get_host_info()
    main_function = get_command(scaffolding, command_name)
    pre_runs = [pr for ing in sorted_ingredients for pr in ing.pre_run_hooks]
    post_runs = [pr for ing in sorted_ingredients for pr in ing.post_run_hooks]

    run = Run(config, config_modifications, main_function,
              copy(experiment.observers), root_logger, run_logger,
              experiment_info, host_info, pre_runs, post_runs,
              experiment.captured_out_filter)

    if hasattr(main_function, 'unobserved'):
        run.unobserved = main_function.unobserved

    run.force = force

    for scaffold in scaffolding.values():
        scaffold.finalize_initialization(run=run)

    return run