예제 #1
0
def create_run(experiment,
               command_name,
               config_updates=None,
               named_configs=(),
               force=False):

    sorted_ingredients = gather_ingredients_topological(experiment)
    scaffolding = create_scaffolding(experiment, sorted_ingredients)

    # --------- configuration process -------------------
    distribute_named_configs(scaffolding, named_configs)
    config_updates = config_updates or {}
    config_updates = convert_to_nested_dict(config_updates)
    root_logger, run_logger = initialize_logging(experiment, scaffolding)

    past_paths = set()
    for scaffold in scaffolding.values():
        scaffold.pick_relevant_config_updates(config_updates, past_paths)
        past_paths.add(scaffold.path)
        scaffold.gather_fallbacks()
        scaffold.set_up_config()

        # update global config
        config = get_configuration(scaffolding)
        # run config hooks
        config_updates = scaffold.run_config_hooks(config, config_updates,
                                                   command_name, run_logger)

    for scaffold in reversed(list(scaffolding.values())):
        scaffold.set_up_seed()  # partially recursive

    config = get_configuration(scaffolding)
    config_modifications = get_config_modifications(scaffolding)

    # ----------------------------------------------------

    experiment_info = experiment.get_experiment_info()
    host_info = get_host_info()
    main_function = get_command(scaffolding, command_name)
    pre_runs = [pr for ing in sorted_ingredients for pr in ing.pre_run_hooks]
    post_runs = [pr for ing in sorted_ingredients for pr in ing.post_run_hooks]

    run = Run(config, config_modifications, main_function,
              copy(experiment.observers), root_logger, run_logger,
              experiment_info, host_info, pre_runs, post_runs,
              experiment.captured_out_filter)

    if hasattr(main_function, 'unobserved'):
        run.unobserved = main_function.unobserved

    run.force = force

    for scaffold in scaffolding.values():
        scaffold.finalize_initialization(run=run)

    return run
예제 #2
0
def distribute_config_updates(scaffolding, config_updates):
    if config_updates is None:
        return
    nested_config_updates = convert_to_nested_dict(config_updates)
    for path, value in iterate_flattened(nested_config_updates):
        for prefix, suffix in reversed(list(iter_path_splits(path))):
            if prefix in scaffolding:
                set_by_dotted_path(scaffolding[prefix].config_updates, suffix,
                                   value)
                break
예제 #3
0
def test_convert_to_nested_dict_nested():
    dotted_dict = {"a.b": {"foo.bar": 8}, "a.b.foo.baz": 7}
    assert convert_to_nested_dict(dotted_dict) == {
        "a": {
            "b": {
                "foo": {
                    "bar": 8,
                    "baz": 7
                }
            }
        }
    }
예제 #4
0
def create_run(experiment, command_name, config_updates=None,
               named_configs=(), force=False):

    sorted_ingredients = gather_ingredients_topological(experiment)
    scaffolding = create_scaffolding(experiment, sorted_ingredients)

    # --------- configuration process -------------------
    distribute_named_configs(scaffolding, named_configs)
    config_updates = config_updates or {}
    config_updates = convert_to_nested_dict(config_updates)
    root_logger, run_logger = initialize_logging(experiment, scaffolding)

    past_paths = set()
    for scaffold in scaffolding.values():
        scaffold.pick_relevant_config_updates(config_updates, past_paths)
        past_paths.add(scaffold.path)
        scaffold.gather_fallbacks()
        scaffold.set_up_config()

        # update global config
        config = get_configuration(scaffolding)
        # run config hooks
        config_updates = scaffold.run_config_hooks(config, config_updates,
                                                   command_name, run_logger)

    for scaffold in reversed(list(scaffolding.values())):
        scaffold.set_up_seed()  # partially recursive

    config = get_configuration(scaffolding)
    config_modifications = get_config_modifications(scaffolding)

    # ----------------------------------------------------

    experiment_info = experiment.get_experiment_info()
    host_info = get_host_info()
    main_function = get_command(scaffolding, command_name)
    pre_runs = [pr for ing in sorted_ingredients for pr in ing.pre_run_hooks]
    post_runs = [pr for ing in sorted_ingredients for pr in ing.post_run_hooks]

    run = Run(config, config_modifications, main_function,
              experiment.observers, root_logger, run_logger, experiment_info,
              host_info, pre_runs, post_runs)

    if hasattr(main_function, 'unobserved'):
        run.unobserved = main_function.unobserved

    run.force = force

    for scaffold in scaffolding.values():
        scaffold.finalize_initialization(run=run)

    return run
예제 #5
0
파일: test_utils.py 프로젝트: talpay/sacred
def test_convert_to_nested_dict_nested():
    dotted_dict = {'a.b': {'foo.bar': 8}, 'a.b.foo.baz': 7}
    assert convert_to_nested_dict(dotted_dict) == \
        {'a': {'b': {'foo': {'bar': 8, 'baz': 7}}}}
예제 #6
0
파일: test_utils.py 프로젝트: talpay/sacred
def test_convert_to_nested_dict():
    dotted_dict = {'foo.bar': 8, 'foo.baz': 7}
    assert convert_to_nested_dict(dotted_dict) == {'foo': {'bar': 8, 'baz': 7}}
예제 #7
0
def test_convert_to_nested_dict():
    dotted_dict = {"foo.bar": 8, "foo.baz": 7}
    assert convert_to_nested_dict(dotted_dict) == {"foo": {"bar": 8, "baz": 7}}
예제 #8
0
def test_convert_to_nested_dict_nested():
    dotted_dict = {'a.b': {'foo.bar': 8}, 'a.b.foo.baz': 7}
    assert convert_to_nested_dict(dotted_dict) == \
        {'a': {'b': {'foo': {'bar': 8, 'baz': 7}}}}
예제 #9
0
def test_convert_to_nested_dict():
    dotted_dict = {'foo.bar': 8, 'foo.baz': 7}
    assert convert_to_nested_dict(dotted_dict) == {'foo': {'bar': 8, 'baz': 7}}
예제 #10
0
def create_run(experiment,
               command_name,
               config_updates=None,
               named_configs=(),
               force=False,
               log_level=None):

    sorted_ingredients = gather_ingredients_topological(experiment)
    scaffolding = create_scaffolding(experiment, sorted_ingredients)
    # get all split non-empty prefixes sorted from deepest to shallowest
    prefixes = sorted([s.split('.') for s in scaffolding if s != ''],
                      reverse=True,
                      key=lambda p: len(p))

    # --------- configuration process -------------------

    # Phase 1: Config updates
    config_updates = config_updates or {}
    config_updates = convert_to_nested_dict(config_updates)
    root_logger, run_logger = initialize_logging(experiment, scaffolding,
                                                 log_level)
    distribute_config_updates(prefixes, scaffolding, config_updates)

    # Phase 2: Named Configs
    for ncfg in named_configs:
        scaff, cfg_name = get_scaffolding_and_config_name(ncfg, scaffolding)
        scaff.gather_fallbacks()
        ncfg_updates = scaff.run_named_config(cfg_name)
        distribute_presets(prefixes, scaffolding, ncfg_updates)
        for ncfg_key, value in iterate_flattened(ncfg_updates):
            set_by_dotted_path(config_updates,
                               join_paths(scaff.path, ncfg_key), value)

    distribute_config_updates(prefixes, scaffolding, config_updates)

    # Phase 3: Normal config scopes
    for scaffold in scaffolding.values():
        scaffold.gather_fallbacks()
        scaffold.set_up_config()

        # update global config
        config = get_configuration(scaffolding)
        # run config hooks
        config_hook_updates = scaffold.run_config_hooks(
            config, command_name, run_logger)
        recursive_update(scaffold.config, config_hook_updates)

    # Phase 4: finalize seeding
    for scaffold in reversed(list(scaffolding.values())):
        scaffold.set_up_seed()  # partially recursive

    config = get_configuration(scaffolding)
    config_modifications = get_config_modifications(scaffolding)

    # ----------------------------------------------------

    experiment_info = experiment.get_experiment_info()
    host_info = get_host_info()
    main_function = get_command(scaffolding, command_name)
    pre_runs = [pr for ing in sorted_ingredients for pr in ing.pre_run_hooks]
    post_runs = [pr for ing in sorted_ingredients for pr in ing.post_run_hooks]

    run = Run(config, config_modifications, main_function,
              copy(experiment.observers), root_logger, run_logger,
              experiment_info, host_info, pre_runs, post_runs,
              experiment.captured_out_filter)

    if hasattr(main_function, 'unobserved'):
        run.unobserved = main_function.unobserved

    run.force = force

    for scaffold in scaffolding.values():
        scaffold.finalize_initialization(run=run)

    return run
예제 #11
0
def create_run(experiment, command_name, config_updates=None,
               named_configs=(), force=False):

    sorted_ingredients = gather_ingredients_topological(experiment)
    scaffolding = create_scaffolding(experiment, sorted_ingredients)
    # get all split non-empty prefixes sorted from deepest to shallowest
    prefixes = sorted([s.split('.') for s in scaffolding if s != ''],
                      reverse=True, key=lambda p: len(p))

    # --------- configuration process -------------------

    # Phase 1: Config updates
    config_updates = config_updates or {}
    config_updates = convert_to_nested_dict(config_updates)
    root_logger, run_logger = initialize_logging(experiment, scaffolding)
    distribute_config_updates(prefixes, scaffolding, config_updates)

    # Phase 2: Named Configs
    for ncfg in named_configs:
        scaff, cfg_name = get_scaffolding_and_config_name(ncfg, scaffolding)
        scaff.gather_fallbacks()
        ncfg_updates = scaff.run_named_config(cfg_name)
        distribute_presets(prefixes, scaffolding, ncfg_updates)
        for ncfg_key, value in iterate_flattened(ncfg_updates):
            set_by_dotted_path(config_updates,
                               join_paths(scaff.path, ncfg_key),
                               value)

    distribute_config_updates(prefixes, scaffolding, config_updates)

    # Phase 3: Normal config scopes
    for scaffold in scaffolding.values():
        scaffold.gather_fallbacks()
        scaffold.set_up_config()

        # update global config
        config = get_configuration(scaffolding)
        # run config hooks
        config_updates = scaffold.run_config_hooks(config, config_updates,
                                                   command_name, run_logger)

    # Phase 4: finalize seeding
    for scaffold in reversed(list(scaffolding.values())):
        scaffold.set_up_seed()  # partially recursive

    config = get_configuration(scaffolding)
    config_modifications = get_config_modifications(scaffolding)

    # ----------------------------------------------------

    experiment_info = experiment.get_experiment_info()
    host_info = get_host_info()
    main_function = get_command(scaffolding, command_name)
    pre_runs = [pr for ing in sorted_ingredients for pr in ing.pre_run_hooks]
    post_runs = [pr for ing in sorted_ingredients for pr in ing.post_run_hooks]

    run = Run(config, config_modifications, main_function,
              copy(experiment.observers), root_logger, run_logger,
              experiment_info, host_info, pre_runs, post_runs,
              experiment.captured_out_filter)

    if hasattr(main_function, 'unobserved'):
        run.unobserved = main_function.unobserved

    run.force = force

    for scaffold in scaffolding.values():
        scaffold.finalize_initialization(run=run)

    return run