Beispiel #1
0
    def setup(
        self,
        config: DictConfig,
        config_loader: ConfigLoader,
        task_function: TaskFunction,
    ) -> None:
        from hydra.core.plugins import Plugins

        self.config = config

        self.launcher = Plugins.instance().instantiate_launcher(
            config=config, config_loader=config_loader, task_function=task_function
        )
Beispiel #2
0
 def setup(
     self,
     config: DictConfig,
     config_loader: ConfigLoader,
     task_function: TaskFunction,
 ) -> None:
     self.job_idx = 0
     self.config = config
     self.config_loader = config_loader
     self.launcher = Plugins.instance().instantiate_launcher(
         config=config,
         config_loader=config_loader,
         task_function=task_function)
Beispiel #3
0
 def setup(
     self,
     config,
     config_loader,
     task_function,
 ):
     self.job_idx = 0
     self.config = config
     self.config_loader = config_loader
     self.launcher = Plugins.instance().instantiate_launcher(
         config=config,
         config_loader=config_loader,
         task_function=task_function)
Beispiel #4
0
 def setup(
     self,
     config: DictConfig,
     config_loader: ConfigLoader,
     task_function: TaskFunction,
 ) -> None:
     self.config = config
     self.config_loader = config_loader
     self.launcher = Plugins.instance().instantiate_launcher(
         config=config,
         config_loader=config_loader,
         task_function=task_function)
     self.sweep_dir = config.hydra.sweep.dir
Beispiel #5
0
 def setup(
     self,
     *,
     hydra_context: HydraContext,
     task_function: TaskFunction,
     config: DictConfig,
 ) -> None:
     self.job_idx = 0
     self.config = config
     self.hydra_context = hydra_context
     self.launcher = Plugins.instance().instantiate_launcher(
         hydra_context=hydra_context,
         task_function=task_function,
         config=config)
Beispiel #6
0
    def _print_plugins(self) -> None:
        assert log is not None
        self._log_header(header="Installed Hydra Plugins", filler="*")
        all_plugins = {p.__name__ for p in Plugins.instance().discover()}
        for plugin_type in [
            ConfigSource,
            CompletionPlugin,
            Launcher,
            Sweeper,
            SearchPathPlugin,
        ]:
            # Mypy false positive?
            plugins = Plugins.instance().discover(plugin_type)  # type: ignore
            if len(plugins) > 0:
                Hydra._log_header(header=f"{plugin_type.__name__}:", prefix="\t")
                for plugin in plugins:
                    log.debug("\t\t{}".format(plugin.__name__))
                    all_plugins.remove(plugin.__name__)

        if len(all_plugins) > 0:
            Hydra._log_header(header="Generic plugins: ", prefix="\t")
            for plugin_name in all_plugins:
                log.debug("\t\t{}".format(plugin_name))
Beispiel #7
0
def _get_completion_help() -> str:
    from hydra.core.plugins import Plugins
    from hydra.plugins.completion_plugin import CompletionPlugin

    completion_plugins = Plugins.instance().discover(CompletionPlugin)
    completion_info: List[str] = []
    for plugin_cls in completion_plugins:
        assert issubclass(plugin_cls, CompletionPlugin)
        for cmd in ["install", "uninstall"]:
            head = f"{plugin_cls.provides().capitalize()} - {cmd.capitalize()}:"
            completion_info.append(head)
            completion_info.append(plugin_cls.help(cmd).format(_get_exec_command()))
        completion_info.append("")
    completion_help = "\n".join([f"    {x}" if x else x for x in completion_info])
    return completion_help
Beispiel #8
0
def test_setup_plugins(
    monkeypatch: Any, plugin: Union[Launcher, Sweeper], config: DictConfig
) -> None:
    task_function = Mock(spec=TaskFunction)
    config_loader = ConfigLoaderImpl(config_search_path=create_config_search_path(None))
    hydra_context = HydraContext(config_loader=config_loader, callbacks=Callbacks())
    plugin_instance = Plugins.instance()
    monkeypatch.setattr(Plugins, "check_usage", lambda _: None)
    monkeypatch.setattr(plugin_instance, "_instantiate", lambda _: plugin)

    msg = "setup() got an unexpected keyword argument 'hydra_context'"
    with raises(TypeError, match=re.escape(msg)):
        if isinstance(plugin, Launcher):
            Plugins.instance().instantiate_launcher(
                hydra_context=hydra_context,
                task_function=task_function,
                config=config,
            )
        else:
            Plugins.instance().instantiate_sweeper(
                hydra_context=hydra_context,
                task_function=task_function,
                config=config,
            )
Beispiel #9
0
    def get_shell_to_plugin_map(
        config_loader: ConfigLoader,
    ) -> DefaultDict[str, List[CompletionPlugin]]:
        shell_to_plugin: DefaultDict[str, List[CompletionPlugin]] = defaultdict(list)
        for clazz in Plugins.instance().discover(CompletionPlugin):
            assert issubclass(clazz, CompletionPlugin)
            plugin = clazz(config_loader)
            shell_to_plugin[plugin.provides()].append(plugin)

        for shell, plugins in shell_to_plugin.items():
            if len(plugins) > 1:
                lst = ",".join([type(plugin).__name__ for plugin in plugins])
                raise ValueError(f"Multiple plugins installed for {shell} : {lst}")

        return shell_to_plugin
Beispiel #10
0
    def setup(
        self,
        *,
        hydra_context: HydraContext,
        task_function: TaskFunction,
        config: DictConfig,
    ) -> None:
        from hydra.core.plugins import Plugins

        self.hydra_context = hydra_context
        self.config = config

        self.launcher = Plugins.instance().instantiate_launcher(
            hydra_context=hydra_context,
            task_function=task_function,
            config=config,
        )
Beispiel #11
0
def create_config_search_path(search_path_dir: Optional[str]) -> ConfigSearchPath:
    from hydra.core.plugins import Plugins
    from hydra.plugins.search_path_plugin import SearchPathPlugin

    search_path = ConfigSearchPathImpl()
    search_path.append("hydra", "pkg://hydra.conf")

    if search_path_dir is not None:
        search_path.append("main", search_path_dir)

    search_path_plugins = Plugins.instance().discover(SearchPathPlugin)
    for spp in search_path_plugins:
        plugin = spp()
        assert isinstance(plugin, SearchPathPlugin)
        plugin.manipulate_search_path(search_path)

    search_path.append("schema", "structured://")

    return search_path
Beispiel #12
0
 def multirun(
     self,
     config_name: Optional[str],
     task_function: TaskFunction,
     overrides: List[str],
 ) -> Any:
     # Initial config is loaded without strict (individual job configs may have strict).
     cfg = self.compose_config(
         config_name=config_name,
         overrides=overrides,
         strict=False,
         with_log_configuration=True,
     )
     HydraConfig.instance().set_config(cfg)
     sweeper = Plugins.instance().instantiate_sweeper(
         config=cfg, config_loader=self.config_loader, task_function=task_function
     )
     task_overrides = cfg.hydra.overrides.task
     return sweeper.sweep(arguments=task_overrides)
Beispiel #13
0
    def multirun(
        self,
        config_name: Optional[str],
        task_function: TaskFunction,
        overrides: List[str],
        with_log_configuration: bool = True,
    ) -> Any:
        cfg = self.compose_config(
            config_name=config_name,
            overrides=overrides,
            with_log_configuration=with_log_configuration,
            run_mode=RunMode.MULTIRUN,
        )

        sweeper = Plugins.instance().instantiate_sweeper(
            config=cfg, config_loader=self.config_loader, task_function=task_function
        )
        task_overrides = OmegaConf.to_container(cfg.hydra.overrides.task, resolve=False)
        assert isinstance(task_overrides, list)
        return sweeper.sweep(arguments=task_overrides)
Beispiel #14
0
    def _print_plugins_profiling_info(self, top_n: int) -> None:
        assert log is not None
        stats = Plugins.instance().get_stats()
        if stats is None:
            return

        items = list(stats.modules_import_time.items())
        # hide anything that took less than 5ms
        filtered = filter(lambda x: x[1] > 0.0005, items)
        sorted_items = sorted(filtered, key=lambda x: x[1], reverse=True)

        top_n = max(len(sorted_items), top_n)
        box: List[List[str]] = [["Module", "Sec"]]

        for item in sorted_items[0:top_n]:
            box.append([item[0], f"{item[1]:.3f}"])
        padding = get_column_widths(box)

        log.debug("")
        self._log_header(header="Profiling information", filler="*")
        self._log_header(
            header=f"Total plugins scan time : {stats.total_time:.3f} seconds",
            filler="-",
        )

        header = f"| {box[0][0].ljust(padding[0])} | {box[0][1].ljust(padding[1])} |"
        self._log_header(
            header=header,
            filler="-",
        )
        del box[0]

        for row in box:
            a = row[0].ljust(padding[0])
            b = row[1].ljust(padding[1])
            log.debug(f"| {a} | {b} |")

        self._log_footer(header=header, filler="-")
Beispiel #15
0
 def multirun(
     self,
     config_name: Optional[str],
     task_function: TaskFunction,
     overrides: List[str],
     with_log_configuration: bool = True,
 ) -> Any:
     # Initial config is loaded without strict (individual job configs may have strict).
     cfg = self.compose_config(
         config_name=config_name,
         overrides=overrides,
         strict=False,
         with_log_configuration=with_log_configuration,
         run_mode=RunMode.MULTIRUN,
     )
     HydraConfig.instance().set_config(cfg)
     sweeper = Plugins.instance().instantiate_sweeper(
         config=cfg,
         config_loader=self.config_loader,
         task_function=task_function)
     task_overrides = OmegaConf.to_container(cfg.hydra.overrides.task,
                                             resolve=False)
     assert isinstance(task_overrides, list)
     return sweeper.sweep(arguments=task_overrides)
Beispiel #16
0
                        'env': env,
                        'seed': seed,
                        'agent.horizon': horizon,
                        'learn_temp.init_targ_entr': init_targ_entr,
                        'learn_temp.final_targ_entr': final_targ_entr,
                        'learn_temp.entr_decay_factor': gamma
                    }
                    overrides_i = [f'{k}={v}' for k, v in overrides_i.items()]
                    overrides.append(overrides_i)
        random.shuffle(overrides)
        # self.validate_batch_is_legal(overrides) # Can take a long time
        returns = self.launcher.launch(overrides, initial_job_idx=self.job_idx)
        self.job_idx += len(returns)


@dataclass
class SVGSweeperConf:
    _target_: str = "svg.sweeper.SVGSweeper"


# Hacks for a non-standard plugin
Plugins.is_in_toplevel_plugins_module = lambda x, y: True
Plugins.instance().class_name_to_class['svg.sweeper.SVGSweeper'] = SVGSweeper

ConfigStore.instance().store(
    group="hydra/sweeper",
    name="svg",
    node=SVGSweeperConf,
    provider="svg",
)
Beispiel #17
0
def test_discovery() -> None:
    # Tests that this plugin can be discovered via the plugins subsystem when looking for Launchers
    assert JoblibLauncher.__name__ in [
        x.__name__ for x in Plugins.instance().discover(Launcher)
    ]
Beispiel #18
0
def test_register_bad_plugin() -> None:
    class NotAPlugin:
        ...

    with raises(ValueError, match="Not a valid Hydra Plugin"):
        Plugins.instance().register(NotAPlugin)  # type: ignore
Beispiel #19
0
def test_discovery() -> None:
    """
    Tests that this plugin can be discovered via the plugins subsystem when looking for Sweeper
    :return:
    """
    assert AxSweeper.__name__ in [x.__name__ for x in Plugins.discover(Sweeper)]
def test_discovery() -> None:
    # Tests that this plugin can be discovered via the plugins subsystem when looking at all Plugins
    assert ExampleSearchPathPlugin.__name__ in [
        x.__name__ for x in Plugins.instance().discover(SearchPathPlugin)
    ]
Beispiel #21
0
def test_discover(plugin_type: Type[Plugin], expected: List[str]) -> None:
    plugins = Plugins.instance().discover(plugin_type)
    expected_classes = [get_class(c) for c in expected]
    for ex in expected_classes:
        assert ex in plugins
def test_discovery() -> None:
    # Tests that this plugin can be discovered via the plugins subsystem when looking at the Sweeper plugins
    assert ExampleSweeper.__name__ in [
        x.__name__ for x in Plugins.instance().discover(Sweeper)
    ]
def test_discovery() -> None:
    assert OptunaSweeper.__name__ in [
        x.__name__ for x in Plugins.instance().discover(Sweeper)
    ]
def test_discovery() -> None:
    # Test that this config source is discoverable when looking at config sources
    assert ConfigSourceExample.__name__ in [
        x.__name__ for x in Plugins.instance().discover(ConfigSource)
    ]
 def test_config_repository_exists(self, restore_singletons: Any, path: str) -> None:
     Plugins.instance()  # initializes
     config_search_path = create_config_search_path(path)
     repo = ConfigRepository(config_search_path=config_search_path)
     assert repo.exists("dataset/imagenet.yaml")
     assert not repo.exists("not_found.yaml")
def test_discovery() -> None:
    assert core.NevergradSweeper.__name__ in [
        x.__name__ for x in Plugins.instance().discover(Sweeper)
    ]
Beispiel #27
0
def register_example_plugin() -> None:
    """The Hydra user should call this function before invoking @hydra.main"""
    Plugins.instance().register(ExampleRegisteredPlugin)
Beispiel #28
0
from typing import List, Optional

import pytest

from hydra._internal.config_repository import ConfigRepository
from hydra._internal.config_search_path_impl import ConfigSearchPathImpl
from hydra._internal.core_plugins.file_config_source import FileConfigSource
from hydra._internal.core_plugins.package_config_source import PackageConfigSource
from hydra.core.object_type import ObjectType
from hydra.core.plugins import Plugins
from hydra.test_utils.config_source_common_tests import ConfigSourceTestSuite
from hydra.test_utils.test_utils import chdir_hydra_root

chdir_hydra_root()

Plugins.register_config_sources()


@pytest.mark.parametrize(
    "type_, path",
    [
        (FileConfigSource,
         "file://tests/test_apps/config_source_test_configs"),
        (PackageConfigSource,
         "pkg://tests.test_apps.config_source_test_configs"),
    ],
)
class TestCoreConfigSources(ConfigSourceTestSuite):
    pass