def run_hydra(
args_parser: argparse.ArgumentParser,
task_function: TaskFunction,
config_path: Optional[str],
config_name: Optional[str],
strict: Optional[bool],
) -> None:
from hydra.core.global_hydra import GlobalHydra
from .hydra import Hydra
args = args_parser.parse_args()
if args.config_name is not None:
config_name = args.config_name
if args.config_path is not None:
config_path = args.config_path
(
calling_file,
calling_module,
task_name,
) = detect_calling_file_or_module_from_task_function(task_function)
config_dir, config_name = split_config_path(config_path, config_name)
search_path = create_automatic_config_search_path(calling_file,
calling_module,
config_dir)
hydra = run_and_report(lambda: Hydra.create_main_hydra2(
task_name=task_name, config_search_path=search_path, strict=strict))
try:
if args.help:
hydra.app_help(config_name=config_name,
args_parser=args_parser,
args=args)
sys.exit(0)
if args.hydra_help:
hydra.hydra_help(config_name=config_name,
args_parser=args_parser,
args=args)
sys.exit(0)
has_show_cfg = args.cfg is not None
num_commands = (args.run + has_show_cfg + args.multirun +
args.shell_completion + args.info)
if num_commands > 1:
raise ValueError(
"Only one of --run, --multirun, -cfg, --info and --shell_completion can be specified"
)
if num_commands == 0:
args.run = True
if args.run:
run_and_report(lambda: hydra.run(
config_name=config_name,
task_function=task_function,
overrides=args.overrides,
))
elif args.multirun:
run_and_report(lambda: hydra.multirun(
config_name=config_name,
task_function=task_function,
overrides=args.overrides,
))
elif args.cfg:
run_and_report(lambda: hydra.show_cfg(
config_name=config_name,
overrides=args.overrides,
cfg_type=args.cfg,
package=args.package,
))
elif args.shell_completion:
run_and_report(lambda: hydra.shell_completion(
config_name=config_name, overrides=args.overrides))
elif args.info:
hydra.show_info(config_name=config_name, overrides=args.overrides)
else:
sys.stderr.write("Command not specified\n")
sys.exit(1)
finally:
GlobalHydra.instance().clear()
def _run_hydra(
args_parser: argparse.ArgumentParser,
task_function: TaskFunction,
config_path: Optional[str],
config_name: Optional[str],
strict: Optional[bool],
) -> None:
from hydra.core.global_hydra import GlobalHydra
from .hydra import Hydra
args = args_parser.parse_args()
if args.config_name is not None:
config_name = args.config_name
if args.config_path is not None:
config_path = args.config_path
(
calling_file,
calling_module,
task_name,
) = detect_calling_file_or_module_from_task_function(task_function)
config_dir, config_name = split_config_path(config_path, config_name)
search_path = create_automatic_config_search_path(calling_file,
calling_module,
config_dir)
def add_conf_dir() -> None:
if args.config_dir is not None:
abs_config_dir = os.path.abspath(args.config_dir)
if not os.path.isdir(abs_config_dir):
raise SearchPathException(
f"Additional config directory '{abs_config_dir}' not found"
)
search_path.prepend(
provider="command-line",
path=f"file://{abs_config_dir}",
anchor=SearchPathQuery(provider="schema"),
)
run_and_report(add_conf_dir)
hydra = run_and_report(lambda: Hydra.create_main_hydra2(
task_name=task_name, config_search_path=search_path, strict=strict))
try:
if args.help:
hydra.app_help(config_name=config_name,
args_parser=args_parser,
args=args)
sys.exit(0)
if args.hydra_help:
hydra.hydra_help(config_name=config_name,
args_parser=args_parser,
args=args)
sys.exit(0)
has_show_cfg = args.cfg is not None
num_commands = (args.run + has_show_cfg + args.multirun +
args.shell_completion + (args.info is not None))
if num_commands > 1:
raise ValueError(
"Only one of --run, --multirun, -cfg, --info and --shell_completion can be specified"
)
if num_commands == 0:
args.run = True
if args.run:
run_and_report(lambda: hydra.run(
config_name=config_name,
task_function=task_function,
overrides=args.overrides,
))
elif args.multirun:
run_and_report(lambda: hydra.multirun(
config_name=config_name,
task_function=task_function,
overrides=args.overrides,
))
elif args.cfg:
run_and_report(lambda: hydra.show_cfg(
config_name=config_name,
overrides=args.overrides,
cfg_type=args.cfg,
package=args.package,
))
elif args.shell_completion:
run_and_report(lambda: hydra.shell_completion(
config_name=config_name, overrides=args.overrides))
elif args.info:
hydra.show_info(args.info,
config_name=config_name,
overrides=args.overrides)
else:
sys.stderr.write("Command not specified\n")
sys.exit(1)
finally:
GlobalHydra.instance().clear()
def convert_namespace_to_omegaconf(args: Namespace) -> DictConfig:
"""Convert a flat argparse.Namespace to a structured DictConfig."""
# Here we are using field values provided in args to override counterparts inside config object
overrides, deletes = override_module_args(args)
# configs will be in fairseq/config after installation
config_path = os.path.join("..", "config")
GlobalHydra.instance().clear()
with initialize(config_path=config_path):
try:
composed_cfg = compose("config", overrides=overrides, strict=False)
except:
logger.error("Error when composing. Overrides: " + str(overrides))
raise
for k in deletes:
composed_cfg[k] = None
cfg = OmegaConf.create(
OmegaConf.to_container(composed_cfg, resolve=True, enum_to_str=True))
# hack to be able to set Namespace in dict config. this should be removed when we update to newer
# omegaconf version that supports object flags, or when we migrate all existing models
from omegaconf import _utils
old_primitive = _utils.is_primitive_type
_utils.is_primitive_type = lambda _: True
if cfg.task is None and getattr(args, "task", None):
cfg.task = Namespace(**vars(args))
from fairseq.tasks import TASK_REGISTRY
_set_legacy_defaults(cfg.task, TASK_REGISTRY[args.task])
cfg.task._name = args.task
if cfg.model is None and getattr(args, "arch", None):
cfg.model = Namespace(**vars(args))
from fairseq.models import ARCH_MODEL_REGISTRY
_set_legacy_defaults(cfg.model, ARCH_MODEL_REGISTRY[args.arch])
cfg.model._name = args.arch
if cfg.optimizer is None and getattr(args, "optimizer", None):
cfg.optimizer = Namespace(**vars(args))
from fairseq.optim import OPTIMIZER_REGISTRY
_set_legacy_defaults(cfg.optimizer, OPTIMIZER_REGISTRY[args.optimizer])
cfg.optimizer._name = args.optimizer
if cfg.lr_scheduler is None and getattr(args, "lr_scheduler", None):
cfg.lr_scheduler = Namespace(**vars(args))
from fairseq.optim.lr_scheduler import LR_SCHEDULER_REGISTRY
_set_legacy_defaults(cfg.lr_scheduler,
LR_SCHEDULER_REGISTRY[args.lr_scheduler])
cfg.lr_scheduler._name = args.lr_scheduler
if cfg.criterion is None and getattr(args, "criterion", None):
cfg.criterion = Namespace(**vars(args))
from fairseq.criterions import CRITERION_REGISTRY
_set_legacy_defaults(cfg.criterion, CRITERION_REGISTRY[args.criterion])
cfg.criterion._name = args.criterion
_utils.is_primitive_type = old_primitive
OmegaConf.set_struct(cfg, True)
return cfg
def setUpClass(cls) -> None:
if GlobalHydra.instance().is_initialized():
GlobalHydra.instance().clear()
initialize_config_module(config_module="vissl.config")
