def launch_jobs(temp_dir: str) -> None:
runs = []
with open(os.path.join(temp_dir, JOB_SPEC_PICKLE), "rb") as f:
job_spec = pickle.load(f) # nosec
singleton_state = job_spec["singleton_state"]
sweep_configs = job_spec["sweep_configs"]
task_function = job_spec["task_function"]
instance_id = _get_instance_id()
sweep_dir = None
for sweep_config in sweep_configs:
with open_dict(sweep_config):
sweep_config.hydra.job.id = (
f"{instance_id}_{sweep_config.hydra.job.num}"
)
setup_globals()
Singleton.set_state(singleton_state)
HydraConfig.instance().set_config(sweep_config)
ray_init_cfg = sweep_config.hydra.launcher.ray_init_cfg
ray_remote_cfg = sweep_config.hydra.launcher.ray_remote_cfg
if not sweep_dir:
sweep_dir = Path(str(HydraConfig.get().sweep.dir))
sweep_dir.mkdir(parents=True, exist_ok=True)
start_ray(ray_init_cfg)
ray_obj = launch_job_on_ray(
ray_remote_cfg, sweep_config, task_function, singleton_state
)
runs.append(ray_obj)
result = [ray.get(run) for run in runs]
_dump_job_return(result, temp_dir)
def execute_job(
idx: int,
overrides: Sequence[str],
config_loader: ConfigLoader,
config: DictConfig,
task_function: TaskFunction,
singleton_state: Dict[Any, Any],
) -> JobReturn:
"""Calls `run_job` in parallel
"""
setup_globals()
Singleton.set_state(singleton_state)
sweep_config = config_loader.load_sweep_config(config, list(overrides))
with open_dict(sweep_config):
sweep_config.hydra.job.id = "{}_{}".format(sweep_config.hydra.job.name,
idx)
sweep_config.hydra.job.num = idx
HydraConfig.instance().set_config(sweep_config)
ret = run_job(
config=sweep_config,
task_function=task_function,
job_dir_key="hydra.sweep.dir",
job_subdir_key="hydra.sweep.subdir",
)
return ret
def run_maze_job(hydra_overrides: Dict[str, str], config_module: str, config_name: str) -> DictConfig:
"""Runs rollout with the given config overrides using maze_run.
:param hydra_overrides: Config overrides for hydra.
:param config_module: The config module.
:param config_name: The name of the default config.
"""
with initialize_config_module(config_module=config_module):
# Config is relative to a module
# For the HydraConfig init below, we need the hydra key there as well (=> return_hydra_config=True)
cfg = compose(config_name=config_name,
overrides=[key + "=" + str(val) for key, val in hydra_overrides.items()],
return_hydra_config=True)
# Init the HydraConfig: This is when Hydra actually creates the output dir and changes into it
# (otherwise we only have the config object, but not the full run environment)
HydraConfig.instance().set_config(cfg)
# For the rollout itself, the Hydra config should not be there anymore
with open_dict(cfg):
del cfg["hydra"]
# Run the rollout
maze_run(cfg)
return cfg
def dispatch_job(
idx: int,
overrides: Sequence[str],
config_loader: ConfigLoader,
config: DictConfig,
task_function: TaskFunction,
singleton_state: Dict[Any, Any],
) -> JobReturn:
"""Calls `run_job` in parallel
Note that Joblib's default backend runs isolated Python processes, see
https://joblib.readthedocs.io/en/latest/parallel.html#shared-memory-semantics
"""
setup_globals()
Singleton.set_state(singleton_state)
log.info("\t#{} : {}".format(idx, " ".join(filter_overrides(overrides))))
sweep_config = config_loader.load_sweep_config(config, list(overrides))
with open_dict(sweep_config):
sweep_config.hydra.job.id = "{}_{}".format(sweep_config.hydra.job.name,
idx)
sweep_config.hydra.job.num = idx
HydraConfig.instance().set_config(sweep_config)
ret = run_job(
config=sweep_config,
task_function=task_function,
job_dir_key="hydra.sweep.dir",
job_subdir_key="hydra.sweep.subdir",
)
return ret
def launch(
self, job_overrides: Sequence[Sequence[str]], initial_job_idx: int
) -> Sequence[JobReturn]:
setup_globals()
assert self.config is not None
assert self.task_function is not None
assert self.config_loader is not None
configure_log(self.config.hydra.hydra_logging, self.config.hydra.verbose)
sweep_dir = self.config.hydra.sweep.dir
Path(str(sweep_dir)).mkdir(parents=True, exist_ok=True)
log.info(f"Launching {len(job_overrides)} jobs locally")
runs: List[JobReturn] = []
for idx, overrides in enumerate(job_overrides):
idx = initial_job_idx + idx
lst = " ".join(filter_overrides(overrides))
log.info(f"\t#{idx} : {lst}")
sweep_config = self.config_loader.load_sweep_config(
self.config, list(overrides)
)
with open_dict(sweep_config):
sweep_config.hydra.job.id = idx
sweep_config.hydra.job.num = idx
HydraConfig.instance().set_config(sweep_config)
ret = run_job(
config=sweep_config,
task_function=self.task_function,
job_dir_key="hydra.sweep.dir",
job_subdir_key="hydra.sweep.subdir",
)
runs.append(ret)
configure_log(self.config.hydra.hydra_logging, self.config.hydra.verbose)
return runs
def multirun(
self,
config_name: Optional[str],
task_function: TaskFunction,
overrides: List[str],
with_log_configuration: bool = True,
) -> Any:
# Initial config is loaded without strict (individual job configs may have strict).
cfg = self.compose_config(
config_name=config_name,
overrides=overrides,
strict=False,
with_log_configuration=with_log_configuration,
run_mode=RunMode.MULTIRUN,
)
HydraConfig.instance().set_config(cfg)
sweeper = Plugins.instance().instantiate_sweeper(
config=cfg,
config_loader=self.config_loader,
task_function=task_function)
task_overrides = OmegaConf.to_container(cfg.hydra.overrides.task,
resolve=False)
assert isinstance(task_overrides, list)
return sweeper.sweep(arguments=task_overrides)
def launch(
self, job_overrides: Sequence[Sequence[str]], initial_job_idx: int
) -> Sequence[JobReturn]:
setup_globals()
assert self.hydra_context is not None
assert self.config is not None
assert self.task_function is not None
configure_log(self.config.hydra.hydra_logging, self.config.hydra.verbose)
sweep_dir = self.config.hydra.sweep.dir
Path(str(sweep_dir)).mkdir(parents=True, exist_ok=True)
log.info("Launching {} jobs on slurm".format(len(job_overrides)))
runs: List[JobReturn] = []
for idx, overrides in enumerate(job_overrides):
idx = initial_job_idx + idx
lst = " ".join(filter_overrides(overrides))
log.info(f"\t#{idx} : {lst}")
sweep_config = self.hydra_context.config_loader.load_sweep_config(
self.config, list(overrides)
)
with open_dict(sweep_config):
sweep_config.hydra.job.id = idx
sweep_config.hydra.job.num = idx
HydraConfig.instance().set_config(sweep_config)
log.info("\tJob name : {}".format(slurm_utils.resolve_name(sweep_config.slurm.job_name)))
slurm_utils.write_slurm(sweep_config)
slurm_utils.write_sh(sweep_config, " ".join(filter_overrides(overrides)))
slurm_utils.launch_job(sweep_config)
configure_log(self.config.hydra.hydra_logging, self.config.hydra.verbose)
if sweep_config.wait:
time.sleep(1)
return runs
def _print_search_path(self, config_name: Optional[str],
overrides: List[str]) -> None:
assert log is not None
log.debug("")
self._log_header(header="Config search path", filler="*")
box: List[List[str]] = [["Provider", "Search path"]]
cfg = self.compose_config(
config_name=config_name,
overrides=overrides,
run_mode=RunMode.RUN,
with_log_configuration=False,
)
HydraConfig.instance().set_config(cfg)
cfg = self.get_sanitized_cfg(cfg, cfg_type="hydra")
sources = cfg.hydra.runtime.config_sources
for sp in sources:
box.append([sp.provider, f"{sp.schema}://{sp.path}"])
provider_pad, search_path_pad = get_column_widths(box)
header = "| {} | {} |".format("Provider".ljust(provider_pad),
"Search path".ljust(search_path_pad))
self._log_header(header=header, filler="-")
for source in sources:
log.debug("| {} | {} |".format(
source.provider.ljust(provider_pad),
f"{source.schema}://{source.path}".ljust(search_path_pad),
))
self._log_footer(header=header, filler="-")
def launch(self, job_overrides: Sequence[Sequence[str]],
initial_job_idx: int) -> Sequence[JobReturn]:
"""
:param job_overrides: a List of List<String>, where each inner list is the arguments for one job run.
:param initial_job_idx: Initial job idx in batch.
:return: an array of return values from run_job with indexes corresponding to the input list indexes.
"""
setup_globals()
assert self.config is not None
assert self.config_loader is not None
assert self.task_function is not None
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
sweep_dir = Path(str(self.config.hydra.sweep.dir))
sweep_dir.mkdir(parents=True, exist_ok=True)
log.info(
f"Example Launcher(foo={self.foo}, bar={self.bar}) is launching {len(job_overrides)} jobs locally"
)
log.info(f"Sweep output dir : {sweep_dir}")
runs = []
for idx, overrides in enumerate(job_overrides):
idx = initial_job_idx + idx
lst = " ".join(filter_overrides(overrides))
log.info(f"\t#{idx} : {lst}")
sweep_config = self.config_loader.load_sweep_config(
self.config, list(overrides))
with open_dict(sweep_config):
# This typically coming from the underlying scheduler (SLURM_JOB_ID for instance)
# In that case, it will not be available here because we are still in the main process.
# but instead should be populated remotely before calling the task_function.
sweep_config.hydra.job.id = f"job_id_for_{idx}"
sweep_config.hydra.job.num = idx
HydraConfig.instance().set_config(sweep_config)
# If your launcher is executing code in a different process, it is important to restore
# the singleton state in the new process.
# To do this, you will likely need to serialize the singleton state along with the other
# parameters passed to the child process.
# happening on this process (executing launcher)
state = Singleton.get_state()
# happening on the spawned process (executing task_function in run_job)
Singleton.set_state(state)
ret = run_job(
config=sweep_config,
task_function=self.task_function,
job_dir_key="hydra.sweep.dir",
job_subdir_key="hydra.sweep.subdir",
)
runs.append(ret)
# reconfigure the logging subsystem for Hydra as the run_job call configured it for the Job.
# This is needed for launchers that calls run_job in the same process and not spawn a new one.
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
return runs
def setup_multirun():
from hydra.core.hydra_config import HydraConfig
if 'num' in HydraConfig.get().job:
job_num = HydraConfig.get().job.num % torch.cuda.device_count()
gpu = job_num % torch.cuda.device_count()
torch.cuda.set_device(gpu)
print(f'Job number {job_num:2d}')
print(f'Setting active GPU to {gpu}')
def run_job(
config: DictConfig,
task_function: TaskFunction,
job_dir_key: str,
job_subdir_key: Optional[str],
configure_logging: bool = True,
) -> "JobReturn":
old_cwd = os.getcwd()
working_dir = str(OmegaConf.select(config, job_dir_key))
orig_hydra_cfg = HydraConfig.instance().cfg
if job_subdir_key is not None:
# evaluate job_subdir_key lazily.
# this is running on the client side in sweep and contains things such as job:id which
# are only available there.
subdir = str(OmegaConf.select(config, job_subdir_key))
working_dir = os.path.join(working_dir, subdir)
try:
ret = JobReturn()
ret.working_dir = working_dir
task_cfg = copy.deepcopy(config)
hydra_cfg = OmegaConf.masked_copy(task_cfg, "hydra")
# maintain parent to preserve interpolation links from hydra_cfg to job_cfg
hydra_cfg._set_parent(task_cfg)
with read_write(task_cfg):
with open_dict(task_cfg):
del task_cfg["hydra"]
HydraConfig.instance().cfg = hydra_cfg # type: ignore
ret.cfg = task_cfg
ret.hydra_cfg = hydra_cfg
overrides = OmegaConf.to_container(config.hydra.overrides.task)
assert isinstance(overrides, list)
ret.overrides = overrides
# handle output directories here
Path(str(working_dir)).mkdir(parents=True, exist_ok=True)
os.chdir(working_dir)
if configure_logging:
configure_log(config.hydra.job_logging, config.hydra.verbose)
if config.hydra.output_subdir is not None:
hydra_output = Path(config.hydra.output_subdir)
_save_config(task_cfg, "config.yaml", hydra_output)
_save_config(hydra_cfg, "hydra.yaml", hydra_output)
_save_config(config.hydra.overrides.task, "overrides.yaml",
hydra_output)
with env_override(hydra_cfg.hydra.job.env_set):
ret.return_value = task_function(task_cfg)
ret.task_name = JobRuntime.instance().get("name")
_flush_loggers()
return ret
finally:
HydraConfig.instance().cfg = orig_hydra_cfg
os.chdir(old_cwd)
def get_original_cwd() -> str:
"""
:return: the original working directory the Hydra application was launched from
"""
if not HydraConfig.initialized():
raise ValueError(
"get_original_cwd() must only be used after HydraConfig is initialized"
)
ret = HydraConfig.get().runtime.cwd
assert ret is not None and isinstance(ret, str)
return ret
def test_foo(restore_singletons: Any) -> Any:
utils.setup_globals()
config_loader = ConfigLoaderImpl(
config_search_path=create_config_search_path(
"pkg://hydra.test_utils.configs"))
cfg = config_loader.load_configuration(
config_name="accessing_hydra_config", overrides=[])
HydraConfig.instance().set_config(cfg)
with open_dict(cfg):
del cfg["hydra"]
assert cfg.job_name == "UNKNOWN_NAME"
assert cfg.config_name == "accessing_hydra_config"
def _run_job(
sweep_config: DictConfig,
task_function: TaskFunction,
singleton_state: Dict[Any, Any],
) -> JobReturn:
setup_globals()
Singleton.set_state(singleton_state)
HydraConfig.instance().set_config(sweep_config)
return run_job(
config=sweep_config,
task_function=task_function,
job_dir_key="hydra.sweep.dir",
job_subdir_key="hydra.sweep.subdir",
)
def launch(self, job_overrides: Sequence[Sequence[str]],
initial_job_idx: int) -> Sequence[JobReturn]:
"""
:param job_overrides: a List of List<String>, where each inner list is the arguments for one job run.
:param initial_job_idx: Initial job idx in batch.
:return: an array of return values from run_job with indexes corresponding to the input list indexes.
"""
setup_globals()
assert self.config is not None
assert self.config_loader is not None
assert self.task_function is not None
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
sweep_dir = Path(str(self.config.hydra.sweep.dir))
sweep_dir.mkdir(parents=True, exist_ok=True)
log.info(
"Example Launcher(foo={}, bar={}) is launching {} jobs locally".
format(self.foo, self.bar, len(job_overrides)))
log.info("Sweep output dir : {}".format(sweep_dir))
runs = []
for idx, overrides in enumerate(job_overrides):
idx = initial_job_idx + idx
log.info("\t#{} : {}".format(idx, " ".join(
filter_overrides(overrides))))
sweep_config = self.config_loader.load_sweep_config(
self.config, list(overrides))
with open_dict(sweep_config):
# This typically coming from the underlying scheduler (SLURM_JOB_ID for instance)
# In that case, it will not be available here because we are still in the main process.
# but instead should be populated remotely before calling the task_function.
sweep_config.hydra.job.id = "job_id_for_{}".format(idx)
sweep_config.hydra.job.num = idx
HydraConfig.instance().set_config(sweep_config)
ret = run_job(
config=sweep_config,
task_function=self.task_function,
job_dir_key="hydra.sweep.dir",
job_subdir_key="hydra.sweep.subdir",
)
runs.append(ret)
# reconfigure the logging subsystem for Hydra as the run_job call configured it for the Job.
# This is needed for launchers that calls run_job in the same process and not spawn a new one.
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
return runs
def hydra_instance() -> Union[Hydra, GlobalHydra]:
"Provide Hydra/GlobalHydra instance for compose"
if HydraConfig.initialized():
yield GlobalHydra.instance()
hydra_init = initialize(config_path="../peddet/conf")
yield hydra_init
GlobalHydra.instance().clear()
def get_class_dist(labels_file="78-classes_train.json",
str2id_file="78-classes_labels.json"):
labels_dir = "./datasets/20bn-sth-sth-v2/labels/"
if HydraConfig.initialized():
labels_dir = os.path.join(hydra.utils.get_original_cwd(), labels_dir)
data_info_path = os.path.join(labels_dir, labels_file)
else:
labels_dir = os.path.join(os.getcwd(), labels_dir)
data_info_path = os.path.abspath(os.path.join(labels_dir, labels_file))
data_info_frame = pd.read_json(data_info_path)
ids_frame = pd.read_json(os.path.join(labels_dir, str2id_file),\
typ='series')
data_info_frame["template"] = data_info_frame["template"].str.replace(
"[", "")
data_info_frame["template"] = data_info_frame["template"].str.replace(
"]", "")
#classes present in labels_file (78)
classes = ids_frame[data_info_frame["template"]]
unique, counts = np.unique(classes, return_counts=True)
old2new = map_orig2new()
#str: new id
classes = ids_frame[unique].replace(old2new)
#returns
unique_str = ids_frame[unique]
unique_new_ids = [old2new[i] for i in unique]
#new_ids: count
new_ids_counts = dict(zip(unique_new_ids, counts))
#str_count:
str_counts = dict(zip(unique_str, counts))
#new_ids in order of the dataframe
class_labels = list(classes[data_info_frame["template"]])
return new_ids_counts, str_counts, class_labels
def _get_job_id() -> int:
try:
return HydraConfig.get().job.id
except MissingMandatoryValue:
# The numeric job ID is missing if not in a multirun context. In that
# case, there can only be a single run.
return 0
def test_configuration_set_via_cmd_and_default_config(
sweep_runner: TSweepRunner, ) -> None:
sweep = sweep_runner(
calling_file="tests/test_ax_sweeper_plugin.py",
calling_module=None,
task_function=quadratic,
config_path="config",
config_name="config.yaml",
overrides=[
"hydra/launcher=basic",
"hydra.sweeper.params.ax_config.max_trials=2",
"hydra.sweeper.params.ax_config.early_stop.max_epochs_without_improvement=2",
"quadratic=basic",
"quadratic.x=-5:-2",
"quadratic.y=-1:1",
],
)
with sweep:
ax_config = HydraConfig.get().sweeper.params.ax_config
assert ax_config.max_trials == 2
assert ax_config.early_stop.max_epochs_without_improvement == 2
assert ax_config.experiment.minimize is True
assert sweep.returns is None
returns = OmegaConf.load(f"{sweep.temp_dir}/optimization_results.yaml")
assert isinstance(returns, DictConfig)
best_parameters = returns["ax"]
assert "quadratic_x" in best_parameters
assert "quadratic_y" in best_parameters
def setup_globals() -> None:
# please add documentation when you add a new resolver
OmegaConf.register_new_resolver(
"now",
lambda pattern: datetime.now().strftime(pattern),
use_cache=True,
replace=True,
)
OmegaConf.register_new_resolver(
"hydra",
lambda path: OmegaConf.select(cast(DictConfig, HydraConfig.get()), path
),
replace=True,
)
vi = sys.version_info
version_dict = {
"major": f"{vi[0]}",
"minor": f"{vi[0]}.{vi[1]}",
"micro": f"{vi[0]}.{vi[1]}.{vi[2]}",
}
OmegaConf.register_new_resolver(
"python_version",
lambda level="minor": version_dict.get(level),
replace=True)
def test_configuration_set_via_cmd_and_default_config(
sweep_runner: TSweepRunner, # noqa: F811
) -> None:
sweep = sweep_runner(
calling_file=os.path.dirname(os.path.abspath(__file__)),
calling_module=None,
task_function=quadratic,
config_path="tests/config",
config_name="default_quadratic.yaml",
overrides=[
"hydra/sweeper=ax",
"hydra/launcher=basic",
"hydra.sweeper.params.ax_config.client.random_seed=1",
"hydra.sweeper.params.ax_config.max_trials=2",
"hydra.sweeper.params.ax_config.early_stop.max_epochs_without_improvement=2",
"quadratic.x=-5:-2",
"quadratic.y=-1:1",
],
)
with sweep:
ax_config = HydraConfig.instance().hydra.sweeper.params.ax_config
assert ax_config.max_trials == 2
assert ax_config.early_stop.max_epochs_without_improvement == 2
assert ax_config.experiment.minimize is True
assert sweep.returns is None
returns = OmegaConf.load(f"{sweep.temp_dir}/optimization_results.yaml")
assert isinstance(returns, DictConfig)
best_parameters = returns["ax"]
assert "quadratic.x" in best_parameters
assert "quadratic.y" in best_parameters
def error_checks(trainer: 'pytorch_lightning.Trainer',
cfg: Optional[Union[DictConfig, Dict]] = None):
"""
Checks that the passed trainer is compliant with NeMo and exp_manager's passed configuration. Checks that:
- Throws error when hydra has changed the working directory. This causes issues with lightning's DDP
- Throws error when trainer has loggers defined but create_tensorboard_logger or create_WandB_logger is True
- Prints error messages when 1) run on multi-node and not Slurm, and 2) run on multi-gpu without DDP
"""
if HydraConfig.initialized() and get_original_cwd() != os.getcwd():
raise ValueError(
"Hydra changed the working directory. This interferes with ExpManger's functionality. Please pass "
"hydra.run.dir=. to your python script.")
if trainer.logger is not None and (cfg.create_tensorboard_logger
or cfg.create_wandb_logger):
raise LoggerMisconfigurationError(
"The pytorch lightning trainer that was passed to exp_manager contained a logger, and either "
f"create_tensorboard_logger: {cfg.create_tensorboard_logger} or create_wandb_logger: "
f"{cfg.create_wandb_logger} was set to True. These can only be used if trainer does not already have a"
" logger.")
if trainer.num_nodes > 1 and not check_slurm(trainer):
logging.error(
"You are running multi-node training without SLURM handling the processes."
" Please note that this is not tested in NeMo and could result in errors."
)
if trainer.num_gpus > 1 and not isinstance(
trainer.accelerator.training_type_plugin, DDPPlugin):
logging.error(
"You are running multi-gpu without ddp.Please note that this is not tested in NeMo and could result in "
"errors.")
def run(
self,
config_name: Optional[str],
task_function: TaskFunction,
overrides: List[str],
) -> JobReturn:
cfg = self.compose_config(config_name=config_name,
overrides=overrides,
with_log_configuration=True)
HydraConfig.instance().set_config(cfg)
return run_job(
config=cfg,
task_function=task_function,
job_dir_key="hydra.run.dir",
job_subdir_key=None,
)
def launch(
launcher: RayAWSLauncher,
job_overrides: Sequence[Sequence[str]],
initial_job_idx: int,
) -> Sequence[JobReturn]:
setup_globals()
assert launcher.config is not None
assert launcher.config_loader is not None
assert launcher.task_function is not None
setup_commands = launcher.env_setup.commands
with read_write(setup_commands):
setup_commands.extend([
f"pip install {package}=={version}"
for package, version in launcher.env_setup.pip_packages.items()
])
setup_commands.extend(launcher.ray_cfg.cluster.setup_commands)
with read_write(launcher.ray_cfg.cluster):
launcher.ray_cfg.cluster.setup_commands = setup_commands
configure_log(launcher.config.hydra.hydra_logging,
launcher.config.hydra.verbose)
log.info(f"Ray Launcher is launching {len(job_overrides)} jobs, ")
with tempfile.TemporaryDirectory() as local_tmp_dir:
sweep_configs = []
for idx, overrides in enumerate(job_overrides):
idx = initial_job_idx + idx
ostr = " ".join(filter_overrides(overrides))
log.info(f"\t#{idx} : {ostr}")
sweep_config = launcher.config_loader.load_sweep_config(
launcher.config, list(overrides))
with open_dict(sweep_config):
# job.id will be set on the EC2 instance before running the job.
sweep_config.hydra.job.num = idx
sweep_configs.append(sweep_config)
_pickle_jobs(
tmp_dir=local_tmp_dir,
sweep_configs=sweep_configs, # type: ignore
task_function=launcher.task_function,
singleton_state=Singleton.get_state(),
)
with tempfile.NamedTemporaryFile(suffix=".yaml", delete=False) as f:
with open(f.name, "w") as file:
OmegaConf.save(config=launcher.ray_cfg.cluster,
f=file.name,
resolve=True)
launcher.ray_yaml_path = f.name
log.info(
f"Saving RayClusterConf in a temp yaml file: {launcher.ray_yaml_path}."
)
return launch_jobs(launcher, local_tmp_dir,
Path(HydraConfig.get().sweep.dir))
def __enter__(self):
# do nothing if hydra is not initialized
if not HydraConfig.initialized() or not self.input_dir:
return
self.hydra_out_dir = os.getcwd()
original_dir = os.path.join(hydra.utils.get_original_cwd(), self.input_dir)
print(f"Switching load directory to {original_dir}")
os.chdir(original_dir)
def test_to_absolute_path(orig_cwd: str, path: str, expected: str) -> None:
# normalize paths to current OS
orig_cwd = str(Path(orig_cwd))
path = str(Path(path))
expected = str(Path(expected))
cfg = OmegaConf.create({"hydra": {"runtime": {"cwd": orig_cwd}}})
assert isinstance(cfg, DictConfig)
HydraConfig().set_config(cfg)
assert utils.to_absolute_path(path) == expected
def app_help(self, config_name: Optional[str], args_parser: ArgumentParser,
args: Any) -> None:
cfg = self.compose_config(
config_name=config_name,
overrides=args.overrides,
run_mode=RunMode.RUN,
with_log_configuration=True,
)
HydraConfig.instance().set_config(cfg)
help_cfg = cfg.hydra.help
clean_cfg = copy.deepcopy(cfg)
clean_cfg = self.get_sanitized_cfg(clean_cfg, "job")
help_text = self.get_help(help_cfg,
clean_cfg,
args_parser,
resolve=args.resolve)
print(help_text)
def _print_config_info(self, config_name: Optional[str],
overrides: List[str]) -> None:
assert log is not None
self._print_search_path(config_name=config_name, overrides=overrides)
self._print_defaults_tree(config_name=config_name, overrides=overrides)
self._print_defaults_list(config_name=config_name, overrides=overrides)
cfg = run_and_report(lambda: self.compose_config(
config_name=config_name,
overrides=overrides,
run_mode=RunMode.RUN,
with_log_configuration=False,
))
HydraConfig.instance().set_config(cfg)
self._log_header(header="Config", filler="*")
with flag_override(cfg, ["struct", "readonly"], [False, False]):
del cfg["hydra"]
log.info(OmegaConf.to_yaml(cfg))
def execute_job(
idx: int,
overrides: Sequence[str],
config_loader: ConfigLoader,
config: DictConfig,
task_function: TaskFunction,
singleton_state: Dict[Any, Any],
cmd_prefix: str,
tsp_prefix: str,
) -> JobReturn:
"""Calls `run_job` in parallel
"""
setup_globals()
Singleton.set_state(singleton_state)
lst = " ".join(overrides)
sweep_config = config_loader.load_sweep_config(config, list(overrides))
with open_dict(sweep_config):
sweep_config.hydra.job.id = "{}_{}".format(sweep_config.hydra.job.name,
idx)
sweep_config.hydra.job.num = idx
HydraConfig.instance().set_config(sweep_config)
def tsp_task_function(task_cfg):
working_dir = os.getcwd()
cmd = f"{cmd_prefix} {lst}"
log.info(f"\t#{idx} : {lst}")
cmd = f"cd {hydra.utils.get_original_cwd()} && {cmd} hydra.run.dir={working_dir}"
job_id = int(subprocess.check_output(cmd, shell=True).rstrip())
log.info(
f"Submitted {idx} to TaskSpooler. View logs: {tsp_prefix} -t {job_id}"
)
return job_id
ret = run_job(
config=sweep_config,
task_function=tsp_task_function,
job_dir_key="hydra.sweep.dir",
job_subdir_key="hydra.sweep.subdir",
)
ret.id = ret.return_value
return ret
def multirun(
self,
config_name: Optional[str],
task_function: TaskFunction,
overrides: List[str],
) -> Any:
# Initial config is loaded without strict (individual job configs may have strict).
cfg = self.compose_config(
config_name=config_name,
overrides=overrides,
strict=False,
with_log_configuration=True,
)
HydraConfig.instance().set_config(cfg)
sweeper = Plugins.instance().instantiate_sweeper(
config=cfg, config_loader=self.config_loader, task_function=task_function
)
task_overrides = cfg.hydra.overrides.task
return sweeper.sweep(arguments=task_overrides)
