def launch(self, job_overrides):
setup_globals()
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
sweep_dir = self.config.hydra.sweep.dir
Path(str(sweep_dir)).mkdir(parents=True, exist_ok=True)
log.info("Launching {} jobs locally".format(len(job_overrides)))
runs = []
for idx, overrides in enumerate(job_overrides):
log.info("\t#{} : {}".format(idx, " ".join(
filter_overrides(overrides))))
sweep_config = self.config_loader.load_sweep_config(
self.config, list(overrides))
with open_dict(sweep_config):
sweep_config.hydra.job.id = idx
sweep_config.hydra.job.num = idx
HydraConfig().set_config(sweep_config)
ret = run_job(
config=sweep_config,
task_function=self.task_function,
job_dir_key="hydra.sweep.dir",
job_subdir_key="hydra.sweep.subdir",
)
runs.append(ret)
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
return runs
def _run(self, config_file, task_function, overrides):
if is_multirun:
raise RuntimeError(
"Performing single run with multiple overrides in hydra "
"(use '-m' for multirun): %s" % str(overrides))
cfg = self.compose_config(config_file=config_file,
overrides=overrides,
strict=strict,
with_log_configuration=True)
HydraConfig().set_config(cfg)
return run_job(
config=cfg,
task_function=task_function,
job_dir_key="hydra.run.dir",
job_subdir_key=None,
)
def run(
self,
config_file: Optional[str],
task_function: TaskFunction,
overrides: List[str],
) -> JobReturn:
cfg = self.compose_config(config_file=config_file,
overrides=overrides,
with_log_configuration=True)
HydraConfig().set_config(cfg)
return run_job(
config=cfg,
task_function=task_function,
job_dir_key="hydra.run.dir",
job_subdir_key=None,
)
def launch(self,
job_overrides: Sequence[Sequence[str]]) -> Sequence[JobReturn]:
"""
:param job_overrides: a List of List<String>, where each inner list is the arguments for one job run.
:return: an array of return values from run_job with indexes corresponding to the input list indexes.
"""
setup_globals()
assert self.config is not None
assert self.config_loader is not None
assert self.task_function is not None
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
sweep_dir = Path(str(self.config.hydra.sweep.dir))
sweep_dir.mkdir(parents=True, exist_ok=True)
log.info(
"Example Launcher(foo={}, bar={}) is launching {} jobs locally".
format(self.foo, self.bar, len(job_overrides)))
log.info("Sweep output dir : {}".format(sweep_dir))
runs = []
for idx, overrides in enumerate(job_overrides):
log.info("\t#{} : {}".format(idx, " ".join(
filter_overrides(overrides))))
sweep_config = self.config_loader.load_sweep_config(
self.config, list(overrides))
with open_dict(sweep_config):
# This typically coming from the underlying scheduler (SLURM_JOB_ID for instance)
# In that case, it will not be available here because we are still in the main process.
# but instead should be populated remotely before calling the task_function.
sweep_config.hydra.job.id = "job_id_for_{}".format(idx)
sweep_config.hydra.job.num = idx
HydraConfig().set_config(sweep_config)
ret = run_job(
config=sweep_config,
task_function=self.task_function,
job_dir_key="hydra.sweep.dir",
job_subdir_key="hydra.sweep.subdir",
)
runs.append(ret)
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
return runs
def run_task(job):
idx, overrides = job
LOGGER.info("\t#{} : {}".format(
idx, " ".join(filter_overrides(overrides))))
sweep_config = self.config_loader.load_sweep_config(
self.config, list(overrides))
with open_dict(sweep_config):
# id is concatenated overrides here
sweep_config.hydra.job.id = '_'.join(sorted(overrides))
sweep_config.hydra.job.num = idx
HydraConfig().set_config(sweep_config)
ret = run_job(
config=sweep_config,
task_function=self.task_function,
job_dir_key="hydra.sweep.dir",
job_subdir_key="hydra.sweep.subdir",
)
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
return (idx, ret)
def launch(*args, **kwargs):
setup_globals()
run_job(*args, **kwargs)