def launch(self, job_overrides):
setup_globals()
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
sweep_dir = self.config.hydra.sweep.dir
Path(str(sweep_dir)).mkdir(parents=True, exist_ok=True)
log.info("Launching {} jobs locally".format(len(job_overrides)))
runs = []
for idx, overrides in enumerate(job_overrides):
log.info("\t#{} : {}".format(idx, " ".join(
filter_overrides(overrides))))
sweep_config = self.config_loader.load_sweep_config(
self.config, list(overrides))
with open_dict(sweep_config):
sweep_config.hydra.job.id = idx
sweep_config.hydra.job.num = idx
HydraConfig().set_config(sweep_config)
ret = run_job(
config=sweep_config,
task_function=self.task_function,
job_dir_key="hydra.sweep.dir",
job_subdir_key="hydra.sweep.subdir",
)
runs.append(ret)
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
return runs
def launch(self,
job_overrides: Sequence[Sequence[str]]) -> Sequence[JobReturn]:
"""
:param job_overrides: a List of List<String>, where each inner list is the arguments for one job run.
:return: an array of return values from run_job with indexes corresponding to the input list indexes.
"""
setup_globals()
assert self.config is not None
assert self.config_loader is not None
assert self.task_function is not None
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
sweep_dir = Path(str(self.config.hydra.sweep.dir))
sweep_dir.mkdir(parents=True, exist_ok=True)
log.info(
"Example Launcher(foo={}, bar={}) is launching {} jobs locally".
format(self.foo, self.bar, len(job_overrides)))
log.info("Sweep output dir : {}".format(sweep_dir))
runs = []
for idx, overrides in enumerate(job_overrides):
log.info("\t#{} : {}".format(idx, " ".join(
filter_overrides(overrides))))
sweep_config = self.config_loader.load_sweep_config(
self.config, list(overrides))
with open_dict(sweep_config):
# This typically coming from the underlying scheduler (SLURM_JOB_ID for instance)
# In that case, it will not be available here because we are still in the main process.
# but instead should be populated remotely before calling the task_function.
sweep_config.hydra.job.id = "job_id_for_{}".format(idx)
sweep_config.hydra.job.num = idx
HydraConfig().set_config(sweep_config)
ret = run_job(
config=sweep_config,
task_function=self.task_function,
job_dir_key="hydra.sweep.dir",
job_subdir_key="hydra.sweep.subdir",
)
runs.append(ret)
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
return runs
def __init__(self, task_name: str, config_loader: ConfigLoader) -> None:
"""
:param task_name: task name
:param config_loader: config loader
"""
setup_globals()
self.config_loader = config_loader
for source in config_loader.get_sources():
# if specified, make sure main config search path exists
if source.provider == "main":
if not source.exists(""):
raise MissingConfigException(
missing_cfg_file=source.path,
message="Primary config dir not found: {}".format(
source.path),
)
JobRuntime().set("name", task_name)
def launch(self, job_overrides):
setup_globals()
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
sweep_dir = self.config.hydra.sweep.dir
Path(str(sweep_dir)).mkdir(parents=True, exist_ok=True)
LOGGER.info("Launching {} jobs locally".format(len(job_overrides)))
def run_task(job):
idx, overrides = job
LOGGER.info("\t#{} : {}".format(
idx, " ".join(filter_overrides(overrides))))
sweep_config = self.config_loader.load_sweep_config(
self.config, list(overrides))
with open_dict(sweep_config):
# id is concatenated overrides here
sweep_config.hydra.job.id = '_'.join(sorted(overrides))
sweep_config.hydra.job.num = idx
HydraConfig().set_config(sweep_config)
ret = run_job(
config=sweep_config,
task_function=self.task_function,
job_dir_key="hydra.sweep.dir",
job_subdir_key="hydra.sweep.subdir",
)
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
return (idx, ret)
if self.ncpu > 1:
jobs = list(enumerate(job_overrides))
runs = sorted([
ret for ret in MPI(
jobs=jobs, func=run_task, ncpu=int(self.ncpu), batch=1)
])
runs = [i[1] for i in runs]
else:
runs = [run_task(job)[1] for job in enumerate(job_overrides)]
return runs
def launch(*args, **kwargs):
setup_globals()
run_job(*args, **kwargs)