def launch(self, job_overrides):
setup_globals()
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
sweep_dir = self.config.hydra.sweep.dir
Path(str(sweep_dir)).mkdir(parents=True, exist_ok=True)
log.info("Launching {} jobs locally".format(len(job_overrides)))
runs = []
for idx, overrides in enumerate(job_overrides):
log.info("\t#{} : {}".format(idx, " ".join(
filter_overrides(overrides))))
sweep_config = self.config_loader.load_sweep_config(
self.config, list(overrides))
with open_dict(sweep_config):
sweep_config.hydra.job.id = idx
sweep_config.hydra.job.num = idx
HydraConfig().set_config(sweep_config)
ret = run_job(
config=sweep_config,
task_function=self.task_function,
job_dir_key="hydra.sweep.dir",
job_subdir_key="hydra.sweep.subdir",
)
runs.append(ret)
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
return runs
def launch(self, job_overrides):
"""
:param job_overrides: a List of List<String>, where each inner list is the arguments for one job run.
:return: an array of return values from run_job with indexes corresponding to the input list indexes.
"""
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
sweep_dir = Path(str(self.config.hydra.sweep.dir))
sweep_dir.mkdir(parents=True, exist_ok=True)
log.info("Ray Launcher is launching {} jobs locally".format(
len(job_overrides)))
log.info("Sweep output dir : {}".format(sweep_dir))
runs = []
for idx, overrides in enumerate(job_overrides):
log.info("\t#{} : {}".format(idx, " ".join(
filter_overrides(overrides))))
sweep_config = self.config_loader.load_sweep_config(
self.config, list(overrides))
with open_dict(sweep_config):
# This typically coming from the underlying scheduler (SLURM_JOB_ID for instance)
# In that case, it will not be available here because we are still in the main process.
# but instead should be populated remotely before calling the task_function.
sweep_config.hydra.job.id = idx
sweep_config.hydra.job.num = idx
HydraConfig().set_config(sweep_config)
ray_remote_cfg = get_key(self.config, 'ray.remote')
if ray_remote_cfg:
run_job_ray = ray.remote(**ray_remote_cfg)(launch)
else:
run_job_ray = ray.remote(launch)
ret = run_job_ray.remote(
config=sweep_config,
task_function=self.task_function,
job_dir_key="hydra.sweep.dir",
job_subdir_key="hydra.sweep.subdir",
)
runs.append(ret)
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
return [ray.get(run) for run in runs]
def launch(self,
job_overrides: Sequence[Sequence[str]]) -> Sequence[JobReturn]:
"""
:param job_overrides: a List of List<String>, where each inner list is the arguments for one job run.
:return: an array of return values from run_job with indexes corresponding to the input list indexes.
"""
setup_globals()
assert self.config is not None
assert self.config_loader is not None
assert self.task_function is not None
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
sweep_dir = Path(str(self.config.hydra.sweep.dir))
sweep_dir.mkdir(parents=True, exist_ok=True)
log.info(
"Example Launcher(foo={}, bar={}) is launching {} jobs locally".
format(self.foo, self.bar, len(job_overrides)))
log.info("Sweep output dir : {}".format(sweep_dir))
runs = []
for idx, overrides in enumerate(job_overrides):
log.info("\t#{} : {}".format(idx, " ".join(
filter_overrides(overrides))))
sweep_config = self.config_loader.load_sweep_config(
self.config, list(overrides))
with open_dict(sweep_config):
# This typically coming from the underlying scheduler (SLURM_JOB_ID for instance)
# In that case, it will not be available here because we are still in the main process.
# but instead should be populated remotely before calling the task_function.
sweep_config.hydra.job.id = "job_id_for_{}".format(idx)
sweep_config.hydra.job.num = idx
HydraConfig().set_config(sweep_config)
ret = run_job(
config=sweep_config,
task_function=self.task_function,
job_dir_key="hydra.sweep.dir",
job_subdir_key="hydra.sweep.subdir",
)
runs.append(ret)
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
return runs
def run_task(job):
idx, overrides = job
LOGGER.info("\t#{} : {}".format(
idx, " ".join(filter_overrides(overrides))))
sweep_config = self.config_loader.load_sweep_config(
self.config, list(overrides))
with open_dict(sweep_config):
# id is concatenated overrides here
sweep_config.hydra.job.id = '_'.join(sorted(overrides))
sweep_config.hydra.job.num = idx
HydraConfig().set_config(sweep_config)
ret = run_job(
config=sweep_config,
task_function=self.task_function,
job_dir_key="hydra.sweep.dir",
job_subdir_key="hydra.sweep.subdir",
)
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
return (idx, ret)