def execute_job(
idx: int,
overrides: Sequence[str],
config_loader: ConfigLoader,
config: DictConfig,
task_function: TaskFunction,
singleton_state: Dict[Any, Any],
) -> JobReturn:
"""Calls `run_job` in parallel
"""
setup_globals()
Singleton.set_state(singleton_state)
sweep_config = config_loader.load_sweep_config(config, list(overrides))
with open_dict(sweep_config):
sweep_config.hydra.job.id = "{}_{}".format(sweep_config.hydra.job.name,
idx)
sweep_config.hydra.job.num = idx
HydraConfig.instance().set_config(sweep_config)
ret = run_job(
config=sweep_config,
task_function=task_function,
job_dir_key="hydra.sweep.dir",
job_subdir_key="hydra.sweep.subdir",
)
return ret
def __call__(
self,
sweep_overrides: List[str],
job_dir_key: str,
job_num: int,
job_id: str,
singleton_state: Dict[type, "Singleton"],
):
Singleton.set_state(singleton_state)
configure_log(self.config.hydra.job_logging, self.config.hydra.verbose)
setup_globals()
sweep_config = self.config_loader.load_sweep_config(
self.config, sweep_overrides)
with open_dict(sweep_config.hydra.job) as job:
# Populate new job variables
if "SLURM_JOB_ID" in os.environ:
job.id = os.environ["SLURM_JOB_ID"]
else:
job.id = job_id
sweep_config.hydra.job.num = job_num
return run_job(
config=sweep_config,
task_function=self.task_function,
job_dir_key=job_dir_key,
job_subdir_key="hydra.sweep.subdir",
)
def __call__(
self,
sweep_overrides: List[str],
job_dir_key: str,
job_num: int,
job_id: str,
singleton_state: Dict[type, Singleton],
) -> JobReturn:
# lazy import to ensure plugin discovery remains fast
import submitit
assert self.config_loader is not None
assert self.config is not None
assert self.task_function is not None
Singleton.set_state(singleton_state)
setup_globals()
sweep_config = self.config_loader.load_sweep_config(
self.config, sweep_overrides)
with open_dict(sweep_config.hydra.job) as job:
# Populate new job variables
job.id = submitit.JobEnvironment().job_id # type: ignore
sweep_config.hydra.job.num = job_num
return run_job(
config=sweep_config,
task_function=self.task_function,
job_dir_key=job_dir_key,
job_subdir_key="hydra.sweep.subdir",
)
def dispatch_job(
idx: int,
overrides: Sequence[str],
config_loader: ConfigLoader,
config: DictConfig,
task_function: TaskFunction,
singleton_state: Dict[Any, Any],
) -> JobReturn:
"""Calls `run_job` in parallel
Note that Joblib's default backend runs isolated Python processes, see
https://joblib.readthedocs.io/en/latest/parallel.html#shared-memory-semantics
"""
setup_globals()
Singleton.set_state(singleton_state)
log.info("\t#{} : {}".format(idx, " ".join(filter_overrides(overrides))))
sweep_config = config_loader.load_sweep_config(config, list(overrides))
with open_dict(sweep_config):
sweep_config.hydra.job.id = "{}_{}".format(sweep_config.hydra.job.name,
idx)
sweep_config.hydra.job.num = idx
HydraConfig.instance().set_config(sweep_config)
ret = run_job(
config=sweep_config,
task_function=task_function,
job_dir_key="hydra.sweep.dir",
job_subdir_key="hydra.sweep.subdir",
)
return ret
def hydra_restore_singletons() -> None:
"""
Restore singletons state after the function returns
"""
state = copy.deepcopy(Singleton.get_state())
yield
Singleton.set_state(state)
def launch_jobs(temp_dir: str) -> None:
runs = []
with open(os.path.join(temp_dir, JOB_SPEC_PICKLE), "rb") as f:
job_spec = pickle.load(f) # nosec
singleton_state = job_spec["singleton_state"]
sweep_configs = job_spec["sweep_configs"]
task_function = job_spec["task_function"]
instance_id = _get_instance_id()
sweep_dir = None
for sweep_config in sweep_configs:
with open_dict(sweep_config):
sweep_config.hydra.job.id = (
f"{instance_id}_{sweep_config.hydra.job.num}"
)
setup_globals()
Singleton.set_state(singleton_state)
HydraConfig.instance().set_config(sweep_config)
ray_init_cfg = sweep_config.hydra.launcher.ray_init_cfg
ray_remote_cfg = sweep_config.hydra.launcher.ray_remote_cfg
if not sweep_dir:
sweep_dir = Path(str(HydraConfig.get().sweep.dir))
sweep_dir.mkdir(parents=True, exist_ok=True)
start_ray(ray_init_cfg)
ray_obj = launch_job_on_ray(
ray_remote_cfg, sweep_config, task_function, singleton_state
)
runs.append(ray_obj)
result = [ray.get(run) for run in runs]
_dump_job_return(result, temp_dir)
def launch(self, job_overrides: Sequence[Sequence[str]],
initial_job_idx: int) -> Sequence[JobReturn]:
"""
:param job_overrides: a List of List<String>, where each inner list is the arguments for one job run.
:param initial_job_idx: Initial job idx in batch.
:return: an array of return values from run_job with indexes corresponding to the input list indexes.
"""
setup_globals()
assert self.config is not None
assert self.hydra_context is not None
assert self.task_function is not None
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
sweep_dir = Path(str(self.config.hydra.sweep.dir))
sweep_dir.mkdir(parents=True, exist_ok=True)
log.info(
f"Example Launcher(foo={self.foo}, bar={self.bar}) is launching {len(job_overrides)} jobs locally"
)
log.info(f"Sweep output dir : {sweep_dir}")
runs = []
for idx, overrides in enumerate(job_overrides):
idx = initial_job_idx + idx
lst = " ".join(filter_overrides(overrides))
log.info(f"\t#{idx} : {lst}")
sweep_config = self.hydra_context.config_loader.load_sweep_config(
self.config, list(overrides))
with open_dict(sweep_config):
# This typically coming from the underlying scheduler (SLURM_JOB_ID for instance)
# In that case, it will not be available here because we are still in the main process.
# but instead should be populated remotely before calling the task_function.
sweep_config.hydra.job.id = f"job_id_for_{idx}"
sweep_config.hydra.job.num = idx
# If your launcher is executing code in a different process, it is important to restore
# the singleton state in the new process.
# To do this, you will likely need to serialize the singleton state along with the other
# parameters passed to the child process.
# happening on this process (executing launcher)
state = Singleton.get_state()
# happening on the spawned process (executing task_function in run_job)
Singleton.set_state(state)
ret = run_job(
hydra_context=self.hydra_context,
task_function=self.task_function,
config=sweep_config,
job_dir_key="hydra.sweep.dir",
job_subdir_key="hydra.sweep.subdir",
)
runs.append(ret)
# reconfigure the logging subsystem for Hydra as the run_job call configured it for the Job.
# This is needed for launchers that calls run_job in the same process and not spawn a new one.
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
return runs
def restore_singletons() -> Any:
"""
A fixture to restore singletons state after this the function.
This is useful for functions that are making a one-off change to singlestons that should not effect
other tests
"""
state = copy.deepcopy(Singleton.get_state())
yield
Singleton.set_state(state)
def hydra_restore_singletons() -> None:
"""
Restore singletons state after the function returns
"""
state = copy.deepcopy(Singleton.get_state())
resolvers = copy.deepcopy(BaseContainer._resolvers)
yield
Singleton.set_state(state)
BaseContainer._resolvers = resolvers
def test_restore_singleton_state_hack() -> None:
"""
This is a hack that allow us to undo changes to the ConfigStore.
During this test, the config store is being modified in Python imports.
Python imports can only run once, so clearing the state during the tests will break
The tests because it will not be reinitialized.
A solution is to undo the changes after the last test.
The reason this logic is in a test is that if it's outside it's being executed during
Pytest's test collection phase, which is before the tests are dunning - so it does not solve the problem.
"""
Singleton.set_state(state)
def _run_job(
sweep_config: DictConfig,
task_function: TaskFunction,
singleton_state: Dict[Any, Any],
) -> JobReturn:
setup_globals()
Singleton.set_state(singleton_state)
HydraConfig.instance().set_config(sweep_config)
return run_job(
config=sweep_config,
task_function=task_function,
job_dir_key="hydra.sweep.dir",
job_subdir_key="hydra.sweep.subdir",
)
def launch(
self, job_overrides: Sequence[Sequence[str]], initial_job_idx: int
) -> Sequence[JobReturn]:
# lazy import to ensure plugin discovery remains fast
import submitit
num_jobs = len(job_overrides)
assert num_jobs > 0
params = self.params
# build executor
init_params = {"folder": self.params["submitit_folder"]}
specific_init_keys = {"max_num_timeout"}
init_params.update(
**{
f"{self._EXECUTOR}_{x}": y
for x, y in params.items()
if x in specific_init_keys
}
)
init_keys = specific_init_keys | {"submitit_folder"}
executor = submitit.AutoExecutor(cluster=self._EXECUTOR, **init_params)
# specify resources/parameters
baseparams = set(dataclasses.asdict(BaseTarget()).keys())
params = {
x if x in baseparams else f"{self._EXECUTOR}_{x}": y
for x, y in params.items()
if x not in init_keys
}
executor.update_parameters(**params)
log.info(
f"Submitit '{self._EXECUTOR}' sweep output dir : "
f"{self.config.hydra.sweep.dir}"
)
sweep_dir = Path(str(self.config.hydra.sweep.dir))
sweep_dir.mkdir(parents=True, exist_ok=True)
if "mode" in self.config.hydra.sweep:
mode = int(str(self.config.hydra.sweep.mode), 8)
os.chmod(sweep_dir, mode=mode)
params = []
for idx, overrides in enumerate(job_overrides):
idx = initial_job_idx + idx
lst = " ".join(filter_overrides(overrides))
log.info(f"\t#{idx} : {lst}")
params.append(
(
list(overrides),
"hydra.sweep.dir",
idx,
f"job_id_for_{idx}",
Singleton.get_state(),
)
)
jobs = executor.map_array(self, *zip(*params))
return [j.results()[0] for j in jobs]
def launch(
launcher: RayAWSLauncher,
job_overrides: Sequence[Sequence[str]],
initial_job_idx: int,
) -> Sequence[JobReturn]:
setup_globals()
assert launcher.config is not None
assert launcher.config_loader is not None
assert launcher.task_function is not None
setup_commands = launcher.env_setup.commands
with read_write(setup_commands):
setup_commands.extend([
f"pip install {package}=={version}"
for package, version in launcher.env_setup.pip_packages.items()
])
setup_commands.extend(launcher.ray_cfg.cluster.setup_commands)
with read_write(launcher.ray_cfg.cluster):
launcher.ray_cfg.cluster.setup_commands = setup_commands
configure_log(launcher.config.hydra.hydra_logging,
launcher.config.hydra.verbose)
log.info(f"Ray Launcher is launching {len(job_overrides)} jobs, ")
with tempfile.TemporaryDirectory() as local_tmp_dir:
sweep_configs = []
for idx, overrides in enumerate(job_overrides):
idx = initial_job_idx + idx
ostr = " ".join(filter_overrides(overrides))
log.info(f"\t#{idx} : {ostr}")
sweep_config = launcher.config_loader.load_sweep_config(
launcher.config, list(overrides))
with open_dict(sweep_config):
# job.id will be set on the EC2 instance before running the job.
sweep_config.hydra.job.num = idx
sweep_configs.append(sweep_config)
_pickle_jobs(
tmp_dir=local_tmp_dir,
sweep_configs=sweep_configs, # type: ignore
task_function=launcher.task_function,
singleton_state=Singleton.get_state(),
)
with tempfile.NamedTemporaryFile(suffix=".yaml", delete=False) as f:
with open(f.name, "w") as file:
OmegaConf.save(config=launcher.ray_cfg.cluster,
f=file.name,
resolve=True)
launcher.ray_yaml_path = f.name
log.info(
f"Saving RayClusterConf in a temp yaml file: {launcher.ray_yaml_path}."
)
return launch_jobs(launcher, local_tmp_dir,
Path(HydraConfig.get().sweep.dir))
def launch(
self, job_overrides: Sequence[Sequence[str]], initial_job_idx: int
) -> Sequence[JobReturn]:
# lazy import to ensure plugin discovery remains fast
import submitit
num_jobs = len(job_overrides)
assert num_jobs > 0
# make sure you don't change inplace
queue_parameters = self.queue_parameters.copy()
OmegaConf.set_struct(queue_parameters, True)
if self.queue == "auto":
max_num_timeout = self.queue_parameters.auto.max_num_timeout
with open_dict(queue_parameters):
del queue_parameters.auto["max_num_timeout"]
executor = submitit.AutoExecutor(
folder=self.folder, max_num_timeout=max_num_timeout
)
elif self.queue == "slurm":
max_num_timeout = self.queue_parameters.slurm.max_num_timeout
with open_dict(queue_parameters):
del queue_parameters.slurm["max_num_timeout"]
executor = submitit.SlurmExecutor(
folder=self.folder, max_num_timeout=max_num_timeout
)
elif self.queue == "local":
executor = submitit.LocalExecutor(folder=self.folder)
else:
raise RuntimeError("Unsupported queue type {}".format(self.queue))
executor.update_parameters(**queue_parameters[self.queue])
log.info("Sweep output dir : {}".format(self.config.hydra.sweep.dir))
sweep_dir = Path(str(self.config.hydra.sweep.dir))
sweep_dir.mkdir(parents=True, exist_ok=True)
if "mode" in self.config.hydra.sweep:
mode = int(str(self.config.hydra.sweep.mode), 8)
os.chmod(sweep_dir, mode=mode)
params = []
for idx, overrides in enumerate(job_overrides):
idx = initial_job_idx + idx
lst = " ".join(filter_overrides(overrides))
log.info(f"\t#{idx} : {lst}")
params.append(
(
list(overrides),
"hydra.sweep.dir",
idx,
f"job_id_for_{idx}",
Singleton.get_state(),
)
)
jobs = executor.map_array(self, *zip(*params))
return [j.results()[0] for j in jobs]
def launch(self, job_overrides: Sequence[Sequence[str]],
initial_job_idx: int) -> Sequence[JobReturn]:
# lazy import to ensure plugin discovery remains fast
import submitit
num_jobs = len(job_overrides)
assert num_jobs > 0
# make sure you don't change inplace
queue_parameters = self.queue_parameters.copy()
OmegaConf.set_struct(queue_parameters, True)
executors = {
QueueType.auto: submitit.AutoExecutor,
QueueType.slurm: submitit.SlurmExecutor,
QueueType.local: submitit.LocalExecutor,
}
init_parameters = {
"cluster", "max_num_timeout", "slurm_max_num_timeout"
}
executor = executors[self.queue](
folder=self.folder,
**{
x: y
for x, y in queue_parameters[self.queue.value].items()
if x in init_parameters
},
)
executor.update_parameters(
**{
x: y
for x, y in queue_parameters[self.queue.value].items()
if x not in init_parameters
})
log.info("Submitit '{}' sweep output dir : {}".format(
self.queue.value, self.config.hydra.sweep.dir))
sweep_dir = Path(str(self.config.hydra.sweep.dir))
sweep_dir.mkdir(parents=True, exist_ok=True)
if "mode" in self.config.hydra.sweep:
mode = int(str(self.config.hydra.sweep.mode), 8)
os.chmod(sweep_dir, mode=mode)
params = []
for idx, overrides in enumerate(job_overrides):
idx = initial_job_idx + idx
lst = " ".join(filter_overrides(overrides))
log.info(f"\t#{idx} : {lst}")
params.append((
list(overrides),
"hydra.sweep.dir",
idx,
f"job_id_for_{idx}",
Singleton.get_state(),
))
jobs = executor.map_array(self, *zip(*params))
return [j.results()[0] for j in jobs]
def execute_job(
idx: int,
overrides: Sequence[str],
config_loader: ConfigLoader,
config: DictConfig,
task_function: TaskFunction,
singleton_state: Dict[Any, Any],
cmd_prefix: str,
tsp_prefix: str,
) -> JobReturn:
"""Calls `run_job` in parallel
"""
setup_globals()
Singleton.set_state(singleton_state)
lst = " ".join(overrides)
sweep_config = config_loader.load_sweep_config(config, list(overrides))
with open_dict(sweep_config):
sweep_config.hydra.job.id = "{}_{}".format(sweep_config.hydra.job.name,
idx)
sweep_config.hydra.job.num = idx
HydraConfig.instance().set_config(sweep_config)
def tsp_task_function(task_cfg):
working_dir = os.getcwd()
cmd = f"{cmd_prefix} {lst}"
log.info(f"\t#{idx} : {lst}")
cmd = f"cd {hydra.utils.get_original_cwd()} && {cmd} hydra.run.dir={working_dir}"
job_id = int(subprocess.check_output(cmd, shell=True).rstrip())
log.info(
f"Submitted {idx} to TaskSpooler. View logs: {tsp_prefix} -t {job_id}"
)
return job_id
ret = run_job(
config=sweep_config,
task_function=tsp_task_function,
job_dir_key="hydra.sweep.dir",
job_subdir_key="hydra.sweep.subdir",
)
ret.id = ret.return_value
return ret
def launch(
launcher: RayAWSLauncher,
job_overrides: Sequence[Sequence[str]],
initial_job_idx: int,
) -> Sequence[JobReturn]:
setup_globals()
assert launcher.config is not None
assert launcher.hydra_context is not None
assert launcher.task_function is not None
setup_commands = launcher.env_setup.commands
packages = filter(
lambda x: x[1] is not None, launcher.env_setup.pip_packages.items()
)
with read_write(setup_commands):
setup_commands.extend(
[f"pip install {package}=={version}" for package, version in packages]
)
setup_commands.extend(launcher.ray_cfg.cluster.setup_commands)
with read_write(launcher.ray_cfg.cluster):
launcher.ray_cfg.cluster.setup_commands = setup_commands
configure_log(launcher.config.hydra.hydra_logging, launcher.config.hydra.verbose)
logging_config = OmegaConf.to_container(
launcher.logging, resolve=True, enum_to_str=True
)
sdk.configure_logging(**logging_config)
log.info(f"Ray Launcher is launching {len(job_overrides)} jobs, ")
with tempfile.TemporaryDirectory() as local_tmp_dir:
sweep_configs = []
for idx, overrides in enumerate(job_overrides):
idx = initial_job_idx + idx
ostr = " ".join(filter_overrides(overrides))
log.info(f"\t#{idx} : {ostr}")
sweep_config = launcher.hydra_context.config_loader.load_sweep_config(
launcher.config, list(overrides)
)
with open_dict(sweep_config):
# job.id will be set on the EC2 instance before running the job.
sweep_config.hydra.job.num = idx
sweep_configs.append(sweep_config)
_pickle_jobs(
tmp_dir=local_tmp_dir,
hydra_context=launcher.hydra_context,
sweep_configs=sweep_configs, # type: ignore
task_function=launcher.task_function,
singleton_state=Singleton.get_state(),
)
return launch_jobs(
launcher, local_tmp_dir, Path(launcher.config.hydra.sweep.dir)
)
def launch(
launcher: JoblibLauncher,
job_overrides: Sequence[Sequence[str]],
initial_job_idx: int,
) -> Sequence[JobReturn]:
"""
:param job_overrides: a List of List<String>, where each inner list is the arguments for one job run.
:param initial_job_idx: Initial job idx in batch.
:return: an array of return values from run_job with indexes corresponding to the input list indexes.
"""
setup_globals()
assert launcher.config is not None
assert launcher.task_function is not None
assert launcher.hydra_context is not None
configure_log(launcher.config.hydra.hydra_logging, launcher.config.hydra.verbose)
sweep_dir = Path(str(launcher.config.hydra.sweep.dir))
sweep_dir.mkdir(parents=True, exist_ok=True)
# Joblib's backend is hard-coded to loky since the threading
# backend is incompatible with Hydra
joblib_cfg = launcher.joblib
joblib_cfg["backend"] = "loky"
process_joblib_cfg(joblib_cfg)
log.info(
"Joblib.Parallel({}) is launching {} jobs".format(
",".join([f"{k}={v}" for k, v in joblib_cfg.items()]),
len(job_overrides),
)
)
log.info("Launching jobs, sweep output dir : {}".format(sweep_dir))
for idx, overrides in enumerate(job_overrides):
log.info("\t#{} : {}".format(idx, " ".join(filter_overrides(overrides))))
singleton_state = Singleton.get_state()
runs = Parallel(**joblib_cfg)(
delayed(execute_job)(
initial_job_idx + idx,
overrides,
launcher.hydra_context,
launcher.config,
launcher.task_function,
singleton_state,
)
for idx, overrides in enumerate(job_overrides)
)
assert isinstance(runs, List)
for run in runs:
assert isinstance(run, JobReturn)
return runs
def launch(self, job_overrides: Sequence[Sequence[str]],
initial_job_idx: int) -> Sequence[JobReturn]:
"""
:param job_overrides: a List of List<String>, where each inner list is the arguments for one job run.
:param initial_job_idx: Initial job idx in batch.
:return: an array of return values from run_job with indexes corresponding to the input list indexes.
"""
setup_globals()
assert self.config is not None
assert self.config_loader is not None
assert self.task_function is not None
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
sweep_dir = Path(str(self.config.hydra.sweep.dir))
sweep_dir.mkdir(parents=True, exist_ok=True)
log.info(
f"TaskSpooler Launcher is launching {len(job_overrides)} jobs locally"
)
log.info(f"Sweep output dir : {sweep_dir}")
runs = []
singleton_state = Singleton.get_state()
for idx, overrides in enumerate(job_overrides):
overrides = list(overrides)
overrides.extend(self.hydra_overrides)
overrides = tuple(overrides)
ret = execute_job(
initial_job_idx + idx,
overrides,
self.config_loader,
self.config,
self.task_function,
singleton_state,
self.cmd_prefix,
self.tsp_prefix,
)
runs.append(ret)
time.sleep(self.time_between_submit)
assert isinstance(runs, List)
for run in runs:
assert isinstance(run, JobReturn)
if self.tail_jobs:
Parallel(n_jobs=len(job_overrides), backend='threading')(
delayed(self.tail_job)(run.return_value) for run in runs)
return runs
def launch(
launcher: RayLocalLauncher,
job_overrides: Sequence[Sequence[str]],
initial_job_idx: int,
) -> Sequence[JobReturn]:
setup_globals()
assert launcher.config is not None
assert launcher.config_loader is not None
assert launcher.task_function is not None
configure_log(launcher.config.hydra.hydra_logging, launcher.config.hydra.verbose)
sweep_dir = Path(str(launcher.config.hydra.sweep.dir))
sweep_dir.mkdir(parents=True, exist_ok=True)
log.info(
f"Ray Launcher is launching {len(job_overrides)} jobs, "
f"sweep output dir: {sweep_dir}"
)
start_ray(launcher.ray_init_cfg)
runs = []
for idx, overrides in enumerate(job_overrides):
idx = initial_job_idx + idx
ostr = " ".join(filter_overrides(overrides))
log.info(f"\t#{idx} : {ostr}")
sweep_config = launcher.config_loader.load_sweep_config(
launcher.config, list(overrides)
)
with open_dict(sweep_config):
# This typically coming from the underlying scheduler (SLURM_JOB_ID for instance)
# In that case, it will not be available here because we are still in the main process.
# but instead should be populated remotely before calling the task_function.
sweep_config.hydra.job.id = f"job_id_for_{idx}"
sweep_config.hydra.job.num = idx
ray_obj = launch_job_on_ray(
launcher.ray_remote_cfg,
sweep_config,
launcher.task_function,
Singleton.get_state(),
)
runs.append(ray_obj)
return [ray.get(run) for run in runs]
def launch(self,
job_overrides: Sequence[Sequence[str]]) -> Sequence[JobReturn]:
"""
:param job_overrides: a List of List<String>, where each inner list is the arguments for one job run.
:return: an array of return values from run_job with indexes corresponding to the input list indexes.
"""
setup_globals()
assert self.config is not None
assert self.config_loader is not None
assert self.task_function is not None
configure_log(self.config.hydra.hydra_logging,
self.config.hydra.verbose)
sweep_dir = Path(str(self.config.hydra.sweep.dir))
sweep_dir.mkdir(parents=True, exist_ok=True)
log.info("Joblib.Parallel({}) is launching {} jobs".format(
",".join([f"{k}={v}" for k, v in self.joblib.items()]),
len(job_overrides),
))
log.info("Launching jobs, sweep output dir : {}".format(sweep_dir))
singleton_state = Singleton.get_state()
for idx, overrides in enumerate(job_overrides):
log.info("\t#{} : {}".format(idx, " ".join(
filter_overrides(overrides))))
runs = Parallel(**self.joblib)(delayed(execute_job)(
idx,
overrides,
self.config_loader,
self.config,
self.task_function,
singleton_state,
) for idx, overrides in enumerate(job_overrides))
assert isinstance(runs, List)
for run in runs:
assert isinstance(run, JobReturn)
return runs
def launch(
launcher: RQLauncher,
job_overrides: Sequence[Sequence[str]],
initial_job_idx: int,
) -> JobReturn:
"""
:param job_overrides: a List of List<String>, where each inner list is the arguments for one job run.
:param initial_job_idx: Initial job idx in batch.
:return: an array of return values from run_job with indexes corresponding to the input list indexes.
"""
setup_globals()
assert launcher.config is not None
assert launcher.config_loader is not None
assert launcher.task_function is not None
configure_log(launcher.config.hydra.hydra_logging,
launcher.config.hydra.verbose)
sweep_dir = Path(str(launcher.config.hydra.sweep.dir))
sweep_dir.mkdir(parents=True, exist_ok=True)
# RQ configuration
rq_cfg = launcher.rq
# Redis configuration
is_async = not rq_cfg.redis.mock
if is_async:
connection = Redis(
host=rq_cfg.redis.host,
port=rq_cfg.redis.port,
db=rq_cfg.redis.db,
password=rq_cfg.redis.password,
)
else:
log.info("Running in synchronous mode")
connection = FakeStrictRedis()
queue = Queue(
name=rq_cfg.queue,
connection=connection,
is_async=is_async,
serializer=cloudpickle,
)
# Enqueue jobs
jobs = []
singleton_state = Singleton.get_state()
log.info(
f"RQ Launcher is enqueuing {len(job_overrides)} job(s) in queue : {rq_cfg.queue}"
)
log.info("Sweep output dir : {}".format(sweep_dir))
if not sweep_dir.is_absolute():
log.warn(
"Using relative sweep dir: Please be aware that dir will be relative to where workers are started from."
)
for idx, overrides in enumerate(job_overrides):
description = " ".join(filter_overrides(overrides))
enqueue_keywords = OmegaConf.to_container(rq_cfg.enqueue, resolve=True)
if enqueue_keywords["job_timeout"] is None:
enqueue_keywords["job_timeout"] = -1
if enqueue_keywords["result_ttl"] is None:
enqueue_keywords["result_ttl"] = -1
if enqueue_keywords["failure_ttl"] is None:
enqueue_keywords["failure_ttl"] = -1
if enqueue_keywords["job_id"] is None:
enqueue_keywords["job_id"] = str(uuid.uuid4())
if enqueue_keywords["description"] is None:
enqueue_keywords["description"] = description
sweep_config = launcher.config_loader.load_sweep_config(
launcher.config, list(overrides))
with open_dict(sweep_config):
sweep_config.hydra.job.id = enqueue_keywords["job_id"]
sweep_config.hydra.job.num = initial_job_idx + idx
job = queue.enqueue(
execute_job,
sweep_config=sweep_config,
task_function=launcher.task_function,
singleton_state=singleton_state,
**enqueue_keywords,
)
jobs.append(job)
log.info(f"Enqueued {job.get_id()}")
log.info(f"\t#{idx+1} : {description}")
log.info("Finished enqueuing")
if rq_cfg.stop_after_enqueue:
raise StopAfterEnqueue
log.info(f"Polling job statuses every {rq_cfg.wait_polling} sec")
while True:
job_ids_done = [
job.get_id() for job in jobs
if job.get_status() in ["finished", "failed"]
]
if len(job_ids_done) == len(jobs):
break
else:
time.sleep(rq_cfg.wait_polling)
runs = []
for job in jobs:
result = job.result if job.result is not None else None
runs.append(result)
assert isinstance(runs, List)
for run in runs:
assert isinstance(run, JobReturn)
return runs
def instance(*args: Any, **kwargs: Any) -> "GlobalHydra":
return Singleton.instance(GlobalHydra, *args, **kwargs) # type: ignore
def test_singleton_get_state(hydra_restore_singletons: Any) -> None:
s = Singleton.get_state()
assert Plugins not in s["instances"]
assert Plugins in Singleton._instances
Singleton.set_state(s)
assert Plugins in Singleton._instances
from hydra._internal.core_plugins.file_config_source import FileConfigSource
from hydra._internal.core_plugins.importlib_resources_config_source import (
ImportlibResourcesConfigSource, )
from hydra._internal.core_plugins.structured_config_source import StructuredConfigSource
from hydra.core.default_element import GroupDefault, InputDefault
from hydra.core.plugins import Plugins
from hydra.core.singleton import Singleton
from hydra.plugins.config_source import ConfigSource
from hydra.test_utils.config_source_common_tests import ConfigSourceTestSuite
from hydra.test_utils.test_utils import chdir_hydra_root
chdir_hydra_root()
# Manually save and restore singletons to work around an issue with things added to the config store via importing.
# restoring is done in test_restore_singleton_state_hack(), which must be the last test in this file.
state = copy.deepcopy(Singleton.get_state())
@pytest.mark.parametrize(
"type_, path",
[
pytest.param(
FileConfigSource,
"file://tests/test_apps/config_source_test/dir",
id="FileConfigSource",
),
pytest.param(
ImportlibResourcesConfigSource,
"pkg://tests.test_apps.config_source_test.dir",
id="ImportlibResourcesConfigSource",
),
def instance(*args: Any, **kwargs: Any) -> "SourcesRegistry":
return Singleton.instance(SourcesRegistry, *args,
**kwargs) # type: ignore
def instance(*args: Any, **kwargs: Any) -> "Plugins":
ret = Singleton.instance(Plugins, *args, **kwargs)
assert isinstance(ret, Plugins)
return ret
def instance(*args: Any, **kwargs: Any) -> "ConfigStore":
return Singleton.instance(ConfigStore, *args, **kwargs) # type: ignore
def instance(*args: Any, **kwargs: Any) -> "HydraConfig":
return Singleton.instance(HydraConfig, *args, **kwargs) # type: ignore
