def execute_job(
idx: int,
overrides: Sequence[str],
config_loader: ConfigLoader,
config: DictConfig,
task_function: TaskFunction,
singleton_state: Dict[Any, Any],
) -> JobReturn:
"""Calls `run_job` in parallel
"""
setup_globals()
Singleton.set_state(singleton_state)
sweep_config = config_loader.load_sweep_config(config, list(overrides))
with open_dict(sweep_config):
sweep_config.hydra.job.id = "{}_{}".format(sweep_config.hydra.job.name,
idx)
sweep_config.hydra.job.num = idx
HydraConfig.instance().set_config(sweep_config)
ret = run_job(
config=sweep_config,
task_function=task_function,
job_dir_key="hydra.sweep.dir",
job_subdir_key="hydra.sweep.subdir",
)
return ret
def dispatch_job(
idx: int,
overrides: Sequence[str],
config_loader: ConfigLoader,
config: DictConfig,
task_function: TaskFunction,
singleton_state: Dict[Any, Any],
) -> JobReturn:
"""Calls `run_job` in parallel
Note that Joblib's default backend runs isolated Python processes, see
https://joblib.readthedocs.io/en/latest/parallel.html#shared-memory-semantics
"""
setup_globals()
Singleton.set_state(singleton_state)
log.info("\t#{} : {}".format(idx, " ".join(filter_overrides(overrides))))
sweep_config = config_loader.load_sweep_config(config, list(overrides))
with open_dict(sweep_config):
sweep_config.hydra.job.id = "{}_{}".format(sweep_config.hydra.job.name,
idx)
sweep_config.hydra.job.num = idx
HydraConfig.instance().set_config(sweep_config)
ret = run_job(
config=sweep_config,
task_function=task_function,
job_dir_key="hydra.sweep.dir",
job_subdir_key="hydra.sweep.subdir",
)
return ret
def __init__(self, task_name: str, config_loader: ConfigLoader) -> None:
"""
:param task_name: task name
:param config_loader: config loader
"""
setup_globals()
self.config_loader = config_loader
for source in config_loader.get_sources():
# if specified, make sure main config search path exists
if source.provider == "main":
if not source.exists(""):
raise MissingConfigException(
missing_cfg_file=source.path,
message=f"Primary config dir not found: {source}",
)
JobRuntime().set("name", task_name)
def execute_job(
idx: int,
overrides: Sequence[str],
config_loader: ConfigLoader,
config: DictConfig,
task_function: TaskFunction,
singleton_state: Dict[Any, Any],
cmd_prefix: str,
tsp_prefix: str,
) -> JobReturn:
"""Calls `run_job` in parallel
"""
setup_globals()
Singleton.set_state(singleton_state)
lst = " ".join(overrides)
sweep_config = config_loader.load_sweep_config(config, list(overrides))
with open_dict(sweep_config):
sweep_config.hydra.job.id = "{}_{}".format(sweep_config.hydra.job.name,
idx)
sweep_config.hydra.job.num = idx
HydraConfig.instance().set_config(sweep_config)
def tsp_task_function(task_cfg):
working_dir = os.getcwd()
cmd = f"{cmd_prefix} {lst}"
log.info(f"\t#{idx} : {lst}")
cmd = f"cd {hydra.utils.get_original_cwd()} && {cmd} hydra.run.dir={working_dir}"
job_id = int(subprocess.check_output(cmd, shell=True).rstrip())
log.info(
f"Submitted {idx} to TaskSpooler. View logs: {tsp_prefix} -t {job_id}"
)
return job_id
ret = run_job(
config=sweep_config,
task_function=tsp_task_function,
job_dir_key="hydra.sweep.dir",
job_subdir_key="hydra.sweep.subdir",
)
ret.id = ret.return_value
return ret