def execute_job( idx: int, overrides: Sequence[str], config_loader: ConfigLoader, config: DictConfig, task_function: TaskFunction, singleton_state: Dict[Any, Any], ) -> JobReturn: """Calls `run_job` in parallel """ setup_globals() Singleton.set_state(singleton_state) sweep_config = config_loader.load_sweep_config(config, list(overrides)) with open_dict(sweep_config): sweep_config.hydra.job.id = "{}_{}".format(sweep_config.hydra.job.name, idx) sweep_config.hydra.job.num = idx HydraConfig.instance().set_config(sweep_config) ret = run_job( config=sweep_config, task_function=task_function, job_dir_key="hydra.sweep.dir", job_subdir_key="hydra.sweep.subdir", ) return ret
def dispatch_job( idx: int, overrides: Sequence[str], config_loader: ConfigLoader, config: DictConfig, task_function: TaskFunction, singleton_state: Dict[Any, Any], ) -> JobReturn: """Calls `run_job` in parallel Note that Joblib's default backend runs isolated Python processes, see https://joblib.readthedocs.io/en/latest/parallel.html#shared-memory-semantics """ setup_globals() Singleton.set_state(singleton_state) log.info("\t#{} : {}".format(idx, " ".join(filter_overrides(overrides)))) sweep_config = config_loader.load_sweep_config(config, list(overrides)) with open_dict(sweep_config): sweep_config.hydra.job.id = "{}_{}".format(sweep_config.hydra.job.name, idx) sweep_config.hydra.job.num = idx HydraConfig.instance().set_config(sweep_config) ret = run_job( config=sweep_config, task_function=task_function, job_dir_key="hydra.sweep.dir", job_subdir_key="hydra.sweep.subdir", ) return ret
def __init__(self, task_name: str, config_loader: ConfigLoader) -> None: """ :param task_name: task name :param config_loader: config loader """ setup_globals() self.config_loader = config_loader for source in config_loader.get_sources(): # if specified, make sure main config search path exists if source.provider == "main": if not source.exists(""): raise MissingConfigException( missing_cfg_file=source.path, message=f"Primary config dir not found: {source}", ) JobRuntime().set("name", task_name)
def execute_job( idx: int, overrides: Sequence[str], config_loader: ConfigLoader, config: DictConfig, task_function: TaskFunction, singleton_state: Dict[Any, Any], cmd_prefix: str, tsp_prefix: str, ) -> JobReturn: """Calls `run_job` in parallel """ setup_globals() Singleton.set_state(singleton_state) lst = " ".join(overrides) sweep_config = config_loader.load_sweep_config(config, list(overrides)) with open_dict(sweep_config): sweep_config.hydra.job.id = "{}_{}".format(sweep_config.hydra.job.name, idx) sweep_config.hydra.job.num = idx HydraConfig.instance().set_config(sweep_config) def tsp_task_function(task_cfg): working_dir = os.getcwd() cmd = f"{cmd_prefix} {lst}" log.info(f"\t#{idx} : {lst}") cmd = f"cd {hydra.utils.get_original_cwd()} && {cmd} hydra.run.dir={working_dir}" job_id = int(subprocess.check_output(cmd, shell=True).rstrip()) log.info( f"Submitted {idx} to TaskSpooler. View logs: {tsp_prefix} -t {job_id}" ) return job_id ret = run_job( config=sweep_config, task_function=tsp_task_function, job_dir_key="hydra.sweep.dir", job_subdir_key="hydra.sweep.subdir", ) ret.id = ret.return_value return ret