def launch(self, job_overrides): setup_globals() configure_log(self.config.hydra.hydra_logging, self.config.hydra.verbose) sweep_dir = self.config.hydra.sweep.dir Path(str(sweep_dir)).mkdir(parents=True, exist_ok=True) log.info("Launching {} jobs locally".format(len(job_overrides))) runs = [] for idx, overrides in enumerate(job_overrides): log.info("\t#{} : {}".format(idx, " ".join( filter_overrides(overrides)))) sweep_config = self.config_loader.load_sweep_config( self.config, list(overrides)) with open_dict(sweep_config): sweep_config.hydra.job.id = idx sweep_config.hydra.job.num = idx HydraConfig().set_config(sweep_config) ret = run_job( config=sweep_config, task_function=self.task_function, job_dir_key="hydra.sweep.dir", job_subdir_key="hydra.sweep.subdir", ) runs.append(ret) configure_log(self.config.hydra.hydra_logging, self.config.hydra.verbose) return runs
def _run(self, config_file, task_function, overrides): if is_multirun: raise RuntimeError( "Performing single run with multiple overrides in hydra " "(use '-m' for multirun): %s" % str(overrides)) cfg = self.compose_config(config_file=config_file, overrides=overrides, strict=strict, with_log_configuration=True) HydraConfig().set_config(cfg) return run_job( config=cfg, task_function=task_function, job_dir_key="hydra.run.dir", job_subdir_key=None, )
def run( self, config_file: Optional[str], task_function: TaskFunction, overrides: List[str], ) -> JobReturn: cfg = self.compose_config(config_file=config_file, overrides=overrides, with_log_configuration=True) HydraConfig().set_config(cfg) return run_job( config=cfg, task_function=task_function, job_dir_key="hydra.run.dir", job_subdir_key=None, )
def launch(self, job_overrides: Sequence[Sequence[str]]) -> Sequence[JobReturn]: """ :param job_overrides: a List of List<String>, where each inner list is the arguments for one job run. :return: an array of return values from run_job with indexes corresponding to the input list indexes. """ setup_globals() assert self.config is not None assert self.config_loader is not None assert self.task_function is not None configure_log(self.config.hydra.hydra_logging, self.config.hydra.verbose) sweep_dir = Path(str(self.config.hydra.sweep.dir)) sweep_dir.mkdir(parents=True, exist_ok=True) log.info( "Example Launcher(foo={}, bar={}) is launching {} jobs locally". format(self.foo, self.bar, len(job_overrides))) log.info("Sweep output dir : {}".format(sweep_dir)) runs = [] for idx, overrides in enumerate(job_overrides): log.info("\t#{} : {}".format(idx, " ".join( filter_overrides(overrides)))) sweep_config = self.config_loader.load_sweep_config( self.config, list(overrides)) with open_dict(sweep_config): # This typically coming from the underlying scheduler (SLURM_JOB_ID for instance) # In that case, it will not be available here because we are still in the main process. # but instead should be populated remotely before calling the task_function. sweep_config.hydra.job.id = "job_id_for_{}".format(idx) sweep_config.hydra.job.num = idx HydraConfig().set_config(sweep_config) ret = run_job( config=sweep_config, task_function=self.task_function, job_dir_key="hydra.sweep.dir", job_subdir_key="hydra.sweep.subdir", ) runs.append(ret) configure_log(self.config.hydra.hydra_logging, self.config.hydra.verbose) return runs
def run_task(job): idx, overrides = job LOGGER.info("\t#{} : {}".format( idx, " ".join(filter_overrides(overrides)))) sweep_config = self.config_loader.load_sweep_config( self.config, list(overrides)) with open_dict(sweep_config): # id is concatenated overrides here sweep_config.hydra.job.id = '_'.join(sorted(overrides)) sweep_config.hydra.job.num = idx HydraConfig().set_config(sweep_config) ret = run_job( config=sweep_config, task_function=self.task_function, job_dir_key="hydra.sweep.dir", job_subdir_key="hydra.sweep.subdir", ) configure_log(self.config.hydra.hydra_logging, self.config.hydra.verbose) return (idx, ret)
def launch(*args, **kwargs): setup_globals() run_job(*args, **kwargs)