def compose_config( self, config_file: Optional[str], overrides: List[str], strict: Optional[bool] = None, with_log_configuration: bool = False, ) -> DictConfig: """ :param self: :param config_file: :param overrides: :param with_log_configuration: True to configure logging subsystem from the loaded config :param strict: None for default behavior (default to true for config file, false if no config file). otherwise forces specific behavior. :return: """ cfg = self.config_loader.load_configuration(config_file=config_file, overrides=overrides, strict=strict) with open_dict(cfg): from .. import __version__ cfg.hydra.runtime.version = __version__ cfg.hydra.runtime.cwd = os.getcwd() if with_log_configuration: configure_log(cfg.hydra.hydra_logging, cfg.hydra.verbose) global log log = logging.getLogger(__name__) self._print_debug_info() return cfg
def launch(self, job_overrides): setup_globals() configure_log(self.config.hydra.hydra_logging, self.config.hydra.verbose) sweep_dir = self.config.hydra.sweep.dir Path(str(sweep_dir)).mkdir(parents=True, exist_ok=True) log.info("Launching {} jobs locally".format(len(job_overrides))) runs = [] for idx, overrides in enumerate(job_overrides): log.info("\t#{} : {}".format(idx, " ".join( filter_overrides(overrides)))) sweep_config = self.config_loader.load_sweep_config( self.config, list(overrides)) with open_dict(sweep_config): sweep_config.hydra.job.id = idx sweep_config.hydra.job.num = idx HydraConfig().set_config(sweep_config) ret = run_job( config=sweep_config, task_function=self.task_function, job_dir_key="hydra.sweep.dir", job_subdir_key="hydra.sweep.subdir", ) runs.append(ret) configure_log(self.config.hydra.hydra_logging, self.config.hydra.verbose) return runs
def launch(self, job_overrides): """ :param job_overrides: a List of List<String>, where each inner list is the arguments for one job run. :return: an array of return values from run_job with indexes corresponding to the input list indexes. """ configure_log(self.config.hydra.hydra_logging, self.config.hydra.verbose) sweep_dir = Path(str(self.config.hydra.sweep.dir)) sweep_dir.mkdir(parents=True, exist_ok=True) log.info("Ray Launcher is launching {} jobs locally".format( len(job_overrides))) log.info("Sweep output dir : {}".format(sweep_dir)) runs = [] for idx, overrides in enumerate(job_overrides): log.info("\t#{} : {}".format(idx, " ".join( filter_overrides(overrides)))) sweep_config = self.config_loader.load_sweep_config( self.config, list(overrides)) with open_dict(sweep_config): # This typically coming from the underlying scheduler (SLURM_JOB_ID for instance) # In that case, it will not be available here because we are still in the main process. # but instead should be populated remotely before calling the task_function. sweep_config.hydra.job.id = idx sweep_config.hydra.job.num = idx HydraConfig().set_config(sweep_config) ray_remote_cfg = get_key(self.config, 'ray.remote') if ray_remote_cfg: run_job_ray = ray.remote(**ray_remote_cfg)(launch) else: run_job_ray = ray.remote(launch) ret = run_job_ray.remote( config=sweep_config, task_function=self.task_function, job_dir_key="hydra.sweep.dir", job_subdir_key="hydra.sweep.subdir", ) runs.append(ret) configure_log(self.config.hydra.hydra_logging, self.config.hydra.verbose) return [ray.get(run) for run in runs]
def launch(self, job_overrides: Sequence[Sequence[str]]) -> Sequence[JobReturn]: """ :param job_overrides: a List of List<String>, where each inner list is the arguments for one job run. :return: an array of return values from run_job with indexes corresponding to the input list indexes. """ setup_globals() assert self.config is not None assert self.config_loader is not None assert self.task_function is not None configure_log(self.config.hydra.hydra_logging, self.config.hydra.verbose) sweep_dir = Path(str(self.config.hydra.sweep.dir)) sweep_dir.mkdir(parents=True, exist_ok=True) log.info( "Example Launcher(foo={}, bar={}) is launching {} jobs locally". format(self.foo, self.bar, len(job_overrides))) log.info("Sweep output dir : {}".format(sweep_dir)) runs = [] for idx, overrides in enumerate(job_overrides): log.info("\t#{} : {}".format(idx, " ".join( filter_overrides(overrides)))) sweep_config = self.config_loader.load_sweep_config( self.config, list(overrides)) with open_dict(sweep_config): # This typically coming from the underlying scheduler (SLURM_JOB_ID for instance) # In that case, it will not be available here because we are still in the main process. # but instead should be populated remotely before calling the task_function. sweep_config.hydra.job.id = "job_id_for_{}".format(idx) sweep_config.hydra.job.num = idx HydraConfig().set_config(sweep_config) ret = run_job( config=sweep_config, task_function=self.task_function, job_dir_key="hydra.sweep.dir", job_subdir_key="hydra.sweep.subdir", ) runs.append(ret) configure_log(self.config.hydra.hydra_logging, self.config.hydra.verbose) return runs
def run_task(job): idx, overrides = job LOGGER.info("\t#{} : {}".format( idx, " ".join(filter_overrides(overrides)))) sweep_config = self.config_loader.load_sweep_config( self.config, list(overrides)) with open_dict(sweep_config): # id is concatenated overrides here sweep_config.hydra.job.id = '_'.join(sorted(overrides)) sweep_config.hydra.job.num = idx HydraConfig().set_config(sweep_config) ret = run_job( config=sweep_config, task_function=self.task_function, job_dir_key="hydra.sweep.dir", job_subdir_key="hydra.sweep.subdir", ) configure_log(self.config.hydra.hydra_logging, self.config.hydra.verbose) return (idx, ret)
def launch(self, job_overrides): setup_globals() configure_log(self.config.hydra.hydra_logging, self.config.hydra.verbose) sweep_dir = self.config.hydra.sweep.dir Path(str(sweep_dir)).mkdir(parents=True, exist_ok=True) LOGGER.info("Launching {} jobs locally".format(len(job_overrides))) def run_task(job): idx, overrides = job LOGGER.info("\t#{} : {}".format( idx, " ".join(filter_overrides(overrides)))) sweep_config = self.config_loader.load_sweep_config( self.config, list(overrides)) with open_dict(sweep_config): # id is concatenated overrides here sweep_config.hydra.job.id = '_'.join(sorted(overrides)) sweep_config.hydra.job.num = idx HydraConfig().set_config(sweep_config) ret = run_job( config=sweep_config, task_function=self.task_function, job_dir_key="hydra.sweep.dir", job_subdir_key="hydra.sweep.subdir", ) configure_log(self.config.hydra.hydra_logging, self.config.hydra.verbose) return (idx, ret) if self.ncpu > 1: jobs = list(enumerate(job_overrides)) runs = sorted([ ret for ret in MPI( jobs=jobs, func=run_task, ncpu=int(self.ncpu), batch=1) ]) runs = [i[1] for i in runs] else: runs = [run_task(job)[1] for job in enumerate(job_overrides)] return runs
from hydra.experimental import initialize, compose from hydra.plugins.common.utils import configure_log from core_lib.helpers.generate_data import generate_email, generate_random_string from examples.combined_core_lib.core_lib.combined_core_lib import CombineCoreLib import pymysql # # Docker # from core_lib.client.solr_client import SolrClient from examples.test_core_lib.core_lib.data_layers.data.db.user import User pymysql.install_as_MySQLdb() configure_log(None, True) current_path = os.path.join(os.path.dirname(os.path.realpath(__file__))) test_output_path = os.path.normpath(os.path.join(current_path, 'test_output')) example_path = os.path.normpath(os.path.join(current_path, '../', 'examples')) combined_core_lib_path = os.path.join(example_path, 'combined_core_lib') from pathlib import Path env_path = Path(os.path.join(current_path, 'test_data', 'load_env.env')) load_dotenv(dotenv_path=env_path) # # Config # config_directory = os.path.join(combined_core_lib_path, 'config')