def shell_completion(self, config_name: Optional[str], overrides: List[str]) -> None: subcommands = ["install", "uninstall", "query"] arguments = OmegaConf.from_dotlist(overrides) num_commands = sum(1 for key in subcommands if arguments[key] is not None) if num_commands != 1: raise ValueError( f"Expecting one subcommand from {subcommands} to be set") shell_to_plugin = self.get_shell_to_plugin_map(self.config_loader) def find_plugin(cmd: str) -> CompletionPlugin: if cmd not in shell_to_plugin: lst = "\n".join(["\t" + x for x in shell_to_plugin.keys()]) raise ValueError( f"No completion plugin for '{cmd}' found, available : \n{lst}" ) return shell_to_plugin[cmd][0] if arguments.install is not None: plugin = find_plugin(arguments.install) plugin.install() elif arguments.uninstall is not None: plugin = find_plugin(arguments.uninstall) plugin.uninstall() elif arguments.query is not None: plugin = find_plugin(arguments.query) plugin.query(config_name=config_name)
def sweep_2_jobs(sweep_runner, overrides): """ Runs a sweep with two jobs """ overrides.append("a=0,1") sweep = sweep_runner( calling_file=None, calling_module="hydra.test_utils.a_module", config_path="configs/compose.yaml", overrides=overrides, ) base = OmegaConf.create({"foo": 10, "bar": 100, "a": 0}) with sweep: temp_dir = Path(sweep.temp_dir) assert len(sweep.returns[0]) == 2 for i in range(2): job_ret = sweep.returns[0][i] expected_conf = OmegaConf.merge( base, OmegaConf.from_dotlist(job_ret.overrides) ) assert job_ret.overrides == ["a={}".format(i)] assert job_ret.cfg == expected_conf assert job_ret.hydra_cfg.hydra.job.name == "a_module" verify_dir_outputs(job_ret, job_ret.overrides) path = temp_dir / str(i) assert path.exists(), "'{}' does not exist, dirs: {}".format( path, [x for x in temp_dir.iterdir() if x.is_dir()] )
def shell_completion(self, overrides): subcommands = ["install", "uninstall", "query"] arguments = OmegaConf.from_dotlist(overrides) num_commands = sum(1 for key in subcommands if arguments[key] is not None) if num_commands != 1: raise ValueError( "Expecting one subcommand from {} to be set".format( subcommands)) shell_to_plugin = self.get_shell_to_plugin_map(self.config_loader) def find_plugin(cmd): if cmd not in shell_to_plugin: raise ValueError( "No completion plugin for '{}' found, available : \n{}". format( cmd, "\n".join(["\t" + x for x in shell_to_plugin.keys()]))) return shell_to_plugin[cmd][0] if arguments.install is not None: plugin = find_plugin(arguments.install) plugin.install() elif arguments.uninstall is not None: plugin = find_plugin(arguments.uninstall) plugin.uninstall() elif arguments.query is not None: plugin = find_plugin(arguments.query) plugin.query()
def process_config_arguments(config_file=None, config_override=None, output_dir=None): """Process configuration arguments. Args: config_file (str, optional): Path to SuperBench config file. Defaults to None. config_override (str, optional): Extra arguments to override config_file, following [Hydra syntax](https://hydra.cc/docs/advanced/override_grammar/basic). Defaults to None. output_dir (str, optional): Path to output directory. Defaults to None. Returns: DictConfig: SuperBench config object. str: Dir for output. Raises: CLIError: If input arguments are invalid. """ config_file = check_argument_file('config_file', config_file) # SuperBench config sb_config = get_sb_config(config_file) if config_override: sb_config_from_override = OmegaConf.from_dotlist(config_override) sb_config = OmegaConf.merge(sb_config, sb_config_from_override) # Create output directory sb_output_dir = create_sb_output_dir(output_dir) return sb_config, sb_output_dir
def not_sweeping_hydra_overrides( sweep_runner: TSweepRunner, overrides: List[str], task_function: Optional[TaskFunction], ) -> None: """ Runs a sweep with two jobs """ overrides.extend(["a=0,1", "hydra.verbose=true,false"]) sweep = sweep_runner( calling_file=None, calling_module="hydra.test_utils.a_module", task_function=task_function, config_path="configs", config_name="compose.yaml", overrides=overrides, strict=None, ) base = OmegaConf.create({"foo": 10, "bar": 100}) with sweep: assert sweep.returns is not None assert len(sweep.returns[0]) == 2 for i in range(2): job_ret = sweep.returns[0][i] expected_conf = OmegaConf.merge( base, OmegaConf.from_dotlist(job_ret.overrides) ) assert job_ret.overrides == [f"a={i}"] assert job_ret.cfg == expected_conf verify_dir_outputs(job_ret, job_ret.overrides)
def test_readonly_from_cli() -> None: c = OmegaConf.create({"foo": {"bar": [1]}}) assert isinstance(c, DictConfig) OmegaConf.set_readonly(c, True) cli = OmegaConf.from_dotlist(["foo.bar=[2]"]) with raises(ReadonlyConfigError, match="foo.bar"): OmegaConf.merge(c, cli)
def main(): args = get_args() weight_file = args.weight_file margin = args.margin image_dir = args.image_dir if not weight_file: weight_file = get_file("EfficientNetB3_224_weights.11-3.44.hdf5", pretrained_model, cache_subdir="pretrained_models", file_hash=modhash, cache_dir=str(Path(__file__).resolve().parent)) # for face detection detector = dlib.get_frontal_face_detector() # load model and weights model_name, img_size = Path(weight_file).stem.split("_")[:2] img_size = int(img_size) cfg = OmegaConf.from_dotlist([f"model.model_name={model_name}", f"model.img_size={img_size}"]) model = get_model(cfg) model.load_weights(weight_file) image_generator = yield_images_from_dir(image_dir) if image_dir else yield_images() for img in image_generator: input_img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) img_h, img_w, _ = np.shape(input_img) # detect faces using dlib detector detected = detector(input_img, 1) faces = np.empty((len(detected), img_size, img_size, 3)) if len(detected) > 0: for i, d in enumerate(detected): x1, y1, x2, y2, w, h = d.left(), d.top(), d.right() + 1, d.bottom() + 1, d.width(), d.height() xw1 = max(int(x1 - margin * w), 0) yw1 = max(int(y1 - margin * h), 0) xw2 = min(int(x2 + margin * w), img_w - 1) yw2 = min(int(y2 + margin * h), img_h - 1) faces[i] = cv2.resize(img[yw1:yw2 + 1, xw1:xw2 + 1], (img_size, img_size)) # if you want to capture roi then uncomment below line cv2.imwrite(f"results/face-{i}.jpg", faces[i]) # predict ages and genders of the detected faces results = model.predict(faces) predicted_genders = results[0] ages = np.arange(0, 101).reshape(101, 1) predicted_ages = results[1].dot(ages).flatten() # draw results for i, d in enumerate(detected): label = "{}, {}".format(int(predicted_ages[i]), "M" if predicted_genders[i][0] < 0.5 else "F") cv2.rectangle(img, (d.left(), d.top()), (d.right(), d.bottom()), (255, 0, 0), 2) draw_label(img, (d.left(), d.top()), label) cv2.imshow("result", img) key = cv2.waitKey(-1) if image_dir else cv2.waitKey(30) if key == 27: # ESC break
def test_readonly_from_cli() -> None: c = OmegaConf.create({"foo": {"bar": [1]}}) assert isinstance(c, DictConfig) OmegaConf.set_readonly(c, True) cli = OmegaConf.from_dotlist(["foo.bar=[2]"]) cfg2 = OmegaConf.merge(c, cli) assert OmegaConf.is_readonly(c) assert OmegaConf.is_readonly(cfg2)
def test_deepcopy_and_merge_and_flags() -> None: c1 = OmegaConf.create( {"dataset": {"name": "imagenet", "path": "/datasets/imagenet"}, "defaults": []} ) OmegaConf.set_struct(c1, True) c2 = copy.deepcopy(c1) with raises(ConfigKeyError): OmegaConf.merge(c2, OmegaConf.from_dotlist(["dataset.bad_key=yes"]))
def test_timestep_operation(self, mock_data_manager): start_dt = datetime.now() training_interval = timedelta(days=1) mock_env = mock.MagicMock() conf = OmegaConf.from_dotlist( ["project.tensorboard_path=/tmp/test_tb/"]) mock_app = mock.MagicMock() mock_app.timing_data = TimingData(start_dt=start_dt, training_interval=training_interval) mock_app.env = mock_env mock_app.config = conf mock_env.env_id_cols = ["env_id_1", "env_id_2"] mock_env.ts_id_col = "ts_1" mock_env.obs_cols = ["obs_1", "obs_2"] mock_timestep = [{ "env_id_1": 1, "env_id_2": 2, "ts_1": 1, "discount": 1.0, "obs_1": 1, "obs_2": 2, "action": 1, "reward": 0.0, "step_type": 0 }] mock_timestep_df = self.spark.createDataFrame(mock_timestep) metadata_dict = {"available_data": [("test_data", 0)]} mock_env.build_time_steps = mock.MagicMock( return_value=mock_timestep_df) mock_data_manager.get_latest.return_value = metadata_dict run_id = 1 operation = BuildTimestepOperation(mock_app, mock_data_manager) operation.run(run_id) mock_data_manager.get_latest.assert_any_call(DATANAME.RUN_CONTEXT, run_id) expected_start_dt = start_dt expected_end_dt = start_dt + training_interval mock_env.build_time_steps.assert_called_with(expected_start_dt, expected_end_dt) expected_metadata = { "available_data": [("test_data", 0), (DATANAME.TIMESTEP, run_id)] } calls = [ call(mock_timestep_df, DATANAME.TIMESTEP, run_id), call(expected_metadata, DATANAME.RUN_CONTEXT, run_id) ] mock_data_manager.store.assert_has_calls(calls, any_order=False)
def temp_override(self, override_dict): old_config = self.config try: dotlist = [f"{k}={v}" for k, v in override_dict.items()] update = OmegaConf.from_dotlist(dotlist) self.config = OmegaConf.merge(self.config, update) yield finally: self.config = old_config
def main(): args = get_args() weight_file = args.weight_file if not weight_file: weight_file = get_file("EfficientNetB3_224_weights.11-3.44.hdf5", pretrained_model, cache_subdir="pretrained_models", file_hash=modhash, cache_dir=os.path.dirname( os.path.abspath(__file__))) # load model and weights model_name, img_size = Path(weight_file).stem.split("_")[:2] img_size = int(img_size) cfg = OmegaConf.from_dotlist( [f"model.model_name={model_name}", f"model.img_size={img_size}"]) model = get_model(cfg) model.load_weights(weight_file) dataset_root = Path(__file__).parent.joinpath("megaage_asian") gt_valid_path = dataset_root.joinpath("file_names.txt") image_paths = [] with open(str(gt_valid_path)) as f: reader = f.read().strip().split('\n') for temp_path in reader: pre, post = temp_path.split('/') post = '_'.join(post.split('_')[1:]) image_paths.append(pre + '/' + post) real_ages = [ int(temp_path.split('/')[-1].split('_')[1]) for temp_path in reader ] batch_size = 8 faces = np.empty((batch_size, img_size, img_size, 3)) ages = [] for i, image_path in tqdm(enumerate(image_paths)): try: cv2.resize(cv2.imread(str(image_path), 1), (img_size, img_size)) faces[i % batch_size] = cv2.resize(cv2.imread(str(image_path), 1), (img_size, img_size)) if (i + 1) % batch_size == 0 or i == len(image_paths) - 1: results = model.predict(faces) ages_out = np.arange(0, 101).reshape(101, 1) predicted_ages = results[1].dot(ages_out).flatten() ages += list(predicted_ages) # len(ages) can be larger than len(image_names) due to the last batch, but it's ok. except Exception as e: continue real_abs_error = 0.0 for i, real_age in enumerate(real_ages): real_abs_error += abs(ages[i] - real_age) print("MAE Real: {}".format(real_abs_error / len(real_ages)))
def tokenizer_args_and_cfg( input_file: str, outdir: str) -> Tuple[argparse.Namespace, OmegaConf]: args = arguments().parse_args( ["--input-file", input_file, "--out-dir", outdir]) config = [ "vocab.max_size=44", "vocab.lowercase=True", "training.max_seq_length=12", ] cfg = OmegaConf.from_dotlist(config) return args, cfg
def __setattr__(self, name, value): if name == "config": object.__setattr__(self, name, value) try: # Can only set attribute if already exists object.__getattribute__(self, name) object.__setattr__(self, name, value) except AttributeError: dotlist = [f"{name}={value}"] update = OmegaConf.from_dotlist(dotlist) self.config = OmegaConf.merge(self.config, update)
def estimate_age_and_gender(image_path): pretrained_model = "https://github.com/yu4u/age-gender-estimation/releases/download/v0.6/EfficientNetB3_224_weights.11-3.44.hdf5" modhash = '6d7f7b7ced093a8b3ef6399163da6ece' weight_file = get_file("EfficientNetB3_224_weights.11-3.44.hdf5", pretrained_model, cache_subdir="pretrained_models", file_hash=modhash, cache_dir=str(Path(__file__).resolve().parent)) margin = 0.4 # for face detection detector = dlib.get_frontal_face_detector() # load model and weights model_name, img_size = Path(weight_file).stem.split("_")[:2] img_size = int(img_size) cfg = OmegaConf.from_dotlist( [f"model.model_name={model_name}", f"model.img_size={img_size}"]) model = get_model(cfg) model.load_weights(weight_file) img = cv2.imread(image_path[0], 1) input_img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) img_h, img_w, _ = np.shape(input_img) # detect faces using dlib detector detected = detector(input_img, 1) faces = np.empty((len(detected), img_size, img_size, 3)) if len(detected) > 0: for i, d in enumerate(detected): x1, y1, x2, y2, w, h = d.left(), d.top( ), d.right() + 1, d.bottom() + 1, d.width(), d.height() xw1 = max(int(x1 - margin * w), 0) yw1 = max(int(y1 - margin * h), 0) xw2 = min(int(x2 + margin * w), img_w - 1) yw2 = min(int(y2 + margin * h), img_h - 1) cv2.rectangle(img, (x1, y1), (x2, y2), (255, 0, 0), 2) # cv2.rectangle(img, (xw1, yw1), (xw2, yw2), (255, 0, 0), 2) faces[i] = cv2.resize(img[yw1:yw2 + 1, xw1:xw2 + 1], (img_size, img_size)) # predict ages and genders of the detected faces results = model.predict(faces) predicted_genders = results[0] ages = np.arange(0, 101).reshape(101, 1) predicted_ages = results[1].dot(ages).flatten() # label = "{}, {}".format(int(predicted_ages[i]), # "M" if predicted_genders[i][0] < 0.5 else "F") return (int(predicted_ages[i]), "M" if predicted_genders[i][0] < 0.5 else "F")
def parse_config(self, name: str, additional: list): if name.endswith(".yaml"): name = name[:-5] # read configuration path = os.path.join(self.config_folder, name + ".yaml") conf = OmegaConf.load(path) # merge with additional settings conf = OmegaConf.merge(conf, OmegaConf.from_dotlist(additional)) # load requested configs, merge, then remove load list if "load" in conf: loads = [] for name in conf.get("load", []): loads.append( OmegaConf.load( os.path.join(self.config_folder, name + ".yaml"))) conf = OmegaConf.merge(*loads, conf) conf.pop("load") # for each part load configuration if is requested for part_name in conf.parts: if "load" in conf.parts[part_name]: load_path = conf.parts[part_name]["load"] if "@" in load_path: load_path, checkpoint = load_path.split("@") else: checkpoint = "latest" model_name, src_part_name = load_path.split(".") src_model_cfg = OmegaConf.load( os.path.join(self.model_folder, model_name + ".yaml")) part_cfg = src_model_cfg.parts[src_part_name] conf.parts[part_name] = OmegaConf.merge( part_cfg, conf.parts[part_name]) conf.parts[part_name].pop("load") # if weights are not specified load also weights if "weights" not in conf.parts[part_name]: conf.parts[part_name][ "weights"] = f"{model_name}.{src_part_name}@{checkpoint}" # Fill mandatory parameters with defaults conf["batch_size"] = conf.get("batch_size", 16) conf["epochs"] = conf.get("epochs", 10) conf["validation_batch_size"] = conf.get("validation_batch_size", conf.batch_size * 2) return conf
def __init__(self, device='cpu'): pretrained_model = "https://github.com/leolani/cltl-face-all/releases/download/v0.0/EfficientNetB3_224_weights.11-3.44.hdf5" modhash = '6d7f7b7ced093a8b3ef6399163da6ece' weight_file = get_file("EfficientNetB3_224_weights.11-3.44.hdf5", pretrained_model, cache_subdir="pretrained_models", file_hash=modhash, cache_dir=str(Path(__file__).resolve().parent)) model_name, img_size = Path(weight_file).stem.split("_")[:2] self.img_size = int(img_size) cfg = OmegaConf.from_dotlist( [f"model.model_name={model_name}", f"model.img_size={img_size}"]) self.model = get_model(cfg) self.model.load_weights(weight_file)
def load_config(cfg_path: Optional[str] = None, default_cfg_path: str = 'configs/default.yaml', update_dotlist: Optional[List[str]] = None) -> DictConfig: config = OmegaConf.load(default_cfg_path) if cfg_path is not None: optional_config = OmegaConf.load(cfg_path) config = OmegaConf.merge(config, optional_config) if update_dotlist is not None: update_config = OmegaConf.from_dotlist(update_dotlist) config = OmegaConf.merge(config, update_config) OmegaConf.set_readonly(config, True) return config
def load_yaml( file_path: Union[str, Path], overrides: Optional[List[str]] = None, ) -> Dict[str, Any]: overrides = overrides or [] file_config = OmegaConf.load(file_path) overrides_config = OmegaConf.from_dotlist(overrides) config = OmegaConf.merge(file_config, overrides_config) dictconfig = OmegaConf.to_container(config) if not isinstance(dictconfig, dict): raise ConfigurationError(f"Config shoud be a dict: {file_path}") return dictconfig
def main(): args = get_args() weight_file = args.weight_file if not weight_file: weight_file = get_file("EfficientNetB3_224_weights.11-3.44.hdf5", pretrained_model, cache_subdir="pretrained_models", file_hash=modhash, cache_dir=os.path.dirname(os.path.abspath(__file__))) # load model and weights model_name, img_size = Path(weight_file).stem.split("_")[:2] img_size = int(img_size) cfg = OmegaConf.from_dotlist([f"model.model_name={model_name}", f"model.img_size={img_size}"]) model = get_model(cfg) model.load_weights(weight_file) dataset_root = Path(__file__).parent.joinpath("appa-real", "appa-real-release") validation_image_dir = dataset_root.joinpath("valid") gt_valid_path = dataset_root.joinpath("gt_avg_valid.csv") image_paths = list(validation_image_dir.glob("*_face.jpg")) batch_size = 8 faces = np.empty((batch_size, img_size, img_size, 3)) ages = [] image_names = [] for i, image_path in tqdm(enumerate(image_paths)): faces[i % batch_size] = cv2.resize(cv2.imread(str(image_path), 1), (img_size, img_size)) image_names.append(image_path.name[:-9]) if (i + 1) % batch_size == 0 or i == len(image_paths) - 1: results = model.predict(faces) ages_out = np.arange(0, 101).reshape(101, 1) predicted_ages = results[1].dot(ages_out).flatten() ages += list(predicted_ages) # len(ages) can be larger than len(image_names) due to the last batch, but it's ok. name2age = {image_names[i]: ages[i] for i in range(len(image_names))} df = pd.read_csv(str(gt_valid_path)) appa_abs_error = 0.0 real_abs_error = 0.0 for i, row in df.iterrows(): appa_abs_error += abs(name2age[row.file_name] - row.apparent_age_avg) real_abs_error += abs(name2age[row.file_name] - row.real_age) print("MAE Apparent: {}".format(appa_abs_error / len(image_names))) print("MAE Real: {}".format(real_abs_error / len(image_names)))
def omegaconf_parse(cls): parser = argparse.ArgumentParser(fromfile_prefix_chars="@") parser.add_argument( "--configs", nargs="*", default=[], help="Configs to load", ) parser.add_argument( "--values", nargs="*", default=[], help="Dot values to change configs", ) args, _unknown = parser.parse_known_args() configs = [OmegaConf.structured(cls)] for path in args.configs: configs.append(OmegaConf.load(path)) configs.append(OmegaConf.from_dotlist(args.values)) omega_config = OmegaConf.merge(*configs) res = cls.parse_from_dict(OmegaConf.to_container(omega_config)) return res
def sweep_2_jobs( sweep_runner: TSweepRunner, overrides: List[str], task_function: Optional[TaskFunction], ) -> None: """ Runs a sweep with two jobs """ job_overrides = ["a=0,1"] overrides.extend(job_overrides) sweep = sweep_runner( calling_file=None, calling_module="hydra.test_utils.a_module", task_function=task_function, config_path="configs/compose.yaml", config_name=None, overrides=overrides, ) base = OmegaConf.create({"foo": 10, "bar": 100, "a": 0}) with sweep: assert sweep.temp_dir is not None assert sweep.returns is not None temp_dir = Path(sweep.temp_dir) multirun_cfg_path = Path(temp_dir) / "multirun.yaml" assert multirun_cfg_path.exists() multirun_cfg = OmegaConf.load(multirun_cfg_path) assert multirun_cfg.hydra.overrides.task == job_overrides assert len(sweep.returns[0]) == 2 for i in range(2): job_ret = sweep.returns[0][i] expected_conf = OmegaConf.merge( base, OmegaConf.from_dotlist(job_ret.overrides) ) assert job_ret.overrides == [f"a={i}"] assert job_ret.cfg == expected_conf assert job_ret.hydra_cfg.hydra.job.name == "a_module", ( "Unexpected job name: " + job_ret.hydra_cfg.hydra.job.name ) verify_dir_outputs(job_ret, job_ret.overrides) path = temp_dir / str(i) lst = [x for x in temp_dir.iterdir() if x.is_dir()] assert path.exists(), f"'{path}' does not exist, dirs: {lst}"
def run_ray_experiment(ray_cfg, cfg): """trains a model based on the base config and the one generated for this experiment Parameters ---------- ray_cfg : DictConfig hparam study configuration cfg : DictConfig base configuration with all non-tuned hyperparameters and information """ ray_cfg = OmegaConf.from_dotlist(dict_to_dotlist(ray_cfg)) cfg = OmegaConf.merge(cfg, ray_cfg) if cfg.notes is None: cfg.notes = f'{cfg.tune.name}_{tune.get_trial_id()}' else: cfg.notes += f'{cfg.tune.name}_{tune.get_trial_id()}' feature_extractor_train(cfg)
def training_args_cfg(): with InputData("train") as train_file: with InputData("valid") as valid_file: with folder() as tokenizer_dir: tok, prefix = train_tokenizer((train_file, tokenizer_dir)) training_args = training.arguments().parse_args([ "--tokenizer-path", f"{prefix}.model", "--train-path", str(train_file), "--valid-path", str(valid_file), "--train-batch-size", "1", "--eval-batch-size", "1", "--epochs", "1", "--max-items", "1", ]) training_config = [ "training.max_seq_length=4", "training.masked_lm_prob=0.1", "training.weight_decay=0.0", "training.learning_rate=5e-05", "seed=42", "model.name=test", "model.hidden_size=312", "model.embedding_size=64", "model.initializer_range=0.02", "model.intermediate_size=312", "model.max_position_embeddings=128", "model.num_attention_heads=4", "vocab.lowercase=False", "vocab.max_size=10", ] training_cfg = OmegaConf.from_dotlist(training_config) yield training_args, training_cfg, tok
def override_config(argv: List[str], config_dir, config) -> OmegaConf: """ This function overrides the config and the subconfigs from the argvs """ valid_args = [] for arg in argv: if check_valid_arg(arg): valid_args.append(arg) # get all subconfig overrides config_subconfig_dirs = [] if "subconfigs" in config: config_subconfig_dirs = [ list(subconfig)[0] for subconfig in config["subconfigs"] ] elif "defaults" in config: config_subconfig_dirs = [ list(subconfig)[0] for subconfig in config["defaults"] ] other_args = [] valid_subconfigs = [] for arg in valid_args: # check if the argument is overriding the subconfig subconfig_dir = arg.split("=")[0] if "." not in subconfig_dir: # check if subconfig_dir is defined in config_subconfig_dirs if subconfig_dir in config_subconfig_dirs: subconfig_name = arg.split("=")[1] valid_subconfigs.append({subconfig_dir: subconfig_name}) continue other_args.append(arg) # get the root subconfigs overrided config = merge_subconfigs(config_dir, valid_subconfigs, config) # override the rest config arg_config = OmegaConf.from_dotlist(other_args) config = OmegaConf.merge(config, arg_config) return config
def test_apply_omegaconf_overrides(): conf = OmegaConf.from_dotlist([ "a.aa.aaa=[1, 2, 3, 4]", "a.aa.bbb=2", "a.bb.aaa='100'", "a.bb.bbb=4" ]) overrides = "a.aa.aaa=[1, 3, 5] a.aa.bbb=3" new_conf = apply_omegaconf_overrides(conf, overrides.split()) assert new_conf.a.aa.aaa == [1, 3, 5] assert new_conf.a.aa.bbb == 3 new_conf2 = apply_omegaconf_overrides(conf, { "a.aa.aaa": [1, 3, 5, 7], "a.aa.bbb": 4 }) assert new_conf2.a.aa.aaa == [1, 3, 5, 7] assert new_conf2.a.aa.bbb == 4 with pytest.raises(KeyError): new_conf3 = apply_omegaconf_overrides(conf, { "a.aa.aaaaaa": [1, 3, 5, 7], "a.aa.bbb": 4 })
def parse(self, override_config=None, verbose=True): if not self.initialized: parser = argparse.ArgumentParser() parser = self.initialize(parser) opt, unknown = parser.parse_known_args() if override_config is not None: # prioritize override_config if it exists base_config = OmegaConf.load(override_config.base_config) # also override the argparse so that checkpoint paths get loaded properly if 'resume_from_path' in override_config.keys(): opt.resume_from_path = override_config['resume_from_path'] else: # otherwise grab the base config file from cli args, or argparse default base_config = OmegaConf.load(opt.base_config) override_config = OmegaConf.create() # make empty config # config from previosuly trained model checkpoint checkpoint_config, opt = self.get_config_from_checkpoint(opt) # config from argparse argparse_config = OmegaConf.create(vars(opt)) # unknown args from command line (will override base config file) # should be in dot-list format: https://omegaconf.readthedocs.io/en/2.1_branch/usage.html#from-a-dot-list cli_config = OmegaConf.from_dotlist(unknown) # configs to the right take priority over configs to the left during merge config = OmegaConf.merge(base_config, checkpoint_config, argparse_config, cli_config, override_config) if verbose: print('') print('----------------- Config ---------------\n') print(OmegaConf.to_yaml(config)) print('------------------- End ----------------\n') print('') return config
def not_sweeping_hydra_overrides(sweep_runner, overrides): """ Runs a sweep with two jobs """ overrides.extend(["a=0,1", "hydra.foo=1,2,3"]) sweep = sweep_runner( calling_file=None, calling_module="hydra.test_utils.a_module", config_path="configs/compose.yaml", overrides=overrides, ) base = OmegaConf.create({"foo": 10, "bar": 100}) with sweep: assert len(sweep.returns[0]) == 2 for i in range(2): job_ret = sweep.returns[0][i] expected_conf = OmegaConf.merge( base, OmegaConf.from_dotlist(job_ret.overrides)) assert job_ret.overrides == ["a={}".format(i)] assert job_ret.cfg == expected_conf verify_dir_outputs(job_ret, job_ret.overrides)
def parse_extra_arg(*args: t.Text) -> t.Tuple[DictConfig, t.Dict]: """ Parse extra arguments on a command line. These arguments correspond to: - MLCube runtime arguments. These start with `-P` prefix and are translated to a nested dictionaries structure using `.` as a separator. For instance, `-Pdocker.image=mlcommons/mnist:0.0.1` translates to python dictionary {'docker': {'image': 'mlcommons/mnist:0.0.1'}}. - Task arguments are all other arguments that do not star with `-P`. These arguments are input/output arguments of tasks. Args: args: List of arguments that have not been parsed before. Returns: Tuple of two dictionaries: (mlcube_arguments, task_arguments). """ mlcube_args = OmegaConf.from_dotlist( [arg[2:] for arg in args if arg.startswith('-P')]) task_args = [ arg.split('=') for arg in args if not arg.startswith('-P') ] task_args = {arg[0]: arg[1] for arg in task_args} return mlcube_args, task_args
def _build_opt_list(self, opts): opts_dot_list = self._convert_to_dot_list(opts) return OmegaConf.from_dotlist(opts_dot_list)