def shell_completion(self, config_name: Optional[str],
overrides: List[str]) -> None:
subcommands = ["install", "uninstall", "query"]
arguments = OmegaConf.from_dotlist(overrides)
num_commands = sum(1 for key in subcommands
if arguments[key] is not None)
if num_commands != 1:
raise ValueError(
f"Expecting one subcommand from {subcommands} to be set")
shell_to_plugin = self.get_shell_to_plugin_map(self.config_loader)
def find_plugin(cmd: str) -> CompletionPlugin:
if cmd not in shell_to_plugin:
lst = "\n".join(["\t" + x for x in shell_to_plugin.keys()])
raise ValueError(
f"No completion plugin for '{cmd}' found, available : \n{lst}"
)
return shell_to_plugin[cmd][0]
if arguments.install is not None:
plugin = find_plugin(arguments.install)
plugin.install()
elif arguments.uninstall is not None:
plugin = find_plugin(arguments.uninstall)
plugin.uninstall()
elif arguments.query is not None:
plugin = find_plugin(arguments.query)
plugin.query(config_name=config_name)
def sweep_2_jobs(sweep_runner, overrides):
"""
Runs a sweep with two jobs
"""
overrides.append("a=0,1")
sweep = sweep_runner(
calling_file=None,
calling_module="hydra.test_utils.a_module",
config_path="configs/compose.yaml",
overrides=overrides,
)
base = OmegaConf.create({"foo": 10, "bar": 100, "a": 0})
with sweep:
temp_dir = Path(sweep.temp_dir)
assert len(sweep.returns[0]) == 2
for i in range(2):
job_ret = sweep.returns[0][i]
expected_conf = OmegaConf.merge(
base, OmegaConf.from_dotlist(job_ret.overrides)
)
assert job_ret.overrides == ["a={}".format(i)]
assert job_ret.cfg == expected_conf
assert job_ret.hydra_cfg.hydra.job.name == "a_module"
verify_dir_outputs(job_ret, job_ret.overrides)
path = temp_dir / str(i)
assert path.exists(), "'{}' does not exist, dirs: {}".format(
path, [x for x in temp_dir.iterdir() if x.is_dir()]
)
def shell_completion(self, overrides):
subcommands = ["install", "uninstall", "query"]
arguments = OmegaConf.from_dotlist(overrides)
num_commands = sum(1 for key in subcommands
if arguments[key] is not None)
if num_commands != 1:
raise ValueError(
"Expecting one subcommand from {} to be set".format(
subcommands))
shell_to_plugin = self.get_shell_to_plugin_map(self.config_loader)
def find_plugin(cmd):
if cmd not in shell_to_plugin:
raise ValueError(
"No completion plugin for '{}' found, available : \n{}".
format(
cmd,
"\n".join(["\t" + x for x in shell_to_plugin.keys()])))
return shell_to_plugin[cmd][0]
if arguments.install is not None:
plugin = find_plugin(arguments.install)
plugin.install()
elif arguments.uninstall is not None:
plugin = find_plugin(arguments.uninstall)
plugin.uninstall()
elif arguments.query is not None:
plugin = find_plugin(arguments.query)
plugin.query()
def process_config_arguments(config_file=None, config_override=None, output_dir=None):
"""Process configuration arguments.
Args:
config_file (str, optional): Path to SuperBench config file. Defaults to None.
config_override (str, optional): Extra arguments to override config_file,
following [Hydra syntax](https://hydra.cc/docs/advanced/override_grammar/basic). Defaults to None.
output_dir (str, optional): Path to output directory. Defaults to None.
Returns:
DictConfig: SuperBench config object.
str: Dir for output.
Raises:
CLIError: If input arguments are invalid.
"""
config_file = check_argument_file('config_file', config_file)
# SuperBench config
sb_config = get_sb_config(config_file)
if config_override:
sb_config_from_override = OmegaConf.from_dotlist(config_override)
sb_config = OmegaConf.merge(sb_config, sb_config_from_override)
# Create output directory
sb_output_dir = create_sb_output_dir(output_dir)
return sb_config, sb_output_dir
def not_sweeping_hydra_overrides(
sweep_runner: TSweepRunner,
overrides: List[str],
task_function: Optional[TaskFunction],
) -> None:
"""
Runs a sweep with two jobs
"""
overrides.extend(["a=0,1", "hydra.verbose=true,false"])
sweep = sweep_runner(
calling_file=None,
calling_module="hydra.test_utils.a_module",
task_function=task_function,
config_path="configs",
config_name="compose.yaml",
overrides=overrides,
strict=None,
)
base = OmegaConf.create({"foo": 10, "bar": 100})
with sweep:
assert sweep.returns is not None
assert len(sweep.returns[0]) == 2
for i in range(2):
job_ret = sweep.returns[0][i]
expected_conf = OmegaConf.merge(
base, OmegaConf.from_dotlist(job_ret.overrides)
)
assert job_ret.overrides == [f"a={i}"]
assert job_ret.cfg == expected_conf
verify_dir_outputs(job_ret, job_ret.overrides)
def test_readonly_from_cli() -> None:
c = OmegaConf.create({"foo": {"bar": [1]}})
assert isinstance(c, DictConfig)
OmegaConf.set_readonly(c, True)
cli = OmegaConf.from_dotlist(["foo.bar=[2]"])
with raises(ReadonlyConfigError, match="foo.bar"):
OmegaConf.merge(c, cli)
def main():
args = get_args()
weight_file = args.weight_file
margin = args.margin
image_dir = args.image_dir
if not weight_file:
weight_file = get_file("EfficientNetB3_224_weights.11-3.44.hdf5", pretrained_model, cache_subdir="pretrained_models",
file_hash=modhash, cache_dir=str(Path(__file__).resolve().parent))
# for face detection
detector = dlib.get_frontal_face_detector()
# load model and weights
model_name, img_size = Path(weight_file).stem.split("_")[:2]
img_size = int(img_size)
cfg = OmegaConf.from_dotlist([f"model.model_name={model_name}", f"model.img_size={img_size}"])
model = get_model(cfg)
model.load_weights(weight_file)
image_generator = yield_images_from_dir(image_dir) if image_dir else yield_images()
for img in image_generator:
input_img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img_h, img_w, _ = np.shape(input_img)
# detect faces using dlib detector
detected = detector(input_img, 1)
faces = np.empty((len(detected), img_size, img_size, 3))
if len(detected) > 0:
for i, d in enumerate(detected):
x1, y1, x2, y2, w, h = d.left(), d.top(), d.right() + 1, d.bottom() + 1, d.width(), d.height()
xw1 = max(int(x1 - margin * w), 0)
yw1 = max(int(y1 - margin * h), 0)
xw2 = min(int(x2 + margin * w), img_w - 1)
yw2 = min(int(y2 + margin * h), img_h - 1)
faces[i] = cv2.resize(img[yw1:yw2 + 1, xw1:xw2 + 1], (img_size, img_size))
# if you want to capture roi then uncomment below line
cv2.imwrite(f"results/face-{i}.jpg", faces[i])
# predict ages and genders of the detected faces
results = model.predict(faces)
predicted_genders = results[0]
ages = np.arange(0, 101).reshape(101, 1)
predicted_ages = results[1].dot(ages).flatten()
# draw results
for i, d in enumerate(detected):
label = "{}, {}".format(int(predicted_ages[i]),
"M" if predicted_genders[i][0] < 0.5 else "F")
cv2.rectangle(img, (d.left(), d.top()), (d.right(), d.bottom()), (255, 0, 0), 2)
draw_label(img, (d.left(), d.top()), label)
cv2.imshow("result", img)
key = cv2.waitKey(-1) if image_dir else cv2.waitKey(30)
if key == 27: # ESC
break
def test_readonly_from_cli() -> None:
c = OmegaConf.create({"foo": {"bar": [1]}})
assert isinstance(c, DictConfig)
OmegaConf.set_readonly(c, True)
cli = OmegaConf.from_dotlist(["foo.bar=[2]"])
cfg2 = OmegaConf.merge(c, cli)
assert OmegaConf.is_readonly(c)
assert OmegaConf.is_readonly(cfg2)
def test_deepcopy_and_merge_and_flags() -> None:
c1 = OmegaConf.create(
{"dataset": {"name": "imagenet", "path": "/datasets/imagenet"}, "defaults": []}
)
OmegaConf.set_struct(c1, True)
c2 = copy.deepcopy(c1)
with raises(ConfigKeyError):
OmegaConf.merge(c2, OmegaConf.from_dotlist(["dataset.bad_key=yes"]))
def test_timestep_operation(self, mock_data_manager):
start_dt = datetime.now()
training_interval = timedelta(days=1)
mock_env = mock.MagicMock()
conf = OmegaConf.from_dotlist(
["project.tensorboard_path=/tmp/test_tb/"])
mock_app = mock.MagicMock()
mock_app.timing_data = TimingData(start_dt=start_dt,
training_interval=training_interval)
mock_app.env = mock_env
mock_app.config = conf
mock_env.env_id_cols = ["env_id_1", "env_id_2"]
mock_env.ts_id_col = "ts_1"
mock_env.obs_cols = ["obs_1", "obs_2"]
mock_timestep = [{
"env_id_1": 1,
"env_id_2": 2,
"ts_1": 1,
"discount": 1.0,
"obs_1": 1,
"obs_2": 2,
"action": 1,
"reward": 0.0,
"step_type": 0
}]
mock_timestep_df = self.spark.createDataFrame(mock_timestep)
metadata_dict = {"available_data": [("test_data", 0)]}
mock_env.build_time_steps = mock.MagicMock(
return_value=mock_timestep_df)
mock_data_manager.get_latest.return_value = metadata_dict
run_id = 1
operation = BuildTimestepOperation(mock_app, mock_data_manager)
operation.run(run_id)
mock_data_manager.get_latest.assert_any_call(DATANAME.RUN_CONTEXT,
run_id)
expected_start_dt = start_dt
expected_end_dt = start_dt + training_interval
mock_env.build_time_steps.assert_called_with(expected_start_dt,
expected_end_dt)
expected_metadata = {
"available_data": [("test_data", 0), (DATANAME.TIMESTEP, run_id)]
}
calls = [
call(mock_timestep_df, DATANAME.TIMESTEP, run_id),
call(expected_metadata, DATANAME.RUN_CONTEXT, run_id)
]
mock_data_manager.store.assert_has_calls(calls, any_order=False)
def temp_override(self, override_dict):
old_config = self.config
try:
dotlist = [f"{k}={v}" for k, v in override_dict.items()]
update = OmegaConf.from_dotlist(dotlist)
self.config = OmegaConf.merge(self.config, update)
yield
finally:
self.config = old_config
def main():
args = get_args()
weight_file = args.weight_file
if not weight_file:
weight_file = get_file("EfficientNetB3_224_weights.11-3.44.hdf5",
pretrained_model,
cache_subdir="pretrained_models",
file_hash=modhash,
cache_dir=os.path.dirname(
os.path.abspath(__file__)))
# load model and weights
model_name, img_size = Path(weight_file).stem.split("_")[:2]
img_size = int(img_size)
cfg = OmegaConf.from_dotlist(
[f"model.model_name={model_name}", f"model.img_size={img_size}"])
model = get_model(cfg)
model.load_weights(weight_file)
dataset_root = Path(__file__).parent.joinpath("megaage_asian")
gt_valid_path = dataset_root.joinpath("file_names.txt")
image_paths = []
with open(str(gt_valid_path)) as f:
reader = f.read().strip().split('\n')
for temp_path in reader:
pre, post = temp_path.split('/')
post = '_'.join(post.split('_')[1:])
image_paths.append(pre + '/' + post)
real_ages = [
int(temp_path.split('/')[-1].split('_')[1]) for temp_path in reader
]
batch_size = 8
faces = np.empty((batch_size, img_size, img_size, 3))
ages = []
for i, image_path in tqdm(enumerate(image_paths)):
try:
cv2.resize(cv2.imread(str(image_path), 1), (img_size, img_size))
faces[i % batch_size] = cv2.resize(cv2.imread(str(image_path), 1),
(img_size, img_size))
if (i + 1) % batch_size == 0 or i == len(image_paths) - 1:
results = model.predict(faces)
ages_out = np.arange(0, 101).reshape(101, 1)
predicted_ages = results[1].dot(ages_out).flatten()
ages += list(predicted_ages)
# len(ages) can be larger than len(image_names) due to the last batch, but it's ok.
except Exception as e:
continue
real_abs_error = 0.0
for i, real_age in enumerate(real_ages):
real_abs_error += abs(ages[i] - real_age)
print("MAE Real: {}".format(real_abs_error / len(real_ages)))
def tokenizer_args_and_cfg(
input_file: str, outdir: str) -> Tuple[argparse.Namespace, OmegaConf]:
args = arguments().parse_args(
["--input-file", input_file, "--out-dir", outdir])
config = [
"vocab.max_size=44",
"vocab.lowercase=True",
"training.max_seq_length=12",
]
cfg = OmegaConf.from_dotlist(config)
return args, cfg
def __setattr__(self, name, value):
if name == "config":
object.__setattr__(self, name, value)
try:
# Can only set attribute if already exists
object.__getattribute__(self, name)
object.__setattr__(self, name, value)
except AttributeError:
dotlist = [f"{name}={value}"]
update = OmegaConf.from_dotlist(dotlist)
self.config = OmegaConf.merge(self.config, update)
def estimate_age_and_gender(image_path):
pretrained_model = "https://github.com/yu4u/age-gender-estimation/releases/download/v0.6/EfficientNetB3_224_weights.11-3.44.hdf5"
modhash = '6d7f7b7ced093a8b3ef6399163da6ece'
weight_file = get_file("EfficientNetB3_224_weights.11-3.44.hdf5",
pretrained_model,
cache_subdir="pretrained_models",
file_hash=modhash,
cache_dir=str(Path(__file__).resolve().parent))
margin = 0.4
# for face detection
detector = dlib.get_frontal_face_detector()
# load model and weights
model_name, img_size = Path(weight_file).stem.split("_")[:2]
img_size = int(img_size)
cfg = OmegaConf.from_dotlist(
[f"model.model_name={model_name}", f"model.img_size={img_size}"])
model = get_model(cfg)
model.load_weights(weight_file)
img = cv2.imread(image_path[0], 1)
input_img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img_h, img_w, _ = np.shape(input_img)
# detect faces using dlib detector
detected = detector(input_img, 1)
faces = np.empty((len(detected), img_size, img_size, 3))
if len(detected) > 0:
for i, d in enumerate(detected):
x1, y1, x2, y2, w, h = d.left(), d.top(
), d.right() + 1, d.bottom() + 1, d.width(), d.height()
xw1 = max(int(x1 - margin * w), 0)
yw1 = max(int(y1 - margin * h), 0)
xw2 = min(int(x2 + margin * w), img_w - 1)
yw2 = min(int(y2 + margin * h), img_h - 1)
cv2.rectangle(img, (x1, y1), (x2, y2), (255, 0, 0), 2)
# cv2.rectangle(img, (xw1, yw1), (xw2, yw2), (255, 0, 0), 2)
faces[i] = cv2.resize(img[yw1:yw2 + 1, xw1:xw2 + 1],
(img_size, img_size))
# predict ages and genders of the detected faces
results = model.predict(faces)
predicted_genders = results[0]
ages = np.arange(0, 101).reshape(101, 1)
predicted_ages = results[1].dot(ages).flatten()
# label = "{}, {}".format(int(predicted_ages[i]),
# "M" if predicted_genders[i][0] < 0.5 else "F")
return (int(predicted_ages[i]),
"M" if predicted_genders[i][0] < 0.5 else "F")
def parse_config(self, name: str, additional: list):
if name.endswith(".yaml"):
name = name[:-5]
# read configuration
path = os.path.join(self.config_folder, name + ".yaml")
conf = OmegaConf.load(path)
# merge with additional settings
conf = OmegaConf.merge(conf, OmegaConf.from_dotlist(additional))
# load requested configs, merge, then remove load list
if "load" in conf:
loads = []
for name in conf.get("load", []):
loads.append(
OmegaConf.load(
os.path.join(self.config_folder, name + ".yaml")))
conf = OmegaConf.merge(*loads, conf)
conf.pop("load")
# for each part load configuration if is requested
for part_name in conf.parts:
if "load" in conf.parts[part_name]:
load_path = conf.parts[part_name]["load"]
if "@" in load_path:
load_path, checkpoint = load_path.split("@")
else:
checkpoint = "latest"
model_name, src_part_name = load_path.split(".")
src_model_cfg = OmegaConf.load(
os.path.join(self.model_folder, model_name + ".yaml"))
part_cfg = src_model_cfg.parts[src_part_name]
conf.parts[part_name] = OmegaConf.merge(
part_cfg, conf.parts[part_name])
conf.parts[part_name].pop("load")
# if weights are not specified load also weights
if "weights" not in conf.parts[part_name]:
conf.parts[part_name][
"weights"] = f"{model_name}.{src_part_name}@{checkpoint}"
# Fill mandatory parameters with defaults
conf["batch_size"] = conf.get("batch_size", 16)
conf["epochs"] = conf.get("epochs", 10)
conf["validation_batch_size"] = conf.get("validation_batch_size",
conf.batch_size * 2)
return conf
def __init__(self, device='cpu'):
pretrained_model = "https://github.com/leolani/cltl-face-all/releases/download/v0.0/EfficientNetB3_224_weights.11-3.44.hdf5"
modhash = '6d7f7b7ced093a8b3ef6399163da6ece'
weight_file = get_file("EfficientNetB3_224_weights.11-3.44.hdf5",
pretrained_model,
cache_subdir="pretrained_models",
file_hash=modhash,
cache_dir=str(Path(__file__).resolve().parent))
model_name, img_size = Path(weight_file).stem.split("_")[:2]
self.img_size = int(img_size)
cfg = OmegaConf.from_dotlist(
[f"model.model_name={model_name}", f"model.img_size={img_size}"])
self.model = get_model(cfg)
self.model.load_weights(weight_file)
def load_config(cfg_path: Optional[str] = None,
default_cfg_path: str = 'configs/default.yaml',
update_dotlist: Optional[List[str]] = None) -> DictConfig:
config = OmegaConf.load(default_cfg_path)
if cfg_path is not None:
optional_config = OmegaConf.load(cfg_path)
config = OmegaConf.merge(config, optional_config)
if update_dotlist is not None:
update_config = OmegaConf.from_dotlist(update_dotlist)
config = OmegaConf.merge(config, update_config)
OmegaConf.set_readonly(config, True)
return config
def load_yaml(
file_path: Union[str, Path],
overrides: Optional[List[str]] = None,
) -> Dict[str, Any]:
overrides = overrides or []
file_config = OmegaConf.load(file_path)
overrides_config = OmegaConf.from_dotlist(overrides)
config = OmegaConf.merge(file_config, overrides_config)
dictconfig = OmegaConf.to_container(config)
if not isinstance(dictconfig, dict):
raise ConfigurationError(f"Config shoud be a dict: {file_path}")
return dictconfig
def main():
args = get_args()
weight_file = args.weight_file
if not weight_file:
weight_file = get_file("EfficientNetB3_224_weights.11-3.44.hdf5", pretrained_model, cache_subdir="pretrained_models",
file_hash=modhash, cache_dir=os.path.dirname(os.path.abspath(__file__)))
# load model and weights
model_name, img_size = Path(weight_file).stem.split("_")[:2]
img_size = int(img_size)
cfg = OmegaConf.from_dotlist([f"model.model_name={model_name}", f"model.img_size={img_size}"])
model = get_model(cfg)
model.load_weights(weight_file)
dataset_root = Path(__file__).parent.joinpath("appa-real", "appa-real-release")
validation_image_dir = dataset_root.joinpath("valid")
gt_valid_path = dataset_root.joinpath("gt_avg_valid.csv")
image_paths = list(validation_image_dir.glob("*_face.jpg"))
batch_size = 8
faces = np.empty((batch_size, img_size, img_size, 3))
ages = []
image_names = []
for i, image_path in tqdm(enumerate(image_paths)):
faces[i % batch_size] = cv2.resize(cv2.imread(str(image_path), 1), (img_size, img_size))
image_names.append(image_path.name[:-9])
if (i + 1) % batch_size == 0 or i == len(image_paths) - 1:
results = model.predict(faces)
ages_out = np.arange(0, 101).reshape(101, 1)
predicted_ages = results[1].dot(ages_out).flatten()
ages += list(predicted_ages)
# len(ages) can be larger than len(image_names) due to the last batch, but it's ok.
name2age = {image_names[i]: ages[i] for i in range(len(image_names))}
df = pd.read_csv(str(gt_valid_path))
appa_abs_error = 0.0
real_abs_error = 0.0
for i, row in df.iterrows():
appa_abs_error += abs(name2age[row.file_name] - row.apparent_age_avg)
real_abs_error += abs(name2age[row.file_name] - row.real_age)
print("MAE Apparent: {}".format(appa_abs_error / len(image_names)))
print("MAE Real: {}".format(real_abs_error / len(image_names)))
def omegaconf_parse(cls):
parser = argparse.ArgumentParser(fromfile_prefix_chars="@")
parser.add_argument(
"--configs", nargs="*", default=[], help="Configs to load",
)
parser.add_argument(
"--values", nargs="*", default=[], help="Dot values to change configs",
)
args, _unknown = parser.parse_known_args()
configs = [OmegaConf.structured(cls)]
for path in args.configs:
configs.append(OmegaConf.load(path))
configs.append(OmegaConf.from_dotlist(args.values))
omega_config = OmegaConf.merge(*configs)
res = cls.parse_from_dict(OmegaConf.to_container(omega_config))
return res
def sweep_2_jobs(
sweep_runner: TSweepRunner,
overrides: List[str],
task_function: Optional[TaskFunction],
) -> None:
"""
Runs a sweep with two jobs
"""
job_overrides = ["a=0,1"]
overrides.extend(job_overrides)
sweep = sweep_runner(
calling_file=None,
calling_module="hydra.test_utils.a_module",
task_function=task_function,
config_path="configs/compose.yaml",
config_name=None,
overrides=overrides,
)
base = OmegaConf.create({"foo": 10, "bar": 100, "a": 0})
with sweep:
assert sweep.temp_dir is not None
assert sweep.returns is not None
temp_dir = Path(sweep.temp_dir)
multirun_cfg_path = Path(temp_dir) / "multirun.yaml"
assert multirun_cfg_path.exists()
multirun_cfg = OmegaConf.load(multirun_cfg_path)
assert multirun_cfg.hydra.overrides.task == job_overrides
assert len(sweep.returns[0]) == 2
for i in range(2):
job_ret = sweep.returns[0][i]
expected_conf = OmegaConf.merge(
base, OmegaConf.from_dotlist(job_ret.overrides)
)
assert job_ret.overrides == [f"a={i}"]
assert job_ret.cfg == expected_conf
assert job_ret.hydra_cfg.hydra.job.name == "a_module", (
"Unexpected job name: " + job_ret.hydra_cfg.hydra.job.name
)
verify_dir_outputs(job_ret, job_ret.overrides)
path = temp_dir / str(i)
lst = [x for x in temp_dir.iterdir() if x.is_dir()]
assert path.exists(), f"'{path}' does not exist, dirs: {lst}"
def run_ray_experiment(ray_cfg, cfg):
"""trains a model based on the base config and the one generated for this experiment
Parameters
----------
ray_cfg : DictConfig
hparam study configuration
cfg : DictConfig
base configuration with all non-tuned hyperparameters and information
"""
ray_cfg = OmegaConf.from_dotlist(dict_to_dotlist(ray_cfg))
cfg = OmegaConf.merge(cfg, ray_cfg)
if cfg.notes is None:
cfg.notes = f'{cfg.tune.name}_{tune.get_trial_id()}'
else:
cfg.notes += f'{cfg.tune.name}_{tune.get_trial_id()}'
feature_extractor_train(cfg)
def training_args_cfg():
with InputData("train") as train_file:
with InputData("valid") as valid_file:
with folder() as tokenizer_dir:
tok, prefix = train_tokenizer((train_file, tokenizer_dir))
training_args = training.arguments().parse_args([
"--tokenizer-path",
f"{prefix}.model",
"--train-path",
str(train_file),
"--valid-path",
str(valid_file),
"--train-batch-size",
"1",
"--eval-batch-size",
"1",
"--epochs",
"1",
"--max-items",
"1",
])
training_config = [
"training.max_seq_length=4",
"training.masked_lm_prob=0.1",
"training.weight_decay=0.0",
"training.learning_rate=5e-05",
"seed=42",
"model.name=test",
"model.hidden_size=312",
"model.embedding_size=64",
"model.initializer_range=0.02",
"model.intermediate_size=312",
"model.max_position_embeddings=128",
"model.num_attention_heads=4",
"vocab.lowercase=False",
"vocab.max_size=10",
]
training_cfg = OmegaConf.from_dotlist(training_config)
yield training_args, training_cfg, tok
def override_config(argv: List[str], config_dir, config) -> OmegaConf:
"""
This function overrides the config and the subconfigs from the argvs
"""
valid_args = []
for arg in argv:
if check_valid_arg(arg):
valid_args.append(arg)
# get all subconfig overrides
config_subconfig_dirs = []
if "subconfigs" in config:
config_subconfig_dirs = [
list(subconfig)[0] for subconfig in config["subconfigs"]
]
elif "defaults" in config:
config_subconfig_dirs = [
list(subconfig)[0] for subconfig in config["defaults"]
]
other_args = []
valid_subconfigs = []
for arg in valid_args:
# check if the argument is overriding the subconfig
subconfig_dir = arg.split("=")[0]
if "." not in subconfig_dir:
# check if subconfig_dir is defined in config_subconfig_dirs
if subconfig_dir in config_subconfig_dirs:
subconfig_name = arg.split("=")[1]
valid_subconfigs.append({subconfig_dir: subconfig_name})
continue
other_args.append(arg)
# get the root subconfigs overrided
config = merge_subconfigs(config_dir, valid_subconfigs, config)
# override the rest config
arg_config = OmegaConf.from_dotlist(other_args)
config = OmegaConf.merge(config, arg_config)
return config
def test_apply_omegaconf_overrides():
conf = OmegaConf.from_dotlist([
"a.aa.aaa=[1, 2, 3, 4]", "a.aa.bbb=2", "a.bb.aaa='100'", "a.bb.bbb=4"
])
overrides = "a.aa.aaa=[1, 3, 5] a.aa.bbb=3"
new_conf = apply_omegaconf_overrides(conf, overrides.split())
assert new_conf.a.aa.aaa == [1, 3, 5]
assert new_conf.a.aa.bbb == 3
new_conf2 = apply_omegaconf_overrides(conf, {
"a.aa.aaa": [1, 3, 5, 7],
"a.aa.bbb": 4
})
assert new_conf2.a.aa.aaa == [1, 3, 5, 7]
assert new_conf2.a.aa.bbb == 4
with pytest.raises(KeyError):
new_conf3 = apply_omegaconf_overrides(conf, {
"a.aa.aaaaaa": [1, 3, 5, 7],
"a.aa.bbb": 4
})
def parse(self, override_config=None, verbose=True):
if not self.initialized:
parser = argparse.ArgumentParser()
parser = self.initialize(parser)
opt, unknown = parser.parse_known_args()
if override_config is not None:
# prioritize override_config if it exists
base_config = OmegaConf.load(override_config.base_config)
# also override the argparse so that checkpoint paths get loaded properly
if 'resume_from_path' in override_config.keys():
opt.resume_from_path = override_config['resume_from_path']
else:
# otherwise grab the base config file from cli args, or argparse default
base_config = OmegaConf.load(opt.base_config)
override_config = OmegaConf.create() # make empty config
# config from previosuly trained model checkpoint
checkpoint_config, opt = self.get_config_from_checkpoint(opt)
# config from argparse
argparse_config = OmegaConf.create(vars(opt))
# unknown args from command line (will override base config file)
# should be in dot-list format: https://omegaconf.readthedocs.io/en/2.1_branch/usage.html#from-a-dot-list
cli_config = OmegaConf.from_dotlist(unknown)
# configs to the right take priority over configs to the left during merge
config = OmegaConf.merge(base_config, checkpoint_config, argparse_config, cli_config, override_config)
if verbose:
print('')
print('----------------- Config ---------------\n')
print(OmegaConf.to_yaml(config))
print('------------------- End ----------------\n')
print('')
return config
def not_sweeping_hydra_overrides(sweep_runner, overrides):
"""
Runs a sweep with two jobs
"""
overrides.extend(["a=0,1", "hydra.foo=1,2,3"])
sweep = sweep_runner(
calling_file=None,
calling_module="hydra.test_utils.a_module",
config_path="configs/compose.yaml",
overrides=overrides,
)
base = OmegaConf.create({"foo": 10, "bar": 100})
with sweep:
assert len(sweep.returns[0]) == 2
for i in range(2):
job_ret = sweep.returns[0][i]
expected_conf = OmegaConf.merge(
base, OmegaConf.from_dotlist(job_ret.overrides))
assert job_ret.overrides == ["a={}".format(i)]
assert job_ret.cfg == expected_conf
verify_dir_outputs(job_ret, job_ret.overrides)
def parse_extra_arg(*args: t.Text) -> t.Tuple[DictConfig, t.Dict]:
""" Parse extra arguments on a command line.
These arguments correspond to:
- MLCube runtime arguments. These start with `-P` prefix and are translated to a nested dictionaries
structure using `.` as a separator. For instance, `-Pdocker.image=mlcommons/mnist:0.0.1` translates to
python dictionary {'docker': {'image': 'mlcommons/mnist:0.0.1'}}.
- Task arguments are all other arguments that do not star with `-P`. These arguments are input/output
arguments of tasks.
Args:
args: List of arguments that have not been parsed before.
Returns:
Tuple of two dictionaries: (mlcube_arguments, task_arguments).
"""
mlcube_args = OmegaConf.from_dotlist(
[arg[2:] for arg in args if arg.startswith('-P')])
task_args = [
arg.split('=') for arg in args if not arg.startswith('-P')
]
task_args = {arg[0]: arg[1] for arg in task_args}
return mlcube_args, task_args
def _build_opt_list(self, opts):
opts_dot_list = self._convert_to_dot_list(opts)
return OmegaConf.from_dotlist(opts_dot_list)
