def test_readonly(src, func, expectation):
c = OmegaConf.create(src)
OmegaConf.set_readonly(c, True)
with expectation:
func(c)
assert c == src
class TableModelConfig:
train_data_folds: List[str] = MISSING
val_data_folds: List[str] = MISSING
train_feats: List[str] = MISSING
target: str = MISSING
model_name: str = MISSING
classifier_model: str = MISSING
model_type: Optional[str] = "CatBoostRegressor"
construct_params: Dict[Any, Any] = field(default_factory=dict)
train_params: Dict[Any, Any] = field(default_factory=dict)
#%%
train_config_path = util.get_my_data_dir(
) / os.environ["TRAIN_CONFIG_FILENAME"]
conf = OmegaConf.create(TableModelConfig(**OmegaConf.load(train_config_path)))
#%%
train_df, val_df, = io.load_dataset(conf.train_data_folds, conf.val_data_folds)
from catboost import CatBoostClassifier
classifier = CatBoostClassifier()
classifier.load_model(conf.classifier_model)
t = [l for l in conf.train_feats if l != "rank_class"]
train_df["rank_class"] = classifier.predict(train_df[t])
val_df["rank_class"] = classifier.predict(val_df[t])
# %%
import pandas as pd
train_data_concat = pd.concat([train_df, val_df], axis=0, ignore_index=True)
#%%
def create(obj: Any, _parent_: Container) -> Any:
"""Create a config object from `obj`, similar to `OmegaConf.create`"""
from omegaconf import OmegaConf
assert isinstance(_parent_, BaseContainer)
return OmegaConf.create(obj, parent=_parent_)
def test_class_instantiate(conf, expected):
conf = OmegaConf.create(conf)
obj = utils.instantiate(conf)
assert obj == expected
def _mock_model_config():
conf = {'temp_file': None}
conf = OmegaConf.create({'model': conf})
OmegaConf.set_struct(conf, True)
return conf
def __init__(self,
model_name,
config,
vocab_file,
model_parallel_size=None,
model_parallel_rank=None):
super().__init__()
self._model_parallel_size = model_parallel_size
self._model_parallel_rank = model_parallel_rank
self._restore_path = None
self._app_state = None
self._model_name = model_name
if not os.path.exists(vocab_file):
raise ValueError(f'Vocab file not found at {vocab_file}')
config["vocab_file"] = vocab_file
config['tokenizer_type'] = 'BertWordPieceLowerCase'
config['lazy_mpu_init'] = True
config['onnx_safe'] = True
# if 'model_parallel_size' in config:
if self._model_parallel_size is not None:
app_state = AppState()
self._app_state = app_state
# must be set for model parallel megatron-lm
os.environ["WORLD_SIZE"] = str(app_state.world_size)
os.environ["RANK"] = str(self._model_parallel_rank)
# used to set model_parallel_size in megatron-lm argparser
def _update_model_parallel_arg(parser):
parser.set_defaults(
model_parallel_size=self._model_parallel_size)
return parser
extra_args_provider = _update_model_parallel_arg
else:
extra_args_provider = None
# Initialize part of Megatron global state that is needed for its constructor.
# We set 'lazy_mpu_init' flag on to make Megatron do only the initialization that does not depend
# on ddp be initialized yet (and we don't want Megatron to initialize DDP itself either)
# and to return a hook for us to call after PTL has torch.distributed initialized.
# We call this hook during .forward
# TODO: can we call this hook using the PTL hook .setup()
self._lazy_init_fn = initialize_megatron(
extra_args_provider=extra_args_provider,
args_defaults=config,
ignore_unknown_args=True)
# read Megatron arguments back
args = get_args()
logging.info(f'Megatron-lm argparse args: {args}')
self.language_model, self._language_model_key = get_language_model(
attention_mask_func=bert_attention_mask_func,
num_tokentypes=2,
add_pooler=False)
self.config = OmegaConf.create(config)
# key used for checkpoints
self._hidden_size = self.language_model.hidden_size
def __init__(self):
self.conf = OmegaConf.create()
self.set("name", "UNKNOWN_NAME")
def test_supported_chars() -> None:
supported_chars = "%_-abc123."
c = OmegaConf.create(dict(dir1="${copy:" + supported_chars + "}"))
OmegaConf.register_resolver("copy", lambda x: x)
assert c.dir1 == supported_chars
def test_interpolation_in_list_key_error() -> None:
# Test that a KeyError is thrown if an str_interpolation key is not available
c = OmegaConf.create(["${10}"])
with pytest.raises(KeyError):
c[0]
def test_non_container_interpolation() -> None:
cfg = OmegaConf.create(dict(foo=0, bar="${foo.baz}"))
with pytest.raises(ConfigKeyError):
cfg.bar
def test_register_resolver_1(restore_resolvers: Any) -> None:
OmegaConf.register_resolver("plus_10", lambda x: int(x) + 10)
c = OmegaConf.create({"k": "${plus_10:990}"})
assert type(c.k) == int
assert c.k == 1000
def test_save_illegal_type() -> None:
with pytest.raises(TypeError):
OmegaConf.save(OmegaConf.create(), 1000) # type: ignore
def test_readonly_list_change_item():
c = OmegaConf.create([1, 2, 3])
OmegaConf.set_readonly(c, True)
with raises(ReadonlyConfigError, match="[1]"):
c[1] = 10
assert c == [1, 2, 3]
def test_readonly_list_append():
c = OmegaConf.create([])
OmegaConf.set_readonly(c, True)
with raises(ReadonlyConfigError, match="[0]"):
c.append(10)
assert c == []
def test_custom_resolver_param_supported_chars() -> None:
supported_chars = "abc123_/:-\\+.$%*@"
c = OmegaConf.create({"dir1": "${copy:" + supported_chars + "}"})
OmegaConf.register_new_resolver("copy", lambda x: x)
assert c.dir1 == supported_chars
def test_unsupported_interpolation_type() -> None:
c = OmegaConf.create({"foo": "${wrong_type:ref}"})
with pytest.raises(ValueError):
c.foo
def create() -> DictConfig:
return OmegaConf.create({"invalid": f"${{ab{c}de}}"})
def test_incremental_dict_with_interpolation() -> None:
conf = OmegaConf.create()
conf.a = 1
conf.b = OmegaConf.create()
conf.b.c = "${a}"
assert conf.b.c == conf.a # type:ignore
def qufpn_config(min_level, max_level, weight_method=None):
"""A dynamic quad fpn config that can adapt to different min/max levels.
It extends the idea of BiFPN, and has four paths:
(up_down -> bottom_up) + (bottom_up -> up_down).
Paper: https://ieeexplore.ieee.org/document/9225379
Ref code: From contribution to TF EfficientDet
https://github.com/google/automl/blob/eb74c6739382e9444817d2ad97c4582dbe9a9020/efficientdet/keras/fpn_configs.py
"""
p = OmegaConf.create()
weight_method = weight_method or 'fastattn'
quad_method = 'fastattn'
num_levels = max_level - min_level + 1
node_ids = {min_level + i: [i] for i in range(num_levels)}
level_last_id = lambda level: node_ids[level][-1]
level_all_ids = lambda level: node_ids[level]
level_first_id = lambda level: node_ids[level][0]
id_cnt = itertools.count(num_levels)
p.nodes = []
for i in range(max_level - 1, min_level - 1, -1):
# top-down path 1.
p.nodes.append({
'reduction':
1 << i,
'inputs_offsets': [level_last_id(i),
level_last_id(i + 1)],
'weight_method':
weight_method
})
node_ids[i].append(next(id_cnt))
node_ids[max_level].append(node_ids[max_level][-1])
for i in range(min_level + 1, max_level):
# bottom-up path 2.
p.nodes.append({
'reduction':
1 << i,
'inputs_offsets':
level_all_ids(i) + [level_last_id(i - 1)],
'weight_method':
weight_method
})
node_ids[i].append(next(id_cnt))
i = max_level
p.nodes.append({
'reduction':
1 << i,
'inputs_offsets': [level_first_id(i)] + [level_last_id(i - 1)],
'weight_method':
weight_method
})
node_ids[i].append(next(id_cnt))
node_ids[min_level].append(node_ids[min_level][-1])
for i in range(min_level + 1, max_level + 1, 1):
# bottom-up path 3.
p.nodes.append({
'reduction':
1 << i,
'inputs_offsets': [
level_first_id(i),
level_last_id(i -
1) if i != min_level + 1 else level_first_id(i -
1)
],
'weight_method':
weight_method
})
node_ids[i].append(next(id_cnt))
node_ids[min_level].append(node_ids[min_level][-1])
for i in range(max_level - 1, min_level, -1):
# top-down path 4.
p.nodes.append({
'reduction':
1 << i,
'inputs_offsets':
[node_ids[i][0]] + [node_ids[i][-1]] + [level_last_id(i + 1)],
'weight_method':
weight_method
})
node_ids[i].append(next(id_cnt))
i = min_level
p.nodes.append({
'reduction': 1 << i,
'inputs_offsets': [node_ids[i][0]] + [level_last_id(i + 1)],
'weight_method': weight_method
})
node_ids[i].append(next(id_cnt))
node_ids[max_level].append(node_ids[max_level][-1])
# NOTE: the order of the quad path is reversed from the original, my code expects the output of
# each FPN repeat to be same as input from backbone, in order of increasing reductions
for i in range(min_level, max_level + 1):
# quad-add path.
p.nodes.append({
'reduction': 1 << i,
'inputs_offsets': [node_ids[i][2], node_ids[i][4]],
'weight_method': quad_method
})
node_ids[i].append(next(id_cnt))
return p
import fairseq #don't required
import torch
from omegaconf import OmegaConf
from desed_task.nnet.WRNN_2_ import WRNN
import pdb
model = torch.load('/home1/irteam/users/koo/trained2.pt')
big_cfg = OmegaConf.create(vars(vars(model['args'])['w2v_args']))
sed = WRNN(big_cfg)
pdb.set_trace()
sed.w2v.load_state_dict(model['model'])
def test_class_instantiate_passthrough(conf, expected):
conf = OmegaConf.create(conf)
obj = utils.instantiate(conf, 10, d=40)
assert obj == expected
def load_checkpoint_to_cpu(path, arg_overrides=None, load_on_all_ranks=False):
"""Loads a checkpoint to CPU (with upgrading for backward compatibility).
If doing single-GPU training or if the checkpoint is only being loaded by at
most one process on each node (current default behavior is for only rank 0
to read the checkpoint from disk), load_on_all_ranks should be False to
avoid errors from torch.distributed not having been initialized or
torch.distributed.barrier() hanging.
If all processes on each node may be loading the checkpoint
simultaneously, load_on_all_ranks should be set to True to avoid I/O
conflicts.
There's currently no support for > 1 but < all processes loading the
checkpoint on each node.
"""
local_path = PathManager.get_local_path(path)
# The locally cached file returned by get_local_path() may be stale for
# remote files that are periodically updated/overwritten (ex:
# checkpoint_last.pt) - so we remove the local copy, sync across processes
# (if needed), and then download a fresh copy.
if local_path != path and PathManager.path_requires_pathmanager(path):
try:
os.remove(local_path)
except FileNotFoundError:
# With potentially multiple processes removing the same file, the
# file being missing is benign (missing_ok isn't available until
# Python 3.8).
pass
if load_on_all_ranks:
torch.distributed.barrier()
local_path = PathManager.get_local_path(path)
with open(local_path, "rb") as f:
state = torch.load(f, map_location=torch.device("cpu"))
if "args" in state and state[
"args"] is not None and arg_overrides is not None:
args = state["args"]
for arg_name, arg_val in arg_overrides.items():
setattr(args, arg_name, arg_val)
if "cfg" in state and state["cfg"] is not None:
# hack to be able to set Namespace in dict config. this should be removed when we update to newer
# omegaconf version that supports object flags, or when we migrate all existing models
from omegaconf import _utils
old_primitive = _utils.is_primitive_type
_utils.is_primitive_type = lambda _: True
state["cfg"] = OmegaConf.create(state["cfg"])
_utils.is_primitive_type = old_primitive
OmegaConf.set_struct(state["cfg"], True)
if arg_overrides is not None:
overwrite_args_by_name(state["cfg"], arg_overrides)
state = _upgrade_state_dict(state)
return state
def __init__(self, cfg: DictConfig, trainer: Trainer = None):
# Get global rank and total number of GPU workers for IterableDataset partitioning, if applicable
self.global_rank = 0
self.world_size = 0
self.local_rank = 0
if trainer is not None:
self.global_rank = (trainer.node_rank *
trainer.num_gpus) + trainer.local_rank
self.world_size = trainer.num_nodes * trainer.num_gpus
self.local_rank = trainer.local_rank
super().__init__(cfg=cfg, trainer=trainer)
schema = OmegaConf.structured(Wav2VecEncoderModelConfig)
if isinstance(cfg, dict):
cfg = OmegaConf.create(cfg)
elif not isinstance(cfg, DictConfig):
raise ValueError(
f"cfg was type: {type(cfg)}. Expected either a dict or a DictConfig"
)
cfg = OmegaConf.create(OmegaConf.to_container(cfg, resolve=True))
cfg = OmegaConf.merge(schema, cfg)
feature_enc_layers = cfg.conv_feature_encoder.conv_feature_layers
self.embed = feature_enc_layers[-1][
0] # Select last conv output layer dimension
self.feature_extractor = ConvFeatureEncoder(
conv_layers=feature_enc_layers,
mode=cfg.conv_feature_encoder.extractor_mode,
conv_bias=cfg.conv_feature_encoder.conv_bias,
)
encoder_embed_dim = cfg.transformer_encoder.encoder.embedding_dim
self.post_extract_proj = (nn.Linear(self.embed, encoder_embed_dim)
if self.embed != encoder_embed_dim
and not cfg.quantizer.quantize_input else
None)
self.mask_cfg = cfg.masking
self.dropout_input = nn.Dropout(cfg.dropout_input)
self.dropout_features = nn.Dropout(cfg.dropout_features)
self.feature_grad_mult = cfg.feature_grad_mult
self.quantizer = None
self.input_quantizer = None
self.n_negatives = cfg.n_negatives
self.cross_sample_negatives = cfg.cross_sample_negatives
self.codebook_negatives = cfg.codebook_negatives
self.negatives_from_everywhere = cfg.negatives_from_everywhere
final_dim = cfg.final_dim if cfg.final_dim > 0 else encoder_embed_dim
self.final_dim = final_dim
if cfg.quantizer.quantize_targets:
vq_dim = cfg.quantizer.latent_dim if cfg.quantizer.latent_dim > 0 else final_dim
self.quantizer = GumbelVectorQuantizer(
dim=self.embed,
num_vars=cfg.quantizer.latent_vars,
temp=cfg.quantizer.latent_temp,
groups=cfg.quantizer.latent_groups,
combine_groups=False,
vq_dim=vq_dim,
time_first=True,
)
self.project_q = nn.Linear(vq_dim, final_dim)
else:
self.project_q = nn.Linear(self.embed, final_dim)
if cfg.quantizer.quantize_input:
if cfg.quantizer.same_quantizer and self.quantizer is not None:
vq_dim = final_dim
self.input_quantizer = self.quantizer
else:
vq_dim = cfg.quantizer.latent_dim if cfg.quantizer.latent_dim > 0 else encoder_embed_dim
self.input_quantizer = GumbelVectorQuantizer(
dim=self.embed,
num_vars=cfg.quantizer.latent_vars,
temp=cfg.quantizer.latent_temp,
groups=cfg.quantizer.latent_groups,
combine_groups=False,
vq_dim=vq_dim,
time_first=True,
)
self.project_inp = nn.Linear(vq_dim, encoder_embed_dim)
self.mask_emb = nn.Parameter(
torch.FloatTensor(encoder_embed_dim).uniform_())
self.encoder = Wav2VecTransformerEncoder(cfg.transformer_encoder)
self.layer_norm = nn.LayerNorm(self.embed)
self.target_glu = None
if cfg.target_glu:
self.target_glu = nn.Sequential(
nn.Linear(final_dim, final_dim * 2), nn.GLU())
self.final_proj = nn.Linear(encoder_embed_dim, final_dim)
self.loss = Wav2VecLoss(
feature_loss_weight=cfg.loss.feature_loss_weight,
prob_ppl_weight=cfg.loss.prob_ppl_weight,
logit_temp=cfg.logit_temp,
)
result = trainer.fit(model)
assert result == 1
assert trainer.custom_kwarg == 'custom'
assert trainer.fast_dev_run
# when we pass in an unknown arg, the base class should complain
with pytest.raises(
TypeError,
match=r"__init__\(\) got an unexpected keyword argument 'abcdefg'"
):
TrainerSubclass(abcdefg='unknown_arg')
@pytest.mark.parametrize('trainer_params', [
OmegaConf.create({
'max_epochs': 1,
'gpus': 1
}),
OmegaConf.create({
'max_epochs': 1,
'gpus': [0]
}),
])
@pytest.mark.skipif(not torch.cuda.is_available(),
reason="test requires GPU machine")
def test_trainer_omegaconf(trainer_params):
Trainer(**trainer_params)
def test_trainer_pickle(tmpdir):
trainer = Trainer(
max_epochs=1,
def __init__(self) -> None:
self.conf: DictConfig = OmegaConf.create()
self.set("name", "UNKNOWN_NAME")
model_type: str = MISSING
model_name: str = MISSING
train_params: Dict[Any, Any] = field(default_factory=dict)
#%%
try:
yaml_path = sys.argv[1]
except IndexError:
print(
"usage: python train_decision_tree.py <path to configguraiton yaml>",
file=sys.stderr,
)
sys.exit(-1)
conf = OmegaConf.create(TableModelConfig(**OmegaConf.load(yaml_path)))
#%%
def make_sklearn_api_model(train_df, conf):
from importlib import import_module
module = import_module(conf.model_module)
model = getattr(module, conf.model_type)(**conf.train_params)
model.fit(train_df[conf.train_feats], train_df[conf.target])
return model
def mse(model, x, y) -> float:
diff = y - model.predict(x)
return float(diff @ diff) / len(y)
def resolve_test_dataloaders(model: 'ModelPT'):
"""
Helper method that operates on the ModelPT class to automatically support
multiple dataloaders for the test set.
It does so by first resolving the path to one/more data files via `resolve_dataset_name_from_cfg()`.
If this resolution fails, it assumes the data loader is prepared to manually support / not support
multiple data loaders and simply calls the appropriate setup method.
If resolution succeeds:
Checks if provided path is to a single file or a list of files.
If a single file is provided, simply tags that file as such and loads it via the setup method.
If multiple files are provided:
Inject a new manifest path at index "i" into the resolved key.
Calls the appropriate setup method to set the data loader.
Collects the initialized data loader in a list and preserves it.
Once all data loaders are processed, assigns the list of loaded loaders to the ModelPT.
Finally assigns a list of unique names resolved from the file paths to the ModelPT.
Args:
model: ModelPT subclass, which requires >=1 Test Dataloaders to be setup.
"""
cfg = copy.deepcopy(model._cfg)
dataloaders = []
# process test_loss_idx
if 'test_dl_idx' in cfg.test_ds:
cfg = OmegaConf.to_container(cfg)
test_dl_idx = cfg['test_ds'].pop('test_dl_idx')
cfg = OmegaConf.create(cfg)
else:
test_dl_idx = 0
# Set val_loss_idx
model._test_dl_idx = test_dl_idx
ds_key = resolve_dataset_name_from_cfg(cfg.test_ds)
if ds_key is None:
logging.debug("Could not resolve file path from provided config - {}. "
"Disabling support for multi-dataloaders.".format(
cfg.test_ds))
model.setup_test_data(cfg.test_ds)
return
ds_values = cfg.test_ds[ds_key]
if isinstance(ds_values, (list, tuple, ListConfig)):
for ds_value in ds_values:
cfg.test_ds[ds_key] = ds_value
model.setup_test_data(cfg.test_ds)
dataloaders.append(model._test_dl)
model._test_dl = dataloaders
model._test_names = [parse_dataset_as_name(ds) for ds in ds_values]
unique_names_check(name_list=model._test_names)
return
else:
model.setup_test_data(cfg.test_ds)
model._test_names = [parse_dataset_as_name(ds_values)]
unique_names_check(name_list=model._test_names)
BASE_TEST_CFG = OmegaConf.create({
# Standard data types.
"str": "hi",
"int": 123,
"float": 1.2,
"dict": {
"a": 0,
"b": {
"c": 1
}
},
"list": [x - 1 for x in range(11)],
"null": None,
# Special cases.
"x@y": 123, # @ in name
"$x$y$z$": 456, # $ in name (beginning, middle and end)
"0": 0, # integer name
"FalsE": {
"TruE": True
}, # bool name
"None": {
"null": 1
}, # null-like name
"1": {
"2": 12
}, # dot-path with int keys
# Used in nested interpolations.
"str_test": "test",
"ref_str": "str",
"options": {
"a": "A",
"b": "B"
},
"choice": "a",
"rel_opt": ".options",
})
def _get_kwargs(
config: Union[DictConfig, ListConfig],
root: bool = True,
**kwargs: Any,
) -> Any:
from hydra.utils import instantiate
assert OmegaConf.is_config(config)
if OmegaConf.is_list(config):
assert isinstance(config, ListConfig)
return [
_get_kwargs(x, root=False) if OmegaConf.is_config(x) else x for x in config
]
assert OmegaConf.is_dict(config), "Input config is not an OmegaConf DictConfig"
recursive = _is_recursive(config, kwargs)
overrides = OmegaConf.create(kwargs, flags={"allow_objects": True})
config.merge_with(overrides)
final_kwargs = OmegaConf.create(flags={"allow_objects": True})
final_kwargs._set_parent(config._get_parent())
final_kwargs._set_flag("readonly", False)
final_kwargs._set_flag("struct", False)
if recursive:
for k, v in config.items_ex(resolve=False):
if OmegaConf.is_none(v):
final_kwargs[k] = v
elif _is_target(v):
final_kwargs[k] = instantiate(v)
elif OmegaConf.is_dict(v):
d = OmegaConf.create({}, flags={"allow_objects": True})
for key, value in v.items_ex(resolve=False):
if _is_target(value):
d[key] = instantiate(value)
elif OmegaConf.is_config(value):
d[key] = _get_kwargs(value, root=False)
else:
d[key] = value
d._metadata.object_type = v._metadata.object_type
final_kwargs[k] = d
elif OmegaConf.is_list(v):
lst = OmegaConf.create([], flags={"allow_objects": True})
for x in v:
if _is_target(x):
lst.append(instantiate(x))
elif OmegaConf.is_config(x):
lst.append(_get_kwargs(x, root=False))
lst[-1]._metadata.object_type = x._metadata.object_type
else:
lst.append(x)
final_kwargs[k] = lst
else:
final_kwargs[k] = v
else:
for k, v in config.items_ex(resolve=False):
final_kwargs[k] = v
final_kwargs._set_flag("readonly", None)
final_kwargs._set_flag("struct", None)
final_kwargs._set_flag("allow_objects", None)
if not root:
# This is tricky, since the root kwargs is exploded anyway we can treat is as an untyped dict
# the motivation is that the object type is used as an indicator to treat the object differently during
# conversion to a primitive container in some cases
final_kwargs._metadata.object_type = config._metadata.object_type
return final_kwargs
def test_readonly_from_cli():
c = OmegaConf.create({"foo": {"bar": [1]}})
OmegaConf.set_readonly(c, True)
cli = OmegaConf.from_dotlist(["foo.bar=[2]"])
with raises(ReadonlyConfigError, match="foo.bar"):
OmegaConf.merge(c, cli)
