def get_from_params(
cls,
heads_params: Dict,
encoder_params: Dict = None,
embedders_params: Dict = None,
in_features: int = None,
) -> "Hydra":
heads_params_ = deepcopy(heads_params)
encoder_params_ = deepcopy(encoder_params)
embedders_params_ = deepcopy(embedders_params)
def _get_normalization_keyword(dct: Dict):
return dct.pop(Hydra._normalize_keyword, False) \
if dct is not None \
else False
if encoder_params_ is not None:
normalize_embeddings: bool = \
_get_normalization_keyword(encoder_params_)
encoder = SequentialNet(**encoder_params_)
in_features = encoder_params_["hiddens"][-1]
if normalize_embeddings:
encoder = nn.Sequential(encoder, Normalize())
else:
assert in_features is not None
encoder = None
heads = Hydra.parse_head_params(head_params=heads_params_,
in_features=in_features)
assert isinstance(heads, nn.ModuleDict)
embedders = {}
if embedders_params_ is not None:
for key, head_params in embedders_params_.items():
if isinstance(head_params, int):
head_params = {"num_embeddings": head_params}
normalize_ = head_params.pop(Hydra._normalize_keyword, False)
block = [("embedding",
nn.Embedding(
embedding_dim=in_features,
**head_params,
))]
if normalize_:
block.append(("normalize", Normalize()))
block = OrderedDict(block)
block = nn.Sequential(block)
embedders[key] = block
embedders = nn.ModuleDict(embedders)
net = cls(
heads=heads,
encoder=encoder,
embedders=embedders,
)
return net
def get_from_params(
cls,
heads_params: Dict,
encoder_params: Dict = None,
embedders_params: Dict = None,
in_features: int = None,
) -> "Hydra":
"""@TODO: Docs. Contribution is welcome."""
heads_params_copy = deepcopy(heads_params)
encoder_params_copy = deepcopy(encoder_params)
embedders_params_copy = deepcopy(embedders_params)
def _get_normalization_keyword(dct: Dict):
return (dct.pop(Hydra.normalize_keyword, False)
if dct is not None else False)
if encoder_params_copy is not None:
normalize_embeddings: bool = _get_normalization_keyword(
encoder_params_copy)
encoder = SequentialNet(**encoder_params_copy)
in_features = encoder_params_copy["hiddens"][-1]
if normalize_embeddings:
encoder = nn.Sequential(encoder, Normalize())
else:
assert in_features is not None
encoder = None
heads = Hydra.parse_head_params(head_params=heads_params_copy,
in_features=in_features)
assert isinstance(heads, nn.ModuleDict)
embedders = {}
if embedders_params_copy is not None:
for key, head_params in embedders_params_copy.items():
if isinstance(head_params, int):
head_params = {"num_embeddings": head_params}
need_normalize = head_params.pop(Hydra.normalize_keyword,
False)
block = [(
"embedding",
nn.Embedding(
embedding_dim=in_features,
**head_params,
),
)]
if need_normalize:
block.append(("normalize", Normalize()))
block = OrderedDict(block)
block = nn.Sequential(block)
embedders[key] = block
embedders = nn.ModuleDict(embedders)
net = cls(heads=heads, encoder=encoder, embedders=embedders)
return net
def parse_head_params(
head_params: Dict,
in_features: int,
is_leaf: bool = False,
) -> Union[nn.Module, nn.ModuleDict]:
"""@TODO: Docs. Contribution is welcome."""
if is_leaf:
if isinstance(head_params, int):
head_params = {"hiddens": [head_params]}
normalize = head_params.pop(Hydra.normalize_keyword, False)
head_params["hiddens"].insert(0, in_features)
output = [("net", SequentialNet(**head_params))]
if normalize:
output.append(("normalize", Normalize()))
output = OrderedDict(output)
output = nn.Sequential(output)
else:
output = {}
hidden_params = head_params.pop(Hydra.hidden_keyword, None)
if hidden_params is not None:
in_features = (hidden_params if isinstance(hidden_params, int)
else hidden_params["hiddens"][-1])
output[Hydra.hidden_keyword] = Hydra.parse_head_params(
head_params=hidden_params,
in_features=in_features,
is_leaf=True,
)
for head_branch_name, head_branch_params in head_params.items():
output[head_branch_name] = Hydra.parse_head_params(
head_params=head_branch_params,
in_features=in_features,
is_leaf=not head_branch_name.startswith(
Hydra.parent_keyword),
)
output = nn.ModuleDict(output)
return output
def test_config4():
config_path = Path(__file__).absolute().parent / "config4.yml"
config4 = utils.load_config(config_path)["model_params"]
with pytest.raises(AssertionError):
hydra = Hydra.get_from_params(**config4)
config4["in_features"] = 16
hydra = Hydra.get_from_params(**config4)
config4_ = copy.deepcopy(config4)
_pop_normalization(config4_)
heads_params = config4_["heads_params"]
heads_params["head1"]["hiddens"].insert(0, 16)
heads_params["_head2"]["_hidden"]["hiddens"].insert(0, 16)
heads_params["_head2"]["head2_1"]["hiddens"].insert(0, 16)
heads_params["_head2"]["_head2_2"]["_hidden"]["hiddens"].insert(0, 16)
heads_params["_head2"]["_head2_2"]["head2_2_1"]["hiddens"].insert(0, 16)
net = nn.ModuleDict({
"encoder":
nn.Sequential(),
"heads":
nn.ModuleDict({
"head1":
nn.Sequential(
OrderedDict([
("net", SequentialNet(**heads_params["head1"])),
])),
"_head2":
nn.ModuleDict({
"_hidden":
nn.Sequential(
OrderedDict([
("net",
SequentialNet(**heads_params["_head2"]["_hidden"]))
])),
"head2_1":
nn.Sequential(
OrderedDict([
("net",
SequentialNet(**heads_params["_head2"]["head2_1"])),
("normalize", Normalize()),
])),
"_head2_2":
nn.ModuleDict({
"_hidden":
nn.Sequential(
OrderedDict([("net",
SequentialNet(**heads_params["_head2"]
["_head2_2"]["_hidden"]))
])),
"head2_2_1":
nn.Sequential(
OrderedDict([("net",
SequentialNet(**heads_params["_head2"]
["_head2_2"]["head2_2_1"]))
])),
})
})
})
})
_check_named_parameters(hydra.encoder, net["encoder"])
_check_named_parameters(hydra.heads, net["heads"])
assert hydra.embedders == {}
input_ = torch.rand(1, 16)
output_kv = hydra(input_)
assert (input_ == output_kv["features"]).sum().item() == 16
assert (input_ == output_kv["embeddings"]).sum().item() == 16
kv_keys = [
"features",
"embeddings",
"head1",
"_head2/",
"_head2/head2_1",
"_head2/_head2_2/",
"_head2/_head2_2/head2_2_1",
]
_check_lists(output_kv.keys(), kv_keys)
with pytest.raises(KeyError):
output_kv = hydra(input_, target1=torch.ones(1, 2).long())
with pytest.raises(KeyError):
output_kv = hydra(input_, target2=torch.ones(1, 2).long())
with pytest.raises(KeyError):
output_kv = hydra(input_,
target1=torch.ones(1, 2).long(),
target2=torch.ones(1, 2).long())
output_tuple = hydra.forward_tuple(input_)
assert len(output_tuple) == 5
assert (output_tuple[0] == output_kv["features"]).sum().item() == 16
assert (output_tuple[1] == output_kv["embeddings"]).sum().item() == 16
def test_config3():
config_path = Path(__file__).absolute().parent / "config3.yml"
config3 = utils.load_config(config_path)["model_params"]
hydra = Hydra.get_from_params(**config3)
config3_ = copy.deepcopy(config3)
_pop_normalization(config3_)
encoder_params = config3_["encoder_params"]
heads_params = config3_["heads_params"]
heads_params["head1"]["hiddens"].insert(0, 16)
heads_params["_head2"]["_hidden"]["hiddens"].insert(0, 16)
heads_params["_head2"]["head2_1"]["hiddens"].insert(0, 16)
heads_params["_head2"]["_head2_2"]["_hidden"]["hiddens"].insert(0, 16)
heads_params["_head2"]["_head2_2"]["head2_2_1"]["hiddens"].insert(0, 16)
net = nn.ModuleDict({
"encoder":
SequentialNet(**encoder_params),
"embedders":
nn.ModuleDict({
"target1":
nn.Sequential(
OrderedDict([
("embedding",
nn.Embedding(embedding_dim=16, num_embeddings=2)),
("normalize", Normalize()),
])),
"target2":
nn.Sequential(
OrderedDict([
("embedding",
nn.Embedding(embedding_dim=16, num_embeddings=2)),
])),
}),
"heads":
nn.ModuleDict({
"head1":
nn.Sequential(
OrderedDict([
("net", SequentialNet(**heads_params["head1"])),
])),
"_head2":
nn.ModuleDict({
"_hidden":
nn.Sequential(
OrderedDict([
("net",
SequentialNet(**heads_params["_head2"]["_hidden"]))
])),
"head2_1":
nn.Sequential(
OrderedDict([
("net",
SequentialNet(**heads_params["_head2"]["head2_1"])),
("normalize", Normalize()),
])),
"_head2_2":
nn.ModuleDict({
"_hidden":
nn.Sequential(
OrderedDict([("net",
SequentialNet(**heads_params["_head2"]
["_head2_2"]["_hidden"]))
])),
"head2_2_1":
nn.Sequential(
OrderedDict([("net",
SequentialNet(**heads_params["_head2"]
["_head2_2"]["head2_2_1"]))
])),
})
})
})
})
_check_named_parameters(hydra.encoder, net["encoder"])
_check_named_parameters(hydra.heads, net["heads"])
_check_named_parameters(hydra.embedders, net["embedders"])
input_ = torch.rand(1, 16)
output_kv = hydra(input_)
assert (input_ == output_kv["features"]).sum().item() == 16
kv_keys = [
"features",
"embeddings",
"head1",
"_head2/",
"_head2/head2_1",
"_head2/_head2_2/",
"_head2/_head2_2/head2_2_1",
]
_check_lists(output_kv.keys(), kv_keys)
output_kv = hydra(input_, target1=torch.ones(1, 2).long())
kv_keys = [
"features",
"embeddings",
"head1",
"_head2/",
"_head2/head2_1",
"_head2/_head2_2/",
"_head2/_head2_2/head2_2_1",
"target1_embeddings",
]
_check_lists(output_kv.keys(), kv_keys)
output_kv = hydra(input_, target2=torch.ones(1, 2).long())
kv_keys = [
"features",
"embeddings",
"head1",
"_head2/",
"_head2/head2_1",
"_head2/_head2_2/",
"_head2/_head2_2/head2_2_1",
"target2_embeddings",
]
_check_lists(output_kv.keys(), kv_keys)
output_kv = hydra(input_,
target1=torch.ones(1, 2).long(),
target2=torch.ones(1, 2).long())
kv_keys = [
"features",
"embeddings",
"head1",
"_head2/",
"_head2/head2_1",
"_head2/_head2_2/",
"_head2/_head2_2/head2_2_1",
"target1_embeddings",
"target2_embeddings",
]
_check_lists(output_kv.keys(), kv_keys)
output_tuple = hydra.forward_tuple(input_)
assert len(output_tuple) == 5
assert (output_tuple[0] == output_kv["features"]).sum().item() == 16
assert (output_tuple[1] == output_kv["embeddings"]).sum().item() == 16
def test_config1():
config1 = {
"encoder_params": {
"hiddens": [16, 16],
"layer_fn": {
"module": "Linear",
"bias": False
},
"norm_fn": "LayerNorm",
},
"heads_params": {
"head1": {
"hiddens": [2],
"layer_fn": {
"module": "Linear",
"bias": True
},
},
"_head2": {
"_hidden": {
"hiddens": [16],
"layer_fn": {
"module": "Linear",
"bias": False
},
},
"head2_1": {
"hiddens": [32],
"layer_fn": {
"module": "Linear",
"bias": True
},
"normalize_output": True
},
"_head2_2": {
"_hidden": {
"hiddens": [16, 16, 16],
"layer_fn": {
"module": "Linear",
"bias": False
},
},
"head2_2_1": {
"hiddens": [32],
"layer_fn": {
"module": "Linear",
"bias": True
},
"normalize_output": False,
},
},
},
},
"embedders_params": {
"target1": {
"num_embeddings": 2,
"normalize_output": True,
},
"target2": {
"num_embeddings": 2,
"normalize_output": False,
},
}
}
hydra = Hydra.get_from_params(**config1)
config1_ = copy.deepcopy(config1)
_pop_normalization(config1_)
encoder_params = config1_["encoder_params"]
heads_params = config1_["heads_params"]
heads_params["head1"]["hiddens"].insert(0, 16)
heads_params["_head2"]["_hidden"]["hiddens"].insert(0, 16)
heads_params["_head2"]["head2_1"]["hiddens"].insert(0, 16)
heads_params["_head2"]["_head2_2"]["_hidden"]["hiddens"].insert(0, 16)
heads_params["_head2"]["_head2_2"]["head2_2_1"]["hiddens"].insert(0, 16)
net = nn.ModuleDict({
"encoder":
SequentialNet(**encoder_params),
"embedders":
nn.ModuleDict({
"target1":
nn.Sequential(
OrderedDict([
("embedding",
nn.Embedding(embedding_dim=16, num_embeddings=2)),
("normalize", Normalize()),
])),
"target2":
nn.Sequential(
OrderedDict([
("embedding",
nn.Embedding(embedding_dim=16, num_embeddings=2)),
])),
}),
"heads":
nn.ModuleDict({
"head1":
nn.Sequential(
OrderedDict([
("net", SequentialNet(**heads_params["head1"])),
])),
"_head2":
nn.ModuleDict({
"_hidden":
nn.Sequential(
OrderedDict([
("net",
SequentialNet(**heads_params["_head2"]["_hidden"]))
])),
"head2_1":
nn.Sequential(
OrderedDict([
("net",
SequentialNet(**heads_params["_head2"]["head2_1"])),
("normalize", Normalize()),
])),
"_head2_2":
nn.ModuleDict({
"_hidden":
nn.Sequential(
OrderedDict([("net",
SequentialNet(**heads_params["_head2"]
["_head2_2"]["_hidden"]))
])),
"head2_2_1":
nn.Sequential(
OrderedDict([("net",
SequentialNet(**heads_params["_head2"]
["_head2_2"]["head2_2_1"]))
])),
})
})
})
})
_check_named_parameters(hydra.encoder, net["encoder"])
_check_named_parameters(hydra.heads, net["heads"])
_check_named_parameters(hydra.embedders, net["embedders"])
input_ = torch.rand(1, 16)
output_kv = hydra(input_)
assert (input_ == output_kv["features"]).sum().item() == 16
kv_keys = [
"features",
"embeddings",
"head1",
"_head2/",
"_head2/head2_1",
"_head2/_head2_2/",
"_head2/_head2_2/head2_2_1",
]
_check_lists(output_kv.keys(), kv_keys)
output_kv = hydra(input_, target1=torch.ones(1, 2).long())
kv_keys = [
"features",
"embeddings",
"head1",
"_head2/",
"_head2/head2_1",
"_head2/_head2_2/",
"_head2/_head2_2/head2_2_1",
"target1_embeddings",
]
_check_lists(output_kv.keys(), kv_keys)
output_kv = hydra(input_, target2=torch.ones(1, 2).long())
kv_keys = [
"features",
"embeddings",
"head1",
"_head2/",
"_head2/head2_1",
"_head2/_head2_2/",
"_head2/_head2_2/head2_2_1",
"target2_embeddings",
]
_check_lists(output_kv.keys(), kv_keys)
output_kv = hydra(input_,
target1=torch.ones(1, 2).long(),
target2=torch.ones(1, 2).long())
kv_keys = [
"features",
"embeddings",
"head1",
"_head2/",
"_head2/head2_1",
"_head2/_head2_2/",
"_head2/_head2_2/head2_2_1",
"target1_embeddings",
"target2_embeddings",
]
_check_lists(output_kv.keys(), kv_keys)
output_tuple = hydra.forward_tuple(input_)
assert len(output_tuple) == 5
assert (output_tuple[0] == output_kv["features"]).sum().item() == 16
assert (output_tuple[1] == output_kv["embeddings"]).sum().item() == 16
def test_config2():
config2 = {
"in_features": 16,
"heads_params": {
"head1": {
"hiddens": [2],
"layer_fn": {
"module": "Linear",
"bias": True
},
},
"_head2": {
"_hidden": {
"hiddens": [16],
"layer_fn": {
"module": "Linear",
"bias": False
},
},
"head2_1": {
"hiddens": [32],
"layer_fn": {
"module": "Linear",
"bias": True
},
"normalize_output": True
},
"_head2_2": {
"_hidden": {
"hiddens": [16, 16, 16],
"layer_fn": {
"module": "Linear",
"bias": False
},
},
"head2_2_1": {
"hiddens": [32],
"layer_fn": {
"module": "Linear",
"bias": True
},
"normalize_output": False,
},
},
},
},
}
hydra = Hydra.get_from_params(**config2)
config2_ = copy.deepcopy(config2)
_pop_normalization(config2_)
heads_params = config2_["heads_params"]
heads_params["head1"]["hiddens"].insert(0, 16)
heads_params["_head2"]["_hidden"]["hiddens"].insert(0, 16)
heads_params["_head2"]["head2_1"]["hiddens"].insert(0, 16)
heads_params["_head2"]["_head2_2"]["_hidden"]["hiddens"].insert(0, 16)
heads_params["_head2"]["_head2_2"]["head2_2_1"]["hiddens"].insert(0, 16)
net = nn.ModuleDict({
"encoder":
nn.Sequential(),
"heads":
nn.ModuleDict({
"head1":
nn.Sequential(
OrderedDict([
("net", SequentialNet(**heads_params["head1"])),
])),
"_head2":
nn.ModuleDict({
"_hidden":
nn.Sequential(
OrderedDict([
("net",
SequentialNet(**heads_params["_head2"]["_hidden"]))
])),
"head2_1":
nn.Sequential(
OrderedDict([
("net",
SequentialNet(**heads_params["_head2"]["head2_1"])),
("normalize", Normalize()),
])),
"_head2_2":
nn.ModuleDict({
"_hidden":
nn.Sequential(
OrderedDict([("net",
SequentialNet(**heads_params["_head2"]
["_head2_2"]["_hidden"]))
])),
"head2_2_1":
nn.Sequential(
OrderedDict([("net",
SequentialNet(**heads_params["_head2"]
["_head2_2"]["head2_2_1"]))
])),
})
})
})
})
_check_named_parameters(hydra.encoder, net["encoder"])
_check_named_parameters(hydra.heads, net["heads"])
assert hydra.embedders == {}
input_ = torch.rand(1, 16)
output_kv = hydra(input_)
assert (input_ == output_kv["features"]).sum().item() == 16
assert (input_ == output_kv["embeddings"]).sum().item() == 16
kv_keys = [
"features",
"embeddings",
"head1",
"_head2/",
"_head2/head2_1",
"_head2/_head2_2/",
"_head2/_head2_2/head2_2_1",
]
_check_lists(output_kv.keys(), kv_keys)
with pytest.raises(KeyError):
output_kv = hydra(input_, target1=torch.ones(1, 2).long())
with pytest.raises(KeyError):
output_kv = hydra(input_, target2=torch.ones(1, 2).long())
with pytest.raises(KeyError):
output_kv = hydra(input_,
target1=torch.ones(1, 2).long(),
target2=torch.ones(1, 2).long())
output_tuple = hydra.forward_tuple(input_)
assert len(output_tuple) == 5
assert (output_tuple[0] == output_kv["features"]).sum().item() == 16
assert (output_tuple[1] == output_kv["embeddings"]).sum().item() == 16