def test_input_shape_and_dtype(
gru_model: GRUModel,
batch_prev_tkids: torch.Tensor,
):
r"""Input must be long tensor."""
try:
gru_model = gru_model.eval()
gru_model.pred(batch_prev_tkids)
except Exception:
assert False
def test_value_range(
gru_model: GRUModel,
batch_prev_tkids: torch.Tensor,
):
r"""Return values are probabilities."""
gru_model = gru_model.eval()
out = gru_model.pred(batch_prev_tkids)
# Probabilities are values within range [0, 1].
assert torch.all(0 <= out).item()
assert torch.all(out <= 1).item()
# Sum of the probabilities equals to 1.
accum_out = out.sum(dim=-1)
assert torch.allclose(accum_out, torch.ones_like(accum_out))
def test_forward_path(
gru_model: GRUModel,
batch_prev_tkids: torch.Tensor,
):
r"""Parameters used during forward must have gradients."""
# Make sure model has no gradients.
gru_model = gru_model.train()
gru_model.zero_grad()
gru_model(batch_prev_tkids).sum().backward()
assert hasattr(gru_model.emb.weight.grad, 'grad')
assert hasattr(gru_model.pre_hid[0].weight.grad, 'grad')
assert hasattr(gru_model.hid.weight_ih_l0.grad, 'grad')
assert hasattr(gru_model.post_hid[-1].weight.grad, 'grad')
def test_return_shape_and_dtype(
gru_model: GRUModel,
batch_prev_tkids: torch.Tensor,
batch_next_tkids: torch.Tensor,
):
r"""Return float tensor with 0 dimension."""
gru_model = gru_model.eval()
loss = gru_model.loss_fn(
batch_prev_tkids=batch_prev_tkids,
batch_next_tkids=batch_next_tkids,
)
# 0 dimension tensor.
assert loss.shape == torch.Size([])
# Return float tensor.
assert loss.dtype == torch.float
def test_invalid_input_vocab_size(self):
r"""Raise exception when input `vocab_size` is invalid."""
msg1 = (
'Must raise `TypeError` or `ValueError` when input `vocab_size` '
'is invalid.')
msg2 = 'Inconsistent error message.'
examples = (False, 0, 0.0, 1.0, math.nan, -math.nan,
math.inf, -math.inf, 0j, 1j, '', b'', (), [], {}, set(),
object(), lambda x: x, type, None, NotImplemented, ...)
for invalid_input in examples:
with self.assertRaises((TypeError, ValueError),
msg=msg1) as ctx_man:
GRUModel(d_emb=1,
d_hid=1,
dropout=0.1,
num_linear_layers=1,
num_rnn_layers=1,
pad_token_id=0,
vocab_size=invalid_input)
if isinstance(ctx_man.exception, TypeError):
self.assertEqual(ctx_man.exception.args[0],
'`vocab_size` must be an instance of `int`.',
msg=msg2)
else:
self.assertEqual(
ctx_man.exception.args[0],
'`vocab_size` must be bigger than or equal to `1`.',
msg=msg2)
def test_input_shape_and_dtype(
gru_model: GRUModel,
batch_prev_tkids: torch.Tensor,
batch_next_tkids: torch.Tensor,
):
r"""Input tensors must be long tensors and have the same shape.
Same shape is required since we are using teacher forcing.
"""
try:
gru_model = gru_model.eval()
gru_model.loss_fn(
batch_prev_tkids=batch_prev_tkids,
batch_next_tkids=batch_next_tkids,
)
except Exception:
assert False
def test_return_shape_and_dtype(
gru_model: GRUModel,
batch_prev_tkids: torch.Tensor,
):
r"""Return float tensor with correct shape."""
gru_model = gru_model.eval()
out = gru_model.pred(batch_prev_tkids)
# Output float tensor.
assert out.dtype == torch.float
# Input shape: (B, S).
# Output shape: (B, S, V).
assert out.shape == (
batch_prev_tkids.shape[0],
batch_prev_tkids.shape[1],
gru_model.emb.num_embeddings,
)
def test_back_propagation_path(
gru_model: GRUModel,
batch_prev_tkids: torch.Tensor,
batch_next_tkids: torch.Tensor,
):
r"""Gradients with respect to loss must get back propagated."""
# Make sure model has no gradients.
gru_model = gru_model.train()
gru_model.zero_grad()
gru_model.loss_fn(
batch_prev_tkids=batch_prev_tkids,
batch_next_tkids=batch_next_tkids,
).backward()
assert hasattr(gru_model.emb.weight.grad, 'grad')
assert hasattr(gru_model.pre_hid[0].weight.grad, 'grad')
assert hasattr(gru_model.hid.weight_ih_l0.grad, 'grad')
assert hasattr(gru_model.post_hid[-1].weight.grad, 'grad')
def setUp(self):
r"""Setup hyperparameters and construct `GRUModel`."""
self.model_objs = []
cls = self.__class__
for d_emb in cls.d_emb_range:
for d_hid in cls.d_hid_range:
for dropout in cls.dropout_range:
for num_linear_layers in cls.num_linear_layers_range:
for num_rnn_layers in cls.num_rnn_layers_range:
for pad_token_id in cls.pad_token_id_range:
for vocab_size in cls.vocab_size_range:
# skip invalid construct.
if vocab_size <= pad_token_id:
continue
model = GRUModel(
d_emb=d_emb,
d_hid=d_hid,
dropout=dropout,
num_linear_layers=num_linear_layers,
num_rnn_layers=num_rnn_layers,
pad_token_id=pad_token_id,
vocab_size=vocab_size)
self.model_objs.append({
'd_emb':
d_emb,
'd_hid':
d_hid,
'dropout':
dropout,
'model':
model,
'num_linear_layers':
num_linear_layers,
'num_rnn_layers':
num_rnn_layers,
'pad_token_id':
pad_token_id,
'vocab_size':
vocab_size,
})
def gru_model(
tknzr: BaseTknzr,
d_emb: int,
d_hid: int,
n_hid_lyr: int,
n_pre_hid_lyr: int,
n_post_hid_lyr: int,
p_emb: float,
p_hid: float,
) -> GRUModel:
r"""Example ``GRUModel`` instance."""
return GRUModel(
d_emb=d_emb,
d_hid=d_hid,
n_hid_lyr=n_hid_lyr,
n_pre_hid_lyr=n_pre_hid_lyr,
n_post_hid_lyr=n_post_hid_lyr,
p_emb=p_emb,
p_hid=p_hid,
tknzr=tknzr,
)
def test_invalid_input_pad_token_id_and_vocab_size(self):
r"""Raise `ValueError` when input `vocab_size <= pad_token_id`."""
msg1 = (
'Must raise `ValueError` when input `vocab_size <= pad_token_id`.')
msg2 = 'Inconsistent error message.'
examples = ((2, 1), (3, 2), (4, 3), (10, 1))
for pad_token_id, vocab_size in examples:
with self.assertRaises(ValueError, msg=msg1) as ctx_man:
GRUModel(
d_emb=1,
d_hid=1,
dropout=0.1,
num_linear_layers=1,
num_rnn_layers=1,
pad_token_id=pad_token_id,
vocab_size=vocab_size,
)
self.assertEqual(
ctx_man.exception.args[0],
'`pad_token_id` must be smaller than `vocab_size`.',
msg=msg2)
def test_save_and_load(tknzr: BaseTknzr, ckpt: int, exp_name: str,
clean_model):
r"""Saved parameters are the same as loaded."""
model = GRUModel(
d_emb=1,
d_hid=1,
n_hid_lyr=1,
n_pre_hid_lyr=1,
n_post_hid_lyr=1,
p_emb=0.5,
p_hid=0.5,
tknzr=tknzr,
)
# Save model parameters.
model.save(
ckpt=ckpt,
exp_name=exp_name,
)
# Load model parameters.
load_model = GRUModel.load(
ckpt=ckpt,
exp_name=exp_name,
d_emb=1,
d_hid=1,
n_hid_lyr=1,
n_pre_hid_lyr=1,
n_post_hid_lyr=1,
p_emb=0.5,
p_hid=0.5,
tknzr=tknzr,
)
# Ensure parameters are the same.
for p_1, p_2 in zip(model.parameters(), load_model.parameters()):
assert torch.equal(p_1, p_2)