def test_input_shape_and_dtype(
lstm_model: LSTMModel,
batch_prev_tkids: torch.Tensor,
):
r"""Input must be long tensor."""
try:
lstm_model = lstm_model.eval()
lstm_model.pred(batch_prev_tkids)
except Exception:
assert False
def test_forward_path(
lstm_model: LSTMModel,
batch_prev_tkids: torch.Tensor,
):
r"""Parameters used during forward must have gradients."""
# Make sure model has no gradients.
lstm_model = lstm_model.train()
lstm_model.zero_grad()
lstm_model(batch_prev_tkids).sum().backward()
assert hasattr(lstm_model.emb.weight.grad, 'grad')
assert hasattr(lstm_model.pre_hid[0].weight.grad, 'grad')
assert hasattr(lstm_model.hid.weight_ih_l0.grad, 'grad')
assert hasattr(lstm_model.post_hid[-1].weight.grad, 'grad')
def test_value_range(
lstm_model: LSTMModel,
batch_prev_tkids: torch.Tensor,
):
r"""Return values are probabilities."""
lstm_model = lstm_model.eval()
out = lstm_model.pred(batch_prev_tkids)
# Probabilities are values within range [0, 1].
assert torch.all(0 <= out).item()
assert torch.all(out <= 1).item()
# Sum of the probabilities equals to 1.
accum_out = out.sum(dim=-1)
assert torch.allclose(accum_out, torch.ones_like(accum_out))
def setUp(self):
r"""Setup hyperparameters and construct `LSTMModel`."""
self.model_objs = []
cls = self.__class__
for d_emb in cls.d_emb_range:
for d_hid in cls.d_hid_range:
for dropout in cls.dropout_range:
for num_linear_layers in cls.num_linear_layers_range:
for num_rnn_layers in cls.num_rnn_layers_range:
for pad_token_id in cls.pad_token_id_range:
for vocab_size in cls.vocab_size_range:
# skip invalid construct.
if vocab_size <= pad_token_id:
continue
model = LSTMModel(
d_emb=d_emb,
d_hid=d_hid,
dropout=dropout,
num_linear_layers=num_linear_layers,
num_rnn_layers=num_rnn_layers,
pad_token_id=pad_token_id,
vocab_size=vocab_size
)
self.model_objs.append({
'd_emb': d_emb,
'd_hid': d_hid,
'dropout': dropout,
'model': model,
'num_linear_layers': num_linear_layers,
'num_rnn_layers': num_rnn_layers,
'pad_token_id': pad_token_id,
'vocab_size': vocab_size,
})
def test_invalid_input_pad_token_id_and_vocab_size(self):
r"""Raise `ValueError` when input `vocab_size <= pad_token_id`."""
msg1 = (
'Must raise `ValueError` when input `vocab_size <= pad_token_id`.'
)
msg2 = 'Inconsistent error message.'
examples = ((2, 1), (3, 2), (4, 3), (10, 1))
for pad_token_id, vocab_size in examples:
with self.assertRaises(ValueError, msg=msg1) as ctx_man:
LSTMModel(
d_emb=1,
d_hid=1,
dropout=0.1,
num_linear_layers=1,
num_rnn_layers=1,
pad_token_id=pad_token_id,
vocab_size=vocab_size,
)
self.assertEqual(
ctx_man.exception.args[0],
'`pad_token_id` must be smaller than `vocab_size`.',
msg=msg2
)
def test_return_shape_and_dtype(
lstm_model: LSTMModel,
batch_prev_tkids: torch.Tensor,
batch_next_tkids: torch.Tensor,
):
r"""Return float tensor with 0 dimension."""
lstm_model = lstm_model.eval()
loss = lstm_model.loss_fn(
batch_prev_tkids=batch_prev_tkids,
batch_next_tkids=batch_next_tkids,
)
# 0 dimension tensor.
assert loss.shape == torch.Size([])
# Return float tensor.
assert loss.dtype == torch.float
def test_n_pre_hid_lyr(tknzr: BaseTknzr):
r"""``n_pre_hid_lyr`` must be an instance of `int` and be positive."""
# Test case: Type mismatched.
wrong_typed_inputs = [
0.0, 0.1, 1.0, '', (), [], {}, set(), None, ..., NotImplemented,
]
for bad_n_pre_hid_lyr in wrong_typed_inputs:
with pytest.raises(TypeError) as excinfo:
LSTMModel(
d_emb=1,
d_hid=1,
n_hid_lyr=1,
n_pre_hid_lyr=bad_n_pre_hid_lyr,
n_post_hid_lyr=1,
p_emb=0.0,
p_hid=0.0,
tknzr=tknzr,
)
assert (
'`n_pre_hid_lyr` must be an instance of `int`'
in str(excinfo.value)
)
# Test case: Invalid value.
wrong_value_inputs = [
0, -1, -2,
]
for bad_n_pre_hid_lyr in wrong_value_inputs:
with pytest.raises(ValueError) as excinfo:
LSTMModel(
d_emb=1,
d_hid=1,
n_hid_lyr=1,
n_pre_hid_lyr=bad_n_pre_hid_lyr,
n_post_hid_lyr=1,
p_emb=0.0,
p_hid=0.0,
tknzr=tknzr,
)
assert (
'`n_pre_hid_lyr` must be bigger than or equal to `1`'
in str(excinfo.value)
)
def test_input_shape_and_dtype(
lstm_model: LSTMModel,
batch_prev_tkids: torch.Tensor,
batch_next_tkids: torch.Tensor,
):
r"""Input tensors must be long tensors and have the same shape.
Same shape is required since we are using teacher forcing.
"""
try:
lstm_model = lstm_model.eval()
lstm_model.loss_fn(
batch_prev_tkids=batch_prev_tkids,
batch_next_tkids=batch_next_tkids,
)
except Exception:
assert False
def test_return_shape_and_dtype(
lstm_model: LSTMModel,
batch_prev_tkids: torch.Tensor,
):
r"""Return float tensor with correct shape."""
lstm_model = lstm_model.eval()
out = lstm_model.pred(batch_prev_tkids)
# Output float tensor.
assert out.dtype == torch.float
# Input shape: (B, S).
# Output shape: (B, S, V).
assert out.shape == (
batch_prev_tkids.shape[0],
batch_prev_tkids.shape[1],
lstm_model.emb.num_embeddings,
)
def test_p_hid(tknzr: BaseTknzr):
r"""``p_hid`` must be an instance of `float` and must be a probability."""
# Test case: Type mismatched.
wrong_typed_inputs = [
False, True, 0, 1, '', (), [], {}, set(), None, ..., NotImplemented,
]
for bad_p_hid in wrong_typed_inputs:
with pytest.raises(TypeError) as excinfo:
LSTMModel(
d_emb=1,
d_hid=1,
n_hid_lyr=1,
n_pre_hid_lyr=1,
n_post_hid_lyr=1,
p_emb=0.0,
p_hid=bad_p_hid,
tknzr=tknzr,
)
assert '`p_hid` must be an instance of `float`' in str(excinfo.value)
# Test case: Invalid value.
wrong_value_inputs = [
-1.0, -0.1, 1.1, 2.0,
]
for bad_p_hid in wrong_value_inputs:
with pytest.raises(ValueError) as excinfo:
LSTMModel(
d_emb=1,
d_hid=1,
n_hid_lyr=1,
n_pre_hid_lyr=1,
n_post_hid_lyr=1,
p_emb=0.0,
p_hid=bad_p_hid,
tknzr=tknzr,
)
assert (
'`p_hid` must be in the range from `0.0` to `1.0`'
in str(excinfo.value)
)
def test_back_propagation_path(
lstm_model: LSTMModel,
batch_prev_tkids: torch.Tensor,
batch_next_tkids: torch.Tensor,
):
r"""Gradients with respect to loss must get back propagated."""
# Make sure model has no gradients.
lstm_model = lstm_model.train()
lstm_model.zero_grad()
lstm_model.loss_fn(
batch_prev_tkids=batch_prev_tkids,
batch_next_tkids=batch_next_tkids,
).backward()
assert hasattr(lstm_model.emb.weight.grad, 'grad')
assert hasattr(lstm_model.pre_hid[0].weight.grad, 'grad')
assert hasattr(lstm_model.hid.weight_ih_l0.grad, 'grad')
assert hasattr(lstm_model.post_hid[-1].weight.grad, 'grad')
def lstm_model(
tknzr: BaseTknzr,
d_emb: int,
d_hid: int,
n_hid_lyr: int,
n_pre_hid_lyr: int,
n_post_hid_lyr: int,
p_emb: float,
p_hid: float,
) -> LSTMModel:
r"""Example ``LSTMModel`` instance."""
return LSTMModel(
d_emb=d_emb,
d_hid=d_hid,
n_hid_lyr=n_hid_lyr,
n_pre_hid_lyr=n_pre_hid_lyr,
n_post_hid_lyr=n_post_hid_lyr,
p_emb=p_emb,
p_hid=p_hid,
tknzr=tknzr,
)
def test_invalid_input_vocab_size(self):
r"""Raise exception when input `vocab_size` is invalid."""
msg1 = (
'Must raise `TypeError` or `ValueError` when input `vocab_size` '
'is invalid.'
)
msg2 = 'Inconsistent error message.'
examples = (
False, 0, 0.0, 1.0, math.nan, -math.nan, math.inf, -math.inf, 0j,
1j, '', b'', (), [], {}, set(), object(), lambda x: x, type, None,
NotImplemented, ...
)
for invalid_input in examples:
with self.assertRaises(
(TypeError, ValueError),
msg=msg1
) as ctx_man:
LSTMModel(
d_emb=1,
d_hid=1,
dropout=0.1,
num_linear_layers=1,
num_rnn_layers=1,
pad_token_id=0,
vocab_size=invalid_input
)
if isinstance(ctx_man.exception, TypeError):
self.assertEqual(
ctx_man.exception.args[0],
'`vocab_size` must be an instance of `int`.',
msg=msg2
)
else:
self.assertEqual(
ctx_man.exception.args[0],
'`vocab_size` must be bigger than or equal to `1`.',
msg=msg2
)
def test_save_and_load(tknzr: BaseTknzr, ckpt: int, exp_name: str,
clean_model):
r"""Saved parameters are the same as loaded."""
model = LSTMModel(
d_emb=1,
d_hid=1,
n_hid_lyr=1,
n_pre_hid_lyr=1,
n_post_hid_lyr=1,
p_emb=0.5,
p_hid=0.5,
tknzr=tknzr,
)
# Save model parameters.
model.save(
ckpt=ckpt,
exp_name=exp_name,
)
# Load model parameters.
load_model = LSTMModel.load(
ckpt=ckpt,
exp_name=exp_name,
d_emb=1,
d_hid=1,
n_hid_lyr=1,
n_pre_hid_lyr=1,
n_post_hid_lyr=1,
p_emb=0.5,
p_hid=0.5,
tknzr=tknzr,
)
# Ensure parameters are the same.
for p_1, p_2 in zip(model.parameters(), load_model.parameters()):
assert torch.equal(p_1, p_2)