def test_save_load(self):
# Create a model with a downstream task
tw = TextWiser(Embedding.Word(word_option=WordOptions.word2vec,
pretrained='en-turian'),
[
Transformation.SVD(n_components=2),
Transformation.Pool(pool_option=PoolOptions.mean)
],
dtype=torch.float32)
tw.fit(docs)
model = nn.Sequential(tw, nn.Linear(2, 1)).to(device)
# Get results of the model
expected = model(docs)
# Save the model to a temporary file
with NamedTemporaryFile() as file:
torch.save(model.state_dict(), file) # Use string name of the file
# Get rid of the original model
del tw
del model
# Create the same model
tw = TextWiser(Embedding.Word(
word_option=WordOptions.word2vec, pretrained='en-turian'), [
Transformation.SVD(n_components=2),
Transformation.Pool(pool_option=PoolOptions.mean)
],
dtype=torch.float32)
tw.fit()
model = nn.Sequential(tw, nn.Linear(2, 1)).to(device)
# Load the model from file
file.seek(0)
model.load_state_dict(torch.load(file, map_location=device))
# Do predictions with the loaded model
predicted = model(docs)
self.assertTrue(torch.allclose(predicted, expected, atol=1e-6))
def test_num_components(self):
# The natural # of components is 3.
n_components = 2 # Restrict the # of components
tw = TextWiser(Embedding.TfIdf(min_df=2),
Transformation.SVD(n_components=n_components),
dtype=torch.float32)
predicted = tw.fit_transform(docs)
self.assertEqual(predicted.shape[1], n_components)
self._reset_seed()
n_components = 200 # Expand the # of components
tw = TextWiser(Embedding.TfIdf(min_df=2),
Transformation.SVD(n_components=n_components),
dtype=torch.float32)
predicted = tw.fit_transform(docs)
self.assertEqual(predicted.shape[1], n_components)
def test_fine_tuneable(self):
tw = TextWiser(Embedding.TfIdf(min_df=2),
Transformation.SVD(n_components=2),
dtype=torch.float32,
is_finetuneable=True)
tw.fit(docs)
embeddings1 = tw._imp[1].V.data.clone().detach()
# Give a fake task to train embeddings on
# Have a linear layer with a single output after pooling
linear = nn.Linear(2, 1, bias=False)
model = nn.Sequential(tw, linear).to(device).train()
y_pred = model(docs)
# Use ones as the target
y_act = torch.ones_like(y_pred)
# Optimize MSE using SGD
criterion = nn.MSELoss()
optimizer = optim.SGD(model.parameters(), lr=1e-3)
# Calculate the loss & gradients
optimizer.zero_grad()
loss = criterion(y_pred, y_act)
loss.backward()
# The embedding layer should have gradients now
self.assertIsNotNone([p for p in tw._imp[1].named_parameters()
][0][1].grad)
# Update weights
optimizer.step()
# The weights should be updated if fine_tune is true, else it should be the same
self.assertFalse(torch.allclose(embeddings1, tw._imp[1].V.data))
def test_v_in_parameters(self):
n_components = 2 # Restrict the # of components
tw = TextWiser(Embedding.TfIdf(min_df=2),
Transformation.SVD(n_components=n_components),
dtype=torch.float32)
tw.fit(docs)
self.assertIn('_imp.1.V', [p[0] for p in tw.named_parameters()])
def test_list_handling(self):
with warnings.catch_warnings():
warnings.simplefilter("ignore")
tw = TextWiser(Embedding.Word(word_option=WordOptions.word2vec,
pretrained='en-turian'),
Transformation.SVD(n_components=2),
dtype=torch.float32)
predicted = tw.fit_transform(docs)
expected = [
torch.tensor([[-0.9719871283, 0.0947150663],
[-0.3805825114, -1.0427029133],
[-0.6929296255, 0.1793890595],
[0.0000000000, 0.0000000000]],
dtype=torch.float32),
torch.tensor([[-0.9719871283, 0.0947150663],
[-0.3805825114, -1.0427029133],
[-0.7170552015, 0.0105144158],
[-0.9385635853, 0.6596723199],
[0.0000000000, 0.0000000000]],
dtype=torch.float32),
torch.tensor([[-0.8687936068, -0.9333068132],
[-0.6859120131, 0.0732812732],
[-0.9385635853, 0.6596723199],
[0.0000000000, 0.0000000000]],
dtype=torch.float32)
]
for p, e in zip(predicted, expected):
self.assertTrue(torch.allclose(p, e.to(device), atol=1e-6))
def test_fit_transform(self):
tw = TextWiser(Embedding.TfIdf(min_df=2),
Transformation.SVD(n_components=2),
dtype=torch.float32)
expected = torch.tensor(
[[-0.8526761532, 0.5070778131], [-0.9837458134, 0.0636523664],
[-0.7350711226, -0.6733918786]],
dtype=torch.float32)
self._test_fit_transform(tw, expected)
self._reset_seed()
self._test_fit_before_transform(tw, expected)
def test_min_components(self):
with self.assertRaises(ValueError):
TextWiser(Embedding.TfIdf(min_df=2),
Transformation.SVD(n_components=1),
dtype=torch.float32)
def test_finetune_validation(self):
# Nothing is fine-tuneable if dtype is numpy
with self.assertRaises(TypeError):
with warnings.catch_warnings():
warnings.simplefilter("ignore")
TextWiser(Embedding.Word(word_option=WordOptions.word2vec,
pretrained='en_turian'),
dtype=np.float32,
is_finetuneable=True)
# Word2Vec is fine-tuneable
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
TextWiser(Embedding.Word(word_option=WordOptions.word2vec,
pretrained='en-turian'),
dtype=torch.float32,
is_finetuneable=True,
lazy_load=True)
except ValueError:
self.fail("Word2vec is fine tuneable")
# ELMo is not fine-tuneable, and should raise an error
with self.assertRaises(ValueError):
with warnings.catch_warnings():
warnings.simplefilter("ignore")
TextWiser(Embedding.Word(word_option=WordOptions.elmo),
dtype=torch.float32,
is_finetuneable=True,
lazy_load=True)
# TfIdf is not fine-tuneable, and should raise an error
with self.assertRaises(ValueError):
TextWiser(Embedding.TfIdf(),
dtype=torch.float32,
is_finetuneable=True,
lazy_load=True)
# TfIdf is not fine-tuneable, but SVD is
try:
TextWiser(Embedding.TfIdf(),
Transformation.SVD(),
dtype=torch.float32,
is_finetuneable=True,
lazy_load=True)
except ValueError:
self.fail("SVD is fine tuneable")
# LDA cannot propagate gradients, so the whole thing is not fine-tuneable
with self.assertRaises(ValueError):
with warnings.catch_warnings():
warnings.simplefilter("ignore")
TextWiser(Embedding.Word(word_option=WordOptions.word2vec,
pretrained='en'),
Transformation.LDA(),
dtype=torch.float32,
is_finetuneable=True,
lazy_load=True)
schema = {
'concat': [{
'transform': [('word2vec', {
'pretrained': 'en-turian'
}), ('pool', {
'pool_option': 'max'
})]
}, {
'transform': ['tfidf', ('nmf', {
'n_components': 30
})]
}]
}
# Word2Vec is fine-tuneable, therefore the whole schema is fine-tuneable
try:
TextWiser(Embedding.Compound(schema=schema),
dtype=torch.float32,
is_finetuneable=True,
lazy_load=True)
except ValueError:
self.fail(
"Any fine-tuneable weights is enough for the model to be fine-tuneable"
)
# TfIdf is not fine-tuneable, but SVD is
schema = {'transform': ['tfidf', 'svd']}
try:
TextWiser(Embedding.Compound(schema=schema),
dtype=torch.float32,
is_finetuneable=True,
lazy_load=True)
except ValueError:
self.fail("SVD is fine tuneable")