def build_data_loaders(
train_data: List[Instance],
dev_data: List[Instance],
) -> Tuple[DataLoader, DataLoader]:
train_loader = SimpleDataLoader(train_data, 8, shuffle=True)
dev_loader = SimpleDataLoader(dev_data, 8, shuffle=False)
return train_loader, dev_loader
def __init__(self) -> None:
self.config: Config = Config().parse_args(known_only=True)
bert_token_indexers = PretrainedTransformerIndexer(
model_name=self.config.model_name)
bert_tokenizer = PretrainedTransformerTokenizer(
model_name=self.config.model_name)
reader = TextClassificationJsonReader(
token_indexers={"tokens": bert_token_indexers},
tokenizer=bert_tokenizer)
train_instances = list(reader.read(self.config.train_file))
dev_instances = list(reader.read(self.config.dev_file))
test_instances = list(reader.read(self.config.test_file))
self.vocab: Vocabulary = Vocabulary.from_instances(train_instances)
# 2. init the data loader
self.train_data_loader = SimpleDataLoader(train_instances,
self.config.batch_size,
shuffle=True)
self.dev_data_loader = SimpleDataLoader(dev_instances,
self.config.batch_size,
shuffle=False)
self.train_data_loader.index_with(self.vocab)
self.dev_data_loader.index_with(self.vocab)
# 3. init the model
self.model = self.init_model()
self.trainer = self.init_trainer()
def build_data_loaders(train_data: List[Instance], dev_data: List[Instance],
batch_size: int) -> Tuple[DataLoader, DataLoader]:
"""
Creates data loaders which loads data in batches of size batch_size for training and validation
Adapted from https://guide.allennlp.org/training-and-prediction
"""
train_loader = SimpleDataLoader(train_data, batch_size, shuffle=True)
dev_loader = SimpleDataLoader(dev_data, batch_size, shuffle=False)
return train_loader, dev_loader
def test_from_params_in_trainer(self):
# This is more of an integration test, making sure that a bunch of pieces fit together
# correctly, but it matters most for this learning rate scheduler, so we're testing it here.
params = Params({
"num_epochs": 5,
"learning_rate_scheduler": {
"type": "slanted_triangular",
"gradual_unfreezing": True,
"discriminative_fine_tuning": True,
"decay_factor": 0.5,
},
})
# The method called in the logic below only checks the length of this list, not its
# contents, so this should be safe.
instances = [1] * 40
optim = self._get_optimizer()
trainer = Trainer.from_params(
model=self.model,
optimizer=Lazy(lambda **kwargs: optim),
serialization_dir=self.TEST_DIR,
params=params,
data_loader=SimpleDataLoader(instances, batch_size=10),
)
assert isinstance(trainer._learning_rate_scheduler, SlantedTriangular)
# This is what we wrote this test for: to be sure that num_epochs is passed correctly, and
# that num_steps_per_epoch is computed and passed correctly. This logic happens inside of
# `Trainer.from_partial_objects`.
assert trainer._learning_rate_scheduler.num_epochs == 5
assert trainer._learning_rate_scheduler.num_steps_per_epoch == 4
# And we'll do one more to make sure that we can override num_epochs in the scheduler if we
# really want to. Not sure why you would ever want to in this case; this is just testing
# the functionality.
params = Params({
"num_epochs": 5,
"learning_rate_scheduler": {
"type": "slanted_triangular",
"num_epochs": 3,
"gradual_unfreezing": True,
"discriminative_fine_tuning": True,
"decay_factor": 0.5,
},
})
trainer = Trainer.from_params(
model=self.model,
optimizer=Lazy(lambda **kwargs: optim),
serialization_dir=self.TEST_DIR,
params=params,
data_loader=SimpleDataLoader(instances, batch_size=10),
)
assert trainer._learning_rate_scheduler.num_epochs == 3
def build_data_loaders(
config, train_data: List[Instance], dev_data: List[Instance],
test_data: List[Instance]
) -> Tuple[DataLoader, DataLoader, DataLoader]:
train_loader = SimpleDataLoader(train_data,
config.batch_size_for_train,
shuffle=True)
dev_loader = SimpleDataLoader(dev_data,
config.batch_size_for_eval,
shuffle=False)
test_loader = SimpleDataLoader(test_data,
config.batch_size_for_eval,
shuffle=False)
return train_loader, dev_loader, test_loader
def test_batch_of_entirely_empty_lists_works(self):
instances = [self.empty_instance, self.empty_instance]
model = DummyModel(self.vocab)
model.eval()
loader = SimpleDataLoader(instances, 2, vocab=self.vocab)
batch = next(iter(loader))
model.forward(**batch)
def test_can_optimise_model_with_dense_and_sparse_params(self):
optimizer_params = Params({"type": "dense_sparse_adam"})
parameters = [[n, p] for n, p in self.model.named_parameters() if p.requires_grad]
optimizer = Optimizer.from_params(model_parameters=parameters, params=optimizer_params)
for instance in self.instances:
instance.index_fields(self.vocab)
GradientDescentTrainer(self.model, optimizer, SimpleDataLoader(self.instances, 2)).train()
def test_get_inverse_hvp_lissa():
vs = [torch.tensor([1.0, 1.0])]
# create a fake model
vocab = Vocabulary()
params = torch.tensor([1, 2]).float()
model = DummyBilinearModelForTestingIF(vocab, params)
used_params = list(model.parameters())
# create a fake instance: just a matrix
A = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
fake_instance = Instance({"tensors": TensorField(A)})
# wrap fake instance into dataloader
lissa_data_loader = SimpleDataLoader([fake_instance],
batch_size=1,
batches_per_epoch=1)
inverse_hvp = get_inverse_hvp_lissa(
vs=vs,
model=model,
used_params=used_params,
lissa_data_loader=lissa_data_loader,
damping=0.0,
num_samples=1,
scale=1.0,
)
# I tried to increase recursion depth to actually approx the inverse Hessian vector product,
# but I suspect due to extremely small number of data point, the algorithm doesn't work well
# on this toy example
ans = torch.tensor([-1.5, -4.5])
assert torch.equal(inverse_hvp, ans)
def test_trainer_saves_models_at_specified_interval(self):
data_loader = SimpleDataLoader(self.instances, 4)
trainer = GradientDescentTrainer(
self.model,
self.optimizer,
data_loader,
num_epochs=2,
serialization_dir=self.TEST_DIR,
checkpointer=Checkpointer(
serialization_dir=self.TEST_DIR,
model_save_interval=0.0001,
num_serialized_models_to_keep=10,
),
)
trainer.train()
# Now check the serialized files for models saved during the epoch.
prefix = "model_state_epoch_*"
file_names = sorted(glob.glob(os.path.join(self.TEST_DIR, prefix)))
epochs = [
re.search(r"_([0-9\.\-]+)\.th", fname).group(1)
for fname in file_names
]
# We should have checkpoints at the end of each epoch and during each, e.g.
# [0.timestamp, 0, 1.timestamp, 1]
assert len(epochs) == 4
assert epochs[3] == "1"
assert "." in epochs[0]
# Now make certain we can restore from timestamped checkpoint.
# To do so, remove the checkpoint from the end of epoch 1&2, so
# that we are forced to restore from the timestamped checkpoints.
for k in range(2):
os.remove(
os.path.join(self.TEST_DIR,
"model_state_epoch_{}.th".format(k)))
os.remove(
os.path.join(self.TEST_DIR,
"training_state_epoch_{}.th".format(k)))
os.remove(os.path.join(self.TEST_DIR, "best.th"))
restore_trainer = GradientDescentTrainer(
self.model,
self.optimizer,
self.data_loader,
num_epochs=2,
serialization_dir=self.TEST_DIR,
checkpointer=Checkpointer(serialization_dir=self.TEST_DIR,
model_save_interval=0.0001),
)
epoch = restore_trainer._restore_checkpoint()
assert epoch == 2
# One batch per epoch.
assert restore_trainer._batch_num_total == 2
class SlowDataLoader:
data_loader = SimpleDataLoader(self.instances, batch_size=2)
def __iter__(self):
time.sleep(2.5)
return iter(self.data_loader)
def __len__(self):
return len(self.data_loader)
def set_target_device(self, _):
pass
def __init__(
self,
model: Model,
train_data_path: DatasetReaderInput,
train_dataset_reader: DatasetReader,
*,
test_dataset_reader: Optional[DatasetReader] = None,
train_data_loader: Lazy[DataLoader] = Lazy(
SimpleDataLoader.from_dataset_reader),
test_data_loader: Lazy[DataLoader] = Lazy(
SimpleDataLoader.from_dataset_reader),
params_to_freeze: List[str] = None,
cuda_device: int = -1,
lissa_batch_size: int = 8,
damping: float = 3e-3,
num_samples: int = 1,
recursion_depth: Union[float, int] = 0.25,
scale: float = 1e4,
) -> None:
super().__init__(
model=model,
train_data_path=train_data_path,
train_dataset_reader=train_dataset_reader,
test_dataset_reader=test_dataset_reader,
train_data_loader=train_data_loader,
test_data_loader=test_data_loader,
params_to_freeze=params_to_freeze,
cuda_device=cuda_device,
)
self._lissa_dataloader = SimpleDataLoader(
list(self._train_loader.iter_instances()),
lissa_batch_size,
shuffle=True,
vocab=self.vocab,
)
self._lissa_dataloader.set_target_device(self.device)
if isinstance(recursion_depth, float) and recursion_depth > 0.0:
self._lissa_dataloader.batches_per_epoch = int(
len(self._lissa_dataloader) * recursion_depth)
elif isinstance(recursion_depth, int) and recursion_depth > 0:
self._lissa_dataloader.batches_per_epoch = recursion_depth
else:
raise ValueError(
"'recursion_depth' should be a positive int or float")
self._damping = damping
self._num_samples = num_samples
self._recursion_depth = recursion_depth
self._scale = scale
def test_sanity_check_callback(self):
model_with_bias = FakeModelForTestingNormalizationBiasVerification(
use_bias=True)
inst = Instance({"x": TensorField(torch.rand(3, 1, 4))})
data_loader = SimpleDataLoader([inst, inst], 2)
trainer = GradientDescentTrainer(
model_with_bias,
self.optimizer,
data_loader,
num_epochs=1,
serialization_dir=self.TEST_DIR,
callbacks=[SanityChecksCallback(serialization_dir=self.TEST_DIR)],
)
with pytest.raises(SanityCheckError):
trainer.train()
def test_elmo_bilm(self):
# get the raw data
sentences, expected_lm_embeddings = self._load_sentences_embeddings()
# load the test model
elmo_bilm = _ElmoBiLm(self.options_file, self.weight_file)
# Deal with the data.
indexer = ELMoTokenCharactersIndexer()
# For each sentence, first create a TextField, then create an instance
instances = []
for batch in zip(*sentences):
for sentence in batch:
tokens = [Token(token) for token in sentence.split()]
field = TextField(tokens, {"character_ids": indexer})
instance = Instance({"elmo": field})
instances.append(instance)
vocab = Vocabulary()
# Now finally we can iterate through batches.
loader = SimpleDataLoader(instances, 3)
loader.index_with(vocab)
for i, batch in enumerate(loader):
lm_embeddings = elmo_bilm(
batch["elmo"]["character_ids"]["elmo_tokens"])
top_layer_embeddings, mask = remove_sentence_boundaries(
lm_embeddings["activations"][2], lm_embeddings["mask"])
# check the mask lengths
lengths = mask.data.numpy().sum(axis=1)
batch_sentences = [sentences[k][i] for k in range(3)]
expected_lengths = [
len(sentence.split()) for sentence in batch_sentences
]
assert lengths.tolist() == expected_lengths
# get the expected embeddings and compare!
expected_top_layer = [
expected_lm_embeddings[k][i] for k in range(3)
]
for k in range(3):
assert numpy.allclose(
top_layer_embeddings[k, :lengths[k], :].data.numpy(),
expected_top_layer[k],
atol=1.0e-6,
)
def test_trainer_can_log_learning_rates_tensorboard(self):
data_loader = SimpleDataLoader(self.instances, 4)
trainer = GradientDescentTrainer(
self.model,
self.optimizer,
data_loader,
num_epochs=2,
serialization_dir=self.TEST_DIR,
callbacks=[
TensorBoardCallback(
serialization_dir=self.TEST_DIR,
summary_interval=2,
should_log_learning_rate=True,
)
],
)
trainer.train()
def test_regularization(self):
penalty = self.model.get_regularization_penalty()
assert penalty is None
data_loader = SimpleDataLoader(self.instances, batch_size=32)
trainer = GradientDescentTrainer(self.model, None,
data_loader) # optimizer,
# You get a RuntimeError if you call `model.forward` twice on the same inputs.
# The data and config are such that the whole dataset is one batch.
training_batch = next(iter(data_loader))
validation_batch = next(iter(data_loader))
training_loss = trainer.batch_outputs(
training_batch, for_training=True)["loss"].item()
validation_loss = trainer.batch_outputs(
validation_batch, for_training=False)["loss"].item()
# Training loss should have the regularization penalty, but validation loss should not.
numpy.testing.assert_almost_equal(training_loss, validation_loss)
def benchmark_xlmr_mdl():
from allennlp.data import DataLoader
from allennlp.training.util import evaluate
xlmr = load_xlmr_coref_model()
instances = xlmr.dataset_reader.load_dataset(testset)
data_loader = SimpleDataLoader(instances, 1)
data_loader.index_with(xlmr.model.vocab)
start = time.time()
metrics = evaluate(xlmr.model, data_loader)
print('**XLM-R model**')
print_speed_performance(start, num_sentences, num_tokens)
print('Precision : ', metrics['coref_precision'])
print('Recall : ', metrics['coref_recall'])
print('F1 : ', metrics['coref_f1'])
print('Mention Recall : ', metrics['mention_recall'])
def test_trainer_respects_epoch_size_smaller_tnan_total(self):
batches_per_epoch = 1
num_epochs = 2
data_loader_smaller_epoch = SimpleDataLoader(
self.instances,
2,
batches_per_epoch=batches_per_epoch,
)
trainer = GradientDescentTrainer(
self.model,
self.optimizer,
data_loader_smaller_epoch,
validation_data_loader=self.validation_data_loader,
num_epochs=num_epochs,
serialization_dir=self.TEST_DIR,
)
assert trainer._batch_num_total == 0
metrics = trainer.train()
epoch = metrics["epoch"]
assert epoch == num_epochs - 1
assert trainer._batch_num_total == num_epochs * batches_per_epoch
def test_sanity_check_default(self):
model_with_bias = FakeModelForTestingNormalizationBiasVerification(use_bias=True)
inst = Instance({"x": TensorField(torch.rand(3, 1, 4))})
data_loader = SimpleDataLoader([inst, inst], 2)
trainer = GradientDescentTrainer.from_partial_objects(
model_with_bias,
serialization_dir=self.TEST_DIR,
data_loader=data_loader,
num_epochs=1,
)
with pytest.raises(SanityCheckError):
trainer.train()
trainer = GradientDescentTrainer.from_partial_objects(
model_with_bias,
serialization_dir=self.TEST_DIR,
data_loader=data_loader,
num_epochs=1,
run_sanity_checks=False,
)
# Check is not run, so no failure.
trainer.train()
def main(serialization_directory: str,
device: int,
data: str,
prefix: str,
domain: str = None):
"""
serialization_directory : str, required.
The directory containing the serialized weights.
device: int, default = -1
The device to run the evaluation on.
data: str, default = None
The data to evaluate on. By default, we use the validation data from
the original experiment.
prefix: str, default=""
The prefix to prepend to the generated gold and prediction files, to distinguish
different models/data.
domain: str, optional (default = None)
If passed, filters the ontonotes evaluation/test dataset to only contain the
specified domain. This overwrites the domain in the config file from the model,
to allow evaluation on domains other than the one the model was trained on.
"""
config = Params.from_file(
os.path.join(serialization_directory, "config.json"))
if domain is not None:
# Hack to allow evaluation on different domains than the
# model was trained on.
config["dataset_reader"]["domain_identifier"] = domain
prefix = f"{domain}_{prefix}"
else:
config["dataset_reader"].pop("domain_identifier", None)
dataset_reader = DatasetReader.from_params(config["dataset_reader"])
evaluation_data_path = data if data else config["validation_data_path"]
archive = load_archive(os.path.join(serialization_directory,
"model.tar.gz"),
cuda_device=device)
model = archive.model
model.eval()
prediction_file_path = os.path.join(serialization_directory,
prefix + "_predictions.txt")
gold_file_path = os.path.join(serialization_directory,
prefix + "_gold.txt")
prediction_file = open(prediction_file_path, "w+")
gold_file = open(gold_file_path, "w+")
# Load the evaluation data and index it.
print("reading evaluation data from {}".format(evaluation_data_path))
dataset = list(dataset_reader.read(evaluation_data_path))
with torch.autograd.no_grad():
loader = SimpleDataLoader(dataset, 32)
model_predictions: List[List[str]] = []
for batch in Tqdm.tqdm(loader):
batch = move_to_device(batch, device)
result = model(**batch)
predictions = model.decode(result)
model_predictions.extend(predictions["tags"])
for instance, prediction in zip(dataset, model_predictions):
fields = instance.fields
verb_index = fields["metadata"]["verb_index"]
gold_tags = fields["metadata"]["gold_tags"]
sentence = fields["metadata"]["words"]
write_to_conll_eval_file(prediction_file, gold_file, verb_index,
sentence, prediction, gold_tags)
prediction_file.close()
gold_file.close()
train_data, dev_data = read_data(dataset_reader)
vocab = build_vocab(train_data + dev_data)
model = build_model(vocab)
train_loader, dev_loader = build_data_loaders(train_data, dev_data)
train_loader.index_with(vocab)
dev_loader.index_with(vocab)
# You obviously won't want to create a temporary file for your training
# results, but for execution in binder for this guide, we need to do this.
with tempfile.TemporaryDirectory() as serialization_dir:
trainer = build_trainer(model, serialization_dir, train_loader, dev_loader)
trainer.train()
return model, dataset_reader
# We've copied the training loop from an earlier example, with updated model
# code, above in the Setup section. We run the training loop to get a trained
# model.
model, dataset_reader = run_training_loop()
# Now we can evaluate the model on a new dataset.
test_data = list(dataset_reader.read("quick_start/data/movie_review/test.tsv"))
data_loader = SimpleDataLoader(test_data, 8)
data_loader.index_with(model.vocab)
results = evaluate(model, data_loader)
print(results)