def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
stacked_encoder: Seq2SeqEncoder,
span_feedforward: FeedForward,
binary_feature_dim: int,
max_span_width: int,
binary_feature_size: int,
distance_feature_size: int,
ontology_path: str,
embedding_dropout: float = 0.2,
srl_label_namespace: str = "labels",
constit_label_namespace: str = "constit_labels",
fast_mode: bool = True,
loss_type: str = "hamming",
unlabeled_constits: bool = False,
np_pp_constits: bool = False,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super(ScaffoldedFrameSrl, self).__init__(vocab, regularizer)
# Base token-level encoding.
self.text_field_embedder = text_field_embedder
self.embedding_dropout = Dropout(p=embedding_dropout)
# There are exactly 2 binary features for the verb predicate embedding.
self.binary_feature_embedding = Embedding(2, binary_feature_dim)
self.stacked_encoder = stacked_encoder
if text_field_embedder.get_output_dim(
) + binary_feature_dim != stacked_encoder.get_input_dim():
raise ConfigurationError(
"The input dimension of the stacked_encoder must be equal to "
"the output dimension of the text_field_embedder.")
# Span-level encoding.
self.max_span_width = max_span_width
self.span_width_embedding = Embedding(max_span_width,
binary_feature_size)
# Based on the average sentence length in FN train.
self.span_distance_bin = 25
self.span_distance_embedding = Embedding(self.span_distance_bin,
distance_feature_size)
self.span_direction_embedding = Embedding(2, binary_feature_size)
self.span_feedforward = TimeDistributed(span_feedforward)
self.head_scorer = TimeDistributed(
torch.nn.Linear(stacked_encoder.get_output_dim(), 1))
self.num_srl_args = self.vocab.get_vocab_size(srl_label_namespace)
self.not_a_span_tag = self.vocab.get_token_index(
"*", srl_label_namespace)
self.outside_span_tag = self.vocab.get_token_index(
"O", srl_label_namespace)
self.semi_crf = SemiMarkovConditionalRandomField(
num_tags=self.num_srl_args,
max_span_width=max_span_width,
default_tag=self.not_a_span_tag,
outside_span_tag=self.outside_span_tag,
loss_type=loss_type)
# self.crf = ConditionalRandomField(self.num_classes)
self.unlabeled_constits = unlabeled_constits
self.np_pp_constits = np_pp_constits
self.constit_label_namespace = constit_label_namespace
assert not (unlabeled_constits and np_pp_constits)
if unlabeled_constits:
self.num_constit_tags = 2
elif np_pp_constits:
self.num_constit_tags = 3
else:
self.num_constit_tags = self.vocab.get_vocab_size(
constit_label_namespace)
# Topmost MLP.
self.srl_arg_projection_layer = TimeDistributed(
Linear(span_feedforward.get_output_dim(), self.num_srl_args))
self.constit_arg_projection_layer = TimeDistributed(
Linear(span_feedforward.get_output_dim(), self.num_constit_tags))
# Evaluation.
self.metrics = {
"constituents":
NonBioSpanBasedF1Measure(vocab,
tag_namespace=constit_label_namespace,
ignore_classes=["*"]),
"srl":
NonBioSpanBasedF1Measure(vocab,
tag_namespace=srl_label_namespace,
ignore_classes=["O", "*"],
ontology_path=ontology_path)
}
# Mode for the model, if turned on it only evaluates on dev and calculates loss for train.
self.fast_mode = fast_mode
initializer(self)
def from_params(cls, # type: ignore
model: Model,
serialization_dir: str,
iterator: DataIterator,
train_data: Iterable[Instance],
validation_data: Optional[Iterable[Instance]],
params: Params,
validation_iterator: DataIterator = None) -> 'Trainer':
# pylint: disable=arguments-differ
patience = params.pop_int("patience", None)
validation_metric = params.pop("validation_metric", "-loss")
shuffle = params.pop_bool("shuffle", True)
num_epochs = params.pop_int("num_epochs", 20)
cuda_device = parse_cuda_device(params.pop("cuda_device", -1))
grad_norm = params.pop_float("grad_norm", None)
grad_clipping = params.pop_float("grad_clipping", None)
lr_scheduler_params = params.pop("learning_rate_scheduler", None)
momentum_scheduler_params = params.pop("momentum_scheduler", None)
if isinstance(cuda_device, list):
model_device = cuda_device[0]
else:
model_device = cuda_device
if model_device >= 0:
# Moving model to GPU here so that the optimizer state gets constructed on
# the right device.
model = model.cuda(model_device)
parameters = [[n, p] for n, p in model.named_parameters() if p.requires_grad]
optimizer = Optimizer.from_params(parameters, params.pop("optimizer"))
if "moving_average" in params:
moving_average = MovingAverage.from_params(params.pop("moving_average"), parameters=parameters)
else:
moving_average = None
if lr_scheduler_params:
lr_scheduler = LearningRateScheduler.from_params(optimizer, lr_scheduler_params)
else:
lr_scheduler = None
if momentum_scheduler_params:
momentum_scheduler = MomentumScheduler.from_params(optimizer, momentum_scheduler_params)
else:
momentum_scheduler = None
if 'checkpointer' in params:
if 'keep_serialized_model_every_num_seconds' in params or \
'num_serialized_models_to_keep' in params:
raise ConfigurationError(
"Checkpointer may be initialized either from the 'checkpointer' key or from the "
"keys 'num_serialized_models_to_keep' and 'keep_serialized_model_every_num_seconds'"
" but the passed config uses both methods.")
checkpointer = Checkpointer.from_params(params.pop("checkpointer"))
else:
num_serialized_models_to_keep = params.pop_int("num_serialized_models_to_keep", 20)
keep_serialized_model_every_num_seconds = params.pop_int(
"keep_serialized_model_every_num_seconds", None)
checkpointer = Checkpointer(
serialization_dir=serialization_dir,
num_serialized_models_to_keep=num_serialized_models_to_keep,
keep_serialized_model_every_num_seconds=keep_serialized_model_every_num_seconds)
model_save_interval = params.pop_float("model_save_interval", None)
summary_interval = params.pop_int("summary_interval", 100)
histogram_interval = params.pop_int("histogram_interval", None)
should_log_parameter_statistics = params.pop_bool("should_log_parameter_statistics", True)
should_log_learning_rate = params.pop_bool("should_log_learning_rate", False)
log_batch_size_period = params.pop_int("log_batch_size_period", None)
params.assert_empty(cls.__name__)
return cls(model, optimizer, iterator,
train_data, validation_data,
patience=patience,
validation_metric=validation_metric,
validation_iterator=validation_iterator,
shuffle=shuffle,
num_epochs=num_epochs,
serialization_dir=serialization_dir,
cuda_device=cuda_device,
grad_norm=grad_norm,
grad_clipping=grad_clipping,
learning_rate_scheduler=lr_scheduler,
momentum_scheduler=momentum_scheduler,
checkpointer=checkpointer,
model_save_interval=model_save_interval,
summary_interval=summary_interval,
histogram_interval=histogram_interval,
should_log_parameter_statistics=should_log_parameter_statistics,
should_log_learning_rate=should_log_learning_rate,
log_batch_size_period=log_batch_size_period,
moving_average=moving_average)
def _read_embeddings_from_text_file(
file_uri: str,
embedding_dim: int,
vocab: Vocabulary,
namespace: str = "tokens") -> torch.FloatTensor:
"""
Read pre-trained word vectors from an eventually compressed text file, possibly contained
inside an archive with multiple files. The text file is assumed to be utf-8 encoded with
space-separated fields: [word] [dim 1] [dim 2] ...
Lines that contain more numerical tokens than `embedding_dim` raise a warning and are skipped.
The remainder of the docstring is identical to `_read_pretrained_embeddings_file`.
"""
tokens_to_keep = set(
vocab.get_index_to_token_vocabulary(namespace).values())
vocab_size = vocab.get_vocab_size(namespace)
embeddings = {}
# First we read the embeddings from the file, only keeping vectors for the words we need.
logger.info("Reading pretrained embeddings from file")
with EmbeddingsTextFile(file_uri) as embeddings_file:
for line in Tqdm.tqdm(embeddings_file):
token = line.split(" ", 1)[0]
if token in tokens_to_keep:
fields = line.rstrip().split(" ")
if len(fields) - 1 != embedding_dim:
# Sometimes there are funny unicode parsing problems that lead to different
# fields lengths (e.g., a word with a unicode space character that splits
# into more than one column). We skip those lines. Note that if you have
# some kind of long header, this could result in all of your lines getting
# skipped. It's hard to check for that here; you just have to look in the
# embedding_misses_file and at the model summary to make sure things look
# like they are supposed to.
logger.warning(
"Found line with wrong number of dimensions (expected: %d; actual: %d): %s",
embedding_dim,
len(fields) - 1,
line,
)
continue
vector = numpy.asarray(fields[1:], dtype="float32")
embeddings[token] = vector
if not embeddings:
raise ConfigurationError(
"No embeddings of correct dimension found; you probably "
"misspecified your embedding_dim parameter, or didn't "
"pre-populate your Vocabulary")
all_embeddings = numpy.asarray(list(embeddings.values()))
embeddings_mean = float(numpy.mean(all_embeddings))
embeddings_std = float(numpy.std(all_embeddings))
# Now we initialize the weight matrix for an embedding layer, starting with random vectors,
# then filling in the word vectors we just read.
logger.info("Initializing pre-trained embedding layer")
embedding_matrix = torch.FloatTensor(vocab_size, embedding_dim).normal_(
embeddings_mean, embeddings_std)
num_tokens_found = 0
index_to_token = vocab.get_index_to_token_vocabulary(namespace)
for i in range(vocab_size):
token = index_to_token[i]
# If we don't have a pre-trained vector for this word, we'll just leave this row alone,
# so the word has a random initialization.
if token in embeddings:
embedding_matrix[i] = torch.FloatTensor(embeddings[token])
num_tokens_found += 1
else:
logger.debug(
"Token %s was not found in the embedding file. Initialising randomly.",
token)
logger.info("Pretrained embeddings were found for %d out of %d tokens",
num_tokens_found, vocab_size)
return embedding_matrix
def search_learning_rate(
trainer: Trainer,
start_lr: float = 1e-5,
end_lr: float = 10,
num_batches: int = 100,
linear_steps: bool = False,
stopping_factor: float = None,
) -> Tuple[List[float], List[float]]:
"""
Runs training loop on the model using :class:`~allennlp.training.trainer.Trainer`
increasing learning rate from ``start_lr`` to ``end_lr`` recording the losses.
# Parameters
trainer: :class:`~allennlp.training.trainer.Trainer`
start_lr : ``float``
The learning rate to start the search.
end_lr : ``float``
The learning rate upto which search is done.
num_batches : ``int``
Number of batches to run the learning rate finder.
linear_steps : ``bool``
Increase learning rate linearly if False exponentially.
stopping_factor : ``float``
Stop the search when the current loss exceeds the best loss recorded by
multiple of stopping factor. If ``None`` search proceeds till the ``end_lr``
# Returns
(learning_rates, losses) : ``Tuple[List[float], List[float]]``
Returns list of learning rates and corresponding losses.
Note: The losses are recorded before applying the corresponding learning rate
"""
if num_batches <= 10:
raise ConfigurationError(
"The number of iterations for learning rate finder should be greater than 10."
)
trainer.model.train()
train_generator = trainer.iterator(trainer.train_data, shuffle=trainer.shuffle)
train_generator_tqdm = Tqdm.tqdm(train_generator, total=num_batches)
learning_rates = []
losses = []
best = 1e9
if linear_steps:
lr_update_factor = (end_lr - start_lr) / num_batches
else:
lr_update_factor = (end_lr / start_lr) ** (1.0 / num_batches)
for i, batch in enumerate(train_generator_tqdm):
if linear_steps:
current_lr = start_lr + (lr_update_factor * i)
else:
current_lr = start_lr * (lr_update_factor ** i)
for param_group in trainer.optimizer.param_groups:
param_group["lr"] = current_lr
trainer.optimizer.zero_grad()
loss = trainer.batch_loss(batch, for_training=True)
loss.backward()
loss = loss.detach().cpu().item()
if stopping_factor is not None and (math.isnan(loss) or loss > stopping_factor * best):
logger.info(f"Loss ({loss}) exceeds stopping_factor * lowest recorded loss.")
break
trainer.rescale_gradients()
trainer.optimizer.step()
learning_rates.append(current_lr)
losses.append(loss)
if loss < best and i > 10:
best = loss
if i == num_batches:
break
return learning_rates, losses
def __init__(self,
model: Model,
optimizer: torch.optim.Optimizer,
iterator: DataIterator,
train_dataset: Iterable[Instance],
validation_dataset: Optional[Iterable[Instance]] = None,
patience: Optional[int] = None,
validation_metric: str = "-loss",
validation_iterator: DataIterator = None,
shuffle: bool = True,
num_epochs: int = 20,
serialization_dir: Optional[str] = None,
num_serialized_models_to_keep: int = 20,
keep_serialized_model_every_num_seconds: int = None,
checkpointer: Checkpointer = None,
model_save_interval: float = None,
cuda_device: Union[int, List] = -1,
grad_norm: Optional[float] = None,
grad_clipping: Optional[float] = None,
learning_rate_scheduler: Optional[LearningRateScheduler] = None,
momentum_scheduler: Optional[MomentumScheduler] = None,
summary_interval: int = 100,
histogram_interval: int = None,
should_log_parameter_statistics: bool = True,
should_log_learning_rate: bool = False,
log_batch_size_period: Optional[int] = None,
moving_average: Optional[MovingAverage] = None,
email: List = None) -> None:
"""
A trainer for doing supervised learning. It just takes a labeled dataset
and a ``DataIterator``, and uses the supplied ``Optimizer`` to learn the weights
for your model over some fixed number of epochs. You can also pass in a validation
dataset and enable early stopping. There are many other bells and whistles as well.
Parameters
----------
model : ``Model``, required.
An AllenNLP model to be optimized. Pytorch Modules can also be optimized if
their ``forward`` method returns a dictionary with a "loss" key, containing a
scalar tensor representing the loss function to be optimized.
If you are training your model using GPUs, your model should already be
on the correct device. (If you use `Trainer.from_params` this will be
handled for you.)
optimizer : ``torch.nn.Optimizer``, required.
An instance of a Pytorch Optimizer, instantiated with the parameters of the
model to be optimized.
iterator : ``DataIterator``, required.
A method for iterating over a ``Dataset``, yielding padded indexed batches.
train_dataset : ``Dataset``, required.
A ``Dataset`` to train on. The dataset should have already been indexed.
validation_dataset : ``Dataset``, optional, (default = None).
A ``Dataset`` to evaluate on. The dataset should have already been indexed.
patience : Optional[int] > 0, optional (default=None)
Number of epochs to be patient before early stopping: the training is stopped
after ``patience`` epochs with no improvement. If given, it must be ``> 0``.
If None, early stopping is disabled.
validation_metric : str, optional (default="loss")
Validation metric to measure for whether to stop training using patience
and whether to serialize an ``is_best`` model each epoch. The metric name
must be prepended with either "+" or "-", which specifies whether the metric
is an increasing or decreasing function.
validation_iterator : ``DataIterator``, optional (default=None)
An iterator to use for the validation set. If ``None``, then
use the training `iterator`.
shuffle: ``bool``, optional (default=True)
Whether to shuffle the instances in the iterator or not.
num_epochs : int, optional (default = 20)
Number of training epochs.
serialization_dir : str, optional (default=None)
Path to directory for saving and loading model files. Models will not be saved if
this parameter is not passed.
num_serialized_models_to_keep : ``int``, optional (default=20)
Number of previous model checkpoints to retain. Default is to keep 20 checkpoints.
A value of None or -1 means all checkpoints will be kept.
keep_serialized_model_every_num_seconds : ``int``, optional (default=None)
If num_serialized_models_to_keep is not None, then occasionally it's useful to
save models at a given interval in addition to the last num_serialized_models_to_keep.
To do so, specify keep_serialized_model_every_num_seconds as the number of seconds
between permanently saved checkpoints. Note that this option is only used if
num_serialized_models_to_keep is not None, otherwise all checkpoints are kept.
checkpointer : ``Checkpointer``, optional (default=None)
An instance of class Checkpointer to use instead of the default. If a checkpointer is specified,
the arguments num_serialized_models_to_keep and keep_serialized_model_every_num_seconds should
not be specified. The caller is responsible for initializing the checkpointer so that it is
consistent with serialization_dir.
model_save_interval : ``float``, optional (default=None)
If provided, then serialize models every ``model_save_interval``
seconds within single epochs. In all cases, models are also saved
at the end of every epoch if ``serialization_dir`` is provided.
cuda_device : ``Union[int, List[int]]``, optional (default = -1)
An integer or list of integers specifying the CUDA device(s) to use. If -1, the CPU is used.
grad_norm : ``float``, optional, (default = None).
If provided, gradient norms will be rescaled to have a maximum of this value.
grad_clipping : ``float``, optional (default = ``None``).
If provided, gradients will be clipped `during the backward pass` to have an (absolute)
maximum of this value. If you are getting ``NaNs`` in your gradients during training
that are not solved by using ``grad_norm``, you may need this.
learning_rate_scheduler : ``LearningRateScheduler``, optional (default = None)
If specified, the learning rate will be decayed with respect to
this schedule at the end of each epoch (or batch, if the scheduler implements
the ``step_batch`` method). If you use :class:`torch.optim.lr_scheduler.ReduceLROnPlateau`,
this will use the ``validation_metric`` provided to determine if learning has plateaued.
To support updating the learning rate on every batch, this can optionally implement
``step_batch(batch_num_total)`` which updates the learning rate given the batch number.
momentum_scheduler : ``MomentumScheduler``, optional (default = None)
If specified, the momentum will be updated at the end of each batch or epoch
according to the schedule.
summary_interval: ``int``, optional, (default = 100)
Number of batches between logging scalars to tensorboard
histogram_interval : ``int``, optional, (default = ``None``)
If not None, then log histograms to tensorboard every ``histogram_interval`` batches.
When this parameter is specified, the following additional logging is enabled:
* Histograms of model parameters
* The ratio of parameter update norm to parameter norm
* Histogram of layer activations
We log histograms of the parameters returned by
``model.get_parameters_for_histogram_tensorboard_logging``.
The layer activations are logged for any modules in the ``Model`` that have
the attribute ``should_log_activations`` set to ``True``. Logging
histograms requires a number of GPU-CPU copies during training and is typically
slow, so we recommend logging histograms relatively infrequently.
Note: only Modules that return tensors, tuples of tensors or dicts
with tensors as values currently support activation logging.
should_log_parameter_statistics : ``bool``, optional, (default = True)
Whether to send parameter statistics (mean and standard deviation
of parameters and gradients) to tensorboard.
should_log_learning_rate : ``bool``, optional, (default = False)
Whether to send parameter specific learning rate to tensorboard.
log_batch_size_period : ``int``, optional, (default = ``None``)
If defined, how often to log the average batch size.
moving_average: ``MovingAverage``, optional, (default = None)
If provided, we will maintain moving averages for all parameters. During training, we
employ a shadow variable for each parameter, which maintains the moving average. During
evaluation, we backup the original parameters and assign the moving averages to corresponding
parameters. Be careful that when saving the checkpoint, we will save the moving averages of
parameters. This is necessary because we want the saved model to perform as well as the validated
model if we load it later. But this may cause problems if you restart the training from checkpoint.
"""
super().__init__(serialization_dir, cuda_device)
# I am not calling move_to_gpu here, because if the model is
# not already on the GPU then the optimizer is going to be wrong.
self.model = model
self.iterator = iterator
self._validation_iterator = validation_iterator
self.shuffle = shuffle
self.optimizer = optimizer
self.train_data = train_dataset
self._validation_data = validation_dataset
if patience is None: # no early stopping
if validation_dataset:
logger.warning('You provided a validation dataset but patience was set to None, '
'meaning that early stopping is disabled')
elif (not isinstance(patience, int)) or patience <= 0:
raise ConfigurationError('{} is an invalid value for "patience": it must be a positive integer '
'or None (if you want to disable early stopping)'.format(patience))
# For tracking is_best_so_far and should_stop_early
self._metric_tracker = MetricTracker(patience, validation_metric)
# Get rid of + or -
self._validation_metric = validation_metric[1:]
self._num_epochs = num_epochs
if checkpointer is not None:
# We can't easily check if these parameters were passed in, so check against their default values.
# We don't check against serialization_dir since it is also used by the parent class.
if num_serialized_models_to_keep != 20 or \
keep_serialized_model_every_num_seconds is not None:
raise ConfigurationError(
"When passing a custom Checkpointer, you may not also pass in separate checkpointer "
"args 'num_serialized_models_to_keep' or 'keep_serialized_model_every_num_seconds'.")
self._checkpointer = checkpointer
else:
self._checkpointer = Checkpointer(serialization_dir,
keep_serialized_model_every_num_seconds,
num_serialized_models_to_keep)
self._model_save_interval = model_save_interval
self._grad_norm = grad_norm
self._grad_clipping = grad_clipping
self._learning_rate_scheduler = learning_rate_scheduler
self._momentum_scheduler = momentum_scheduler
self._moving_average = moving_average
# We keep the total batch number as an instance variable because it
# is used inside a closure for the hook which logs activations in
# ``_enable_activation_logging``.
self._batch_num_total = 0
self._tensorboard = TensorboardWriter(
get_batch_num_total=lambda: self._batch_num_total,
serialization_dir=serialization_dir,
summary_interval=summary_interval,
histogram_interval=histogram_interval,
should_log_parameter_statistics=should_log_parameter_statistics,
should_log_learning_rate=should_log_learning_rate)
self._log_batch_size_period = log_batch_size_period
self._last_log = 0.0 # time of last logging
# Enable activation logging.
if histogram_interval is not None:
self._tensorboard.enable_activation_logging(self.model)
self._email = email
def _lstm_forward(self,
inputs: PackedSequence,
initial_state: Optional[Tuple[torch.Tensor, torch.Tensor]] = None) -> \
Tuple[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
"""
Parameters
----------
inputs : ``PackedSequence``, required.
A batch first ``PackedSequence`` to run the stacked LSTM over.
initial_state : ``Tuple[torch.Tensor, torch.Tensor]``, optional, (default = None)
A tuple (state, memory) representing the initial hidden state and memory
of the LSTM, with shape (num_layers, batch_size, 2 * hidden_size) and
(num_layers, batch_size, 2 * cell_size) respectively.
Returns
-------
output_sequence : ``torch.FloatTensor``
The encoded sequence of shape (num_layers, batch_size, sequence_length, hidden_size)
final_states: ``Tuple[torch.FloatTensor, torch.FloatTensor]``
The per-layer final (state, memory) states of the LSTM, with shape
(num_layers, batch_size, 2 * hidden_size) and (num_layers, batch_size, 2 * cell_size)
respectively. The last dimension is duplicated because it contains the state/memory
for both the forward and backward layers.
"""
if initial_state is None:
hidden_states: List[Optional[Tuple[torch.Tensor,
torch.Tensor]]] = [None] * len(
self.forward_layers)
elif initial_state[0].size()[0] != len(self.forward_layers):
raise ConfigurationError(
"Initial states were passed to forward() but the number of "
"initial states does not match the number of layers.")
else:
hidden_states = list(
zip(initial_state[0].split(1, 0), initial_state[1].split(1,
0)))
inputs, batch_lengths = pad_packed_sequence(inputs, batch_first=True)
forward_output_sequence = inputs
backward_output_sequence = inputs
final_states = []
sequence_outputs = []
for layer_index, state in enumerate(hidden_states):
forward_layer = getattr(self,
'forward_layer_{}'.format(layer_index))
backward_layer = getattr(self,
'backward_layer_{}'.format(layer_index))
forward_cache = forward_output_sequence
backward_cache = backward_output_sequence
if state is not None:
forward_hidden_state, backward_hidden_state = state[0].split(
self.hidden_size, 2)
forward_memory_state, backward_memory_state = state[1].split(
self.cell_size, 2)
forward_state = (forward_hidden_state, forward_memory_state)
backward_state = (backward_hidden_state, backward_memory_state)
else:
forward_state = None
backward_state = None
forward_output_sequence, forward_state = forward_layer(
forward_output_sequence, batch_lengths, forward_state)
backward_output_sequence, backward_state = backward_layer(
backward_output_sequence, batch_lengths, backward_state)
# Skip connections, just adding the input to the output.
if layer_index != 0:
forward_output_sequence += forward_cache
backward_output_sequence += backward_cache
sequence_outputs.append(
torch.cat([forward_output_sequence, backward_output_sequence],
-1))
# Append the state tuples in a list, so that we can return
# the final states for all the layers.
final_states.append(
(torch.cat([forward_state[0], backward_state[0]], -1),
torch.cat([forward_state[1], backward_state[1]], -1)))
stacked_sequence_outputs: torch.FloatTensor = torch.stack(
sequence_outputs)
# Stack the hidden state and memory for each layer into 2 tensors of shape
# (num_layers, batch_size, hidden_size) and (num_layers, batch_size, cell_size)
# respectively.
final_hidden_states, final_memory_states = zip(*final_states)
final_state_tuple: Tuple[torch.FloatTensor,
torch.FloatTensor] = (torch.cat(
final_hidden_states,
0), torch.cat(final_memory_states, 0))
return stacked_sequence_outputs, final_state_tuple
# Disable some of the more verbose logging statements
logging.getLogger('allennlp.common.params').disabled = True
logging.getLogger('allennlp.nn.initializers').disabled = True
logging.getLogger('allennlp.modules.token_embedders.embedding').setLevel(
logging.INFO)
# Load from archive
archive = load_archive(args.archive_file, args.cuda_device, args.overrides,
args.weights_file)
config = archive.config
prepare_environment(config)
model = archive.model
model.eval()
if not isinstance(model, GraphDependencyParser):
raise ConfigurationError(
"The loaded model seems not to be an am-parser (GraphDependencyParser)"
)
if not args.formalism in model.tasks:
raise ConfigurationError(
f"The model at hand was not trained on {args.formalism} but on {list(model.tasks.keys())}"
)
# Load the evaluation data
# Try to use the validation dataset reader if there is one - otherwise fall back
# to the default dataset_reader used for both training and validation.
validation_dataset_reader_params = config.pop('validation_dataset_reader',
None)
if validation_dataset_reader_params is not None:
dataset_reader = DatasetReader.from_params(
validation_dataset_reader_params)
def _extend(
self,
counter: Dict[str, Dict[str, int]] = None,
min_count: Dict[str, int] = None,
max_vocab_size: Union[int, Dict[str, int]] = None,
non_padded_namespaces: Iterable[str] = DEFAULT_NON_PADDED_NAMESPACES,
pretrained_files: Optional[Dict[str, str]] = None,
only_include_pretrained_words: bool = False,
tokens_to_add: Dict[str, List[str]] = None,
min_pretrained_embeddings: Dict[str, int] = None,
) -> None:
"""
This method can be used for extending already generated vocabulary. It takes same
parameters as Vocabulary initializer. The `_token_to_index` and `_index_to_token`
mappings of calling vocabulary will be retained. It is an inplace operation so None will be
returned.
"""
if not isinstance(max_vocab_size, dict):
int_max_vocab_size = max_vocab_size
max_vocab_size = defaultdict(
lambda: int_max_vocab_size) # type: ignore
min_count = min_count or {}
pretrained_files = pretrained_files or {}
min_pretrained_embeddings = min_pretrained_embeddings or {}
non_padded_namespaces = set(non_padded_namespaces)
counter = counter or {}
tokens_to_add = tokens_to_add or {}
self._retained_counter = counter
# Make sure vocabulary extension is safe.
current_namespaces = {*self._token_to_index}
extension_namespaces = {*counter, *tokens_to_add}
for namespace in current_namespaces & extension_namespaces:
# if new namespace was already present
# Either both should be padded or none should be.
original_padded = not any(
namespace_match(pattern, namespace)
for pattern in self._non_padded_namespaces)
extension_padded = not any(
namespace_match(pattern, namespace)
for pattern in non_padded_namespaces)
if original_padded != extension_padded:
raise ConfigurationError(
"Common namespace {} has conflicting ".format(namespace) +
"setting of padded = True/False. " +
"Hence extension cannot be done.")
# Add new non-padded namespaces for extension
self._token_to_index.add_non_padded_namespaces(non_padded_namespaces)
self._index_to_token.add_non_padded_namespaces(non_padded_namespaces)
self._non_padded_namespaces.update(non_padded_namespaces)
for namespace in counter:
if namespace in pretrained_files:
pretrained_list = _read_pretrained_tokens(
pretrained_files[namespace])
min_embeddings = min_pretrained_embeddings.get(namespace, 0)
if min_embeddings > 0:
tokens_old = tokens_to_add.get(namespace, [])
tokens_new = pretrained_list[:min_embeddings]
tokens_to_add[namespace] = tokens_old + tokens_new
pretrained_set = set(pretrained_list)
else:
pretrained_set = None
token_counts = list(counter[namespace].items())
token_counts.sort(key=lambda x: x[1], reverse=True)
try:
max_vocab = max_vocab_size[namespace]
except KeyError:
max_vocab = None
if max_vocab:
token_counts = token_counts[:max_vocab]
for token, count in token_counts:
if pretrained_set is not None:
if only_include_pretrained_words:
if token in pretrained_set and count >= min_count.get(
namespace, 1):
self.add_token_to_namespace(token, namespace)
elif token in pretrained_set or count >= min_count.get(
namespace, 1):
self.add_token_to_namespace(token, namespace)
elif count >= min_count.get(namespace, 1):
self.add_token_to_namespace(token, namespace)
for namespace, tokens in tokens_to_add.items():
for token in tokens:
self.add_token_to_namespace(token, namespace)
def __init__(
self,
hidden_dim: int = 100,
num_perspectives: int = 20,
share_weights_between_directions: bool = True,
is_forward: bool = None,
with_full_match: bool = True,
with_maxpool_match: bool = True,
with_attentive_match: bool = True,
with_max_attentive_match: bool = True,
) -> None:
super().__init__()
self.hidden_dim = hidden_dim
self.num_perspectives = num_perspectives
self.is_forward = is_forward
self.with_full_match = with_full_match
self.with_maxpool_match = with_maxpool_match
self.with_attentive_match = with_attentive_match
self.with_max_attentive_match = with_max_attentive_match
if not (with_full_match or with_maxpool_match or with_attentive_match
or with_max_attentive_match):
raise ConfigurationError(
"At least one of the matching method should be enabled")
def create_parameter(
): # utility function to create and initialize a parameter
param = nn.Parameter(torch.zeros(num_perspectives, hidden_dim))
torch.nn.init.kaiming_normal_(param)
return param
def share_or_create(
weights_to_share
): # utility function to create or share the weights
return weights_to_share if share_weights_between_directions else create_parameter(
)
output_dim = (
2 # used to calculate total output dimension, 2 is for cosine max and cosine min
)
if with_full_match:
if is_forward is None:
raise ConfigurationError(
"Must specify is_forward to enable full matching")
self.full_match_weights = create_parameter()
self.full_match_weights_reversed = share_or_create(
self.full_match_weights)
output_dim += num_perspectives + 1
if with_maxpool_match:
self.maxpool_match_weights = create_parameter()
output_dim += num_perspectives * 2
if with_attentive_match:
self.attentive_match_weights = create_parameter()
self.attentive_match_weights_reversed = share_or_create(
self.attentive_match_weights)
output_dim += num_perspectives + 1
if with_max_attentive_match:
self.max_attentive_match_weights = create_parameter()
self.max_attentive_match_weights_reversed = share_or_create(
self.max_attentive_match_weights)
output_dim += num_perspectives + 1
self.output_dim = output_dim
def __call__(
self, # type: ignore
batch_verb_indices: List[Optional[int]],
batch_sentences: List[List[str]],
batch_conll_formatted_predicted_tags: List[List[str]],
batch_conll_formatted_gold_tags: List[List[str]],
) -> None:
"""
Parameters
----------
batch_verb_indices : ``List[Optional[int]]``, required.
The indices of the verbal predicate in the sentences which
the gold labels are the arguments for, or None if the sentence
contains no verbal predicate.
batch_sentences : ``List[List[str]]``, required.
The word tokens for each instance in the batch.
batch_conll_formatted_predicted_tags : ``List[List[str]]``, required.
A list of predicted CoNLL-formatted SRL tags (itself a list) to compute score for.
Use allennlp.models.semantic_role_labeler.convert_bio_tags_to_conll_format
to convert from BIO to CoNLL format before passing the tags into the metric,
if applicable.
batch_conll_formatted_gold_tags : ``List[List[str]]``, required.
A list of gold CoNLL-formatted SRL tags (itself a list) to use as a reference.
Use allennlp.models.semantic_role_labeler.convert_bio_tags_to_conll_format
to convert from BIO to CoNLL format before passing the
tags into the metric, if applicable.
"""
if not os.path.exists(self._srl_eval_path):
raise ConfigurationError(
f"srl-eval.pl not found at {self._srl_eval_path}.")
tempdir = tempfile.mkdtemp()
gold_path = os.path.join(tempdir, "gold.txt")
predicted_path = os.path.join(tempdir, "predicted.txt")
with open(predicted_path, "w",
encoding="utf-8") as predicted_file, open(
gold_path, "w", encoding="utf-8") as gold_file:
for verb_index, sentence, predicted_tag_sequence, gold_tag_sequence in zip(
batch_verb_indices,
batch_sentences,
batch_conll_formatted_predicted_tags,
batch_conll_formatted_gold_tags,
):
write_conll_formatted_tags_to_file(
predicted_file,
gold_file,
verb_index,
sentence,
predicted_tag_sequence,
gold_tag_sequence,
)
perl_script_command = [
"perl", self._srl_eval_path, gold_path, predicted_path
]
completed_process = subprocess.run(perl_script_command,
stdout=subprocess.PIPE,
universal_newlines=True,
check=True)
for line in completed_process.stdout.split("\n"):
stripped = line.strip().split()
if len(stripped) == 7:
tag = stripped[0]
# Overall metrics are calculated in get_metric, skip them here.
if tag == "Overall" or tag in self._ignore_classes:
continue
# This line contains results for a span
num_correct = int(stripped[1])
num_excess = int(stripped[2])
num_missed = int(stripped[3])
self._true_positives[tag] += num_correct
self._false_positives[tag] += num_excess
self._false_negatives[tag] += num_missed
shutil.rmtree(tempdir)
def forward(
self, # type: ignore
words: Dict[str, torch.LongTensor],
metadata: List[Dict[str, Any]],
morpho_embedding: torch.FloatTensor = None,
pos_tags: torch.LongTensor = None,
head_tags: torch.LongTensor = None,
head_indices: torch.LongTensor = None,
grammar_values: torch.LongTensor = None,
lemma_indices: torch.LongTensor = None) -> Dict[str, torch.Tensor]:
# pylint: disable=arguments-differ
"""
Parameters
----------
words : Dict[str, torch.LongTensor], required
The output of ``TextField.as_array()``, which should typically be passed directly to a
``TextFieldEmbedder``. This output is a dictionary mapping keys to ``TokenIndexer``
tensors. At its most basic, using a ``SingleIdTokenIndexer`` this is: ``{"tokens":
Tensor(batch_size, sequence_length)}``. This dictionary will have the same keys as were used
for the ``TokenIndexers`` when you created the ``TextField`` representing your
sequence. The dictionary is designed to be passed directly to a ``TextFieldEmbedder``,
which knows how to combine different word representations into a single vector per
token in your input.
pos_tags : ``torch.LongTensor``, required
The output of a ``SequenceLabelField`` containing POS tags.
POS tags are required regardless of whether they are used in the model,
because they are used to filter the evaluation metric to only consider
heads of words which are not punctuation.
metadata : List[Dict[str, Any]], optional (default=None)
A dictionary of metadata for each batch element which has keys:
words : ``List[str]``, required.
The tokens in the original sentence.
pos : ``List[str]``, required.
The dependencies POS tags for each word.
head_tags : torch.LongTensor, optional (default = None)
A torch tensor representing the sequence of integer gold class labels for the arcs
in the dependency parse. Has shape ``(batch_size, sequence_length)``.
head_indices : torch.LongTensor, optional (default = None)
A torch tensor representing the sequence of integer indices denoting the parent of every
word in the dependency parse. Has shape ``(batch_size, sequence_length)``.
Returns
-------
An output dictionary consisting of:
loss : ``torch.FloatTensor``, optional
A scalar loss to be optimised.
arc_loss : ``torch.FloatTensor``
The loss contribution from the unlabeled arcs.
loss : ``torch.FloatTensor``, optional
The loss contribution from predicting the dependency
tags for the gold arcs.
heads : ``torch.FloatTensor``
The predicted head indices for each word. A tensor
of shape (batch_size, sequence_length).
head_types : ``torch.FloatTensor``
The predicted head types for each arc. A tensor
of shape (batch_size, sequence_length).
mask : ``torch.LongTensor``
A mask denoting the padded elements in the batch.
"""
embedded_text_input = self.text_field_embedder(words)
if morpho_embedding is not None:
embedded_text_input = torch.cat(
[embedded_text_input, morpho_embedding], -1)
if grammar_values is not None and self._pos_tag_embedding is not None:
embedded_pos_tags = self._pos_tag_embedding(grammar_values)
embedded_text_input = torch.cat(
[embedded_text_input, embedded_pos_tags], -1)
elif self._pos_tag_embedding is not None:
raise ConfigurationError(
"Model uses a POS embedding, but no POS tags were passed.")
mask = get_text_field_mask(words)
output_dict = self._parse(embedded_text_input, mask, head_tags,
head_indices, grammar_values, lemma_indices)
if self.task_config.task_type == "multitask":
losses = ["arc_nll", "tag_nll", "grammar_nll", "lemma_nll"]
elif self.task_config.task_type == "single":
if self.task_config.params["model"] == "morphology":
losses = ["grammar_nll"]
elif self.task_config.params["model"] == "lemmatization":
losses = ["lemma_nll"]
elif self.task_config.params["model"] == "syntax":
losses = ["arc_nll", "tag_nll"]
else:
assert False, "Unknown model type {}".format(
self.task_config.params["model"])
else:
assert False, "Unknown task type {}".format(
self.task_config.task_type)
output_dict["loss"] = sum(output_dict[loss_name]
for loss_name in losses)
if head_indices is not None and head_tags is not None:
evaluation_mask = self._get_mask_for_eval(mask, pos_tags)
# We calculate attatchment scores for the whole sentence
# but excluding the symbolic ROOT token at the start,
# which is why we start from the second element in the sequence.
self._attachment_scores(output_dict["heads"][:, 1:],
output_dict["head_tags"][:, 1:],
head_indices, head_tags, evaluation_mask)
output_dict["words"] = [meta["words"] for meta in metadata]
if metadata and "pos" in metadata[0]:
output_dict["pos"] = [meta["pos"] for meta in metadata]
return output_dict
def read(self, file_path: str) -> Dataset:
"""
Returns an `Iterable` containing all the instances
in the specified dataset.
If `self.lazy` is False, this calls `self._read()`,
ensures that the result is a list, then returns the resulting list.
If `self.lazy` is True, this returns an object whose
`__iter__` method calls `self._read()` each iteration.
In this case your implementation of `_read()` must also be lazy
(that is, not load all instances into memory at once), otherwise
you will get a `ConfigurationError`.
In either case, the returned `Iterable` can be iterated
over multiple times. It's unlikely you want to override this function,
but if you do your result should likewise be repeatedly iterable.
"""
lazy = getattr(self, "lazy", None)
if lazy is None:
logger.warning(
"DatasetReader.lazy is not set, "
"did you forget to call the superclass constructor?"
)
if self._cache_directory:
cache_file = self._get_cache_location_for_file_path(file_path)
else:
cache_file = None
if lazy:
instances: Iterable[Instance] = _LazyInstances(
lambda: self._read(file_path),
cache_file,
self.deserialize_instance,
self.serialize_instance,
)
if self.max_instances is not None:
instances = itertools.islice(instances, 0, self.max_instances)
else:
# First we read the instances, either from a cache or from the original file.
if cache_file and os.path.exists(cache_file):
instances = self._instances_from_cache_file(cache_file)
else:
instances = self._read(file_path)
if self.max_instances is not None:
if isinstance(instances, list):
instances = instances[: self.max_instances]
else:
instances = itertools.islice(instances, 0, self.max_instances)
# Then some validation.
if not isinstance(instances, list):
instances = [instance for instance in Tqdm.tqdm(instances)]
if not instances:
raise ConfigurationError(
"No instances were read from the given filepath {}. "
"Is the path correct?".format(file_path)
)
# And finally we write to the cache if we need to.
if cache_file and not os.path.exists(cache_file):
logger.info(f"Caching instances to {cache_file}")
self._instances_to_cache_file(cache_file, instances)
instances = AllennlpDataset(instances)
return instances
def read(self, file_path: str):
# Import is here, since it isn't necessary by default.
import nltk
# Holds tuples of (question_text, answer_sentence_id)
questions = []
logger.info("Reading file at %s", file_path)
with open(file_path) as dataset_file:
dataset_json = json.load(dataset_file)
dataset = dataset_json['data']
logger.info("Reading the dataset")
for article in tqdm(dataset):
for paragraph in article['paragraphs']:
paragraph_id = len(self._paragraph_sentences)
self._paragraph_sentences[paragraph_id] = []
context_article = paragraph["context"]
# Split the context_article into a list of sentences.
sentences = nltk.sent_tokenize(context_article)
# Make a dict from span indices to sentence. The end span is
# exclusive, and the start span is inclusive.
span_to_sentence_index = {}
current_index = 0
for sentence in sentences:
sentence_id = len(self._sentence_to_id)
self._sentence_to_id[sentence] = sentence_id
self._id_to_sentence[sentence_id] = sentence
self._sentence_paragraph_map[sentence_id] = paragraph_id
self._paragraph_sentences[paragraph_id].append(sentence_id)
sentence_len = len(sentence)
# Need to add one to the end index to account for the
# trailing space after punctuation that is stripped by NLTK.
span_to_sentence_index[(current_index, current_index +
sentence_len + 1)] = sentence
current_index += sentence_len + 1
for question_answer in paragraph['qas']:
question_text = question_answer["question"].strip()
question_id = len(self._question_to_id)
self._question_to_id[question_text] = question_id
self._id_to_question[question_id] = question_text
# There may be multiple answer annotations, so pick the one
# that occurs the most.
candidate_answer_start_indices: Counter = Counter()
for answer in question_answer["answers"]:
candidate_answer_start_indices[
answer["answer_start"]] += 1
answer_start_index, _ = candidate_answer_start_indices.most_common(
1)[0]
# Get the full sentence corresponding to the answer.
answer_sentence = None
for span_tuple in span_to_sentence_index:
start_span, end_span = span_tuple
if start_span <= answer_start_index and answer_start_index < end_span:
answer_sentence = span_to_sentence_index[
span_tuple]
break
else: # no break
raise ValueError(
"Index of answer start was out of bounds. "
"This should never happen, please raise "
"an issue on GitHub.")
# Now that we have the string of the full sentence, we need to
# search for it in our shuffled list to get the index.
answer_id = self._sentence_to_id[answer_sentence]
# Now we can make the string representation and add this
# to the list of processed_rows.
questions.append((question_id, answer_id))
instances = []
logger.info("Processing questions into training instances")
for question_id, answer_id in tqdm(questions):
sentence_choices, correct_choice = self._get_sentence_choices(
question_id, answer_id)
question_text = self._id_to_question[question_id]
instance = self.text_to_instance(question_text, sentence_choices,
correct_choice)
instances.append(instance)
if not instances:
raise ConfigurationError(
"No instances were read from the given filepath {}. "
"Is the path correct?".format(file_path))
return Dataset(instances)
def to_bioul(tag_sequence: List[str], encoding: str = "IOB1") -> List[str]:
"""
Given a tag sequence encoded with IOB1 labels, recode to BIOUL.
In the IOB1 scheme, I is a token inside a span, O is a token outside
a span and B is the beginning of span immediately following another
span of the same type.
In the BIO scheme, I is a token inside a span, O is a token outside
a span and B is the beginning of a span.
# Parameters
tag_sequence : `List[str]`, required.
The tag sequence encoded in IOB1, e.g. ["I-PER", "I-PER", "O"].
encoding : `str`, optional, (default = `"IOB1"`).
The encoding type to convert from. Must be either "IOB1" or "BIO".
# Returns
bioul_sequence : `List[str]`
The tag sequence encoded in IOB1, e.g. ["B-PER", "L-PER", "O"].
"""
if encoding not in {"IOB1", "BIO"}:
raise ConfigurationError(
f"Invalid encoding {encoding} passed to 'to_bioul'.")
def replace_label(full_label, new_label):
# example: full_label = 'I-PER', new_label = 'U', returns 'U-PER'
parts = list(full_label.partition("-"))
parts[0] = new_label
return "".join(parts)
def pop_replace_append(in_stack, out_stack, new_label):
# pop the last element from in_stack, replace the label, append
# to out_stack
tag = in_stack.pop()
new_tag = replace_label(tag, new_label)
out_stack.append(new_tag)
def process_stack(stack, out_stack):
# process a stack of labels, add them to out_stack
if len(stack) == 1:
# just a U token
pop_replace_append(stack, out_stack, "U")
else:
# need to code as BIL
recoded_stack = []
pop_replace_append(stack, recoded_stack, "L")
while len(stack) >= 2:
pop_replace_append(stack, recoded_stack, "I")
pop_replace_append(stack, recoded_stack, "B")
recoded_stack.reverse()
out_stack.extend(recoded_stack)
# Process the tag_sequence one tag at a time, adding spans to a stack,
# then recode them.
bioul_sequence = []
stack: List[str] = []
for label in tag_sequence:
# need to make a dict like
# token = {'token': 'Matt', "labels": {'conll2003': "B-PER"}
# 'gold': 'I-PER'}
# where 'gold' is the raw value from the CoNLL data set
if label == "O" and len(stack) == 0:
bioul_sequence.append(label)
elif label == "O" and len(stack) > 0:
# need to process the entries on the stack plus this one
process_stack(stack, bioul_sequence)
bioul_sequence.append(label)
elif label[0] == "I":
# check if the previous type is the same as this one
# if it is then append to stack
# otherwise this start a new entity if the type
# is different
if len(stack) == 0:
if encoding == "BIO":
raise InvalidTagSequence(tag_sequence)
stack.append(label)
else:
# check if the previous type is the same as this one
this_type = label.partition("-")[2]
prev_type = stack[-1].partition("-")[2]
if this_type == prev_type:
stack.append(label)
else:
if encoding == "BIO":
raise InvalidTagSequence(tag_sequence)
# a new entity
process_stack(stack, bioul_sequence)
stack.append(label)
elif label[0] == "B":
if len(stack) > 0:
process_stack(stack, bioul_sequence)
stack.append(label)
else:
raise InvalidTagSequence(tag_sequence)
# process the stack
if len(stack) > 0:
process_stack(stack, bioul_sequence)
return bioul_sequence
def from_params(
cls: Type["Step"],
params: Params,
constructor_to_call: Callable[..., "Step"] = None,
constructor_to_inspect: Union[Callable[..., "Step"],
Callable[["Step"], None]] = None,
existing_steps: Optional[Dict[str, "Step"]] = None,
**extras,
) -> "Step":
# Why do we need a custom from_params? Step classes have a run() method that takes all the
# parameters necessary to perform the step. The __init__() method of the step takes those
# same parameters, but each of them could be wrapped in another Step instead of being
# supplied directly. from_params() doesn't know anything about these shenanigans, so
# we have to supply the necessary logic here.
if constructor_to_call is not None:
raise ConfigurationError(
f"{cls.__name__}.from_params cannot be called with a constructor_to_call."
)
if constructor_to_inspect is not None:
raise ConfigurationError(
f"{cls.__name__}.from_params cannot be called with a constructor_to_inspect."
)
if existing_steps is None:
existing_steps = {}
if isinstance(params, str):
params = Params({"type": params})
if not isinstance(params, Params):
raise ConfigurationError(
"from_params was passed a `params` object that was not a `Params`. This probably "
"indicates malformed parameters in a configuration file, where something that "
"should have been a dictionary was actually a list, or something else. "
f"This happened when constructing an object of type {cls}.")
as_registrable = cast(Type[Registrable], cls)
choice = params.pop_choice("type",
choices=as_registrable.list_available(),
default_to_first_choice=True)
subclass, constructor_name = as_registrable.resolve_class_name(choice)
if not issubclass(subclass, Step):
# This can happen if `choice` is a fully qualified name.
raise ConfigurationError(
f"Tried to make a Step of type {choice}, but ended up with a {subclass}."
)
parameters = infer_method_params(subclass, subclass.run)
del parameters["self"]
init_parameters = infer_constructor_params(subclass)
del init_parameters["self"]
del init_parameters["kwargs"]
parameter_overlap = parameters.keys() & init_parameters.keys()
assert len(parameter_overlap) <= 0, (
f"If this assert fails it means that you wrote a Step with a run() method that takes one of the "
f"reserved parameters ({', '.join(init_parameters.keys())})")
parameters.update(init_parameters)
kwargs: Dict[str, Any] = {}
accepts_kwargs = False
for param_name, param in parameters.items():
if param.kind == param.VAR_KEYWORD:
# When a class takes **kwargs we store the fact that the method allows extra keys; if
# we get extra parameters, instead of crashing, we'll just pass them as-is to the
# constructor, and hope that you know what you're doing.
accepts_kwargs = True
continue
explicitly_set = param_name in params
constructed_arg = pop_and_construct_arg(
subclass.__name__,
param_name,
param.annotation,
param.default,
params,
existing_steps=existing_steps,
**extras,
)
# If the param wasn't explicitly set in `params` and we just ended up constructing
# the default value for the parameter, we can just omit it.
# Leaving it in can cause issues with **kwargs in some corner cases, where you might end up
# with multiple values for a single parameter (e.g., the default value gives you lazy=False
# for a dataset reader inside **kwargs, but a particular dataset reader actually hard-codes
# lazy=True - the superclass sees both lazy=True and lazy=False in its constructor).
if explicitly_set or constructed_arg is not param.default:
kwargs[param_name] = constructed_arg
if accepts_kwargs:
kwargs.update(params)
else:
params.assert_empty(subclass.__name__)
return subclass(**kwargs)
def search(self, start_predictions: torch.Tensor, start_state: StateType,
step: StepFunctionType) -> Tuple[torch.Tensor, torch.Tensor]:
"""
Given a starting state and a step function, apply beam search to find the
most likely target sequences.
Notes
-----
If your step function returns `-inf` for some log probabilities
(like if you're using a masked log-softmax) then some of the "best"
sequences returned may also have `-inf` log probability. Specifically
this happens when the beam size is smaller than the number of actions
with finite log probability (non-zero probability) returned by the step function.
Therefore if you're using a mask you may want to check the results from `search`
and potentially discard sequences with non-finite log probability.
# Parameters
start_predictions : `torch.Tensor`
A tensor containing the initial predictions with shape `(batch_size,)`.
Usually the initial predictions are just the index of the "start" token
in the target vocabulary.
start_state : `StateType`
The initial state passed to the `step` function. Each value of the state dict
should be a tensor of shape `(batch_size, *)`, where `*` means any other
number of dimensions.
step : `StepFunctionType`
A function that is responsible for computing the next most likely tokens,
given the current state and the predictions from the last time step.
The function should accept two arguments. The first being a tensor
of shape `(group_size,)`, representing the index of the predicted
tokens from the last time step, and the second being the current state.
The `group_size` will be `batch_size * beam_size`, except in the initial
step, for which it will just be `batch_size`.
The function is expected to return a tuple, where the first element
is a tensor of shape `(group_size, target_vocab_size)` containing
the log probabilities of the tokens for the next step, and the second
element is the updated state. The tensor in the state should have shape
`(group_size, *)`, where `*` means any other number of dimensions.
# Returns
Tuple[torch.Tensor, torch.Tensor]
Tuple of `(predictions, log_probabilities)`, where `predictions`
has shape `(batch_size, beam_size, max_steps)` and `log_probabilities`
has shape `(batch_size, beam_size)`.
"""
batch_size = start_predictions.size()[0]
# List of (batch_size, beam_size) tensors. One for each time step. Does not
# include the start symbols, which are implicit.
predictions: List[torch.Tensor] = []
# List of (batch_size, beam_size) tensors. One for each time step. None for
# the first. Stores the index n for the parent prediction, i.e.
# predictions[t-1][i][n], that it came from.
backpointers: List[torch.Tensor] = []
# Calculate the first timestep. This is done outside the main loop
# because we are going from a single decoder input (the output from the
# encoder) to the top `beam_size` decoder outputs. On the other hand,
# within the main loop we are going from the `beam_size` elements of the
# beam to `beam_size`^2 candidates from which we will select the top
# `beam_size` elements for the next iteration.
# shape: (batch_size, num_classes)
start_class_log_probabilities, state = step(start_predictions,
start_state)
num_classes = start_class_log_probabilities.size()[1]
# Make sure `per_node_beam_size` is not larger than `num_classes`.
if self.per_node_beam_size > num_classes:
raise ConfigurationError(
f"Target vocab size ({num_classes:d}) too small "
f"relative to per_node_beam_size ({self.per_node_beam_size:d}).\n"
f"Please decrease beam_size or per_node_beam_size.")
# shape: (batch_size, beam_size), (batch_size, beam_size)
start_top_log_probabilities, start_predicted_classes = start_class_log_probabilities.topk(
self.beam_size)
if self.beam_size == 1 and (start_predicted_classes
== self._end_index).all():
warnings.warn(
"Empty sequences predicted. You may want to increase the beam size or ensure "
"your step function is working properly.",
RuntimeWarning,
)
return start_predicted_classes.unsqueeze(
-1), start_top_log_probabilities
# The log probabilities for the last time step.
# shape: (batch_size, beam_size)
last_log_probabilities = start_top_log_probabilities
# shape: [(batch_size, beam_size)]
predictions.append(start_predicted_classes)
# Log probability tensor that mandates that the end token is selected.
# shape: (batch_size * beam_size, num_classes)
log_probs_after_end = start_class_log_probabilities.new_full(
(batch_size * self.beam_size, num_classes), float("-inf"))
log_probs_after_end[:, self._end_index] = 0.0
# Set the same state for each element in the beam.
for key, state_tensor in state.items():
_, *last_dims = state_tensor.size()
# shape: (batch_size * beam_size, *)
state[key] = (state_tensor.unsqueeze(1).expand(
batch_size, self.beam_size,
*last_dims).reshape(batch_size * self.beam_size, *last_dims))
for timestep in range(self.max_steps - 1):
# shape: (batch_size * beam_size,)
last_predictions = predictions[-1].reshape(batch_size *
self.beam_size)
# If every predicted token from the last step is `self._end_index`,
# then we can stop early.
if (last_predictions == self._end_index).all():
break
# Take a step. This get the predicted log probs of the next classes
# and updates the state.
# shape: (batch_size * beam_size, num_classes)
class_log_probabilities, state = step(last_predictions, state)
# shape: (batch_size * beam_size, num_classes)
last_predictions_expanded = last_predictions.unsqueeze(-1).expand(
batch_size * self.beam_size, num_classes)
# Here we are finding any beams where we predicted the end token in
# the previous timestep and replacing the distribution with a
# one-hot distribution, forcing the beam to predict the end token
# this timestep as well.
# shape: (batch_size * beam_size, num_classes)
cleaned_log_probabilities = torch.where(
last_predictions_expanded == self._end_index,
log_probs_after_end,
class_log_probabilities,
)
# shape (both): (batch_size * beam_size, per_node_beam_size)
top_log_probabilities, predicted_classes = cleaned_log_probabilities.topk(
self.per_node_beam_size)
# Here we expand the last log probabilities to (batch_size * beam_size, per_node_beam_size)
# so that we can add them to the current log probs for this timestep.
# This lets us maintain the log probability of each element on the beam.
# shape: (batch_size * beam_size, per_node_beam_size)
expanded_last_log_probabilities = (
last_log_probabilities.unsqueeze(2).expand(
batch_size, self.beam_size,
self.per_node_beam_size).reshape(
batch_size * self.beam_size, self.per_node_beam_size))
# shape: (batch_size * beam_size, per_node_beam_size)
summed_top_log_probabilities = top_log_probabilities + expanded_last_log_probabilities
# shape: (batch_size, beam_size * per_node_beam_size)
reshaped_summed = summed_top_log_probabilities.reshape(
batch_size, self.beam_size * self.per_node_beam_size)
# shape: (batch_size, beam_size * per_node_beam_size)
reshaped_predicted_classes = predicted_classes.reshape(
batch_size, self.beam_size * self.per_node_beam_size)
# Keep only the top `beam_size` beam indices.
# shape: (batch_size, beam_size), (batch_size, beam_size)
restricted_beam_log_probs, restricted_beam_indices = reshaped_summed.topk(
self.beam_size)
# Use the beam indices to extract the corresponding classes.
# shape: (batch_size, beam_size)
restricted_predicted_classes = reshaped_predicted_classes.gather(
1, restricted_beam_indices)
predictions.append(restricted_predicted_classes)
# shape: (batch_size, beam_size)
last_log_probabilities = restricted_beam_log_probs
# The beam indices come from a `beam_size * per_node_beam_size` dimension where the
# indices with a common ancestor are grouped together. Hence
# dividing by per_node_beam_size gives the ancestor. (Note that this is integer
# division as the tensor is a LongTensor.)
# shape: (batch_size, beam_size)
backpointer = restricted_beam_indices / self.per_node_beam_size
backpointers.append(backpointer)
# Keep only the pieces of the state tensors corresponding to the
# ancestors created this iteration.
for key, state_tensor in state.items():
_, *last_dims = state_tensor.size()
# shape: (batch_size, beam_size, *)
expanded_backpointer = backpointer.view(
batch_size, self.beam_size,
*([1] * len(last_dims))).expand(batch_size, self.beam_size,
*last_dims)
# shape: (batch_size * beam_size, *)
state[key] = (state_tensor.reshape(
batch_size, self.beam_size,
*last_dims).gather(1, expanded_backpointer).reshape(
batch_size * self.beam_size, *last_dims))
if not torch.isfinite(last_log_probabilities).all():
warnings.warn(
"Infinite log probabilities encountered. Some final sequences may not make sense. "
"This can happen when the beam size is larger than the number of valid (non-zero "
"probability) transitions that the step function produces.",
RuntimeWarning,
)
# Reconstruct the sequences.
# shape: [(batch_size, beam_size, 1)]
reconstructed_predictions = [predictions[-1].unsqueeze(2)]
# shape: (batch_size, beam_size)
cur_backpointers = backpointers[-1]
for timestep in range(len(predictions) - 2, 0, -1):
# shape: (batch_size, beam_size, 1)
cur_preds = predictions[timestep].gather(
1, cur_backpointers).unsqueeze(2)
reconstructed_predictions.append(cur_preds)
# shape: (batch_size, beam_size)
cur_backpointers = backpointers[timestep - 1].gather(
1, cur_backpointers)
# shape: (batch_size, beam_size, 1)
final_preds = predictions[0].gather(1, cur_backpointers).unsqueeze(2)
reconstructed_predictions.append(final_preds)
# shape: (batch_size, beam_size, max_steps)
all_predictions = torch.cat(list(reversed(reconstructed_predictions)),
2)
return all_predictions, last_log_probabilities
def run(self, *, ref: str) -> T: # type: ignore
raise ConfigurationError(
f"Step {self.name} is a RefStep (referring to {ref}). RefSteps cannot be executed. "
"They are only useful while parsing an experiment.")
def forward(
self, # type: ignore
tokens: Dict[str, torch.LongTensor],
spans: torch.LongTensor,
metadata: List[Dict[str, Any]],
pos_tags: Dict[str, torch.LongTensor] = None,
span_labels: torch.LongTensor = None) -> Dict[str, torch.Tensor]:
# pylint: disable=arguments-differ
"""
Parameters
----------
tokens : Dict[str, torch.LongTensor], required
The output of ``TextField.as_array()``, which should typically be passed directly to a
``TextFieldEmbedder``. This output is a dictionary mapping keys to ``TokenIndexer``
tensors. At its most basic, using a ``SingleIdTokenIndexer`` this is: ``{"tokens":
Tensor(batch_size, num_tokens)}``. This dictionary will have the same keys as were used
for the ``TokenIndexers`` when you created the ``TextField`` representing your
sequence. The dictionary is designed to be passed directly to a ``TextFieldEmbedder``,
which knows how to combine different word representations into a single vector per
token in your input.
spans : ``torch.LongTensor``, required.
A tensor of shape ``(batch_size, num_spans, 2)`` representing the
inclusive start and end indices of all possible spans in the sentence.
metadata : List[Dict[str, Any]], required.
A dictionary of metadata for each batch element which has keys:
tokens : ``List[str]``, required.
The original string tokens in the sentence.
gold_tree : ``nltk.Tree``, optional (default = None)
Gold NLTK trees for use in evaluation.
pos_tags : ``List[str]``, optional.
The POS tags for the sentence. These can be used in the
model as embedded features, but they are passed here
in addition for use in constructing the tree.
pos_tags : ``torch.LongTensor``, optional (default = None)
The output of a ``SequenceLabelField`` containing POS tags.
span_labels : ``torch.LongTensor``, optional (default = None)
A torch tensor representing the integer gold class labels for all possible
spans, of shape ``(batch_size, num_spans)``.
Returns
-------
An output dictionary consisting of:
class_probabilities : ``torch.FloatTensor``
A tensor of shape ``(batch_size, num_spans, span_label_vocab_size)``
representing a distribution over the label classes per span.
spans : ``torch.LongTensor``
The original spans tensor.
tokens : ``List[List[str]]``, required.
A list of tokens in the sentence for each element in the batch.
pos_tags : ``List[List[str]]``, required.
A list of POS tags in the sentence for each element in the batch.
num_spans : ``torch.LongTensor``, required.
A tensor of shape (batch_size), representing the lengths of non-padded spans
in ``enumerated_spans``.
loss : ``torch.FloatTensor``, optional
A scalar loss to be optimised.
"""
embedded_text_input = self.text_field_embedder(tokens)
if pos_tags is not None and self.pos_tag_embedding is not None:
embedded_pos_tags = self.pos_tag_embedding(pos_tags)
embedded_text_input = torch.cat(
[embedded_text_input, embedded_pos_tags], -1)
elif self.pos_tag_embedding is not None:
raise ConfigurationError(
"Model uses a POS embedding, but no POS tags were passed.")
mask = get_text_field_mask(tokens)
# Looking at the span start index is enough to know if
# this is padding or not. Shape: (batch_size, num_spans)
span_mask = (spans[:, :, 0] >= 0).squeeze(-1).long()
if span_mask.dim() == 1:
# This happens if you use batch_size 1 and encounter
# a length 1 sentence in PTB, which do exist. -.-
span_mask = span_mask.unsqueeze(-1)
if span_labels is not None and span_labels.dim() == 1:
span_labels = span_labels.unsqueeze(-1)
num_spans = get_lengths_from_binary_sequence_mask(span_mask)
encoded_text = self.encoder(embedded_text_input, mask)
span_representations = self.span_extractor(encoded_text, spans, mask,
span_mask)
if self.feedforward_layer is not None:
span_representations = self.feedforward_layer(span_representations)
logits = self.tag_projection_layer(span_representations)
class_probabilities = last_dim_softmax(logits, span_mask.unsqueeze(-1))
output_dict = {
"class_probabilities": class_probabilities,
"spans": spans,
"tokens": [meta["tokens"] for meta in metadata],
"pos_tags": [meta.get("pos_tags") for meta in metadata],
"num_spans": num_spans
}
if span_labels is not None:
loss = sequence_cross_entropy_with_logits(logits, span_labels,
span_mask)
self.tag_accuracy(class_probabilities, span_labels, span_mask)
output_dict["loss"] = loss
# The evalb score is expensive to compute, so we only compute
# it for the validation and test sets.
batch_gold_trees = [meta.get("gold_tree") for meta in metadata]
if all(batch_gold_trees
) and self._evalb_score is not None and not self.training:
gold_pos_tags: List[List[str]] = [
list(zip(*tree.pos()))[1] for tree in batch_gold_trees
]
predicted_trees = self.construct_trees(
class_probabilities.cpu().data,
spans.cpu().data, num_spans.data, output_dict["tokens"],
gold_pos_tags)
self._evalb_score(predicted_trees, batch_gold_trees)
return output_dict
def __call__(
self,
predictions: torch.Tensor,
gold_labels: torch.Tensor,
mask: Optional[torch.BoolTensor] = None,
):
"""
# Parameters
predictions : `torch.Tensor`, required.
A tensor of predictions of shape (batch_size, ..., num_classes).
gold_labels : `torch.Tensor`, required.
A tensor of integer class label of shape (batch_size, ...). It must be the same
shape as the `predictions` tensor without the `num_classes` dimension.
mask : `torch.BoolTensor`, optional (default = `None`).
A masking tensor the same size as `gold_labels`.
"""
predictions, gold_labels, mask = self.detach_tensors(
predictions, gold_labels, mask)
# Calculate true_positive_sum, true_negative_sum, pred_sum, true_sum
num_classes = predictions.size(-1)
if (gold_labels >= num_classes).any():
raise ConfigurationError(
"A gold label passed to FBetaMeasure contains "
f"an id >= {num_classes}, the number of classes.")
# It means we call this metric at the first time
# when `self._true_positive_sum` is None.
if self._true_positive_sum is None:
self._true_positive_sum = torch.zeros(num_classes,
device=predictions.device)
self._true_sum = torch.zeros(num_classes,
device=predictions.device)
self._pred_sum = torch.zeros(num_classes,
device=predictions.device)
self._total_sum = torch.zeros(num_classes,
device=predictions.device)
if mask is None:
mask = torch.ones_like(gold_labels).bool()
gold_labels = gold_labels.float()
# If the prediction tensor is all zeros, the record is not classified to any of the labels.
pred_mask = predictions.sum(dim=-1) != 0
argmax_predictions = predictions.max(dim=-1)[1].float()
true_positives = (gold_labels == argmax_predictions) & mask & pred_mask
true_positives_bins = gold_labels[true_positives]
# Watch it:
# The total numbers of true positives under all _predicted_ classes are zeros.
if true_positives_bins.shape[0] == 0:
true_positive_sum = torch.zeros(num_classes,
device=predictions.device)
else:
true_positive_sum = torch.bincount(true_positives_bins.long(),
minlength=num_classes).float()
pred_bins = argmax_predictions[mask & pred_mask].long()
# Watch it:
# When the `mask` is all 0, we will get an _empty_ tensor.
if pred_bins.shape[0] != 0:
pred_sum = torch.bincount(pred_bins, minlength=num_classes).float()
else:
pred_sum = torch.zeros(num_classes, device=predictions.device)
gold_labels_bins = gold_labels[mask].long()
if gold_labels.shape[0] != 0:
true_sum = torch.bincount(gold_labels_bins,
minlength=num_classes).float()
else:
true_sum = torch.zeros(num_classes, device=predictions.device)
self._total_sum += mask.sum().to(torch.float)
self._true_positive_sum += dist_reduce_sum(true_positive_sum)
self._pred_sum += dist_reduce_sum(pred_sum)
self._true_sum += dist_reduce_sum(true_sum)
def train_model(
params: Params,
serialization_dir: str,
file_friendly_logging: bool = False,
recover: bool = False,
force: bool = False,
node_rank: int = 0,
include_package: List[str] = None,
dry_run: bool = False,
) -> Optional[Model]:
"""
Trains the model specified in the given [`Params`](../common/params.md#params) object, using the data
and training parameters also specified in that object, and saves the results in `serialization_dir`.
# Parameters
params : `Params`
A parameter object specifying an AllenNLP Experiment.
serialization_dir : `str`
The directory in which to save results and logs.
file_friendly_logging : `bool`, optional (default=`False`)
If `True`, we add newlines to tqdm output, even on an interactive terminal, and we slow
down tqdm's output to only once every 10 seconds.
recover : `bool`, optional (default=`False`)
If `True`, we will try to recover a training run from an existing serialization
directory. This is only intended for use when something actually crashed during the middle
of a run. For continuing training a model on new data, see `Model.from_archive`.
force : `bool`, optional (default=`False`)
If `True`, we will overwrite the serialization directory if it already exists.
node_rank : `int`, optional
Rank of the current node in distributed training
include_package : `List[str]`, optional
In distributed mode, extra packages mentioned will be imported in trainer workers.
dry_run : `bool`, optional (default=`False`)
Do not train a model, but create a vocabulary, show dataset statistics and other training
information.
# Returns
best_model : `Optional[Model]`
The model with the best epoch weights or `None` if in dry run.
"""
training_util.create_serialization_dir(params, serialization_dir, recover,
force)
params.to_file(os.path.join(serialization_dir, CONFIG_NAME))
distributed_params = params.params.pop("distributed", None)
# If distributed isn't in the config and the config contains strictly
# one cuda device, we just run a single training process.
if distributed_params is None:
model = _train_worker(
process_rank=0,
params=params,
serialization_dir=serialization_dir,
file_friendly_logging=file_friendly_logging,
include_package=include_package,
dry_run=dry_run,
)
if not dry_run:
archive_model(serialization_dir)
return model
# Otherwise, we are running multiple processes for training.
else:
# We are careful here so that we can raise a good error if someone
# passed the wrong thing - cuda_devices are required.
device_ids = distributed_params.pop("cuda_devices", None)
multi_device = isinstance(device_ids, list) and len(device_ids) > 1
num_nodes = distributed_params.pop("num_nodes", 1)
if not (multi_device or num_nodes > 1):
raise ConfigurationError(
"Multiple cuda devices/nodes need to be configured to run distributed training."
)
check_for_gpu(device_ids)
master_addr = distributed_params.pop("master_address", "127.0.0.1")
master_port = distributed_params.pop("master_port", 29500)
num_procs = len(device_ids)
world_size = num_nodes * num_procs
logging.info(
f"Switching to distributed training mode since multiple GPUs are configured"
f"Master is at: {master_addr}:{master_port} | Rank of this node: {node_rank} | "
f"Number of workers in this node: {num_procs} | Number of nodes: {num_nodes} | "
f"World size: {world_size}")
# Creating `Vocabulary` objects from workers could be problematic since
# the data loaders in each worker will yield only `rank` specific
# instances. Hence it is safe to construct the vocabulary and write it
# to disk before initializing the distributed context. The workers will
# load the vocabulary from the path specified.
vocab_dir = os.path.join(serialization_dir, "vocabulary")
if recover:
vocab = Vocabulary.from_files(vocab_dir)
else:
vocab = training_util.make_vocab_from_params(
params.duplicate(),
serialization_dir,
print_statistics=dry_run)
params["vocabulary"] = {
"type": "from_files",
"directory": vocab_dir,
"padding_token": vocab._padding_token,
"oov_token": vocab._oov_token,
}
mp.spawn(
_train_worker,
args=(
params.duplicate(),
serialization_dir,
file_friendly_logging,
include_package,
dry_run,
node_rank,
master_addr,
master_port,
world_size,
device_ids,
),
nprocs=num_procs,
)
if dry_run:
return None
else:
archive_model(serialization_dir)
model = Model.load(params, serialization_dir)
return model
def find_learning_rate_model(
params: Params,
serialization_dir: str,
start_lr: float = 1e-5,
end_lr: float = 10,
num_batches: int = 100,
linear_steps: bool = False,
stopping_factor: float = None,
force: bool = False,
) -> None:
"""
Runs learning rate search for given `num_batches` and saves the results in ``serialization_dir``
# Parameters
params : ``Params``
A parameter object specifying an AllenNLP Experiment.
serialization_dir : ``str``
The directory in which to save results.
start_lr : ``float``
Learning rate to start the search.
end_lr : ``float``
Learning rate upto which search is done.
num_batches : ``int``
Number of mini-batches to run Learning rate finder.
linear_steps : ``bool``
Increase learning rate linearly if False exponentially.
stopping_factor : ``float``
Stop the search when the current loss exceeds the best loss recorded by
multiple of stopping factor. If ``None`` search proceeds till the ``end_lr``
force : ``bool``
If True and the serialization directory already exists, everything in it will
be removed prior to finding the learning rate.
"""
create_serialization_dir(params, serialization_dir, recover=False, force=force)
prepare_environment(params)
cuda_device = params.params.get("trainer").get("cuda_device", -1)
check_for_gpu(cuda_device)
distributed_params = params.params.get("distributed", None)
# See https://github.com/allenai/allennlp/issues/3658
assert not distributed_params, "find-lr is not compatible with DistributedDataParallel."
all_datasets = datasets_from_params(params)
datasets_for_vocab_creation = set(params.pop("datasets_for_vocab_creation", all_datasets))
for dataset in datasets_for_vocab_creation:
if dataset not in all_datasets:
raise ConfigurationError(f"invalid 'dataset_for_vocab_creation' {dataset}")
logger.info(
"From dataset instances, %s will be considered for vocabulary creation.",
", ".join(datasets_for_vocab_creation),
)
vocab = Vocabulary.from_params(
params.pop("vocabulary", {}),
instances=(
instance
for key, dataset in all_datasets.items()
for instance in dataset
if key in datasets_for_vocab_creation
),
)
model = Model.from_params(vocab=vocab, params=params.pop("model"))
iterator = DataIterator.from_params(params.pop("iterator"))
iterator.index_with(vocab)
train_data = all_datasets["train"]
trainer_params = params.pop("trainer")
no_grad_regexes = trainer_params.pop("no_grad", ())
for name, parameter in model.named_parameters():
if any(re.search(regex, name) for regex in no_grad_regexes):
parameter.requires_grad_(False)
trainer_choice = trainer_params.pop("type", "default")
if trainer_choice != "default":
raise ConfigurationError("currently find-learning-rate only works with the default Trainer")
trainer = Trainer.from_params(
model=model,
serialization_dir=serialization_dir,
iterator=iterator,
train_data=train_data,
validation_data=None,
params=trainer_params,
validation_iterator=None,
)
logger.info(
f"Starting learning rate search from {start_lr} to {end_lr} in {num_batches} iterations."
)
learning_rates, losses = search_learning_rate(
trainer,
start_lr=start_lr,
end_lr=end_lr,
num_batches=num_batches,
linear_steps=linear_steps,
stopping_factor=stopping_factor,
)
logger.info(f"Finished learning rate search.")
losses = _smooth(losses, 0.98)
_save_plot(learning_rates, losses, os.path.join(serialization_dir, "lr-losses.png"))
def from_partial_objects(
cls,
serialization_dir: str,
local_rank: int,
dataset_reader: DatasetReader,
train_data_path: str,
model: Lazy[Model],
data_loader: Lazy[DataLoader],
trainer: Lazy[Trainer],
vocabulary: Lazy[Vocabulary] = None,
datasets_for_vocab_creation: List[str] = None,
validation_dataset_reader: DatasetReader = None,
validation_data_path: str = None,
validation_data_loader: Lazy[DataLoader] = None,
test_data_path: str = None,
evaluate_on_test: bool = False,
batch_weight_key: str = "",
) -> "TrainModel":
"""
This method is intended for use with our `FromParams` logic, to construct a `TrainModel`
object from a config file passed to the `allennlp train` command. The arguments to this
method are the allowed top-level keys in a configuration file (except for the first three,
which are obtained separately).
You *could* use this outside of our `FromParams` logic if you really want to, but there
might be easier ways to accomplish your goal than instantiating `Lazy` objects. If you are
writing your own training loop, we recommend that you look at the implementation of this
method for inspiration and possibly some utility functions you can call, but you very likely
should not use this method directly.
The `Lazy` type annotations here are a mechanism for building dependencies to an object
sequentially - the `TrainModel` object needs data, a model, and a trainer, but the model
needs to see the data before it's constructed (to create a vocabulary) and the trainer needs
the data and the model before it's constructed. Objects that have sequential dependencies
like this are labeled as `Lazy` in their type annotations, and we pass the missing
dependencies when we call their `construct()` method, which you can see in the code below.
# Parameters
serialization_dir: `str`
The directory where logs and model archives will be saved.
In a typical AllenNLP configuration file, this parameter does not get an entry as a
top-level key, it gets passed in separately.
local_rank: `int`
The process index that is initialized using the GPU device id.
In a typical AllenNLP configuration file, this parameter does not get an entry as a
top-level key, it gets passed in separately.
dataset_reader: `DatasetReader`
The `DatasetReader` that will be used for training and (by default) for validation.
train_data_path: `str`
The file (or directory) that will be passed to `dataset_reader.read()` to construct the
training data.
model: `Lazy[Model]`
The model that we will train. This is lazy because it depends on the `Vocabulary`;
after constructing the vocabulary we call `model.construct(vocab=vocabulary)`.
data_loader: `Lazy[DataLoader]`
The data_loader we use to batch instances from the dataset reader at training and (by
default) validation time. This is lazy because it takes a dataset in it's constructor.
trainer: `Lazy[Trainer]`
The `Trainer` that actually implements the training loop. This is a lazy object because
it depends on the model that's going to be trained.
vocabulary: `Lazy[Vocabulary]`, optional (default=`None`)
The `Vocabulary` that we will use to convert strings in the data to integer ids (and
possibly set sizes of embedding matrices in the `Model`). By default we construct the
vocabulary from the instances that we read.
datasets_for_vocab_creation: `List[str]`, optional (default=`None`)
If you pass in more than one dataset but don't want to use all of them to construct a
vocabulary, you can pass in this key to limit it. Valid entries in the list are
"train", "validation" and "test".
validation_dataset_reader: `DatasetReader`, optional (default=`None`)
If given, we will use this dataset reader for the validation data instead of
`dataset_reader`.
validation_data_path: `str`, optional (default=`None`)
If given, we will use this data for computing validation metrics and early stopping.
validation_data_loader: `Lazy[DataLoader]`, optional (default=`None`)
If given, the data_loader we use to batch instances from the dataset reader at
validation and test time. This is lazy because it takes a dataset in it's constructor.
test_data_path: `str`, optional (default=`None`)
If given, we will use this as test data. This makes it available for vocab creation by
default, but nothing else.
evaluate_on_test: `bool`, optional (default=`False`)
If given, we will evaluate the final model on this data at the end of training. Note
that we do not recommend using this for actual test data in every-day experimentation;
you should only very rarely evaluate your model on actual test data.
batch_weight_key: `str`, optional (default=`""`)
The name of metric used to weight the loss on a per-batch basis. This is only used
during evaluation on final test data, if you've specified `evaluate_on_test=True`.
"""
datasets = training_util.read_all_datasets(
train_data_path=train_data_path,
dataset_reader=dataset_reader,
validation_dataset_reader=validation_dataset_reader,
validation_data_path=validation_data_path,
test_data_path=test_data_path,
)
if datasets_for_vocab_creation:
for key in datasets_for_vocab_creation:
if key not in datasets:
raise ConfigurationError(
f"invalid 'dataset_for_vocab_creation' {key}")
logger.info(
"From dataset instances, %s will be considered for vocabulary creation.",
", ".join(datasets_for_vocab_creation),
)
instance_generator = (instance for key, dataset in datasets.items()
if datasets_for_vocab_creation is None
or key in datasets_for_vocab_creation
for instance in dataset)
vocabulary_ = vocabulary.construct(instances=instance_generator)
if not vocabulary_:
vocabulary_ = Vocabulary.from_instances(instance_generator)
model_ = model.construct(vocab=vocabulary_)
# Initializing the model can have side effect of expanding the vocabulary.
# Save the vocab only in the master. In the degenerate non-distributed
# case, we're trivially the master. In the distributed case this is safe
# to do without worrying about race conditions since saving and loading
# the vocab involves acquiring a file lock.
if common_util.is_master():
vocabulary_path = os.path.join(serialization_dir, "vocabulary")
vocabulary_.save_to_files(vocabulary_path)
for dataset in datasets.values():
dataset.index_with(model_.vocab)
data_loader_ = data_loader.construct(dataset=datasets["train"])
validation_data = datasets.get("validation")
if validation_data is not None:
# Because of the way Lazy[T] works, we can't check it's existence
# _before_ we've tried to construct it. It returns None if it is not
# present, so we try to construct it first, and then afterward back off
# to the data_loader configuration used for training if it returns None.
validation_data_loader_ = validation_data_loader.construct(
dataset=validation_data)
if validation_data_loader_ is None:
validation_data_loader_ = data_loader.construct(
dataset=validation_data)
else:
validation_data_loader_ = None
test_data = datasets.get("test")
if test_data is not None:
test_data_loader = validation_data_loader.construct(
dataset=test_data)
if test_data_loader is None:
test_data_loader = data_loader.construct(dataset=test_data)
else:
test_data_loader = None
# We don't need to pass serialization_dir and local_rank here, because they will have been
# passed through the trainer by from_params already, because they were keyword arguments to
# construct this class in the first place.
trainer_ = trainer.construct(
model=model_,
data_loader=data_loader_,
validation_data_loader=validation_data_loader_,
)
return cls(
serialization_dir=serialization_dir,
model=model_,
trainer=trainer_,
evaluation_data_loader=test_data_loader,
evaluate_on_test=evaluate_on_test,
batch_weight_key=batch_weight_key,
)
def forward(
self, # type: ignore
words: TextFieldTensors,
pos_tags: torch.LongTensor,
metadata: List[Dict[str, Any]],
head_tags: torch.LongTensor = None,
head_indices: torch.LongTensor = None,
) -> Dict[str, torch.Tensor]:
"""
# Parameters
words : TextFieldTensors, required
The output of `TextField.as_array()`, which should typically be passed directly to a
`TextFieldEmbedder`. This output is a dictionary mapping keys to `TokenIndexer`
tensors. At its most basic, using a `SingleIdTokenIndexer` this is : `{"tokens":
Tensor(batch_size, sequence_length)}`. This dictionary will have the same keys as were used
for the `TokenIndexers` when you created the `TextField` representing your
sequence. The dictionary is designed to be passed directly to a `TextFieldEmbedder`,
which knows how to combine different word representations into a single vector per
token in your input.
pos_tags : `torch.LongTensor`, required
The output of a `SequenceLabelField` containing POS tags.
POS tags are required regardless of whether they are used in the model,
because they are used to filter the evaluation metric to only consider
heads of words which are not punctuation.
metadata : List[Dict[str, Any]], optional (default=None)
A dictionary of metadata for each batch element which has keys:
words : `List[str]`, required.
The tokens in the original sentence.
pos : `List[str]`, required.
The dependencies POS tags for each word.
head_tags : torch.LongTensor, optional (default = None)
A torch tensor representing the sequence of integer gold class labels for the arcs
in the dependency parse. Has shape `(batch_size, sequence_length)`.
head_indices : torch.LongTensor, optional (default = None)
A torch tensor representing the sequence of integer indices denoting the parent of every
word in the dependency parse. Has shape `(batch_size, sequence_length)`.
# Returns
An output dictionary consisting of:
loss : `torch.FloatTensor`, optional
A scalar loss to be optimised.
arc_loss : `torch.FloatTensor`
The loss contribution from the unlabeled arcs.
loss : `torch.FloatTensor`, optional
The loss contribution from predicting the dependency
tags for the gold arcs.
heads : `torch.FloatTensor`
The predicted head indices for each word. A tensor
of shape (batch_size, sequence_length).
head_types : `torch.FloatTensor`
The predicted head types for each arc. A tensor
of shape (batch_size, sequence_length).
mask : `torch.LongTensor`
A mask denoting the padded elements in the batch.
"""
embedded_text_input = self.text_field_embedder(words)
if pos_tags is not None and self._pos_tag_embedding is not None:
embedded_pos_tags = self._pos_tag_embedding(pos_tags)
embedded_text_input = torch.cat([embedded_text_input, embedded_pos_tags], -1)
elif self._pos_tag_embedding is not None:
raise ConfigurationError("Model uses a POS embedding, but no POS tags were passed.")
mask = get_text_field_mask(words)
predicted_heads, predicted_head_tags, mask, arc_nll, tag_nll = self._parse(
embedded_text_input, mask, head_tags, head_indices
)
loss = arc_nll + tag_nll
if head_indices is not None and head_tags is not None:
evaluation_mask = self._get_mask_for_eval(mask[:, 1:], pos_tags)
# We calculate attatchment scores for the whole sentence
# but excluding the symbolic ROOT token at the start,
# which is why we start from the second element in the sequence.
self._attachment_scores(
predicted_heads[:, 1:],
predicted_head_tags[:, 1:],
head_indices,
head_tags,
evaluation_mask,
)
output_dict = {
"heads": predicted_heads,
"head_tags": predicted_head_tags,
"arc_loss": arc_nll,
"tag_loss": tag_nll,
"loss": loss,
"mask": mask,
"words": [meta["words"] for meta in metadata],
"pos": [meta["pos"] for meta in metadata],
}
return output_dict
def __init__(self,
model: Model,
optimizer: torch.optim.Optimizer,
train_iterator: DataIterator,
val_iterator: DataIterator,
train_dataset: Iterable[Instance],
validation_dataset: Optional[Iterable[Instance]] = None,
patience: int = 2,
validation_metric: str = "-loss",
num_epochs: int = 20,
serialization_dir: Optional[str] = None,
num_serialized_models_to_keep: int = 2,
keep_serialized_model_every_num_seconds: int = None,
model_save_interval: float = None,
cuda_device: Union[int, List] = -1,
grad_norm: Optional[float] = None,
grad_clipping: Optional[float] = None,
learning_rate_scheduler: Optional[PytorchLRScheduler] = None,
histogram_interval: int = None) -> None:
"""
Parameters
----------
model : ``Model``, required.
An AllenNLP model to be optimized. Pytorch Modules can also be optimized if
their ``forward`` method returns a dictionary with a "loss" key, containing a
scalar tensor representing the loss function to be optimized.
optimizer : ``torch.nn.Optimizer``, required.
An instance of a Pytorch Optimizer, instantiated with the parameters of the
model to be optimized.
iterator : ``DataIterator``, required.
A method for iterating over a ``Dataset``, yielding padded indexed batches.
train_dataset : ``Dataset``, required.
A ``Dataset`` to train on. The dataset should have already been indexed.
validation_dataset : ``Dataset``, optional, (default = None).
A ``Dataset`` to evaluate on. The dataset should have already been indexed.
patience : int, optional (default=2)
Number of epochs to be patient before early stopping.
validation_metric : str, optional (default="loss")
Validation metric to measure for whether to stop training using patience
and whether to serialize an ``is_best`` model each epoch. The metric name
must be prepended with either "+" or "-", which specifies whether the metric
is an increasing or decreasing function.
num_epochs : int, optional (default = 20)
Number of training epochs.
serialization_dir : str, optional (default=None)
Path to directory for saving and loading model files. Models will not be saved if
this parameter is not passed.
num_serialized_models_to_keep : ``int``, optional (default=None)
Number of previous model checkpoints to retain. Default is to keep all checkpoints.
keep_serialized_model_every_num_seconds : ``int``, optional (default=None)
If num_serialized_models_to_keep is not None, then occasionally it's useful to
save models at a given interval in addition to the last num_serialized_models_to_keep.
To do so, specify keep_serialized_model_every_num_seconds as the number of seconds
between permanently saved checkpoints. Note that this option is only used if
num_serialized_models_to_keep is not None, otherwise all checkpoints are kept.
model_save_interval : ``float``, optional (default=None)
If provided, then serialize models every ``model_save_interval``
seconds within single epochs. In all cases, models are also saved
at the end of every epoch if ``serialization_dir`` is provided.
cuda_device : ``int``, optional (default = -1)
An integer specifying the CUDA device to use. If -1, the CPU is used.
grad_norm : ``float``, optional, (default = None).
If provided, gradient norms will be rescaled to have a maximum of this value.
grad_clipping : ``float``, optional (default = ``None``).
If provided, gradients will be clipped `during the backward pass` to have an (absolute)
maximum of this value. If you are getting ``NaNs`` in your gradients during training
that are not solved by using ``grad_norm``, you may need this.
learning_rate_scheduler : ``PytorchLRScheduler``, optional, (default = None)
A Pytorch learning rate scheduler. The learning rate will be decayed with respect to
this schedule at the end of each epoch. If you use
:class:`torch.optim.lr_scheduler.ReduceLROnPlateau`, this will use the ``validation_metric``
provided to determine if learning has plateaued. To support updating the learning
rate on every batch, this can optionally implement ``step_batch(batch_num)`` which
updates the learning rate given the batch number.
histogram_interval : ``int``, optional, (default = ``None``)
If not None, then log histograms to tensorboard every ``histogram_interval`` batches.
When this parameter is specified, the following additional logging is enabled:
* Histograms of model parameters
* The ratio of parameter update norm to parameter norm
* Histogram of layer activations
We log histograms of the parameters returned by
``model.get_parameters_for_histogram_tensorboard_logging``.
The layer activations are logged for any modules in the ``Model`` that have
the attribute ``should_log_activations`` set to ``True``. Logging
histograms requires a number of GPU-CPU copies during training and is typically
slow, so we recommend logging histograms relatively infrequently.
Note: only Modules that return tensors, tuples of tensors or dicts
with tensors as values currently support activation logging.
"""
self._model = model
self._train_iterator = train_iterator
self._val_iterator = val_iterator
self._optimizer = optimizer
self._train_data = train_dataset
self._validation_data = validation_dataset
self._patience = patience
self._num_epochs = num_epochs
self._serialization_dir = serialization_dir
self._num_serialized_models_to_keep = num_serialized_models_to_keep
self._keep_serialized_model_every_num_seconds = keep_serialized_model_every_num_seconds
self._serialized_paths: List[Any] = []
self._last_permanent_saved_checkpoint_time = time.time()
self._model_save_interval = model_save_interval
self._grad_norm = grad_norm
self._batch_grad_norm = None
self._grad_clipping = grad_clipping
self._learning_rate_scheduler = learning_rate_scheduler
increase_or_decrease = validation_metric[0]
if increase_or_decrease not in ["+", "-"]:
raise ConfigurationError("Validation metrics must specify whether they should increase "
"or decrease by pre-pending the metric name with a +/-.")
self._validation_metric = validation_metric[1:]
self._validation_metric_decreases = increase_or_decrease == "-"
if not isinstance(cuda_device, int) and not isinstance(cuda_device, list):
raise ConfigurationError("Expected an int or list for cuda_device, got {}".format(cuda_device))
if isinstance(cuda_device, list):
logger.info(f"WARNING: Multiple GPU support is experimental not recommended for use. "
"In some cases it may lead to incorrect results or undefined behavior.")
self._multiple_gpu = True
self._cuda_devices = cuda_device
# data_parallel will take care of transfering to cuda devices,
# so the iterator keeps data on CPU.
self._iterator_device = -1
else:
self._multiple_gpu = False
self._cuda_devices = [cuda_device]
self._iterator_device = cuda_device
if self._cuda_devices[0] != -1:
self._model = self._model.cuda(self._cuda_devices[0])
# print(self._cuda_devices)
self._log_interval = 10 # seconds
self._summary_interval = 100 # num batches between logging to tensorboard
self._histogram_interval = histogram_interval
self._log_histograms_this_batch = False
self._batch_num_total = 0
self._last_log = 0.0 # time of last logging
if serialization_dir is not None:
train_log = SummaryWriter(os.path.join(serialization_dir, "log", "train"))
validation_log = SummaryWriter(os.path.join(serialization_dir, "log", "validation"))
self._tensorboard = TensorboardWriter(train_log, validation_log)
else:
self._tensorboard = TensorboardWriter()
def train(self) -> Dict[str, Any]:
"""
Trains the supplied model with the supplied parameters.
"""
try:
epoch_counter = self._restore_checkpoint()
except RuntimeError:
traceback.print_exc()
raise ConfigurationError("Could not recover training from the checkpoint. Did you mean to output to "
"a different serialization directory or delete the existing serialization "
"directory?")
training_util.enable_gradient_clipping(self.model, self._grad_clipping)
logger.info("Beginning training.")
train_metrics: Dict[str, float] = {}
val_metrics: Dict[str, float] = {}
this_epoch_val_metric: float = None
metrics: Dict[str, Any] = {}
epochs_trained = 0
training_start_time = time.time()
metrics['best_epoch'] = self._metric_tracker.best_epoch
for key, value in self._metric_tracker.best_epoch_metrics.items():
metrics["best_validation_" + key] = value
for epoch in range(epoch_counter, self._num_epochs):
epoch_start_time = time.time()
train_metrics = self._train_epoch(epoch)
# get peak of memory usage
if 'cpu_memory_MB' in train_metrics:
metrics['peak_cpu_memory_MB'] = max(metrics.get('peak_cpu_memory_MB', 0),
train_metrics['cpu_memory_MB'])
for key, value in train_metrics.items():
if key.startswith('gpu_'):
metrics["peak_"+key] = max(metrics.get("peak_"+key, 0), value)
if self._validation_data is not None:
with torch.no_grad():
# We have a validation set, so compute all the metrics on it.
val_loss, num_batches = self._validation_loss()
val_metrics = training_util.get_metrics(self.model, val_loss, num_batches, reset=True)
# Check validation metric for early stopping
this_epoch_val_metric = val_metrics[self._validation_metric]
self._metric_tracker.add_metric(this_epoch_val_metric)
if self._metric_tracker.should_stop_early():
logger.info("Ran out of patience. Stopping training.")
break
self._tensorboard.log_metrics(train_metrics,
val_metrics=val_metrics,
log_to_console=True,
epoch=epoch + 1) # +1 because tensorboard doesn't like 0
# Create overall metrics dict
training_elapsed_time = time.time() - training_start_time
metrics["training_duration"] = str(datetime.timedelta(seconds=training_elapsed_time))
metrics["training_start_epoch"] = epoch_counter
metrics["training_epochs"] = epochs_trained
metrics["epoch"] = epoch
for key, value in train_metrics.items():
metrics["training_" + key] = value
for key, value in val_metrics.items():
metrics["validation_" + key] = value
if self._metric_tracker.is_best_so_far():
# Update all the best_ metrics.
# (Otherwise they just stay the same as they were.)
metrics['best_epoch'] = epoch
for key, value in val_metrics.items():
metrics["best_validation_" + key] = value
self._metric_tracker.best_epoch_metrics = val_metrics
if self._serialization_dir:
dump_metrics(os.path.join(self._serialization_dir, f'metrics_epoch_{epoch}.json'), metrics)
# The Scheduler API is agnostic to whether your schedule requires a validation metric -
# if it doesn't, the validation metric passed here is ignored.
if self._learning_rate_scheduler:
self._learning_rate_scheduler.step(this_epoch_val_metric, epoch)
if self._momentum_scheduler:
self._momentum_scheduler.step(this_epoch_val_metric, epoch)
self._save_checkpoint(epoch)
epoch_elapsed_time = time.time() - epoch_start_time
logger.info("Epoch duration: %s", datetime.timedelta(seconds=epoch_elapsed_time))
if epoch < self._num_epochs - 1:
training_elapsed_time = time.time() - training_start_time
estimated_time_remaining = training_elapsed_time * \
((self._num_epochs - epoch_counter) / float(epoch - epoch_counter + 1) - 1)
formatted_time = str(datetime.timedelta(seconds=int(estimated_time_remaining)))
logger.info("Estimated training time remaining: %s", formatted_time)
epochs_trained += 1
# make sure pending events are flushed to disk and files are closed properly
self._tensorboard.close()
# Load the best model state before returning
best_model_state = self._checkpointer.best_model_state()
if best_model_state:
self.model.load_state_dict(best_model_state)
# send Email after train
self.sendEmail(str(metrics))
return metrics
def from_params(params: Params,
serialization_dir: str,
recover: bool = False,
cache_directory: str = None,
cache_prefix: str = None) -> 'TrainerPieces':
all_datasets = training_util.datasets_from_params(
params, cache_directory, cache_prefix)
datasets_for_vocab_creation = set(
params.pop("datasets_for_vocab_creation", all_datasets))
for dataset in datasets_for_vocab_creation:
if dataset not in all_datasets:
raise ConfigurationError(
f"invalid 'dataset_for_vocab_creation' {dataset}")
logger.info(
"From dataset instances, %s will be considered for vocabulary creation.",
", ".join(datasets_for_vocab_creation))
if recover and os.path.exists(
os.path.join(serialization_dir, "vocabulary")):
vocab = Vocabulary.from_files(
os.path.join(serialization_dir, "vocabulary"))
params.pop("vocabulary", {})
else:
vocab = Vocabulary.from_params(params.pop(
"vocabulary", {}), (instance
for key, dataset in all_datasets.items()
for instance in dataset
if key in datasets_for_vocab_creation))
model = Model.from_params(vocab=vocab, params=params.pop('model'))
# If vocab extension is ON for training, embedding extension should also be
# done. If vocab and embeddings are already in sync, it would be a no-op.
model.extend_embedder_vocab()
# Initializing the model can have side effect of expanding the vocabulary
vocab.save_to_files(os.path.join(serialization_dir, "vocabulary"))
iterator = DataIterator.from_params(params.pop("iterator"))
iterator.index_with(model.vocab)
validation_iterator_params = params.pop("validation_iterator", None)
if validation_iterator_params:
validation_iterator = DataIterator.from_params(
validation_iterator_params)
validation_iterator.index_with(model.vocab)
else:
validation_iterator = None
train_data = all_datasets['train']
validation_data = all_datasets.get('validation')
test_data = all_datasets.get('test')
trainer_params = params.pop("trainer")
no_grad_regexes = trainer_params.pop("no_grad", ())
for name, parameter in model.named_parameters():
if any(re.search(regex, name) for regex in no_grad_regexes):
parameter.requires_grad_(False)
frozen_parameter_names, tunable_parameter_names = \
get_frozen_and_tunable_parameter_names(model)
logger.info("Following parameters are Frozen (without gradient):")
for name in frozen_parameter_names:
logger.info(name)
logger.info("Following parameters are Tunable (with gradient):")
for name in tunable_parameter_names:
logger.info(name)
return TrainerPieces(model, iterator, train_data, validation_data,
test_data, validation_iterator, trainer_params)
def extend_vocab(
self,
extended_vocab: Vocabulary,
vocab_namespace: str = None,
extension_pretrained_file: str = None,
model_path: str = None,
):
"""
Extends the embedding matrix according to the extended vocabulary.
If extension_pretrained_file is available, it will be used for initializing the new words
embeddings in the extended vocabulary; otherwise we will check if _pretrained_file attribute
is already available. If none is available, they will be initialized with xavier uniform.
# Parameters
extended_vocab : `Vocabulary`
Vocabulary extended from original vocabulary used to construct
this `Embedding`.
vocab_namespace : `str`, (optional, default=None)
In case you know what vocab_namespace should be used for extension, you
can pass it. If not passed, it will check if vocab_namespace used at the
time of `Embedding` construction is available. If so, this namespace
will be used or else extend_vocab will be a no-op.
extension_pretrained_file : `str`, (optional, default=None)
A file containing pretrained embeddings can be specified here. It can be
the path to a local file or an URL of a (cached) remote file. Check format
details in `from_params` of `Embedding` class.
model_path : `str`, (optional, default=None)
Path traversing the model attributes upto this embedding module.
Eg. "_text_field_embedder.token_embedder_tokens". This is only useful
to give a helpful error message when extend_vocab is implicitly called
by train or any other command.
"""
# Caveat: For allennlp v0.8.1 and below, we weren't storing vocab_namespace as an attribute,
# knowing which is necessary at time of embedding vocab extension. So old archive models are
# currently unextendable.
vocab_namespace = vocab_namespace or self._vocab_namespace
if not vocab_namespace:
# It's not safe to default to "tokens" or any other namespace.
logging.info(
"Loading a model trained before embedding extension was implemented; "
"pass an explicit vocab namespace if you want to extend the vocabulary."
)
return
extended_num_embeddings = extended_vocab.get_vocab_size(
vocab_namespace)
if extended_num_embeddings == self.num_embeddings:
# It's already been extended. No need to initialize / read pretrained file in first place (no-op)
return
if extended_num_embeddings < self.num_embeddings:
raise ConfigurationError(
f"Size of namespace, {vocab_namespace} for extended_vocab is smaller than "
f"embedding. You likely passed incorrect vocab or namespace for extension."
)
# Case 1: user passed extension_pretrained_file and it's available.
if extension_pretrained_file and is_url_or_existing_file(
extension_pretrained_file):
# Don't have to do anything here, this is the happy case.
pass
# Case 2: user passed extension_pretrained_file and it's not available
elif extension_pretrained_file:
raise ConfigurationError(
f"You passed pretrained embedding file {extension_pretrained_file} "
f"for model_path {model_path} but it's not available.")
# Case 3: user didn't pass extension_pretrained_file, but pretrained_file attribute was
# saved during training and is available.
elif is_url_or_existing_file(self._pretrained_file):
extension_pretrained_file = self._pretrained_file
# Case 4: no file is available, hope that pretrained embeddings weren't used in the first place and warn
else:
extra_info = (f"Originally pretrained_file was at "
f"{self._pretrained_file}. "
if self._pretrained_file else "")
# It's better to warn here and not give error because there is no way to distinguish between
# whether pretrained-file wasn't used during training or user forgot to pass / passed incorrect
# mapping. Raising an error would prevent fine-tuning in the former case.
logging.warning(
f"Embedding at model_path, {model_path} cannot locate the pretrained_file. "
f"{extra_info} If you are fine-tuning and want to use using pretrained_file for "
f"embedding extension, please pass the mapping by --embedding-sources argument."
)
embedding_dim = self.weight.data.shape[-1]
if not extension_pretrained_file:
extra_num_embeddings = extended_num_embeddings - self.num_embeddings
extra_weight = torch.FloatTensor(extra_num_embeddings,
embedding_dim)
torch.nn.init.xavier_uniform_(extra_weight)
else:
# It's easiest to just reload the embeddings for the entire vocab,
# then only keep the ones we need.
whole_weight = _read_pretrained_embeddings_file(
extension_pretrained_file, embedding_dim, extended_vocab,
vocab_namespace)
extra_weight = whole_weight[self.num_embeddings:, :]
device = self.weight.data.device
extended_weight = torch.cat(
[self.weight.data, extra_weight.to(device)], dim=0)
self.weight = torch.nn.Parameter(
extended_weight, requires_grad=self.weight.requires_grad)
def __init__(
self,
step_name: Optional[str] = None,
cache_results: Optional[bool] = None,
step_format: Optional[Format] = None,
only_if_needed: Optional[bool] = None,
**kwargs,
):
self.logger = cast(AllenNlpLogger,
logging.getLogger(self.__class__.__name__))
if self.VERSION is not None:
assert _version_re.match(
self.VERSION
), f"Invalid characters in version '{self.VERSION}'"
self.kwargs = kwargs
if step_format is None:
self.format = self.FORMAT
else:
self.format = step_format
self.unique_id_cache: Optional[str] = None
if step_name is None:
self.name = self.unique_id()
self.only_if_needed = False
else:
self.name = step_name
self.only_if_needed = True
if only_if_needed is not None:
self.only_if_needed = only_if_needed
if cache_results is True:
if not self.CACHEABLE:
raise ConfigurationError(
f"Step {self.name} is configured to use the cache, but it's not a cacheable step."
)
if not self.DETERMINISTIC:
logger.warning(
f"Step {self.name} is going to be cached despite not being deterministic."
)
self.cache_results = True
elif cache_results is False:
self.cache_results = False
elif cache_results is None:
c = (self.DETERMINISTIC, self.CACHEABLE)
if c == (False, None):
self.cache_results = False
elif c == (True, None):
self.cache_results = True
elif c == (False, False):
self.cache_results = False
elif c == (True, False):
self.cache_results = False
elif c == (False, True):
logger.warning(
f"Step {self.name} is set to be cacheable despite not being deterministic."
)
self.cache_results = True
elif c == (True, True):
self.cache_results = True
else:
assert False, "Step.DETERMINISTIC or step.CACHEABLE are set to an invalid value."
else:
raise ConfigurationError(
f"Step {step_name}'s cache_results parameter is set to an invalid value."
)
self.work_dir_for_run: Optional[
PathLike] = None # This is set only while the run() method runs.
def forward(self, # type: ignore
premise_hypothesis: Dict[str, torch.Tensor] = None,
premise: Dict[str, torch.Tensor] = None,
hypothesis: Dict[str, torch.LongTensor] = None,
dataset: List[str] = None,
label: torch.IntTensor = None,
metadata: List[Dict[str, Any]] = None) -> Dict[str, torch.Tensor]:
# pylint: disable=arguments-differ
"""
Parameters
----------
premise_hypothesis : Dict[str, torch.LongTensor]
Combined in a single text field for BERT encoding
premise : Dict[str, torch.LongTensor]
From a ``TextField``
hypothesis : Dict[str, torch.LongTensor]
From a ``TextField``
label : torch.IntTensor, optional, (default = None)
From a ``LabelField``
dataset : List[str]
Task indicator
metadata : ``List[Dict[str, Any]]``, optional, (default = None)
Metadata containing the original tokenization of the premise and
hypothesis with 'premise_tokens' and 'hypothesis_tokens' keys respectively.
Returns
-------
An output dictionary consisting of:
label_logits : torch.FloatTensor
A tensor of shape ``(batch_size, num_labels)`` representing unnormalised log
probabilities of the entailment label.
label_probs : torch.FloatTensor
A tensor of shape ``(batch_size, num_labels)`` representing probabilities of the
entailment label.
loss : torch.FloatTensor, optional
A scalar loss to be optimised.
"""
taskname = "nli-en"
if dataset is not None: # TODO: hardcoded; used when not multitask reader was used
taskname = dataset[0]
else:
taskname = "nli-en"
if premise_hypothesis is not None:
raise ConfigurationError("you should not combine premise and hypothesis for this model")
if premise is not None and hypothesis is not None:
embedded_premise = self._text_field_embedder(premise, lang=taskname.split("-")[-1])
embedded_hypothesis = self._text_field_embedder(hypothesis, lang=taskname.split("-")[-1])
mask_premise = get_text_field_mask(premise).float()
mask_hypothesis = get_text_field_mask(hypothesis).float()
if self._premise_encoder:
embedded_premise = self._premise_encoder(embedded_premise, mask_premise)
if self._hypothesis_encoder:
embedded_hypothesis = self._hypothesis_encoder(embedded_hypothesis, mask_hypothesis)
projected_premise = self._attend_feedforward(embedded_premise)
projected_hypothesis = self._attend_feedforward(embedded_hypothesis)
# Shape: (batch_size, premise_length, hypothesis_length)
similarity_matrix = self._matrix_attention(projected_premise, projected_hypothesis)
# Shape: (batch_size, premise_length, hypothesis_length)
p2h_attention = masked_softmax(similarity_matrix, mask_hypothesis)
# Shape: (batch_size, premise_length, embedding_dim)
attended_hypothesis = weighted_sum(embedded_hypothesis, p2h_attention)
# Shape: (batch_size, hypothesis_length, premise_length)
h2p_attention = masked_softmax(similarity_matrix.transpose(1, 2).contiguous(), mask_premise)
# Shape: (batch_size, hypothesis_length, embedding_dim)
attended_premise = weighted_sum(embedded_premise, h2p_attention)
premise_compare_input = torch.cat([embedded_premise, attended_hypothesis], dim=-1)
hypothesis_compare_input = torch.cat([embedded_hypothesis, attended_premise], dim=-1)
compared_premise = self._compare_feedforward(premise_compare_input)
compared_premise = compared_premise * mask_premise.unsqueeze(-1)
# Shape: (batch_size, compare_dim)
compared_premise = compared_premise.sum(dim=1)
compared_hypothesis = self._compare_feedforward(hypothesis_compare_input)
compared_hypothesis = compared_hypothesis * mask_hypothesis.unsqueeze(-1)
# Shape: (batch_size, compare_dim)
compared_hypothesis = compared_hypothesis.sum(dim=1)
aggregate_input = torch.cat([compared_premise, compared_hypothesis], dim=-1)
logits = self._aggregate_feedforward(aggregate_input)
probs = torch.nn.functional.softmax(logits, dim=-1)
else:
raise ConfigurationError("One of premise or hypothesis is None. Check your DatasetReader")
output_dict = {"logits": logits,
"probs": probs,
"h2p_attention": h2p_attention,
"p2h_attention": p2h_attention}
if label is not None:
loss = self._loss(logits, label.long().view(-1))
output_dict["loss"] = loss
self._nli_per_lang_acc[taskname](logits, label)
if metadata is not None:
output_dict["premise_tokens"] = [x["premise_tokens"] for x in metadata]
output_dict["hypothesis_tokens"] = [x["hypothesis_tokens"] for x in metadata]
return output_dict
def forward(
self, # type: ignore
tokens: Dict[str, torch.LongTensor],
targets: torch.LongTensor,
target_index: torch.LongTensor,
span_starts: torch.LongTensor,
span_ends: torch.LongTensor,
tags: torch.LongTensor = None,
**kwargs) -> Dict[str, torch.Tensor]:
# pylint: disable=arguments-differ
"""
Parameters
----------
tokens : Dict[str, torch.LongTensor], required
The output of ``TextField.as_array()``, which should typically be passed directly to a
``TextFieldEmbedder``. This output is a dictionary mapping keys to ``TokenIndexer``
tensors. At its most basic, using a ``SingleIdTokenIndexer`` this is: ``{"tokens":
Tensor(batch_size, num_tokens)}``. This dictionary will have the same keys as were used
for the ``TokenIndexers`` when you created the ``TextField`` representing your
sequence. The dictionary is designed to be passed directly to a ``TextFieldEmbedder``,
which knows how to combine different word representations into a single vector per
token in your input.
verb_indicator: torch.LongTensor, required.
An integer ``SequenceFeatureField`` representation of the position of the verb
in the sentence. This should have shape (batch_size, num_tokens) and importantly, can be
all zeros, in the case that the sentence has no verbal predicate.
bio : torch.LongTensor, optional (default = None)
A torch tensor representing the sequence of integer gold class labels
of shape ``(batch_size, num_tokens)``
tags: shape ``(batch_size, num_spans)``
span_starts: shape ``(batch_size, num_spans)``
span_ends: shape ``(batch_size, num_spans)``
Returns
-------
An output dictionary consisting of:
logits : torch.FloatTensor
A tensor of shape ``(batch_size, num_tokens, tag_vocab_size)`` representing
unnormalised log probabilities of the tag classes.
class_probabilities : torch.FloatTensor
A tensor of shape ``(batch_size, num_tokens, tag_vocab_size)`` representing
a distribution of the tag classes per word.
loss : torch.FloatTensor, optional
A scalar loss to be optimised.
"""
embedded_text_input = self.embedding_dropout(
self.text_field_embedder(tokens))
text_mask = util.get_text_field_mask(tokens)
embedded_verb_indicator = self.binary_feature_embedding(targets.long())
# Concatenate the verb feature onto the embedded text. This now
# has shape (batch_size, sequence_length, embedding_dim + binary_feature_dim).
embedded_text_with_verb_indicator = torch.cat(
[embedded_text_input, embedded_verb_indicator], -1)
embedding_dim_with_binary_feature = embedded_text_with_verb_indicator.size(
)[2]
if self.stacked_encoder.get_input_dim(
) != embedding_dim_with_binary_feature:
raise ConfigurationError(
"The SRL model uses an indicator feature, which makes "
"the embedding dimension one larger than the value "
"specified. Therefore, the 'input_dim' of the stacked_encoder "
"must be equal to total_embedding_dim + 1.")
encoded_text = self.stacked_encoder(embedded_text_with_verb_indicator,
text_mask)
batch_size, num_spans = tags.size()
assert num_spans % self.max_span_width == 0
tags = tags.view(batch_size, -1, self.max_span_width)
span_starts = F.relu(span_starts.float()).long().view(batch_size, -1)
span_ends = F.relu(span_ends.float()).long().view(batch_size, -1)
target_index = F.relu(target_index.float()).long().view(batch_size)
# shape (batch_size, sequence_length * max_span_width, embedding_dim)
span_embeddings = span_srl_util.compute_span_representations(
self.max_span_width, encoded_text, target_index, span_starts,
span_ends, self.span_width_embedding,
self.span_direction_embedding, self.span_distance_embedding,
self.span_distance_bin, self.head_scorer)
span_scores = self.span_feedforward(span_embeddings)
# FN-specific parameters.
fn_args = []
for extra_arg in ['frame', 'valid_frame_elements']:
if extra_arg in kwargs and kwargs[extra_arg] is not None:
fn_args.append(kwargs[extra_arg])
if fn_args: # FrameSRL batch.
frame, valid_frame_elements = fn_args
output_dict = self.compute_srl_graph(
span_scores=span_scores,
frame=frame,
valid_frame_elements=valid_frame_elements,
tags=tags,
text_mask=text_mask,
target_index=target_index)
else: # Scaffold batch.
if "span_mask" in kwargs and kwargs["span_mask"] is not None:
span_mask = kwargs["span_mask"]
if "parent_tags" in kwargs and kwargs["parent_tags"] is not None:
parent_tags = kwargs["parent_tags"]
if self.unlabeled_constits:
not_a_constit = self.vocab.get_token_index(
"*", self.constit_label_namespace)
tags = (tags != not_a_constit).float().view(
batch_size, -1, self.max_span_width)
elif self.constit_label_namespace == "parent_labels":
tags = parent_tags.view(batch_size, -1, self.max_span_width)
elif self.np_pp_constits:
tags = self.get_new_tags_np_pp(tags, batch_size)
output_dict = self.compute_constit_graph(span_mask=span_mask,
span_scores=span_scores,
constit_tags=tags,
text_mask=text_mask)
if self.fast_mode and not self.training:
output_dict["loss"] = Variable(torch.FloatTensor([0.00]))
return output_dict
