def test_span_metrics_are_computed_correctly(self):
from allennlp_models.structured_prediction.models.srl import (
convert_bio_tags_to_conll_format, )
batch_verb_indices = [2]
batch_sentences = [["The", "cat", "loves", "hats", "."]]
batch_bio_predicted_tags = [["B-ARG0", "B-ARG1", "B-V", "B-ARG1", "O"]]
batch_conll_predicted_tags = [
convert_bio_tags_to_conll_format(tags)
for tags in batch_bio_predicted_tags
]
batch_bio_gold_tags = [["B-ARG0", "I-ARG0", "B-V", "B-ARG1", "O"]]
batch_conll_gold_tags = [
convert_bio_tags_to_conll_format(tags)
for tags in batch_bio_gold_tags
]
srl_scorer = SrlEvalScorer(ignore_classes=["V"])
srl_scorer(batch_verb_indices, batch_sentences,
batch_conll_predicted_tags, batch_conll_gold_tags)
metrics = srl_scorer.get_metric()
assert len(metrics) == 9
assert_allclose(metrics["precision-ARG0"], 0.0)
assert_allclose(metrics["recall-ARG0"], 0.0)
assert_allclose(metrics["f1-measure-ARG0"], 0.0)
assert_allclose(metrics["precision-ARG1"], 0.5)
assert_allclose(metrics["recall-ARG1"], 1.0)
assert_allclose(metrics["f1-measure-ARG1"], 2 / 3)
assert_allclose(metrics["precision-overall"], 1 / 3)
assert_allclose(metrics["recall-overall"], 1 / 2)
assert_allclose(metrics["f1-measure-overall"],
(2 * (1 / 3) * (1 / 2)) / ((1 / 3) + (1 / 2)))
def __init__(
self,
vocab: Vocabulary,
bert_model: Union[str, BertModel],
embedding_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
label_smoothing: float = None,
ignore_span_metric: bool = False,
srl_eval_path: str = DEFAULT_SRL_EVAL_PATH,
**kwargs,
) -> None:
super().__init__(vocab, **kwargs)
if isinstance(bert_model, str):
self.bert_model = BertModel.from_pretrained(bert_model)
else:
self.bert_model = bert_model
self.num_classes = self.vocab.get_vocab_size("labels")
if srl_eval_path is not None:
# For the span based evaluation, we don't want to consider labels
# for verb, because the verb index is provided to the model.
self.span_metric = SrlEvalScorer(srl_eval_path,
ignore_classes=["V"])
else:
self.span_metric = None
self.tag_projection_layer = Linear(self.bert_model.config.hidden_size,
self.num_classes)
self.embedding_dropout = Dropout(p=embedding_dropout)
self._label_smoothing = label_smoothing
self.ignore_span_metric = ignore_span_metric
initializer(self)
def __init__(
self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
binary_feature_dim: int,
embedding_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
label_smoothing: float = None,
ignore_span_metric: bool = False,
srl_eval_path: str = DEFAULT_SRL_EVAL_PATH,
**kwargs,
) -> None:
super().__init__(vocab, **kwargs)
self.text_field_embedder = text_field_embedder
self.num_classes = self.vocab.get_vocab_size("labels")
if srl_eval_path is not None:
# For the span based evaluation, we don't want to consider labels
# for verb, because the verb index is provided to the model.
self.span_metric = SrlEvalScorer(srl_eval_path,
ignore_classes=["V"])
else:
self.span_metric = None
self.encoder = encoder
# There are exactly 2 binary features for the verb predicate embedding.
self.binary_feature_embedding = Embedding(
num_embeddings=2, embedding_dim=binary_feature_dim)
self.tag_projection_layer = TimeDistributed(
Linear(self.encoder.get_output_dim(), self.num_classes))
self.embedding_dropout = Dropout(p=embedding_dropout)
self._label_smoothing = label_smoothing
self.ignore_span_metric = ignore_span_metric
check_dimensions_match(
text_field_embedder.get_output_dim() + binary_feature_dim,
encoder.get_input_dim(),
"text embedding dim + verb indicator embedding dim",
"encoder input dim",
)
initializer(self)
def __init__(
self,
vocab: Vocabulary,
bert_model: Union[str, AutoModel],
embedding_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
label_smoothing: float = None,
ignore_span_metric: bool = False,
srl_eval_path: str = DEFAULT_SRL_EVAL_PATH,
restrict_frames: bool = False,
restrict_roles: bool = False,
inventory: str = "verbatlas",
**kwargs,
) -> None:
# bypass SrlBert constructor
Model.__init__(self, vocab, **kwargs)
self.lemma_frame_dict = load_lemma_frame(LEMMA_FRAME_PATH)
self.frame_role_dict = load_role_frame(FRAME_ROLE_PATH)
self.restrict_frames = restrict_frames
self.restrict_roles = restrict_roles
self.transformer = AutoModel.from_pretrained(bert_model)
self.frame_criterion = nn.CrossEntropyLoss()
if inventory == "verbatlas":
# add missing labels
frame_list = load_label_list(FRAME_LIST_PATH)
self.vocab.add_tokens_to_namespace(frame_list, "frames_labels")
self.num_classes = self.vocab.get_vocab_size("labels")
self.frame_num_classes = self.vocab.get_vocab_size("frames_labels")
if srl_eval_path is not None:
# For the span based evaluation, we don't want to consider labels
# for verb, because the verb index is provided to the model.
self.span_metric = SrlEvalScorer(srl_eval_path, ignore_classes=["V"])
else:
self.span_metric = None
self.f1_frame_metric = FBetaMeasure(average="micro")
self.tag_projection_layer = nn.Linear(self.transformer.config.hidden_size, self.num_classes)
self.frame_projection_layer = nn.Linear(
self.transformer.config.hidden_size, self.frame_num_classes
)
self.embedding_dropout = nn.Dropout(p=embedding_dropout)
self._label_smoothing = label_smoothing
self.ignore_span_metric = ignore_span_metric
initializer(self)
def __init__(
self,
vocab: Vocabulary,
bert_model: Union[str, Dict[str, Any], BertModel],
embedding_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
label_smoothing: float = None,
ignore_span_metric: bool = False,
srl_eval_path: str = DEFAULT_SRL_EVAL_PATH,
**kwargs,
) -> None:
super().__init__(vocab, **kwargs)
if isinstance(bert_model, str):
self.bert_model = BertModel.from_pretrained(bert_model)
elif isinstance(bert_model, dict):
warnings.warn(
"Initializing BertModel without pretrained weights. This is fine if you're loading "
"from an AllenNLP archive, but not if you're training.",
UserWarning,
)
bert_config = BertConfig.from_dict(bert_model)
self.bert_model = BertModel(bert_config)
else:
self.bert_model = bert_model
self.num_classes = self.vocab.get_vocab_size("labels")
if srl_eval_path is not None:
# For the span based evaluation, we don't want to consider labels
# for verb, because the verb index is provided to the model.
self.span_metric = SrlEvalScorer(srl_eval_path,
ignore_classes=["V"])
else:
self.span_metric = None
self.tag_projection_layer = Linear(self.bert_model.config.hidden_size,
self.num_classes)
self.embedding_dropout = Dropout(p=embedding_dropout)
self._label_smoothing = label_smoothing
self.ignore_span_metric = ignore_span_metric
initializer(self)
def test_distributed_setting_throws_an_error(self):
from allennlp_models.structured_prediction.models.srl import (
convert_bio_tags_to_conll_format, )
batch_verb_indices = [2]
batch_sentences = [["The", "cat", "loves", "hats", "."]]
batch_bio_predicted_tags = [["B-ARG0", "B-ARG1", "B-V", "B-ARG1", "O"]]
batch_conll_predicted_tags = [
convert_bio_tags_to_conll_format(tags)
for tags in batch_bio_predicted_tags
]
batch_bio_gold_tags = [["B-ARG0", "I-ARG0", "B-V", "B-ARG1", "O"]]
batch_conll_gold_tags = [
convert_bio_tags_to_conll_format(tags)
for tags in batch_bio_gold_tags
]
metric_kwargs = {
"batch_verb_indices": [batch_verb_indices, batch_verb_indices],
"batch_sentences": [batch_sentences, batch_sentences],
"batch_conll_formatted_predicted_tags": [
batch_conll_predicted_tags,
batch_conll_predicted_tags,
],
"batch_conll_formatted_gold_tags":
[batch_conll_gold_tags, batch_conll_gold_tags],
}
desired_values = {} # it does not matter, we expect the run to fail.
with pytest.raises(Exception) as exc:
run_distributed_test(
[-1, -1],
global_distributed_metric,
SrlEvalScorer(ignore_classes=["V"]),
metric_kwargs,
desired_values,
exact=True,
)
assert (
"RuntimeError: Distributed aggregation for `SrlEvalScorer` is currently not supported."
in str(exc.value))
class TransformerSrlSpan(SrlBert):
"""
# Parameters
vocab : `Vocabulary`, required
A Vocabulary, required in order to compute sizes for input/output projections.
model : `Union[str, AutoModel]`, required.
A string describing the BERT model to load or an already constructed AutoModel.
initializer : `InitializerApplicator`, optional (default=`InitializerApplicator()`)
Used to initialize the model parameters.
label_smoothing : `float`, optional (default = `0.0`)
Whether or not to use label smoothing on the labels when computing cross entropy loss.
ignore_span_metric : `bool`, optional (default = `False`)
Whether to calculate span loss, which is irrelevant when predicting BIO for Open Information Extraction.
srl_eval_path : `str`, optional (default=`DEFAULT_SRL_EVAL_PATH`)
The path to the srl-eval.pl script. By default, will use the srl-eval.pl included with allennlp,
which is located at allennlp/tools/srl-eval.pl . If `None`, srl-eval.pl is not used.
"""
def __init__(
self,
vocab: Vocabulary,
bert_model: Union[str, AutoModel],
embedding_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
label_smoothing: float = None,
ignore_span_metric: bool = False,
srl_eval_path: str = DEFAULT_SRL_EVAL_PATH,
inventory: str = "verbatlas",
**kwargs,
) -> None:
# bypass SrlBert constructor
Model.__init__(self, vocab, **kwargs)
self.transformer = AutoModel.from_pretrained(bert_model)
self.frame_criterion = nn.CrossEntropyLoss()
if inventory == "verbatlas":
# add missing frame labels
frame_list = load_label_list(FRAME_LIST_PATH)
self.vocab.add_tokens_to_namespace(frame_list, "frames_labels")
# add missing role labels
role_list = load_label_list(ROLE_LIST_PATH)
self.vocab.add_tokens_to_namespace(role_list, "labels")
self.num_classes = self.vocab.get_vocab_size("labels")
self.frame_num_classes = self.vocab.get_vocab_size("frames_labels")
if srl_eval_path is not None:
# For the span based evaluation, we don't want to consider labels
# for verb, because the verb index is provided to the model.
self.span_metric = SrlEvalScorer(srl_eval_path,
ignore_classes=["V"])
else:
self.span_metric = None
self.f1_frame_metric = FBetaMeasure(average="micro")
self.tag_projection_layer = nn.Linear(
self.transformer.config.hidden_size, self.num_classes)
self.frame_projection_layer = nn.Linear(
self.transformer.config.hidden_size, self.frame_num_classes)
self.embedding_dropout = nn.Dropout(p=embedding_dropout)
self._label_smoothing = label_smoothing
self.ignore_span_metric = ignore_span_metric
initializer(self)
def forward( # type: ignore
self,
tokens: TextFieldTensors,
verb_indicator: torch.Tensor,
frame_indicator: torch.Tensor,
metadata: List[Any],
tags: torch.LongTensor = None,
frame_tags: torch.LongTensor = None,
):
"""
# Parameters
tokens : `TextFieldTensors`, required
The output of `TextField.as_array()`, which should typically be passed directly to a
`TextFieldEmbedder`. For this model, this must be a `SingleIdTokenIndexer` which
indexes wordpieces from the BERT vocabulary.
verb_indicator: `torch.LongTensor`, required.
An integer `SequenceFeatureField` representation of the position of the verb
in the sentence. This should have shape (batch_size, num_tokens) and importantly, can be
all zeros, in the case that the sentence has no verbal predicate.
frame_indicator: torch.LongTensor, required.
An integer ``SequenceFeatureField`` representation of the position of the frame
in the sentence. This should have shape (batch_size, num_tokens). Similar to verb_indicator,
but handles bert wordpiece tokenizer by cosnidering a frame only the first subtoken.
tags : `torch.LongTensor`, optional (default = `None`)
A torch tensor representing the sequence of integer gold class labels
of shape `(batch_size, num_tokens)`
frame_tags : torch.LongTensor, optional (default = None)
A torch tensor representing the gold frames
of shape ``(batch_size, num_tokens)``
metadata : `List[Dict[str, Any]]`, optional, (default = `None`)
metadata containg the original words in the sentence, the verb to compute the
frame for, and start offsets for converting wordpieces back to a sequence of words,
under 'words', 'verb' and 'offsets' keys, respectively.
# Returns
An output dictionary consisting of:
logits : `torch.FloatTensor`
A tensor of shape `(batch_size, num_tokens, tag_vocab_size)` representing
unnormalised log probabilities of the tag classes.
class_probabilities : `torch.FloatTensor`
A tensor of shape `(batch_size, num_tokens, tag_vocab_size)` representing
a distribution of the tag classes per word.
loss : `torch.FloatTensor`, optional
A scalar loss to be optimised.
"""
mask = get_text_field_mask(tokens)
input_ids = util.get_token_ids_from_text_field_tensors(tokens)
bert_embeddings, _ = self.transformer(
input_ids=input_ids,
token_type_ids=verb_indicator,
attention_mask=mask,
return_dict=False,
)
# extract embeddings
embedded_text_input = self.embedding_dropout(bert_embeddings)
frame_embeddings = embedded_text_input[frame_indicator == 1]
# get sizes
batch_size, sequence_length, _ = embedded_text_input.size()
# outputs
logits = self.tag_projection_layer(embedded_text_input)
frame_logits = self.frame_projection_layer(frame_embeddings)
reshaped_log_probs = logits.view(-1, self.num_classes)
class_probabilities = F.softmax(reshaped_log_probs, dim=-1).view(
[batch_size, sequence_length, self.num_classes])
frame_probabilities = F.softmax(frame_logits, dim=-1)
# We need to retain the mask in the output dictionary
# so that we can crop the sequences to remove padding
# when we do viterbi inference in self.make_output_human_readable.
output_dict = {
"logits": logits,
"frame_logits": frame_logits,
"class_probabilities": class_probabilities,
"frame_probabilities": frame_probabilities,
"mask": mask,
}
# We add in the offsets here so we can compute the un-wordpieced tags.
words, verbs, offsets = zip(*[(x["words"], x["verb"], x["offsets"])
for x in metadata])
lemmas = [l for x in metadata for l in x["lemmas"]]
output_dict["words"] = list(words)
output_dict["lemma"] = list(lemmas)
output_dict["verb"] = list(verbs)
output_dict["wordpiece_offsets"] = list(offsets)
if tags is not None:
# compute role loss
role_loss = sequence_cross_entropy_with_logits(
logits, tags, mask, label_smoothing=self._label_smoothing)
# compute frame loss
frame_tags_filtered = frame_tags[frame_indicator == 1]
frame_loss = self.frame_criterion(frame_logits,
frame_tags_filtered)
if not self.ignore_span_metric and self.span_metric is not None and not self.training:
batch_verb_indices = [
example_metadata["verb_index"]
for example_metadata in metadata
]
batch_sentences = [
example_metadata["words"] for example_metadata in metadata
]
# Get the BIO tags from make_output_human_readable()
batch_bio_predicted_tags = self.make_output_human_readable(
output_dict).pop("tags")
from allennlp_models.structured_prediction.models.srl import (
convert_bio_tags_to_conll_format, )
batch_conll_predicted_tags = [
convert_bio_tags_to_conll_format(tags)
for tags in batch_bio_predicted_tags
]
batch_bio_gold_tags = [
example_metadata["gold_tags"]
for example_metadata in metadata
]
batch_conll_gold_tags = [
convert_bio_tags_to_conll_format(tags)
for tags in batch_bio_gold_tags
]
self.span_metric(
batch_verb_indices,
batch_sentences,
batch_conll_predicted_tags,
batch_conll_gold_tags,
)
self.f1_frame_metric(frame_logits, frame_tags_filtered)
output_dict["frame_loss"] = frame_loss
output_dict["role_loss"] = role_loss
output_dict["loss"] = (role_loss + frame_loss) / 2
return output_dict
def decode_frames(
self, output_dict: Dict[str,
torch.Tensor]) -> Dict[str, torch.Tensor]:
# frame prediction
frame_probabilities = output_dict["frame_probabilities"]
frame_predictions = frame_probabilities.argmax(
dim=-1).cpu().data.numpy()
output_dict["frame_tags"] = [
self.vocab.get_token_from_index(f, namespace="frames_labels")
for f in frame_predictions
]
output_dict["frame_scores"] = [
fp[f] for f, fp in zip(frame_predictions, frame_probabilities)
]
return output_dict
@overrides
def make_output_human_readable(
self, output_dict: Dict[str,
torch.Tensor]) -> Dict[str, torch.Tensor]:
output_dict = self.decode_frames(output_dict)
output_dict = super().make_output_human_readable(output_dict)
return output_dict
@overrides
def get_metrics(self, reset: bool = False):
if self.ignore_span_metric:
# Return an empty dictionary if ignoring the
# span metric
return {}
else:
metric_dict = self.span_metric.get_metric(reset=reset)
frame_metric_dict = self.f1_frame_metric.get_metric(reset=reset)
# This can be a lot of metrics, as there are 3 per class.
# we only really care about the overall metrics, so we filter for them here.
metric_dict_filtered = {
x.split("-")[0] + "_role": y
for x, y in metric_dict.items() if "overall" in x
}
frame_metric_dict = {
x + "_frame": y
for x, y in frame_metric_dict.items()
}
return {**metric_dict_filtered, **frame_metric_dict}
def _get_label_tokens(self, namespace: str = "labels"):
return self.vocab.get_token_to_index_vocabulary(namespace).keys()
def _get_label_ids(self, namespace: str = "labels"):
return self.vocab.get_index_to_token_vocabulary(namespace).keys()
default_predictor = "transformer_srl"
class SemanticRoleLabeler(Model):
"""
This model performs semantic role labeling using BIO tags using Propbank semantic roles.
Specifically, it is an implementation of [Deep Semantic Role Labeling - What works
and what's next](https://www.aclweb.org/anthology/P17-1044).
This implementation is effectively a series of stacked interleaved LSTMs with highway
connections, applied to embedded sequences of words concatenated with a binary indicator
containing whether or not a word is the verbal predicate to generate predictions for in
the sentence. Additionally, during inference, Viterbi decoding is applied to constrain
the predictions to contain valid BIO sequences.
Specifically, the model expects and outputs IOB2-formatted tags, where the
B- tag is used in the beginning of every chunk (i.e. all chunks start with the B- tag).
# Parameters
vocab : `Vocabulary`, required
A Vocabulary, required in order to compute sizes for input/output projections.
text_field_embedder : `TextFieldEmbedder`, required
Used to embed the `tokens` `TextField` we get as input to the model.
encoder : `Seq2SeqEncoder`
The encoder (with its own internal stacking) that we will use in between embedding tokens
and predicting output tags.
binary_feature_dim : int, required.
The dimensionality of the embedding of the binary verb predicate features.
initializer : `InitializerApplicator`, optional (default=`InitializerApplicator()`)
Used to initialize the model parameters.
label_smoothing : `float`, optional (default = 0.0)
Whether or not to use label smoothing on the labels when computing cross entropy loss.
ignore_span_metric : `bool`, optional (default = False)
Whether to calculate span loss, which is irrelevant when predicting BIO for Open Information Extraction.
srl_eval_path : `str`, optional (default=`DEFAULT_SRL_EVAL_PATH`)
The path to the srl-eval.pl script. By default, will use the srl-eval.pl included with allennlp,
which is located at allennlp/tools/srl-eval.pl . If `None`, srl-eval.pl is not used.
"""
def __init__(
self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
binary_feature_dim: int,
embedding_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
label_smoothing: float = None,
ignore_span_metric: bool = False,
srl_eval_path: str = DEFAULT_SRL_EVAL_PATH,
**kwargs,
) -> None:
super().__init__(vocab, **kwargs)
self.text_field_embedder = text_field_embedder
self.num_classes = self.vocab.get_vocab_size("labels")
if srl_eval_path is not None:
# For the span based evaluation, we don't want to consider labels
# for verb, because the verb index is provided to the model.
self.span_metric = SrlEvalScorer(srl_eval_path,
ignore_classes=["V"])
else:
self.span_metric = None
self.encoder = encoder
# There are exactly 2 binary features for the verb predicate embedding.
self.binary_feature_embedding = Embedding(
num_embeddings=2, embedding_dim=binary_feature_dim)
self.tag_projection_layer = TimeDistributed(
Linear(self.encoder.get_output_dim(), self.num_classes))
self.embedding_dropout = Dropout(p=embedding_dropout)
self._label_smoothing = label_smoothing
self.ignore_span_metric = ignore_span_metric
check_dimensions_match(
text_field_embedder.get_output_dim() + binary_feature_dim,
encoder.get_input_dim(),
"text embedding dim + verb indicator embedding dim",
"encoder input dim",
)
initializer(self)
def forward( # type: ignore
self,
tokens: TextFieldTensors,
verb_indicator: torch.LongTensor,
tags: torch.LongTensor = None,
metadata: List[Dict[str, Any]] = None,
) -> Dict[str, torch.Tensor]:
"""
# Parameters
tokens : TextFieldTensors, required
The output of `TextField.as_array()`, which should typically be passed directly to a
`TextFieldEmbedder`. This output is a dictionary mapping keys to `TokenIndexer`
tensors. At its most basic, using a `SingleIdTokenIndexer` this is : `{"tokens":
Tensor(batch_size, num_tokens)}`. This dictionary will have the same keys as were used
for the `TokenIndexers` when you created the `TextField` representing your
sequence. The dictionary is designed to be passed directly to a `TextFieldEmbedder`,
which knows how to combine different word representations into a single vector per
token in your input.
verb_indicator: torch.LongTensor, required.
An integer `SequenceFeatureField` representation of the position of the verb
in the sentence. This should have shape (batch_size, num_tokens) and importantly, can be
all zeros, in the case that the sentence has no verbal predicate.
tags : torch.LongTensor, optional (default = None)
A torch tensor representing the sequence of integer gold class labels
of shape `(batch_size, num_tokens)`
metadata : `List[Dict[str, Any]]`, optional, (default = None)
metadata containg the original words in the sentence and the verb to compute the
frame for, under 'words' and 'verb' keys, respectively.
# Returns
An output dictionary consisting of:
logits : torch.FloatTensor
A tensor of shape `(batch_size, num_tokens, tag_vocab_size)` representing
unnormalised log probabilities of the tag classes.
class_probabilities : torch.FloatTensor
A tensor of shape `(batch_size, num_tokens, tag_vocab_size)` representing
a distribution of the tag classes per word.
loss : torch.FloatTensor, optional
A scalar loss to be optimised.
"""
embedded_text_input = self.embedding_dropout(
self.text_field_embedder(tokens))
mask = get_text_field_mask(tokens)
embedded_verb_indicator = self.binary_feature_embedding(
verb_indicator.long())
# Concatenate the verb feature onto the embedded text. This now
# has shape (batch_size, sequence_length, embedding_dim + binary_feature_dim).
embedded_text_with_verb_indicator = torch.cat(
[embedded_text_input, embedded_verb_indicator], -1)
batch_size, sequence_length, _ = embedded_text_with_verb_indicator.size(
)
encoded_text = self.encoder(embedded_text_with_verb_indicator, mask)
logits = self.tag_projection_layer(encoded_text)
reshaped_log_probs = logits.view(-1, self.num_classes)
class_probabilities = F.softmax(reshaped_log_probs, dim=-1).view(
[batch_size, sequence_length, self.num_classes])
output_dict = {
"logits": logits,
"class_probabilities": class_probabilities
}
# We need to retain the mask in the output dictionary
# so that we can crop the sequences to remove padding
# when we do viterbi inference in self.make_output_human_readable.
output_dict["mask"] = mask
if tags is not None:
loss = sequence_cross_entropy_with_logits(
logits, tags, mask, label_smoothing=self._label_smoothing)
if not self.ignore_span_metric and self.span_metric is not None and not self.training:
batch_verb_indices = [
example_metadata["verb_index"]
for example_metadata in metadata
]
batch_sentences = [
example_metadata["words"] for example_metadata in metadata
]
# Get the BIO tags from make_output_human_readable()
# TODO (nfliu): This is kind of a hack, consider splitting out part
# of make_output_human_readable() to a separate function.
batch_bio_predicted_tags = self.make_output_human_readable(
output_dict).pop("tags")
batch_conll_predicted_tags = [
convert_bio_tags_to_conll_format(tags)
for tags in batch_bio_predicted_tags
]
batch_bio_gold_tags = [
example_metadata["gold_tags"]
for example_metadata in metadata
]
batch_conll_gold_tags = [
convert_bio_tags_to_conll_format(tags)
for tags in batch_bio_gold_tags
]
self.span_metric(
batch_verb_indices,
batch_sentences,
batch_conll_predicted_tags,
batch_conll_gold_tags,
)
output_dict["loss"] = loss
words, verbs = zip(*[(x["words"], x["verb"]) for x in metadata])
if metadata is not None:
output_dict["words"] = list(words)
output_dict["verb"] = list(verbs)
return output_dict
@overrides
def make_output_human_readable(
self, output_dict: Dict[str,
torch.Tensor]) -> Dict[str, torch.Tensor]:
"""
Does constrained viterbi decoding on class probabilities output in :func:`forward`. The
constraint simply specifies that the output tags must be a valid BIO sequence. We add a
`"tags"` key to the dictionary with the result.
"""
all_predictions = output_dict["class_probabilities"]
sequence_lengths = get_lengths_from_binary_sequence_mask(
output_dict["mask"]).data.tolist()
if all_predictions.dim() == 3:
predictions_list = [
all_predictions[i].detach().cpu()
for i in range(all_predictions.size(0))
]
else:
predictions_list = [all_predictions]
all_tags = []
transition_matrix = self.get_viterbi_pairwise_potentials()
start_transitions = self.get_start_transitions()
for predictions, length in zip(predictions_list, sequence_lengths):
max_likelihood_sequence, _ = viterbi_decode(
predictions[:length],
transition_matrix,
allowed_start_transitions=start_transitions)
tags = [
self.vocab.get_token_from_index(x, namespace="labels")
for x in max_likelihood_sequence
]
all_tags.append(tags)
output_dict["tags"] = all_tags
return output_dict
def get_metrics(self, reset: bool = False):
if self.ignore_span_metric:
# Return an empty dictionary if ignoring the
# span metric
return {}
else:
metric_dict = self.span_metric.get_metric(reset=reset)
# This can be a lot of metrics, as there are 3 per class.
# we only really care about the overall metrics, so we filter for them here.
return {x: y for x, y in metric_dict.items() if "overall" in x}
def get_viterbi_pairwise_potentials(self):
"""
Generate a matrix of pairwise transition potentials for the BIO labels.
The only constraint implemented here is that I-XXX labels must be preceded
by either an identical I-XXX tag or a B-XXX tag. In order to achieve this
constraint, pairs of labels which do not satisfy this constraint have a
pairwise potential of -inf.
# Returns
transition_matrix : torch.Tensor
A (num_labels, num_labels) matrix of pairwise potentials.
"""
all_labels = self.vocab.get_index_to_token_vocabulary("labels")
num_labels = len(all_labels)
transition_matrix = torch.zeros([num_labels, num_labels])
for i, previous_label in all_labels.items():
for j, label in all_labels.items():
# I labels can only be preceded by themselves or
# their corresponding B tag.
if i != j and label[
0] == "I" and not previous_label == "B" + label[1:]:
transition_matrix[i, j] = float("-inf")
return transition_matrix
def get_start_transitions(self):
"""
In the BIO sequence, we cannot start the sequence with an I-XXX tag.
This transition sequence is passed to viterbi_decode to specify this constraint.
# Returns
start_transitions : torch.Tensor
The pairwise potentials between a START token and
the first token of the sequence.
"""
all_labels = self.vocab.get_index_to_token_vocabulary("labels")
num_labels = len(all_labels)
start_transitions = torch.zeros(num_labels)
for i, label in all_labels.items():
if label[0] == "I":
start_transitions[i] = float("-inf")
return start_transitions
default_predictor = "semantic-role-labeling"
def test_srl_eval_correctly_scores_identical_tags(self):
batch_verb_indices = [3, 8, 2, 0]
batch_sentences = [
[
"Mali",
"government",
"officials",
"say",
"the",
"woman",
"'s",
"confession",
"was",
"forced",
".",
],
[
"Mali",
"government",
"officials",
"say",
"the",
"woman",
"'s",
"confession",
"was",
"forced",
".",
],
[
"The",
"prosecution",
"rested",
"its",
"case",
"last",
"month",
"after",
"four",
"months",
"of",
"hearings",
".",
],
["Come", "in", "and", "buy", "."],
]
batch_bio_predicted_tags = [
[
"B-ARG0",
"I-ARG0",
"I-ARG0",
"B-V",
"B-ARG1",
"I-ARG1",
"I-ARG1",
"I-ARG1",
"I-ARG1",
"I-ARG1",
"O",
],
[
"O", "O", "O", "O", "B-ARG1", "I-ARG1", "I-ARG1", "I-ARG1",
"B-V", "B-ARG2", "O"
],
[
"B-ARG0",
"I-ARG0",
"B-V",
"B-ARG1",
"I-ARG1",
"B-ARGM-TMP",
"I-ARGM-TMP",
"B-ARGM-TMP",
"I-ARGM-TMP",
"I-ARGM-TMP",
"I-ARGM-TMP",
"I-ARGM-TMP",
"O",
],
["B-V", "B-AM-DIR", "O", "O", "O"],
]
from allennlp_models.structured_prediction.models.srl import (
convert_bio_tags_to_conll_format, )
batch_conll_predicted_tags = [
convert_bio_tags_to_conll_format(tags)
for tags in batch_bio_predicted_tags
]
batch_bio_gold_tags = [
[
"B-ARG0",
"I-ARG0",
"I-ARG0",
"B-V",
"B-ARG1",
"I-ARG1",
"I-ARG1",
"I-ARG1",
"I-ARG1",
"I-ARG1",
"O",
],
[
"O", "O", "O", "O", "B-ARG1", "I-ARG1", "I-ARG1", "I-ARG1",
"B-V", "B-ARG2", "O"
],
[
"B-ARG0",
"I-ARG0",
"B-V",
"B-ARG1",
"I-ARG1",
"B-ARGM-TMP",
"I-ARGM-TMP",
"B-ARGM-TMP",
"I-ARGM-TMP",
"I-ARGM-TMP",
"I-ARGM-TMP",
"I-ARGM-TMP",
"O",
],
["B-V", "B-AM-DIR", "O", "O", "O"],
]
batch_conll_gold_tags = [
convert_bio_tags_to_conll_format(tags)
for tags in batch_bio_gold_tags
]
srl_scorer = SrlEvalScorer(ignore_classes=["V"])
srl_scorer(batch_verb_indices, batch_sentences,
batch_conll_predicted_tags, batch_conll_gold_tags)
metrics = srl_scorer.get_metric()
assert len(metrics) == 18
assert_allclose(metrics["precision-ARG0"], 1.0)
assert_allclose(metrics["recall-ARG0"], 1.0)
assert_allclose(metrics["f1-measure-ARG0"], 1.0)
assert_allclose(metrics["precision-ARG1"], 1.0)
assert_allclose(metrics["recall-ARG1"], 1.0)
assert_allclose(metrics["f1-measure-ARG1"], 1.0)
assert_allclose(metrics["precision-ARG2"], 1.0)
assert_allclose(metrics["recall-ARG2"], 1.0)
assert_allclose(metrics["f1-measure-ARG2"], 1.0)
assert_allclose(metrics["precision-ARGM-TMP"], 1.0)
assert_allclose(metrics["recall-ARGM-TMP"], 1.0)
assert_allclose(metrics["f1-measure-ARGM-TMP"], 1.0)
assert_allclose(metrics["precision-AM-DIR"], 1.0)
assert_allclose(metrics["recall-AM-DIR"], 1.0)
assert_allclose(metrics["f1-measure-AM-DIR"], 1.0)
assert_allclose(metrics["precision-overall"], 1.0)
assert_allclose(metrics["recall-overall"], 1.0)
assert_allclose(metrics["f1-measure-overall"], 1.0)
def __init__(
self,
vocab: Vocabulary,
bert_model: Union[str, AutoModel],
mismatched_embedder: TokenEmbedder = None,
lp: bool = False,
lpsmap: bool = False,
lpsmap_core_roles_only: bool = True,
validation_inference: bool = True,
batch_size: int = None,
encoder: Seq2SeqEncoder = None,
reinitialize_pos_embedding: bool = False,
embedding_dropout: float = 0.0,
mlp_hidden_size: int = 300,
initializer: InitializerApplicator = InitializerApplicator(),
label_smoothing: float = None,
ignore_span_metric: bool = False,
srl_eval_path: str = DEFAULT_SRL_EVAL_PATH,
label_encoding: str = "BIO",
constrain_crf_decoding: bool = None,
include_start_end_transitions: bool = True,
label_namespace: str = "labels",
**kwargs,
) -> None:
super().__init__(vocab, **kwargs)
if isinstance(bert_model, str):
if mismatched_embedder is None:
self.bert_model = AutoModel.from_pretrained(bert_model)
self.bert_config = AutoConfig.from_pretrained(bert_model)
else:
if mismatched_embedder is None:
self.bert_model = bert_model
self.bert_config = bert_model.config
if reinitialize_pos_embedding:
self.bert_model._init_weights(
self.bert_model.embeddings.position_embeddings)
# self.bert_model._init_weights(self.bert_model.embeddings.token_type_embeddings)
if mismatched_embedder is not None:
self.bert_model = mismatched_embedder
self._label_namespace = label_namespace
self.num_classes = self.vocab.get_vocab_size(label_namespace)
if srl_eval_path is not None:
# For the span based evaluation, we don't want to consider labels
# for verb, because the verb index is provided to the model.
self.span_metric = SrlEvalScorer(srl_eval_path,
ignore_classes=["V"])
else:
self.span_metric = None
if constrain_crf_decoding is None:
constrain_crf_decoding = label_encoding is not None
self.label_encoding = label_encoding
self.constrain_crf_decoding = constrain_crf_decoding
if constrain_crf_decoding:
if not label_encoding:
raise ConfigurationError(
"constrain_crf_decoding is True, but no label_encoding was specified."
)
labels = self.vocab.get_index_to_token_vocabulary(label_namespace)
constraints = allowed_transitions(label_encoding, labels)
else:
constraints = None
self.include_start_end_transitions = include_start_end_transitions
self.crf = ConditionalRandomField(
self.num_classes,
constraints,
include_start_end_transitions=include_start_end_transitions)
self._encoder = encoder
representation_size = self.bert_config.hidden_size
if self.bert_config.type_vocab_size == 1:
representation_size = self.bert_config.hidden_size * 2
if encoder is None:
self.tag_projection_layer = torch.nn.Sequential(
Linear(representation_size, mlp_hidden_size), torch.nn.ReLU(),
Linear(mlp_hidden_size, self.num_classes))
else:
self.tag_projection_layer = torch.nn.Sequential(
Linear(encoder.get_output_dim() * 2, mlp_hidden_size),
torch.nn.ReLU(), Linear(mlp_hidden_size, self.num_classes))
self.embedding_dropout = Dropout(p=embedding_dropout)
self.predicate_embedding = torch.nn.Embedding(num_embeddings=2,
embedding_dim=10)
self._label_smoothing = label_smoothing
self.ignore_span_metric = ignore_span_metric
self._lp = lp
self._lpsmap = lpsmap
self._lpsmap_core_only = lpsmap_core_roles_only
self._val_inference = validation_inference
if self._lpsmap:
self._core_roles = []
for i in range(6):
try:
self._core_roles.append(
self.vocab.get_token_index(
"B-ARG" + str(i), namespace=self._label_namespace))
except:
logger.info("B-ARG" + str(i) + " is not in labels")
self._r_roles = []
self._c_roles = []
for i in range(self.num_classes):
token = self.vocab.get_token_from_index(
i, namespace=self._label_namespace)
if token[:4] == "B-R-" and token[4:] != "ARG1":
try:
base_arg_index = self.vocab.get_token_index(
"B-" + token[4:], namespace=self._label_namespace)
self._r_roles.append((i, base_arg_index))
except:
logger.info("B-" + token[4:] + " is not in labels")
elif token[:4] == "B-C-" and token[4:] != "ARG1":
try:
base_arg_index = self.vocab.get_token_index(
"B-" + token[4:], namespace=self._label_namespace)
self._c_roles.append((i, base_arg_index))
except:
logger.info("B-" + token[4:] + " is not in labels")
# self._core_roles = [index for index in range(self.vocab.get_vocab_size("labels")) if index in [self.vocab.get_token_index("B-ARG"+str(i), namespace="labels") for i in range(3)]]
self.lpsmap = None
if lp:
"""self._layer_list = []
self.length_map = {}
self.lengths = []
for max_sequence_length in [70, 100, 200, 300]:
x = cp.Variable((max_sequence_length, self.vocab.get_vocab_size(namespace="labels")))
S = cp.Parameter((max_sequence_length, self.vocab.get_vocab_size(namespace="labels")))
constraints = [x >= 0, cp.sum(x, axis=1) == 1]
objective = cp.Maximize(cp.sum(cp.multiply(x, S)))
problem = cp.Problem(objective, constraints)
assert problem.is_dpp()
lp_layer = CvxpyLayer(problem, parameters=[S], variables=[x])
self._layer_list.append(lp_layer)
self.length_map[max_sequence_length] = len(self._layer_list)-1
self.lengths.append(max_sequence_length)
self._layer_list = torch.nn.ModuleList(self._layer_list)"""
pass
initializer(self)
class SrlBert(Model):
"""
A BERT based model [Simple BERT Models for Relation Extraction and Semantic Role Labeling (Shi et al, 2019)]
(https://arxiv.org/abs/1904.05255) with some modifications (no additional parameters apart from a linear
classification layer), which is currently the state-of-the-art single model for English PropBank SRL
(Newswire sentences).
# Parameters
vocab : `Vocabulary`, required
A Vocabulary, required in order to compute sizes for input/output projections.
model : `Union[str, BertModel]`, required.
A string describing the BERT model to load or an already constructed BertModel.
initializer : `InitializerApplicator`, optional (default=`InitializerApplicator()`)
Used to initialize the model parameters.
label_smoothing : `float`, optional (default = `0.0`)
Whether or not to use label smoothing on the labels when computing cross entropy loss.
ignore_span_metric : `bool`, optional (default = `False`)
Whether to calculate span loss, which is irrelevant when predicting BIO for Open Information Extraction.
srl_eval_path : `str`, optional (default=`DEFAULT_SRL_EVAL_PATH`)
The path to the srl-eval.pl script. By default, will use the srl-eval.pl included with allennlp,
which is located at allennlp/tools/srl-eval.pl . If `None`, srl-eval.pl is not used.
"""
def __init__(
self,
vocab: Vocabulary,
bert_model: Union[str, AutoModel],
mismatched_embedder: TokenEmbedder = None,
lp: bool = False,
lpsmap: bool = False,
lpsmap_core_roles_only: bool = True,
validation_inference: bool = True,
batch_size: int = None,
encoder: Seq2SeqEncoder = None,
reinitialize_pos_embedding: bool = False,
embedding_dropout: float = 0.0,
mlp_hidden_size: int = 300,
initializer: InitializerApplicator = InitializerApplicator(),
label_smoothing: float = None,
ignore_span_metric: bool = False,
srl_eval_path: str = DEFAULT_SRL_EVAL_PATH,
label_encoding: str = "BIO",
constrain_crf_decoding: bool = None,
include_start_end_transitions: bool = True,
label_namespace: str = "labels",
**kwargs,
) -> None:
super().__init__(vocab, **kwargs)
if isinstance(bert_model, str):
if mismatched_embedder is None:
self.bert_model = AutoModel.from_pretrained(bert_model)
self.bert_config = AutoConfig.from_pretrained(bert_model)
else:
if mismatched_embedder is None:
self.bert_model = bert_model
self.bert_config = bert_model.config
if reinitialize_pos_embedding:
self.bert_model._init_weights(
self.bert_model.embeddings.position_embeddings)
# self.bert_model._init_weights(self.bert_model.embeddings.token_type_embeddings)
if mismatched_embedder is not None:
self.bert_model = mismatched_embedder
self._label_namespace = label_namespace
self.num_classes = self.vocab.get_vocab_size(label_namespace)
if srl_eval_path is not None:
# For the span based evaluation, we don't want to consider labels
# for verb, because the verb index is provided to the model.
self.span_metric = SrlEvalScorer(srl_eval_path,
ignore_classes=["V"])
else:
self.span_metric = None
if constrain_crf_decoding is None:
constrain_crf_decoding = label_encoding is not None
self.label_encoding = label_encoding
self.constrain_crf_decoding = constrain_crf_decoding
if constrain_crf_decoding:
if not label_encoding:
raise ConfigurationError(
"constrain_crf_decoding is True, but no label_encoding was specified."
)
labels = self.vocab.get_index_to_token_vocabulary(label_namespace)
constraints = allowed_transitions(label_encoding, labels)
else:
constraints = None
self.include_start_end_transitions = include_start_end_transitions
self.crf = ConditionalRandomField(
self.num_classes,
constraints,
include_start_end_transitions=include_start_end_transitions)
self._encoder = encoder
representation_size = self.bert_config.hidden_size
if self.bert_config.type_vocab_size == 1:
representation_size = self.bert_config.hidden_size * 2
if encoder is None:
self.tag_projection_layer = torch.nn.Sequential(
Linear(representation_size, mlp_hidden_size), torch.nn.ReLU(),
Linear(mlp_hidden_size, self.num_classes))
else:
self.tag_projection_layer = torch.nn.Sequential(
Linear(encoder.get_output_dim() * 2, mlp_hidden_size),
torch.nn.ReLU(), Linear(mlp_hidden_size, self.num_classes))
self.embedding_dropout = Dropout(p=embedding_dropout)
self.predicate_embedding = torch.nn.Embedding(num_embeddings=2,
embedding_dim=10)
self._label_smoothing = label_smoothing
self.ignore_span_metric = ignore_span_metric
self._lp = lp
self._lpsmap = lpsmap
self._lpsmap_core_only = lpsmap_core_roles_only
self._val_inference = validation_inference
if self._lpsmap:
self._core_roles = []
for i in range(6):
try:
self._core_roles.append(
self.vocab.get_token_index(
"B-ARG" + str(i), namespace=self._label_namespace))
except:
logger.info("B-ARG" + str(i) + " is not in labels")
self._r_roles = []
self._c_roles = []
for i in range(self.num_classes):
token = self.vocab.get_token_from_index(
i, namespace=self._label_namespace)
if token[:4] == "B-R-" and token[4:] != "ARG1":
try:
base_arg_index = self.vocab.get_token_index(
"B-" + token[4:], namespace=self._label_namespace)
self._r_roles.append((i, base_arg_index))
except:
logger.info("B-" + token[4:] + " is not in labels")
elif token[:4] == "B-C-" and token[4:] != "ARG1":
try:
base_arg_index = self.vocab.get_token_index(
"B-" + token[4:], namespace=self._label_namespace)
self._c_roles.append((i, base_arg_index))
except:
logger.info("B-" + token[4:] + " is not in labels")
# self._core_roles = [index for index in range(self.vocab.get_vocab_size("labels")) if index in [self.vocab.get_token_index("B-ARG"+str(i), namespace="labels") for i in range(3)]]
self.lpsmap = None
if lp:
"""self._layer_list = []
self.length_map = {}
self.lengths = []
for max_sequence_length in [70, 100, 200, 300]:
x = cp.Variable((max_sequence_length, self.vocab.get_vocab_size(namespace="labels")))
S = cp.Parameter((max_sequence_length, self.vocab.get_vocab_size(namespace="labels")))
constraints = [x >= 0, cp.sum(x, axis=1) == 1]
objective = cp.Maximize(cp.sum(cp.multiply(x, S)))
problem = cp.Problem(objective, constraints)
assert problem.is_dpp()
lp_layer = CvxpyLayer(problem, parameters=[S], variables=[x])
self._layer_list.append(lp_layer)
self.length_map[max_sequence_length] = len(self._layer_list)-1
self.lengths.append(max_sequence_length)
self._layer_list = torch.nn.ModuleList(self._layer_list)"""
pass
initializer(self)
def forward( # type: ignore
self,
tokens: TextFieldTensors,
verb_indicator: torch.Tensor,
sentence_end: torch.LongTensor,
metadata: List[Any],
tags: torch.LongTensor = None,
offsets: torch.LongTensor = None):
"""
# Parameters
tokens : `TextFieldTensors`, required
The output of `TextField.as_array()`, which should typically be passed directly to a
`TextFieldEmbedder`. For this model, this must be a `SingleIdTokenIndexer` which
indexes wordpieces from the BERT vocabulary.
verb_indicator: `torch.LongTensor`, required.
An integer `SequenceFeatureField` representation of the position of the verb
in the sentence. This should have shape (batch_size, num_tokens) and importantly, can be
all zeros, in the case that the sentence has no verbal predicate.
tags : `torch.LongTensor`, optional (default = `None`)
A torch tensor representing the sequence of integer gold class labels
of shape `(batch_size, num_tokens)`
metadata : `List[Dict[str, Any]]`, optional, (default = `None`)
metadata containing the original words in the sentence, the verb to compute the
frame for, and start offsets for converting wordpieces back to a sequence of words,
under 'words', 'verb' and 'offsets' keys, respectively.
# Returns
An output dictionary consisting of:
logits : `torch.FloatTensor`
A tensor of shape `(batch_size, num_tokens, tag_vocab_size)` representing
unnormalised log probabilities of the tag classes.
class_probabilities : `torch.FloatTensor`
A tensor of shape `(batch_size, num_tokens, tag_vocab_size)` representing
a distribution of the tag classes per word.
loss : `torch.FloatTensor`, optional
A scalar loss to be optimised.
"""
if isinstance(self.bert_model,
PretrainedTransformerMismatchedEmbedder):
encoder_inputs = tokens["tokens"]
if self.bert_config.type_vocab_size > 1:
encoder_inputs["type_ids"] = verb_indicator
encoded_text = self.bert_model(**encoder_inputs)
batch_size = encoded_text.shape[0]
if self.bert_config.type_vocab_size == 1:
verb_embeddings = encoded_text[
torch.arange(batch_size).to(encoded_text.device),
verb_indicator.argmax(1), :]
verb_embeddings = torch.where(
(verb_indicator.sum(1, keepdim=True) > 0).repeat(
1, verb_embeddings.shape[-1]), verb_embeddings,
torch.zeros_like(verb_embeddings))
encoded_text = torch.cat(
(encoded_text, verb_embeddings.unsqueeze(1).repeat(
1, encoded_text.shape[1], 1)),
dim=2)
mask = tokens["tokens"]["mask"]
index = mask.sum(1).argmax().item()
# print(mask.shape, encoded_text.shape, tokens["tokens"]["token_ids"].shape, tags.shape, max([len(x['words']) for x in metadata]), mask.sum(1)[index].item())
# print(tokens["tokens"]["token_ids"][index,:])
else:
mask = get_text_field_mask(tokens)
bert_embeddings, _ = self.bert_model(
input_ids=util.get_token_ids_from_text_field_tensors(tokens),
# token_type_ids=verb_indicator,
attention_mask=mask,
)
batch_size, _ = mask.size()
embedded_text_input = self.embedding_dropout(bert_embeddings)
# Restrict to sentence part
sentence_mask = (torch.arange(mask.shape[1]).unsqueeze(0).repeat(
batch_size, 1).to(mask.device) <
sentence_end.unsqueeze(1).repeat(
1, mask.shape[1])).long()
cutoff = sentence_end.max().item()
if self._encoder is None:
encoded_text = embedded_text_input
mask = sentence_mask[:, :cutoff].contiguous()
encoded_text = encoded_text[:, :cutoff, :]
tags = tags[:, :cutoff].contiguous()
else:
predicate_embeddings = self.predicate_embedding(verb_indicator)
encoder_inputs = torch.cat(
(embedded_text_input, predicate_embeddings), dim=-1)
encoded_text = self._encoder(encoder_inputs,
mask=sentence_mask.bool())
# print(verb_indicator)
predicate_index = (verb_indicator * torch.arange(
start=verb_indicator.shape[-1] - 1, end=-1,
step=-1).to(mask.device).unsqueeze(0).repeat(
batch_size, 1)).argmax(1)
# print(predicate_index)
predicate_hidden = encoded_text[
torch.arange(batch_size).to(mask.device), predicate_index]
predicate_exists, _ = verb_indicator.max(1)
encoded_text = encoded_text[:, :cutoff, :]
tags = tags[:, :cutoff].contiguous()
mask = sentence_mask[:, :cutoff].contiguous()
predicate_exists = predicate_exists.unsqueeze(1).repeat(
1, encoded_text.shape[-1])
predicate_hidden = torch.where(
predicate_exists > 0, predicate_hidden,
torch.zeros_like(predicate_hidden))
encoded_text = torch.cat(
(encoded_text, predicate_hidden.unsqueeze(1).repeat(
1, encoded_text.shape[1], 1)),
dim=-1)
sequence_length = encoded_text.shape[1]
logits = self.tag_projection_layer(encoded_text)
# print(mask, logits)
if self._lp and sequence_length <= 100:
eps = 1e-4
Q = eps * torch.eye(
sequence_length * self.num_classes,
sequence_length * self.num_classes).unsqueeze(0).repeat(
batch_size, 1, 1).to(logits.device).float()
p = logits.view(batch_size, -1)
G = -1 * torch.eye(
sequence_length * self.num_classes).unsqueeze(0).repeat(
batch_size, 1, 1).to(logits.device).float()
h = torch.zeros_like(p)
A = torch.arange(sequence_length *
self.num_classes).unsqueeze(0).repeat(
sequence_length, 1)
A2 = torch.arange(sequence_length).unsqueeze(1).repeat(
1, sequence_length * self.num_classes) * self.num_classes
A = torch.where((A >= A2) & (A < A2 + self.num_classes),
torch.ones_like(A), torch.zeros_like(A))
A = A.unsqueeze(0).repeat(batch_size, 1,
1).to(logits.device).float()
b = torch.ones_like(A[:, :, 0])
probs = QPFunction()(Q, p, torch.autograd.Variable(torch.Tensor()),
torch.autograd.Variable(torch.Tensor()), A, b)
probs = probs.view(batch_size, sequence_length, self.num_classes)
"""logits_shape = logits.shape
logits = torch.where(mask.bool().unsqueeze(-1).repeat(1, 1, logits.shape[-1]), logits, logits-10000)
max_sequence_length = min([l for l in self.lengths if l >= sequence_length])
if max_sequence_length > logits_shape[1]:
logits = torch.cat((logits, torch.zeros((batch_size, max_sequence_length-logits_shape[1], logits_shape[2])).to(logits.device)), dim=1)
lp_layer = self._layer_list[self.length_map[max_sequence_length]]
probs, = lp_layer(logits)
print(torch.isnan(probs).any())
if max_sequence_length > logits_shape[1]:
probs = probs[:,:logits_shape[1],:]"""
logits = (torch.nn.functional.relu(probs) + 1e-4).log()
if self._lpsmap:
if self._lpsmap_core_only:
all_logits = logits
else:
all_logits = torch.cat((logits, 0.5 * torch.ones(
(batch_size, 1, logits.shape[-1])).to(logits.device)),
dim=1)
probs = []
for i in range(batch_size):
if self.constrain_crf_decoding:
unaries = logits[i, :, :].view(-1).cpu()
additionals = self.crf.transitions.view(-1).repeat(
sequence_length) + 10000 * (
self.crf._constraint_mask[:-2, :-2] -
1).view(-1).repeat(sequence_length)
start_transitions = self.crf.start_transitions + 10000 * (
self.crf._constraint_mask[-2, :-2] - 1)
end_transitions = self.crf.start_transitions + 10000 * (
self.crf._constraint_mask[-1, :-2] - 1)
additionals = torch.cat(
(additionals, start_transitions, end_transitions),
dim=0).cpu()
fg = TorchFactorGraph()
x = fg.variable_from(unaries)
f = PFactorSequence()
f.initialize(
[self.num_classes for _ in range(sequence_length)])
factor = TorchOtherFactor(f, x, additionals)
fg.add(factor)
# add budget constraint for each state
for state in self._core_roles:
vars_state = x[state::self.num_classes]
fg.add(AtMostOne(vars_state))
# solve SparseMAP
fg.solve(max_iter=200)
probs.append(
unaries.to(logits.device).view(sequence_length,
self.num_classes))
else:
fg = TorchFactorGraph()
x = fg.variable_from(all_logits[i, :, :].cpu())
for j in range(sequence_length):
fg.add(Xor(x[j, :]))
for j in self._core_roles:
fg.add(AtMostOne(x[:sequence_length, j]))
if not self._lpsmap_core_only:
full_sequence = list(range(sequence_length))
base_roles = set([
second
for (_, second) in self._r_roles + self._c_roles
])
"""for (r_role, base_role) in self._r_roles+self._c_roles:
for j in range(sequence_length):
fg.add(Imply(x[full_sequence+[j],[base_role]*sequence_length+[r_role]], negated=[True]*(sequence_length+1)))"""
for base_role in base_roles:
fg.add(OrOut(x[:, base_role]))
for (r_role,
base_role) in self._r_roles + self._c_roles:
fg.add(OrOut(x[:, r_role]))
fg.add(
Or(x[[sequence_length, sequence_length],
[r_role, base_role]],
negated=[True, False]))
max_iter = 100
if not self._lpsmap_core_only:
max_iter = min(max_iter, 400)
elif (not self.training) and not self._val_inference:
max_iter = min(max_iter, 200)
fg.solve(max_iter=max_iter)
probs.append(x.value[:sequence_length, :].contiguous().to(
logits.device))
class_probabilities = torch.stack(probs)
# class_probabilities = self.lpsmap(logits)
max_seq_length = 200
# if self.lpsmap is None:
"""with torch.no_grad():
# self.lpsmap = LpSparseMap(num_rows=sequence_length, num_cols=self.num_classes, batch_size=batch_size, device=logits.device, constraints=[('xor', ('row', list(range(sequence_length)))), ('budget', ('col', self._core_roles))])
max_iter = 1000
constraint_types = ["xor", "budget"]
constraint_dims = ["row", "col"]
constraint_sets = [list(range(sequence_length)), self._core_roles]
class_probabilities = lpsmap(logits, constraint_types, constraint_dims, constraint_sets, max_iter)
# if max_seq_length > sequence_length:
# logits = torch.cat((logits, -9999.*torch.ones((batch_size, max_seq_length-sequence_length, self.num_classes)).to(logits.device)), dim=1)
# class_probabilities = self.lpsmap.solve(logits, max_iter=max_iter)"""
# logits = (class_probabilities+1e-4).log()
else:
reshaped_log_probs = logits.view(-1, self.num_classes)
class_probabilities = F.softmax(reshaped_log_probs, dim=-1).view(
[batch_size, sequence_length, self.num_classes])
output_dict = {
"logits": logits,
"class_probabilities": class_probabilities
}
# We need to retain the mask in the output dictionary
# so that we can crop the sequences to remove padding
# when we do viterbi inference in self.make_output_human_readable.
output_dict["mask"] = mask
# We add in the offsets here so we can compute the un-wordpieced tags.
words, verbs, offsets = zip(*[(x["words"], x["verb"], x["offsets"])
for x in metadata])
output_dict["words"] = list(words)
output_dict["verb"] = list(verbs)
output_dict["wordpiece_offsets"] = list(offsets)
if tags is not None:
# print(mask.shape, tags.shape, logits.shape, tags.max(), tags.min())
if self._lpsmap:
loss = LpsmapLoss.apply(logits, class_probabilities, tags,
mask)
# tags_1hot = torch.zeros_like(class_probabilities).scatter_(2, tags.unsqueeze(-1), torch.ones_like(class_probabilities))
# loss = -(tags_1hot*class_probabilities*mask.unsqueeze(-1).repeat(1, 1, class_probabilities.shape[-1])).sum()
elif self.constrain_crf_decoding:
loss = -self.crf(logits, tags, mask)
else:
loss = sequence_cross_entropy_with_logits(
logits, tags, mask, label_smoothing=self._label_smoothing)
if not self.ignore_span_metric and self.span_metric is not None and not self.training:
batch_verb_indices = [
example_metadata["verb_index"]
for example_metadata in metadata
]
batch_sentences = [
example_metadata["words"] for example_metadata in metadata
]
# Get the BIO tags from make_output_human_readable()
# TODO (nfliu): This is kind of a hack, consider splitting out part
# of make_output_human_readable() to a separate function.
batch_bio_predicted_tags = self.make_output_human_readable(
output_dict).pop("tags")
from allennlp_models.structured_prediction.models.srl import (
convert_bio_tags_to_conll_format, )
if self.constrain_crf_decoding and not self._lpsmap:
batch_conll_predicted_tags = [
convert_bio_tags_to_conll_format([
self.vocab.get_token_from_index(
tag, namespace=self._label_namespace)
for tag in seq
]) for (seq, _) in self.crf.viterbi_tags(logits, mask)
]
else:
batch_conll_predicted_tags = [
convert_bio_tags_to_conll_format(tags)
for tags in batch_bio_predicted_tags
]
batch_bio_gold_tags = [
example_metadata["gold_tags"]
for example_metadata in metadata
]
# print(batch_bio_gold_tags)
batch_conll_gold_tags = [
convert_bio_tags_to_conll_format(tags)
for tags in batch_bio_gold_tags
]
self.span_metric(
batch_verb_indices,
batch_sentences,
batch_conll_predicted_tags,
batch_conll_gold_tags,
)
output_dict["loss"] = loss
output_dict["gold_tags"] = [x["gold_tags"] for x in metadata]
return output_dict
def lpsmap(self, scores: torch.Tensor):
batch_size, sequence_length, num_classes = scores.shape
C_indices = torch.arange(sequence_length * num_classes).to(
scores.device)
batch_range = torch.arange(batch_size).to(scores.device)
C_indices = torch.cat(
(batch_range.unsqueeze(1).repeat(
1, C_indices.numel()).view(-1).unsqueeze(0),
C_indices.repeat(batch_size).unsqueeze(0).repeat(2, 1)),
dim=0)
C_values = torch.ones_like(C_indices[0, :]).float()
C = torch.sparse.FloatTensor(
C_indices, C_values,
torch.Size([
batch_size, sequence_length * num_classes,
sequence_length * num_classes
])).to(scores.device)
delta = torch.ones_like(scores).view(batch_size, -1, 1)
M = C.clone()
D = torch.bmm(
C, delta) # [batch_size, num_factors*num_variables_per_factor, 1]
D_inverse = 1. / D
D_inverse_per_factor = D_inverse.view(
-1, num_classes,
1) # [batch_size*num_factors, num_variables_per_factor, 1]
# D_block = torch.diag_embed(D)
# D_inverse_block = torch.diag_embed(D_inverse)
# M_tilde = torch.bmm(M, D_inverse).t()
mu = torch.ones_like(scores).view(batch_size, -1, 1)
lambd = torch.zeros(
(batch_size * sequence_length, num_classes, 1)).to(scores.device)
gamma = 1
T = 10
# Reshape D, eta, and C_tilde so that a batch has only the variables for a single factor
eta = scores.view(batch_size * sequence_length, num_classes, 1)
D = D.view(batch_size * sequence_length, num_classes, 1)
D_eta_product = D * eta # [batch_size*num_factors, num_variables_per_factor]
# C_tilde = C_tilde.view(batch_size*sequence_length, num_classes, sequence_length*num_classes)
range_per_factor = torch.arange(num_classes).to(
scores.device).unsqueeze(0).repeat(batch_size * sequence_length,
1).float()
eps = 1e-6
for t in range(T):
Cmu = torch.bmm(C, mu).view(
-1, num_classes,
1) # [batch_size*num_factors, num_variables_per_factor, 1]
eta_tilde = (
D_eta_product - lambd + gamma * D_inverse_per_factor * Cmu) / (
gamma + 1
) # [batch_size*num_factors, num_variables_per_factor, 1]
# Procedure for solving XOR QP according to Duchi et al. 2008
# TODO: adjust this for when Mf_tilde \neq I (i.e. when there are more factors)
eta_tilde_sorted, eta_indices = eta_tilde.view(
batch_size * sequence_length, -1
).sort(
dim=-1, descending=True
) # [batch_size*num_factors, num_variables_per_factor], [batch_size*num_factors, num_variables_per_factor]
eta_tilde_cumsum = torch.cumsum(
eta_tilde_sorted,
dim=1) # [batch_size*num_factors, num_variables_per_factor]
eta_tilde_cumsum = (eta_tilde_cumsum - 1) / (
range_per_factor + 1
) # [batch_size*num_factors, num_variables_per_factor]
rho = ((eta_tilde_sorted - eta_tilde_cumsum > 0).float() *
(1 + range_per_factor)).argmax(
1) # [batch_size*num_factors]
theta = torch.gather(
eta_tilde_cumsum, 1,
rho.unsqueeze(-1)).repeat(1, num_classes).unsqueeze(
-1) #[batch_size*num_factors, num_variables_per_factor, 1]
p = torch.nn.functional.relu(
eta_tilde - theta
) # [batch_size*num_factors, num_variables_per_factor = num_assignments_per_factor, 1]
Mp = torch.bmm(M, p.view(
batch_size, -1,
1)) # [batch_size, num_factors*num_variables_per_factor, 1]
Dinv_Mp = D_inverse * Mp # [batch_size, num_factors*num_variables_per_factor, 1]
Dinv2_Mp = D_inverse * Dinv_Mp # [batch_size, num_factors*num_variables_per_factor, 1]
mu_new = torch.bmm(C.transpose(1, 2),
Dinv2_Mp) # [batch_size, num_variables, 1]
lambd_diff = D_inverse * torch.bmm(
C, mu_new
) - Dinv_Mp # [batch_size, num_factors*num_variables_per_factor, 1]
lambd += gamma * lambd_diff.view(
-1, num_classes,
1) # [batch_size*num_factors, num_variables_per_factor]
mu_diff_norm = torch.norm((mu_new - mu).squeeze(-1),
dim=1) # [batch_size]
lambd_diff_norm = torch.norm(lambd_diff.squeeze(-1),
dim=1) # [batch_size]
if mu_diff_norm.max().item() < eps and lambd_diff_norm.max().item(
) < eps:
break
mu = mu_new
mu = mu.view(batch_size, sequence_length, num_classes)
return mu
@overrides
def make_output_human_readable(
self, output_dict: Dict[str,
torch.Tensor]) -> Dict[str, torch.Tensor]:
"""
Does constrained viterbi decoding on class probabilities output in :func:`forward`. The
constraint simply specifies that the output tags must be a valid BIO sequence. We add a
`"tags"` key to the dictionary with the result.
NOTE: First, we decode a BIO sequence on top of the wordpieces. This is important; viterbi
decoding produces low quality output if you decode on top of word representations directly,
because the model gets confused by the 'missing' positions (which is sensible as it is trained
to perform tagging on wordpieces, not words).
Secondly, it's important that the indices we use to recover words from the wordpieces are the
start_offsets (i.e offsets which correspond to using the first wordpiece of words which are
tokenized into multiple wordpieces) as otherwise, we might get an ill-formed BIO sequence
when we select out the word tags from the wordpiece tags. This happens in the case that a word
is split into multiple word pieces, and then we take the last tag of the word, which might
correspond to, e.g, I-V, which would not be allowed as it is not preceeded by a B tag.
"""
all_predictions = output_dict["class_probabilities"]
sequence_lengths = get_lengths_from_binary_sequence_mask(
output_dict["mask"]).data.tolist()
if all_predictions.dim() == 3:
predictions_list = [
all_predictions[i].detach().cpu()
for i in range(all_predictions.size(0))
]
else:
predictions_list = [all_predictions]
wordpiece_tags = []
word_tags = []
transition_matrix = self.get_viterbi_pairwise_potentials()
start_transitions = self.get_start_transitions()
# **************** Different ********************
# We add in the offsets here so we can compute the un-wordpieced tags.
for predictions, length, offsets in zip(
predictions_list, sequence_lengths,
output_dict["wordpiece_offsets"]):
max_likelihood_sequence, _ = viterbi_decode(
predictions[:length],
transition_matrix,
allowed_start_transitions=start_transitions)
tags = [
self.vocab.get_token_from_index(
x, namespace=self._label_namespace)
for x in max_likelihood_sequence
]
wordpiece_tags.append(tags)
if isinstance(self.bert_model,
PretrainedTransformerMismatchedEmbedder):
word_tags.append(tags)
else:
word_tags.append([tags[i] for i in offsets])
# print(word_tags)
output_dict["wordpiece_tags"] = wordpiece_tags
output_dict["tags"] = word_tags
return output_dict
def get_metrics(self, reset: bool = False):
if self.ignore_span_metric:
# Return an empty dictionary if ignoring the
# span metric
return {}
else:
metric_dict = self.span_metric.get_metric(reset=reset)
# This can be a lot of metrics, as there are 3 per class.
# we only really care about the overall metrics, so we filter for them here.
return {x: y for x, y in metric_dict.items() if "overall" in x}
def get_viterbi_pairwise_potentials(self):
"""
Generate a matrix of pairwise transition potentials for the BIO labels.
The only constraint implemented here is that I-XXX labels must be preceded
by either an identical I-XXX tag or a B-XXX tag. In order to achieve this
constraint, pairs of labels which do not satisfy this constraint have a
pairwise potential of -inf.
# Returns
transition_matrix : `torch.Tensor`
A `(num_labels, num_labels)` matrix of pairwise potentials.
"""
all_labels = self.vocab.get_index_to_token_vocabulary(
self._label_namespace)
num_labels = len(all_labels)
transition_matrix = torch.zeros([num_labels, num_labels])
for i, previous_label in all_labels.items():
for j, label in all_labels.items():
# I labels can only be preceded by themselves or
# their corresponding B tag.
if i != j and label[
0] == "I" and not previous_label == "B" + label[1:]:
transition_matrix[i, j] = float("-inf")
return transition_matrix
def get_start_transitions(self):
"""
In the BIO sequence, we cannot start the sequence with an I-XXX tag.
This transition sequence is passed to viterbi_decode to specify this constraint.
# Returns
start_transitions : `torch.Tensor`
The pairwise potentials between a START token and
the first token of the sequence.
"""
all_labels = self.vocab.get_index_to_token_vocabulary(
self._label_namespace)
num_labels = len(all_labels)
start_transitions = torch.zeros(num_labels)
for i, label in all_labels.items():
if label[0] == "I":
start_transitions[i] = float("-inf")
return start_transitions
default_predictor = "semantic_role_labeling"
class SrlBert(Model):
"""
A BERT based model [Simple BERT Models for Relation Extraction and Semantic Role Labeling (Shi et al, 2019)]
(https://arxiv.org/abs/1904.05255) with some modifications (no additional parameters apart from a linear
classification layer), which is currently the state-of-the-art single model for English PropBank SRL
(Newswire sentences).
# Parameters
vocab : `Vocabulary`, required
A Vocabulary, required in order to compute sizes for input/output projections.
model : `Union[str, BertModel]`, required.
A string describing the BERT model to load or an already constructed BertModel.
initializer : `InitializerApplicator`, optional (default=`InitializerApplicator()`)
Used to initialize the model parameters.
label_smoothing : `float`, optional (default = `0.0`)
Whether or not to use label smoothing on the labels when computing cross entropy loss.
ignore_span_metric : `bool`, optional (default = `False`)
Whether to calculate span loss, which is irrelevant when predicting BIO for Open Information Extraction.
srl_eval_path : `str`, optional (default=`DEFAULT_SRL_EVAL_PATH`)
The path to the srl-eval.pl script. By default, will use the srl-eval.pl included with allennlp,
which is located at allennlp/tools/srl-eval.pl . If `None`, srl-eval.pl is not used.
"""
def __init__(
self,
vocab: Vocabulary,
bert_model: Union[str, BertModel],
embedding_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
label_smoothing: float = None,
ignore_span_metric: bool = False,
srl_eval_path: str = DEFAULT_SRL_EVAL_PATH,
**kwargs,
) -> None:
super().__init__(vocab, **kwargs)
if isinstance(bert_model, str):
self.bert_model = BertModel.from_pretrained(bert_model)
else:
self.bert_model = bert_model
self.num_classes = self.vocab.get_vocab_size("labels")
if srl_eval_path is not None:
# For the span based evaluation, we don't want to consider labels
# for verb, because the verb index is provided to the model.
self.span_metric = SrlEvalScorer(srl_eval_path,
ignore_classes=["V"])
else:
self.span_metric = None
self.tag_projection_layer = Linear(self.bert_model.config.hidden_size,
self.num_classes)
self.embedding_dropout = Dropout(p=embedding_dropout)
self._label_smoothing = label_smoothing
self.ignore_span_metric = ignore_span_metric
initializer(self)
def forward( # type: ignore
self,
tokens: TextFieldTensors,
verb_indicator: torch.Tensor,
metadata: List[Any],
tags: torch.LongTensor = None,
):
"""
# Parameters
tokens : `TextFieldTensors`, required
The output of `TextField.as_array()`, which should typically be passed directly to a
`TextFieldEmbedder`. For this model, this must be a `SingleIdTokenIndexer` which
indexes wordpieces from the BERT vocabulary.
verb_indicator: `torch.LongTensor`, required.
An integer `SequenceFeatureField` representation of the position of the verb
in the sentence. This should have shape (batch_size, num_tokens) and importantly, can be
all zeros, in the case that the sentence has no verbal predicate.
tags : `torch.LongTensor`, optional (default = `None`)
A torch tensor representing the sequence of integer gold class labels
of shape `(batch_size, num_tokens)`
metadata : `List[Dict[str, Any]]`, optional, (default = `None`)
metadata containing the original words in the sentence, the verb to compute the
frame for, and start offsets for converting wordpieces back to a sequence of words,
under 'words', 'verb' and 'offsets' keys, respectively.
# Returns
An output dictionary consisting of:
logits : `torch.FloatTensor`
A tensor of shape `(batch_size, num_tokens, tag_vocab_size)` representing
unnormalised log probabilities of the tag classes.
class_probabilities : `torch.FloatTensor`
A tensor of shape `(batch_size, num_tokens, tag_vocab_size)` representing
a distribution of the tag classes per word.
loss : `torch.FloatTensor`, optional
A scalar loss to be optimised.
"""
mask = get_text_field_mask(tokens)
bert_embeddings, _ = self.bert_model(
input_ids=util.get_token_ids_from_text_field_tensors(tokens),
token_type_ids=verb_indicator,
attention_mask=mask,
)
embedded_text_input = self.embedding_dropout(bert_embeddings)
batch_size, sequence_length, _ = embedded_text_input.size()
logits = self.tag_projection_layer(embedded_text_input)
reshaped_log_probs = logits.view(-1, self.num_classes)
class_probabilities = F.softmax(reshaped_log_probs, dim=-1).view(
[batch_size, sequence_length, self.num_classes])
output_dict = {
"logits": logits,
"class_probabilities": class_probabilities
}
# We need to retain the mask in the output dictionary
# so that we can crop the sequences to remove padding
# when we do viterbi inference in self.make_output_human_readable.
output_dict["mask"] = mask
# We add in the offsets here so we can compute the un-wordpieced tags.
words, verbs, offsets = zip(*[(x["words"], x["verb"], x["offsets"])
for x in metadata])
output_dict["words"] = list(words)
output_dict["verb"] = list(verbs)
output_dict["wordpiece_offsets"] = list(offsets)
if tags is not None:
loss = sequence_cross_entropy_with_logits(
logits, tags, mask, label_smoothing=self._label_smoothing)
if not self.ignore_span_metric and self.span_metric is not None and not self.training:
batch_verb_indices = [
example_metadata["verb_index"]
for example_metadata in metadata
]
batch_sentences = [
example_metadata["words"] for example_metadata in metadata
]
# Get the BIO tags from make_output_human_readable()
# TODO (nfliu): This is kind of a hack, consider splitting out part
# of make_output_human_readable() to a separate function.
batch_bio_predicted_tags = self.make_output_human_readable(
output_dict).pop("tags")
from allennlp_models.structured_prediction.models.srl import (
convert_bio_tags_to_conll_format, )
batch_conll_predicted_tags = [
convert_bio_tags_to_conll_format(tags)
for tags in batch_bio_predicted_tags
]
batch_bio_gold_tags = [
example_metadata["gold_tags"]
for example_metadata in metadata
]
batch_conll_gold_tags = [
convert_bio_tags_to_conll_format(tags)
for tags in batch_bio_gold_tags
]
self.span_metric(
batch_verb_indices,
batch_sentences,
batch_conll_predicted_tags,
batch_conll_gold_tags,
)
output_dict["loss"] = loss
return output_dict
@overrides
def make_output_human_readable(
self, output_dict: Dict[str,
torch.Tensor]) -> Dict[str, torch.Tensor]:
"""
Does constrained viterbi decoding on class probabilities output in :func:`forward`. The
constraint simply specifies that the output tags must be a valid BIO sequence. We add a
`"tags"` key to the dictionary with the result.
NOTE: First, we decode a BIO sequence on top of the wordpieces. This is important; viterbi
decoding produces low quality output if you decode on top of word representations directly,
because the model gets confused by the 'missing' positions (which is sensible as it is trained
to perform tagging on wordpieces, not words).
Secondly, it's important that the indices we use to recover words from the wordpieces are the
start_offsets (i.e offsets which correspond to using the first wordpiece of words which are
tokenized into multiple wordpieces) as otherwise, we might get an ill-formed BIO sequence
when we select out the word tags from the wordpiece tags. This happens in the case that a word
is split into multiple word pieces, and then we take the last tag of the word, which might
correspond to, e.g, I-V, which would not be allowed as it is not preceeded by a B tag.
"""
all_predictions = output_dict["class_probabilities"]
sequence_lengths = get_lengths_from_binary_sequence_mask(
output_dict["mask"]).data.tolist()
if all_predictions.dim() == 3:
predictions_list = [
all_predictions[i].detach().cpu()
for i in range(all_predictions.size(0))
]
else:
predictions_list = [all_predictions]
wordpiece_tags = []
word_tags = []
transition_matrix = self.get_viterbi_pairwise_potentials()
start_transitions = self.get_start_transitions()
# **************** Different ********************
# We add in the offsets here so we can compute the un-wordpieced tags.
for predictions, length, offsets in zip(
predictions_list, sequence_lengths,
output_dict["wordpiece_offsets"]):
max_likelihood_sequence, _ = viterbi_decode(
predictions[:length],
transition_matrix,
allowed_start_transitions=start_transitions)
tags = [
self.vocab.get_token_from_index(x, namespace="labels")
for x in max_likelihood_sequence
]
wordpiece_tags.append(tags)
word_tags.append([tags[i] for i in offsets])
output_dict["wordpiece_tags"] = wordpiece_tags
output_dict["tags"] = word_tags
return output_dict
def get_metrics(self, reset: bool = False):
if self.ignore_span_metric:
# Return an empty dictionary if ignoring the
# span metric
return {}
else:
metric_dict = self.span_metric.get_metric(reset=reset)
# This can be a lot of metrics, as there are 3 per class.
# we only really care about the overall metrics, so we filter for them here.
return {x: y for x, y in metric_dict.items() if "overall" in x}
def get_viterbi_pairwise_potentials(self):
"""
Generate a matrix of pairwise transition potentials for the BIO labels.
The only constraint implemented here is that I-XXX labels must be preceded
by either an identical I-XXX tag or a B-XXX tag. In order to achieve this
constraint, pairs of labels which do not satisfy this constraint have a
pairwise potential of -inf.
# Returns
transition_matrix : `torch.Tensor`
A `(num_labels, num_labels)` matrix of pairwise potentials.
"""
all_labels = self.vocab.get_index_to_token_vocabulary("labels")
num_labels = len(all_labels)
transition_matrix = torch.zeros([num_labels, num_labels])
for i, previous_label in all_labels.items():
for j, label in all_labels.items():
# I labels can only be preceded by themselves or
# their corresponding B tag.
if i != j and label[
0] == "I" and not previous_label == "B" + label[1:]:
transition_matrix[i, j] = float("-inf")
return transition_matrix
def get_start_transitions(self):
"""
In the BIO sequence, we cannot start the sequence with an I-XXX tag.
This transition sequence is passed to viterbi_decode to specify this constraint.
# Returns
start_transitions : `torch.Tensor`
The pairwise potentials between a START token and
the first token of the sequence.
"""
all_labels = self.vocab.get_index_to_token_vocabulary("labels")
num_labels = len(all_labels)
start_transitions = torch.zeros(num_labels)
for i, label in all_labels.items():
if label[0] == "I":
start_transitions[i] = float("-inf")
return start_transitions
default_predictor = "semantic_role_labeling"
def __init__(
self,
vocab: Vocabulary,
bert_model: Union[str, BertModel],
mention_feedforward: FeedForward,
context_layer: Seq2SeqEncoder = None,
embedding_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
max_span_width: int = 30,
feature_size: int = 10,
spans_per_word: float = 100,
label_smoothing: float = None,
ignore_span_metric: bool = False,
srl_eval_path: str = DEFAULT_SRL_EVAL_PATH,
**kwargs,
) -> None:
super().__init__(vocab, **kwargs)
if isinstance(bert_model, str):
self.bert_model = BertModel.from_pretrained(bert_model)
else:
self.bert_model = bert_model
self.num_classes = self.vocab.get_vocab_size("span_labels")
if srl_eval_path is not None:
# For the span based evaluation, we don't want to consider labels
# for verb, because the verb index is provided to the model.
self.span_metric = SrlEvalScorer(srl_eval_path,
ignore_classes=["V"])
else:
self.span_metric = None
self.tag_projection_layer = Linear(self.bert_model.config.hidden_size,
self.num_classes)
self.embedding_dropout = Dropout(p=embedding_dropout)
self._label_smoothing = label_smoothing
self.ignore_span_metric = ignore_span_metric
self._mention_feedforward = TimeDistributed(mention_feedforward)
self._mention_scorer = TimeDistributed(
torch.nn.Linear(mention_feedforward.get_output_dim(), 1))
self._attentive_span_extractor = SelfAttentiveSpanExtractor(
input_dim=self.bert_model.config.hidden_size)
self.span_representation_dim = self._attentive_span_extractor.get_output_dim(
)
self._context_layer = context_layer
if context_layer is not None:
self._endpoint_span_extractor = EndpointSpanExtractor(
context_layer.get_output_dim(),
combination="x,y",
num_width_embeddings=max_span_width,
span_width_embedding_dim=feature_size,
bucket_widths=False,
)
self.span_representation_dim = self._endpoint_span_extractor.get_output_dim(
)
self.hidden_layer = torch.nn.Sequential(
torch.nn.Linear(self.span_representation_dim +
self.bert_model.config.hidden_size,
self.span_representation_dim,
bias=False), torch.nn.ReLU())
self.output_layer = torch.nn.Linear(self.span_representation_dim,
self.num_classes - 1,
bias=False)
self._max_span_width = max_span_width
self._spans_per_word = spans_per_word
self._ce_loss = torch.nn.CrossEntropyLoss(reduction='none')
self._bce_loss = torch.nn.BCEWithLogitsLoss(reduction='none')
initializer(self)
class SrlE2e(Model):
"""
# Parameters
vocab : `Vocabulary`, required
A Vocabulary, required in order to compute sizes for input/output projections.
model : `Union[str, BertModel]`, required.
A string describing the BERT model to load or an already constructed BertModel.
initializer : `InitializerApplicator`, optional (default=`InitializerApplicator()`)
Used to initialize the model parameters.
label_smoothing : `float`, optional (default = `0.0`)
Whether or not to use label smoothing on the labels when computing cross entropy loss.
ignore_span_metric : `bool`, optional (default = `False`)
Whether to calculate span loss, which is irrelevant when predicting BIO for Open Information Extraction.
srl_eval_path : `str`, optional (default=`DEFAULT_SRL_EVAL_PATH`)
The path to the srl-eval.pl script. By default, will use the srl-eval.pl included with allennlp,
which is located at allennlp/tools/srl-eval.pl . If `None`, srl-eval.pl is not used.
"""
def __init__(
self,
vocab: Vocabulary,
bert_model: Union[str, BertModel],
mention_feedforward: FeedForward,
context_layer: Seq2SeqEncoder = None,
embedding_dropout: float = 0.0,
initializer: InitializerApplicator = InitializerApplicator(),
max_span_width: int = 30,
feature_size: int = 10,
spans_per_word: float = 100,
label_smoothing: float = None,
ignore_span_metric: bool = False,
srl_eval_path: str = DEFAULT_SRL_EVAL_PATH,
**kwargs,
) -> None:
super().__init__(vocab, **kwargs)
if isinstance(bert_model, str):
self.bert_model = BertModel.from_pretrained(bert_model)
else:
self.bert_model = bert_model
self.num_classes = self.vocab.get_vocab_size("span_labels")
if srl_eval_path is not None:
# For the span based evaluation, we don't want to consider labels
# for verb, because the verb index is provided to the model.
self.span_metric = SrlEvalScorer(srl_eval_path,
ignore_classes=["V"])
else:
self.span_metric = None
self.tag_projection_layer = Linear(self.bert_model.config.hidden_size,
self.num_classes)
self.embedding_dropout = Dropout(p=embedding_dropout)
self._label_smoothing = label_smoothing
self.ignore_span_metric = ignore_span_metric
self._mention_feedforward = TimeDistributed(mention_feedforward)
self._mention_scorer = TimeDistributed(
torch.nn.Linear(mention_feedforward.get_output_dim(), 1))
self._attentive_span_extractor = SelfAttentiveSpanExtractor(
input_dim=self.bert_model.config.hidden_size)
self.span_representation_dim = self._attentive_span_extractor.get_output_dim(
)
self._context_layer = context_layer
if context_layer is not None:
self._endpoint_span_extractor = EndpointSpanExtractor(
context_layer.get_output_dim(),
combination="x,y",
num_width_embeddings=max_span_width,
span_width_embedding_dim=feature_size,
bucket_widths=False,
)
self.span_representation_dim = self._endpoint_span_extractor.get_output_dim(
)
self.hidden_layer = torch.nn.Sequential(
torch.nn.Linear(self.span_representation_dim +
self.bert_model.config.hidden_size,
self.span_representation_dim,
bias=False), torch.nn.ReLU())
self.output_layer = torch.nn.Linear(self.span_representation_dim,
self.num_classes - 1,
bias=False)
self._max_span_width = max_span_width
self._spans_per_word = spans_per_word
self._ce_loss = torch.nn.CrossEntropyLoss(reduction='none')
self._bce_loss = torch.nn.BCEWithLogitsLoss(reduction='none')
initializer(self)
def forward( # type: ignore
self,
tokens: TextFieldTensors,
verb_indicator: torch.Tensor,
sentence_end: torch.LongTensor,
spans: torch.LongTensor,
span_labels: torch.LongTensor,
metadata: List[Any],
tags: torch.LongTensor = None,
):
"""
# Parameters
tokens : `TextFieldTensors`, required
The output of `TextField.as_array()`, which should typically be passed directly to a
`TextFieldEmbedder`. For this model, this must be a `SingleIdTokenIndexer` which
indexes wordpieces from the BERT vocabulary.
verb_indicator: `torch.LongTensor`, required.
An integer `SequenceFeatureField` representation of the position of the verb
in the sentence. This should have shape (batch_size, num_tokens) and importantly, can be
all zeros, in the case that the sentence has no verbal predicate.
tags : `torch.LongTensor`, optional (default = `None`)
A torch tensor representing the sequence of integer gold class labels
of shape `(batch_size, num_tokens)`
metadata : `List[Dict[str, Any]]`, optional, (default = `None`)
metadata containg the original words in the sentence, the verb to compute the
frame for, and start offsets for converting wordpieces back to a sequence of words,
under 'words', 'verb' and 'offsets' keys, respectively.
# Returns
An output dictionary consisting of:
logits : `torch.FloatTensor`
A tensor of shape `(batch_size, num_tokens, tag_vocab_size)` representing
unnormalised log probabilities of the tag classes.
class_probabilities : `torch.FloatTensor`
A tensor of shape `(batch_size, num_tokens, tag_vocab_size)` representing
a distribution of the tag classes per word.
loss : `torch.FloatTensor`, optional
A scalar loss to be optimised.
"""
mask = get_text_field_mask(tokens)
start = time.time()
bert_embeddings, _ = self.bert_model(
input_ids=util.get_token_ids_from_text_field_tensors(tokens),
# token_type_ids=verb_indicator,
attention_mask=mask,
)
# Shape: (batch_size, num_spans)
span_mask = (spans[:, :, 0] >= 0).squeeze(-1)
# SpanFields return -1 when they are used as padding. As we do
# some comparisons based on span widths when we attend over the
# span representations that we generate from these indices, we
# need them to be <= 0. This is only relevant in edge cases where
# the number of spans we consider after the pruning stage is >= the
# total number of spans, because in this case, it is possible we might
# consider a masked span.
# Shape: (batch_size, num_spans, 2)
spans = F.relu(spans.float()).long()
embedded_text_input = self.embedding_dropout(bert_embeddings)
batch_size, sequence_length, _ = embedded_text_input.size()
# Shape: (batch_size, num_spans, emebedding_size)
attended_span_embeddings = self._attentive_span_extractor(
bert_embeddings, spans)
if self._context_layer is not None:
contextualized_embeddings = self._context_layer(
embedded_text_input, mask)
# Shape: (batch_size, num_spans, 2 * encoding_dim + feature_size)
endpoint_span_embeddings = self._endpoint_span_extractor(
contextualized_embeddings, spans)
# Shape: (batch_size, num_spans, emebedding_size + 2 * encoding_dim + feature_size)
# span_embeddings = torch.cat([endpoint_span_embeddings, attended_span_embeddings], -1)
span_embeddings = endpoint_span_embeddings
else:
span_embeddings = attended_span_embeddings
# Prune based on mention scores.
num_spans_to_keep = int(
math.floor(self._spans_per_word * sequence_length))
num_spans = spans.shape[1]
num_spans_to_keep = min(num_spans_to_keep, num_spans)
# Shape: (batch_size, num_spans)
span_mention_scores = self._mention_scorer(
self._mention_feedforward(span_embeddings)).squeeze(-1)
# Shape: (batch_size, num_spans) for all 3 tensors
top_span_mention_scores, top_span_mask, top_span_indices = util.masked_topk(
span_mention_scores, span_mask, num_spans_to_keep)
verb_index = verb_indicator.argmax(1).unsqueeze(1).unsqueeze(2).repeat(
1, 1, embedded_text_input.shape[-1])
verb_embeddings = torch.gather(embedded_text_input, 1, verb_index)
assert len(
verb_embeddings.shape) == 3 and verb_embeddings.shape[1] == 1
verb_embeddings = verb_embeddings.squeeze(1)
# print(verb_indicator.sum(1, keepdim=True) > 0)
verb_embeddings = torch.where(
(verb_indicator.sum(1, keepdim=True) > 0).repeat(
1, verb_embeddings.shape[-1]), verb_embeddings,
torch.zeros_like(verb_embeddings))
# print(verb_embeddings)
flat_top_span_indices = util.flatten_and_batch_shift_indices(
top_span_indices, spans.shape[1])
span_embeddings = util.batched_index_select(span_embeddings,
top_span_indices,
flat_top_span_indices)
top_spans = util.batched_index_select(spans, top_span_indices,
flat_top_span_indices)
top_span_labels = util.batched_index_select(
span_labels.unsqueeze(-1), top_span_indices,
flat_top_span_indices).squeeze(-1)
concatenated_span_embeddings = torch.cat(
(span_embeddings, verb_embeddings.unsqueeze(1).repeat(
1, span_embeddings.shape[1], 1)),
dim=2)
# print(concatenated_span_embeddings[:,:,:])
hidden = self.hidden_layer(concatenated_span_embeddings)
# print(hidden[1,:,:])
# print(top_span_indices)
# print([[span_mention_scores[i,top_span_indices[i,j]].item() for j in range(top_span_indices.shape[1])] for i in range(top_span_labels.shape[0])])
# print(top_span_mention_scores, self.vocab.get_token_index("O", namespace="span_labels"))
predictions = self.output_layer(hidden)
# predictions += top_span_mention_scores.unsqueeze(-1).repeat(1, 1, self.num_classes-1)
predictions = torch.cat(
(torch.zeros_like(predictions[:, :, :1]), predictions), dim=-1)
# print(top_span_mention_scores.unsqueeze(-1).repeat(1, 1, self.num_classes-1))
output_dict = {}
# We need to retain the mask in the output dictionary
# so that we can crop the sequences to remove padding
# when we do viterbi inference in self.make_output_human_readable.
output_dict["mask"] = mask
# We add in the offsets here so we can compute the un-wordpieced tags.
words, verbs, offsets = zip(*[(x["words"], x["verb"], x["offsets"])
for x in metadata])
output_dict["words"] = list(words)
output_dict["verb"] = list(verbs)
output_dict["wordpiece_offsets"] = list(offsets)
if tags is not None:
loss = (self._ce_loss(predictions.view(-1, predictions.shape[-1]),
top_span_labels.view(-1)) *
top_span_mask.float().view(-1)
).sum() / top_span_mask.float().sum()
# print(top_span_labels)
# print(predictions.argmax(-1))
if not self.ignore_span_metric and self.span_metric is not None and not self.training:
batch_verb_indices = [
example_metadata["verb_index"]
for example_metadata in metadata
]
batch_sentences = [
example_metadata["words"] for example_metadata in metadata
]
# Get the BIO tags from make_output_human_readable()
# TODO (nfliu): This is kind of a hack, consider splitting out part
# of make_output_human_readable() to a separate function.
batch_bio_predicted_tags = self.get_tags(
top_spans, predictions, mask.shape[1], top_span_mask,
output_dict)
from allennlp_models.structured_prediction.models.srl import (
convert_bio_tags_to_conll_format, )
batch_conll_predicted_tags = [
convert_bio_tags_to_conll_format(tags)
for tags in batch_bio_predicted_tags
]
batch_bio_gold_tags = [
example_metadata["gold_tags"]
for example_metadata in metadata
]
# print('G', batch_bio_gold_tags)
batch_conll_gold_tags = [
convert_bio_tags_to_conll_format(tags)
for tags in batch_bio_gold_tags
]
self.span_metric(
batch_verb_indices,
batch_sentences,
batch_conll_predicted_tags,
batch_conll_gold_tags,
)
output_dict["loss"] = loss
return output_dict
def get_tags(self, spans, logits, sequence_length, span_mask, output_dict):
predicted_tag_ids = logits.argmax(2)
predicted_tags = []
for i in range(spans.shape[0]):
sequence = ["O" for _ in range(sequence_length)]
for j in range(spans.shape[1]):
if span_mask[i, j].item() == 0:
continue
tag = predicted_tag_ids[i, j].item()
if tag != self.vocab.get_token_index("O",
namespace="span_labels"):
start = spans[i, j, 0].item()
end = spans[i, j, 1].item()
if all([el == "O" for el in sequence[start:end + 1]]):
tag = self.vocab.get_token_from_index(
tag, namespace="span_labels")
sequence[start] = "B-" + tag
for index in range(start + 1, end + 1):
sequence[index] = "I-" + tag
predicted_tags.append(
[sequence[ind] for ind in output_dict["wordpiece_offsets"][i]])
print(predicted_tags)
return predicted_tags
@overrides
def make_output_human_readable(
self, output_dict: Dict[str,
torch.Tensor]) -> Dict[str, torch.Tensor]:
"""
Does constrained viterbi decoding on class probabilities output in :func:`forward`. The
constraint simply specifies that the output tags must be a valid BIO sequence. We add a
`"tags"` key to the dictionary with the result.
NOTE: First, we decode a BIO sequence on top of the wordpieces. This is important; viterbi
decoding produces low quality output if you decode on top of word representations directly,
because the model gets confused by the 'missing' positions (which is sensible as it is trained
to perform tagging on wordpieces, not words).
Secondly, it's important that the indices we use to recover words from the wordpieces are the
start_offsets (i.e offsets which correspond to using the first wordpiece of words which are
tokenized into multiple wordpieces) as otherwise, we might get an ill-formed BIO sequence
when we select out the word tags from the wordpiece tags. This happens in the case that a word
is split into multiple word pieces, and then we take the last tag of the word, which might
correspond to, e.g, I-V, which would not be allowed as it is not preceeded by a B tag.
"""
all_predictions = output_dict["class_probabilities"]
sequence_lengths = get_lengths_from_binary_sequence_mask(
output_dict["mask"]).data.tolist()
if all_predictions.dim() == 3:
predictions_list = [
all_predictions[i].detach().cpu()
for i in range(all_predictions.size(0))
]
else:
predictions_list = [all_predictions]
wordpiece_tags = []
word_tags = []
transition_matrix = self.get_viterbi_pairwise_potentials()
start_transitions = self.get_start_transitions()
# **************** Different ********************
# We add in the offsets here so we can compute the un-wordpieced tags.
for predictions, length, offsets in zip(
predictions_list, sequence_lengths,
output_dict["wordpiece_offsets"]):
max_likelihood_sequence, _ = viterbi_decode(
predictions[:length],
transition_matrix,
allowed_start_transitions=start_transitions)
tags = [
self.vocab.get_token_from_index(x, namespace="labels")
for x in max_likelihood_sequence
]
wordpiece_tags.append(tags)
word_tags.append([tags[i] for i in offsets])
output_dict["wordpiece_tags"] = wordpiece_tags
output_dict["tags"] = word_tags
return output_dict
def get_metrics(self, reset: bool = False):
if self.ignore_span_metric:
# Return an empty dictionary if ignoring the
# span metric
return {}
else:
metric_dict = self.span_metric.get_metric(reset=reset)
# This can be a lot of metrics, as there are 3 per class.
# we only really care about the overall metrics, so we filter for them here.
return {x: y for x, y in metric_dict.items() if "overall" in x}
def get_viterbi_pairwise_potentials(self):
"""
Generate a matrix of pairwise transition potentials for the BIO labels.
The only constraint implemented here is that I-XXX labels must be preceded
by either an identical I-XXX tag or a B-XXX tag. In order to achieve this
constraint, pairs of labels which do not satisfy this constraint have a
pairwise potential of -inf.
# Returns
transition_matrix : `torch.Tensor`
A `(num_labels, num_labels)` matrix of pairwise potentials.
"""
all_labels = self.vocab.get_index_to_token_vocabulary("labels")
num_labels = len(all_labels)
transition_matrix = torch.zeros([num_labels, num_labels])
for i, previous_label in all_labels.items():
for j, label in all_labels.items():
# I labels can only be preceded by themselves or
# their corresponding B tag.
if i != j and label[
0] == "I" and not previous_label == "B" + label[1:]:
transition_matrix[i, j] = float("-inf")
return transition_matrix
def get_start_transitions(self):
"""
In the BIO sequence, we cannot start the sequence with an I-XXX tag.
This transition sequence is passed to viterbi_decode to specify this constraint.
# Returns
start_transitions : `torch.Tensor`
The pairwise potentials between a START token and
the first token of the sequence.
"""
all_labels = self.vocab.get_index_to_token_vocabulary("labels")
num_labels = len(all_labels)
start_transitions = torch.zeros(num_labels)
for i, label in all_labels.items():
if label[0] == "I":
start_transitions[i] = float("-inf")
return start_transitions
default_predictor = "semantic_role_labeling"