def test_span_metrics_are_computed_correcly_with_prediction_map(self):
# In this example, datapoint1 only has access to ARG1 and V labels,
# whereas datapoint2 only has access to ARG2 and V labels.
# gold_labels = [["O", "B-ARG1", "I-ARG1", "O", "B-V", "O"],
# ["B-ARG2", "I-ARG2", "O", "B-V", "I-V", "O"]]
gold_indices = [[0, 1, 2, 0, 3, 0], [1, 2, 0, 3, 4, 0]]
prediction_map_indices = [[0, 1, 2, 5, 6], [0, 3, 4, 5, 6]]
gold_tensor = torch.Tensor(gold_indices)
prediction_map_tensor = torch.Tensor(prediction_map_indices)
prediction_tensor = torch.rand([2, 6, 5])
prediction_tensor[0, 0, 0] = 1
prediction_tensor[0, 1, 1] = 1 # (True Positive - ARG1
prediction_tensor[0, 2, 2] = 1 # *)
prediction_tensor[0, 3, 0] = 1
prediction_tensor[0, 4, 3] = 1 # (True Positive - V)
prediction_tensor[0, 5, 1] = 1 # (False Positive - ARG1)
prediction_tensor[1, 0, 0] = 1 # (False Negative - ARG2
prediction_tensor[1, 1, 0] = 1 # *)
prediction_tensor[1, 2, 0] = 1
prediction_tensor[1, 3, 3] = 1 # (True Positive - V
prediction_tensor[1, 4, 4] = 1 # *)
prediction_tensor[1, 5, 1] = 1 # (False Positive - ARG2)
metric = SpanBasedF1Measure(self.vocab, "tags")
metric(prediction_tensor,
gold_tensor,
prediction_map=prediction_map_tensor)
assert metric._true_positives["ARG1"] == 1
assert metric._true_positives["ARG2"] == 0
assert metric._true_positives["V"] == 2
assert "O" not in metric._true_positives.keys()
assert metric._false_negatives["ARG1"] == 0
assert metric._false_negatives["ARG2"] == 1
assert metric._false_negatives["V"] == 0
assert "O" not in metric._false_negatives.keys()
assert metric._false_positives["ARG1"] == 1
assert metric._false_positives["ARG2"] == 1
assert metric._false_positives["V"] == 0
assert "O" not in metric._false_positives.keys()
# Check things are accumulating correctly.
metric(prediction_tensor,
gold_tensor,
prediction_map=prediction_map_tensor)
assert metric._true_positives["ARG1"] == 2
assert metric._true_positives["ARG2"] == 0
assert metric._true_positives["V"] == 4
assert "O" not in metric._true_positives.keys()
assert metric._false_negatives["ARG1"] == 0
assert metric._false_negatives["ARG2"] == 2
assert metric._false_negatives["V"] == 0
assert "O" not in metric._false_negatives.keys()
assert metric._false_positives["ARG1"] == 2
assert metric._false_positives["ARG2"] == 2
assert metric._false_positives["V"] == 0
assert "O" not in metric._false_positives.keys()
metric_dict = metric.get_metric()
numpy.testing.assert_almost_equal(metric_dict["recall-ARG2"], 0.0)
numpy.testing.assert_almost_equal(metric_dict["precision-ARG2"], 0.0)
numpy.testing.assert_almost_equal(metric_dict["f1-measure-ARG2"], 0.0)
numpy.testing.assert_almost_equal(metric_dict["recall-ARG1"], 1.0)
numpy.testing.assert_almost_equal(metric_dict["precision-ARG1"], 0.5)
numpy.testing.assert_almost_equal(metric_dict["f1-measure-ARG1"],
0.666666666)
numpy.testing.assert_almost_equal(metric_dict["recall-V"], 1.0)
numpy.testing.assert_almost_equal(metric_dict["precision-V"], 1.0)
numpy.testing.assert_almost_equal(metric_dict["f1-measure-V"], 1.0)
numpy.testing.assert_almost_equal(metric_dict["recall-overall"], 0.75)
numpy.testing.assert_almost_equal(metric_dict["precision-overall"],
0.6)
numpy.testing.assert_almost_equal(metric_dict["f1-measure-overall"],
0.666666666)
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
label_namespace: str = "labels",
constraint_type: str = None,
feedforward: FeedForward = FeedForward(
input_dim=66,
num_layers=100,
hidden_dims=64,
activations=torch.nn.ReLU(),
dropout=0.5),
include_start_end_transitions: bool = True,
dropout: float = None,
verbose_metrics: bool = True,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super().__init__(vocab, regularizer)
self.label_namespace = label_namespace
self.text_field_embedder = text_field_embedder
self.num_tags = self.vocab.get_vocab_size(label_namespace)
self.encoder = encoder
self._verbose_metrics = verbose_metrics
if dropout:
self.dropout = torch.nn.Dropout(dropout)
else:
self.dropout = None
self._feedforward = feedforward
if feedforward is not None:
output_dim = feedforward.get_output_dim()
else:
output_dim = self.encoder.get_output_dim()
self.tag_projection_layer = TimeDistributed(
Linear(output_dim, self.num_tags))
if constraint_type is not None:
labels = self.vocab.get_index_to_token_vocabulary(label_namespace)
constraints = allowed_transitions(constraint_type, labels)
else:
constraints = None
self.crf = ConditionalRandomField(
self.num_tags,
constraints,
include_start_end_transitions=include_start_end_transitions)
self.span_metric = SpanBasedF1Measure(vocab,
tag_namespace=label_namespace,
label_encoding=constraint_type
or "BIO")
check_dimensions_match(text_field_embedder.get_output_dim(),
encoder.get_input_dim(),
"text field embedding dim", "encoder input dim")
if feedforward is not None:
check_dimensions_match(encoder.get_output_dim(),
feedforward.get_input_dim(),
"encoder output dim",
"feedforward input dim")
initializer(self)
def test_span_f1_matches_perl_script_for_continued_arguments(self):
bio_tags = ["B-ARG1", "O", "B-C-ARG1", "B-V", "B-ARGM-ADJ", "O"]
sentence = ["Mark", "and", "Matt", "were", "running", "fast", "."]
gold_indices = [
self.vocab.get_token_index(x, "tags") for x in bio_tags
]
gold_tensor = torch.Tensor([gold_indices])
prediction_tensor = torch.rand(
[1, 6, self.vocab.get_vocab_size("tags")])
mask = torch.LongTensor([[1, 1, 1, 1, 1, 1, 1, 1, 1]])
# Make prediction so that it is exactly correct.
for i, tag_index in enumerate(gold_indices):
prediction_tensor[0, i, tag_index] = 1
metric = SpanBasedF1Measure(self.vocab, "tags")
metric(prediction_tensor, gold_tensor, mask)
metric_dict = metric.get_metric()
# We merged the continued ARG1 label into a single span, so there should
# be exactly 1 true positive for ARG1 and nothing present for C-ARG1
assert metric._true_positives["ARG1"] == 1
# The labels containing continuation references get merged into
# the labels that they continue, so they should never appear in
# the precision/recall counts.
assert "C-ARG1" not in metric._true_positives.keys()
assert metric._true_positives["V"] == 1
assert metric._true_positives["ARGM-ADJ"] == 1
numpy.testing.assert_almost_equal(metric_dict["recall-ARG1"], 1.0)
numpy.testing.assert_almost_equal(metric_dict["precision-ARG1"], 1.0)
numpy.testing.assert_almost_equal(metric_dict["f1-measure-ARG1"], 1.0)
numpy.testing.assert_almost_equal(metric_dict["recall-V"], 1.0)
numpy.testing.assert_almost_equal(metric_dict["precision-V"], 1.0)
numpy.testing.assert_almost_equal(metric_dict["f1-measure-V"], 1.0)
numpy.testing.assert_almost_equal(metric_dict["recall-ARGM-ADJ"], 1.0)
numpy.testing.assert_almost_equal(metric_dict["precision-ARGM-ADJ"],
1.0)
numpy.testing.assert_almost_equal(metric_dict["f1-measure-ARGM-ADJ"],
1.0)
numpy.testing.assert_almost_equal(metric_dict["recall-overall"], 1.0)
numpy.testing.assert_almost_equal(metric_dict["precision-overall"],
1.0)
numpy.testing.assert_almost_equal(metric_dict["f1-measure-overall"],
1.0)
# Check that the number of true positive ARG1 labels is the same as the perl script's output:
gold_file_path = os.path.join(self.TEST_DIR, "gold_conll_eval.txt")
prediction_file_path = os.path.join(self.TEST_DIR,
"prediction_conll_eval.txt")
with open(gold_file_path,
"a+") as gold_file, open(prediction_file_path,
"a+") as prediction_file:
# Use the same bio tags as prediction vs gold to make it obvious by looking
# at the perl script output if something is wrong.
write_to_conll_eval_file(gold_file, prediction_file, 4, sentence,
bio_tags, bio_tags)
# Run the official perl script and collect stdout.
perl_script_command = [
"perl",
str(self.TOOLS_ROOT / "srl-eval.pl"), prediction_file_path,
gold_file_path
]
stdout = subprocess.check_output(perl_script_command,
universal_newlines=True)
stdout_lines = stdout.split("\n")
# Parse the stdout of the perl script to find the ARG1 row (this happens to be line 8).
num_correct_arg1_instances_from_perl_evaluation = int(
[token for token in stdout_lines[8].split(" ") if token][1])
assert num_correct_arg1_instances_from_perl_evaluation == metric._true_positives[
"ARG1"]
def __init__(
self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
label_namespace: str = "labels",
label_encoding: Optional[str] = None,
include_start_end_transitions: bool = True,
calculate_span_f1: bool = None,
dropout: Optional[float] = None,
tcn_level: Optional[int] = None,
tcn_input_size: Optional[int] = None,
kernel_size: Optional[int] = None,
tcn_hidden_size: Optional[int] = None,
verbose_metrics: bool = False,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None,
) -> None:
super().__init__(vocab, regularizer)
self.label_namespace = label_namespace
self.text_field_embedder = text_field_embedder
self.num_tags = self.vocab.get_vocab_size(label_namespace)
self._verbose_metrics = verbose_metrics
if dropout:
self.dropout = torch.nn.Dropout(dropout)
else:
self.dropout = None
self.tcn_level = tcn_level
self.tcn_input_size = tcn_input_size
self.kernel_size = kernel_size
self.tcn_hidden_size = tcn_hidden_size
self.num_channels = [self.tcn_hidden_size] * self.tcn_level
self.tag_projection_layer = TimeDistributed(
Linear(self.tcn_hidden_size, self.num_tags))
if calculate_span_f1 is None:
calculate_span_f1 = label_encoding is not None
self.label_encoding = label_encoding
self.include_start_end_transitions = include_start_end_transitions
self.tcn = tch_layer.TemporalConvNet(self.tcn_input_size,
self.num_channels,
kernel_size,
dropout=dropout)
self.metrics = {
"accuracy": CategoricalAccuracy(),
"accuracy3": CategoricalAccuracy(top_k=3),
}
self.calculate_span_f1 = calculate_span_f1
if calculate_span_f1:
if not label_encoding:
raise ConfigurationError("calculate_span_f1 is True, but "
"no label_encoding was specified.")
self._f1_metric = SpanBasedF1Measure(vocab,
tag_namespace=label_namespace,
label_encoding=label_encoding)
initializer(self)
def __init__(
self,
backbone: ModelBackbone,
labels: List[str],
label_encoding: Optional[str] = "BIOUL",
top_k: int = 1,
dropout: Optional[float] = 0.0,
feedforward: Optional[FeedForwardConfiguration] = None,
) -> None:
super(TokenClassification, self).__init__(backbone)
self._label_encoding = label_encoding
# Convert span labels to tag labels if necessary
# We just check if "O" is in the label list, a necessary tag for IOB/BIOUL schemes, an unlikely label for spans
if "O" not in labels and "o" not in labels:
labels = span_labels_to_tag_labels(labels, self._label_encoding)
# Issue a warning if you have the "O" tag but no other BIO/BIOUL looking tags.
elif not any([
label.lower().startswith(tag) for label in labels
for tag in ["b-", "i-"]
]):
self.__LOGGER.warning(
"We interpreted the 'O' label as tag label, but did not find a 'B' or 'I' tag."
"Make sure your tag labels comply with the BIO/BIOUL tagging scheme."
)
vocabulary.set_labels(self.backbone.vocab, labels)
self.top_k = top_k
self.dropout = torch.nn.Dropout(dropout)
self._feedforward: FeedForward = (
None if not feedforward else feedforward.input_dim(
backbone.encoder.get_output_dim()).compile())
# output layers
self._classifier_input_dim = (self._feedforward.get_output_dim()
if self._feedforward else
backbone.encoder.get_output_dim())
# we want this linear applied to each token in the sequence
self._label_projection_layer = TimeDistributed(
torch.nn.Linear(self._classifier_input_dim, self.num_labels))
constraints = allowed_transitions(
self._label_encoding,
vocabulary.get_index_to_labels_dictionary(self.backbone.vocab),
)
self._crf = ConditionalRandomField(self.num_labels,
constraints,
include_start_end_transitions=True)
self.metrics = {"accuracy": CategoricalAccuracy()}
if self.top_k:
self.metrics.update({
f"accuracy_{self.top_k}":
CategoricalAccuracy(top_k=self.top_k)
})
self.f1_metric = SpanBasedF1Measure(
self.backbone.vocab,
tag_namespace=vocabulary.LABELS_NAMESPACE,
label_encoding=self._label_encoding,
)
self.__all_metrics = [self.f1_metric]
self.__all_metrics.extend(self.metrics.values())
def __init__(self, vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
label_namespace: str = "labels",
label_encoding: Optional[str] = None,
include_start_end_transitions: bool = True,
constrain_crf_decoding: bool = None,
calculate_span_f1: bool = None,
dropout: float = 0.1,
verbose_metrics: bool = False,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super().__init__(vocab, regularizer)
self.label_namespace = label_namespace
self.text_field_embedder = text_field_embedder
self.num_tags = self.vocab.get_vocab_size(label_namespace)
self._verbose_metrics = verbose_metrics
self.dropout = torch.nn.Dropout(dropout)
self.tag_projection_layer = TimeDistributed(
Linear(self.text_field_embedder.get_output_dim(), self.num_tags)
)
# if constrain_crf_decoding and calculate_span_f1 are not
# provided, (i.e., they're None), set them to True
# if label_encoding is provided and False if it isn't.
if constrain_crf_decoding is None:
constrain_crf_decoding = label_encoding is not None
if calculate_span_f1 is None:
calculate_span_f1 = label_encoding is not None
self.label_encoding = label_encoding
if constrain_crf_decoding:
if not label_encoding:
raise ConfigurationError("constrain_crf_decoding is True, but "
"no label_encoding was specified.")
labels = self.vocab.get_index_to_token_vocabulary(label_namespace)
constraints = allowed_transitions(label_encoding, labels)
else:
constraints = None
self.include_start_end_transitions = include_start_end_transitions
self.crf = ConditionalRandomField(
self.num_tags, constraints,
include_start_end_transitions=include_start_end_transitions
)
self.metrics = {
"accuracy": CategoricalAccuracy(),
"accuracy3": CategoricalAccuracy(top_k=3)
}
for index, label in self.vocab.get_index_to_token_vocabulary(label_namespace).items():
self.metrics['F1_' + label] = F1Measure(positive_label=index)
self.calculate_span_f1 = calculate_span_f1
if calculate_span_f1:
if not label_encoding:
raise ConfigurationError("calculate_span_f1 is True, but "
"no label_encoding was specified.")
self._f1_metric = SpanBasedF1Measure(vocab,
tag_namespace=label_namespace,
label_encoding=label_encoding)
initializer(self)
def __init__(
self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
label_namespace: str = "labels",
constraint_type: str = None,
include_start_end_transitions: bool = True,
dropout: float = None,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None,
cuda_device: int = -1,
) -> None:
super().__init__(vocab, regularizer)
self.label_namespace = label_namespace
self.text_field_embedder = text_field_embedder
# This is our trainable parameter that is used as logit of 'O'-tag
self.bias_outside = torch.nn.Parameter(torch.zeros(1) - 4.0,
requires_grad=True)
self.num_tags = self.vocab.get_vocab_size(label_namespace)
# We also train scales in the embedding space for every class assuming that they may be different.
self.scale_classes = torch.nn.Parameter(torch.ones(self.num_tags),
requires_grad=True)
self.encoder = encoder
if dropout:
self.dropout = torch.nn.Dropout(dropout)
else:
self.dropout = None
self.last_layer = TimeDistributed(
Linear(self.encoder.get_output_dim(), 64))
if constraint_type is not None:
labels = self.vocab.get_index_to_token_vocabulary(label_namespace)
constraints = allowed_transitions(constraint_type, labels)
else:
constraints = None
self.crf = ConditionalRandomField(
self.num_tags,
constraints,
include_start_end_transitions=include_start_end_transitions,
)
self.loss = torch.nn.CrossEntropyLoss()
self.cuda_device = cuda_device
if self.cuda_device >= 0:
self.text_field_embedder = self.text_field_embedder.cuda(
self.cuda_device)
self.encoder = self.encoder.cuda(self.cuda_device)
self.last_layer = self.last_layer.cuda(self.cuda_device)
self.elmo_weight = torch.nn.Parameter(torch.ones(1).cuda(
self.cuda_device),
requires_grad=True)
self.span_metric = SpanBasedF1Measure(
vocab,
tag_namespace=label_namespace,
label_encoding=constraint_type or "BIO",
)
check_dimensions_match(
text_field_embedder.get_output_dim(),
encoder.get_input_dim(),
"text field embedding dim",
"encoder input dim",
)
initializer(self)
self.hash = 0
self.number_epoch = 0
def __init__(self,
vocab: Vocabulary,
token_representation_dim: int,
encoder: Optional[Seq2SeqEncoder] = None,
decoder: Optional[Union[FeedForward, str]] = None,
use_crf: bool = False,
constrain_crf_decoding: bool = False,
include_start_end_transitions: bool = True,
label_encoding: Optional[str] = None,
contextualizer: Optional[Contextualizer] = None,
calculate_per_label_f1: bool = False,
calculate_span_f1: bool = False,
calculate_perplexity: bool = False,
loss_average: str = "batch",
pretrained_file: Optional[str] = None,
transfer_contextualizer_from_pretrained_file: bool = False,
transfer_encoder_from_pretrained_file: bool = False,
freeze_encoder: bool = False,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super(Tagger, self).__init__(vocab, regularizer)
self._num_classes = self.vocab.get_vocab_size("labels")
self._token_representation_dim = token_representation_dim
self._contextualizer = contextualizer
if encoder is None:
encoder = PassThroughEncoder(input_dim=token_representation_dim)
self._encoder = encoder
# Load the contextualizer and encoder weights from the
# pretrained_file if applicable
if pretrained_file:
archive = None
if self._contextualizer and transfer_contextualizer_from_pretrained_file:
logger.info("Attempting to load contextualizer weights from "
"pretrained_file at {}".format(pretrained_file))
archive = load_archive(cached_path(pretrained_file))
contextualizer_state = archive.model._contextualizer.state_dict()
contextualizer_layer_num = self._contextualizer._layer_num
logger.info("contextualizer_layer_num {}".format(contextualizer_layer_num))
self._contextualizer.load_state_dict(contextualizer_state)
if contextualizer_layer_num is not None:
logger.info("Setting layer num to {}".format(
contextualizer_layer_num))
self._contextualizer.set_layer_num(contextualizer_layer_num)
else:
self._contextualizer.reset_layer_num()
logger.info("Successfully loaded contextualizer weights!")
if transfer_encoder_from_pretrained_file:
logger.info("Attempting to load encoder weights from "
"pretrained_file at {}".format(pretrained_file))
if archive is None:
archive = load_archive(cached_path(pretrained_file))
encoder_state = archive.model._encoder.state_dict()
self._encoder.load_state_dict(encoder_state)
logger.info("Successfully loaded encoder weights!")
self._freeze_encoder = freeze_encoder
for parameter in self._encoder.parameters():
# If freeze is true, requires_grad should be false and vice versa.
parameter.requires_grad_(not self._freeze_encoder)
if decoder is None or decoder == "linear":
# Create the default decoder (logistic regression) if it is not provided.
decoder = FeedForward.from_params(Params(
{"input_dim": self._encoder.get_output_dim(),
"num_layers": 1,
"hidden_dims": self._num_classes,
"activations": "linear"}))
logger.info("No decoder provided to model, using default "
"decoder: {}".format(decoder))
elif decoder == "mlp":
# Create the MLP decoder
decoder = FeedForward.from_params(Params(
{"input_dim": self._encoder.get_output_dim(),
"num_layers": 2,
"hidden_dims": [1024, self._num_classes],
"activations": ["relu", "linear"]}))
logger.info("Using MLP decoder: {}".format(decoder))
self._decoder = TimeDistributed(decoder)
self._use_crf = use_crf
self._constrain_crf_decoding = constrain_crf_decoding
self._crf = None
if use_crf:
logger.info("Using CRF on top of decoder outputs")
if constrain_crf_decoding:
if label_encoding is None:
raise ConfigurationError(
"constrain_crf_decoding is True, but "
"label_encoding was not provided. label_encoding "
"must be provided.")
logger.info("Constraining CRF decoding with label "
"encoding {}".format(label_encoding))
labels = self.vocab.get_index_to_token_vocabulary("labels")
constraints = allowed_transitions(label_encoding, labels)
else:
constraints = None
self._crf = ConditionalRandomField(
self._num_classes, constraints,
include_start_end_transitions=include_start_end_transitions)
check_dimensions_match(self._token_representation_dim, self._encoder.get_input_dim(),
"dimensionality of token representation", "encoder input dim")
check_dimensions_match(self._encoder.get_output_dim(), self._decoder._module.get_input_dim(),
"encoder output dim", "decoder input dim")
check_dimensions_match(self._decoder._module.get_output_dim(), self._num_classes,
"decoder output dim", "number of classes")
if loss_average not in {"batch", "token"}:
raise ConfigurationError("loss_average is {}, expected one of batch "
"or token".format(loss_average))
self.loss_average = loss_average
self.metrics = {
"accuracy": CategoricalAccuracy(),
"accuracy3": CategoricalAccuracy(top_k=3)
}
self.calculate_perplexity = calculate_perplexity
if calculate_perplexity:
self.metrics["perplexity"] = Perplexity()
self.calculate_per_label_f1 = calculate_per_label_f1
self.calculate_span_f1 = calculate_span_f1
if label_encoding and label_encoding not in ["BIO", "BIOUL", "IOB1"]:
raise ConfigurationError("If not None, label encoding must be one of BIO, BIOUL, "
"or IOB1. Got {}".format(label_encoding))
self.label_encoding = label_encoding
label_metric_name = "label_{}" if self.calculate_per_label_f1 else "_label_{}"
for label_name, label_index in self.vocab._token_to_index["labels"].items():
self.metrics[label_metric_name.format(label_name)] = F1Measure(positive_label=label_index)
if self.calculate_span_f1:
if not self.label_encoding:
raise ConfigurationError("label_encoding must be provided when "
"calculating_span_f1 is true.")
else:
# Set up span-based F1 measure
self.metrics["span_based_f1"] = SpanBasedF1Measure(self.vocab,
tag_namespace="labels",
label_encoding=self.label_encoding)
# Whether to run in error analysis mode or not, see commands.error_analysis
self.error_analysis = False
logger.info("Applying initializer...")
initializer(self)
def __init__(self, vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
intent_encoder: Seq2SeqEncoder = None,
tag_encoder: Seq2SeqEncoder = None,
attention: Attention = None,
attention_function: SimilarityFunction = None,
context_for_intent: bool = True,
context_for_tag: bool = True,
attention_for_intent: bool = True,
attention_for_tag: bool = True,
sequence_label_namespace: str = "labels",
intent_label_namespace: str = "intent_labels",
feedforward: Optional[FeedForward] = None,
label_encoding: Optional[str] = None,
include_start_end_transitions: bool = True,
crf_decoding: bool = False,
constrain_crf_decoding: bool = None,
focal_loss_gamma: float = None,
nongeneral_intent_weight: float = 5.,
num_train_examples: float = None,
calculate_span_f1: bool = None,
dropout: Optional[float] = None,
verbose_metrics: bool = False,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super().__init__(vocab, regularizer)
self.context_for_intent = context_for_intent
self.context_for_tag = context_for_tag
self.attention_for_intent = attention_for_intent
self.attention_for_tag = attention_for_tag
self.sequence_label_namespace = sequence_label_namespace
self.intent_label_namespace = intent_label_namespace
self.text_field_embedder = text_field_embedder
self.num_tags = self.vocab.get_vocab_size(sequence_label_namespace)
self.num_intents = self.vocab.get_vocab_size(intent_label_namespace)
self.encoder = encoder
self.intent_encoder = intent_encoder
self.tag_encoder = intent_encoder
self._feedforward = feedforward
self._verbose_metrics = verbose_metrics
self.rl = False
if attention:
if attention_function:
raise ConfigurationError("You can only specify an attention module or an "
"attention function, but not both.")
self.attention = attention
elif attention_function:
self.attention = LegacyAttention(attention_function)
if dropout:
self.dropout = torch.nn.Dropout(dropout)
else:
self.dropout = None
projection_input_dim = feedforward.get_output_dim() if self._feedforward else self.encoder.get_output_dim()
if self.context_for_intent:
projection_input_dim += self.encoder.get_output_dim()
if self.attention_for_intent:
projection_input_dim += self.encoder.get_output_dim()
self.intent_projection_layer = Linear(projection_input_dim, self.num_intents)
if num_train_examples:
try:
pos_weight = torch.tensor([log10((num_train_examples - self.vocab._retained_counter[intent_label_namespace][t]) /
self.vocab._retained_counter[intent_label_namespace][t]) for i, t in
self.vocab.get_index_to_token_vocabulary(intent_label_namespace).items()])
except:
pos_weight = torch.tensor([1. for i, t in
self.vocab.get_index_to_token_vocabulary(intent_label_namespace).items()])
else:
# pos_weight = torch.tensor([(lambda t: 1. if "general" in t else nongeneral_intent_weight)(t) for i, t in
pos_weight = torch.tensor([(lambda t: nongeneral_intent_weight if "Request" in t else 1.)(t) for i, t in
self.vocab.get_index_to_token_vocabulary(intent_label_namespace).items()])
self.intent_loss = torch.nn.BCEWithLogitsLoss(pos_weight=pos_weight, reduction="none")
tag_projection_input_dim = feedforward.get_output_dim() if self._feedforward else self.encoder.get_output_dim()
if self.context_for_tag:
tag_projection_input_dim += self.encoder.get_output_dim()
if self.attention_for_tag:
tag_projection_input_dim += self.encoder.get_output_dim()
self.tag_projection_layer = TimeDistributed(Linear(tag_projection_input_dim,
self.num_tags))
# if constrain_crf_decoding and calculate_span_f1 are not
# provided, (i.e., they're None), set them to True
# if label_encoding is provided and False if it isn't.
if constrain_crf_decoding is None:
constrain_crf_decoding = label_encoding is not None
if calculate_span_f1 is None:
calculate_span_f1 = label_encoding is not None
self.label_encoding = label_encoding
if constrain_crf_decoding:
if not label_encoding:
raise ConfigurationError("constrain_crf_decoding is True, but "
"no label_encoding was specified.")
labels = self.vocab.get_index_to_token_vocabulary(sequence_label_namespace)
constraints = allowed_transitions(label_encoding, labels)
else:
constraints = None
self.include_start_end_transitions = include_start_end_transitions
if crf_decoding:
self.crf = ConditionalRandomField(
self.num_tags, constraints,
include_start_end_transitions=include_start_end_transitions
)
else:
self.crf = None
self._intent_f1_metric = MultiLabelF1Measure(vocab,
namespace=intent_label_namespace)
self.calculate_span_f1 = calculate_span_f1
if calculate_span_f1:
if not label_encoding:
raise ConfigurationError("calculate_span_f1 is True, but "
"no label_encoding was specified.")
self._f1_metric = SpanBasedF1Measure(vocab,
tag_namespace=sequence_label_namespace,
label_encoding=label_encoding)
self._dai_f1_metric = DialogActItemF1Measure()
check_dimensions_match(text_field_embedder.get_output_dim(), encoder.get_input_dim(),
"text field embedding dim", "encoder input dim")
if feedforward is not None:
check_dimensions_match(encoder.get_output_dim(), feedforward.get_input_dim(),
"encoder output dim", "feedforward input dim")
initializer(self)
def __init__(self,
vocab: Vocabulary,
text_field_embedder: list,
encoder: list,
label_namespace: str = "labels",
constraint_type: str = None,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super().__init__(vocab, regularizer)
self.losses = []
self.copy_parameters = [[] for _ in range(len(encoder))]
self.n_copies = len(self.copy_parameters) - 1
self.label_namespace = label_namespace
self.text_field_embedders = text_field_embedder
# self.text_field_embedder = text_field_embedder[-1]
# for i in range(len(text_field_embedder)):
# self.copy_parameters[i] += [w for w in text_field_embedder[i].parameters()]
self.num_tags = self.vocab.get_vocab_size(label_namespace)
self.encoders = encoder
self.encoder = encoder[-1]
for i in range(len(encoder)):
self.copy_parameters[i] += [w for w in encoder[i].parameters()]
for em in self.text_field_embedders:
em.cuda(1)
self.tag_projection_layers = [
TimeDistributed(
Linear(self.encoders[0].get_output_dim(), self.num_tags))
for _ in range(len(encoder))
]
self.tag_projection_layer = self.tag_projection_layers[-1]
for i in range(len(self.tag_projection_layers)):
self.copy_parameters[i] += [
w for w in self.tag_projection_layers[i].parameters()
]
if constraint_type is not None:
labels = self.vocab.get_index_to_token_vocabulary(label_namespace)
constraints = allowed_transitions(constraint_type, labels)
else:
constraints = None
self.crfs = [
ConditionalRandomField(self.num_tags, constraints)
for _ in range(len(encoder))
]
self.crf = self.crfs[-1]
for i in range(len(self.crfs)):
self.copy_parameters[i] += [w for w in self.crfs[i].parameters()]
self.span_metric = SpanBasedF1Measure(vocab,
tag_namespace=label_namespace)
check_dimensions_match(text_field_embedder[0].get_output_dim(),
encoder[0].get_input_dim(),
"text field embedding dim", "encoder input dim")
initializer(self)
self.optimizers = [
torch.optim.Adam(self.copy_parameters[i], 0.0001)
for i in range(self.n_copies + 1)
]
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
pos_tag_embedding: Embedding = None,
pos_tag_loss: Optional[float] = None,
label_namespace: str = "labels",
encoder: Optional[Seq2SeqEncoder] = None,
feedforward: Optional[FeedForward] = None,
label_encoding: Optional[str] = None,
crf: bool = True,
include_start_end_transitions: bool = True,
constrain_crf_decoding: bool = None,
calculate_span_f1: bool = None,
dropout: Optional[float] = None,
verbose_metrics: bool = False,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super().__init__(vocab, regularizer)
if pos_tag_loss is not None or pos_tag_embedding is not None:
pos_tag_err = (f"Model uses POS tags but the Vocabulary {vocab} "
"does not contain `pos_tags` namespace")
if 'pos_tags' not in vocab._token_to_index:
raise ConfigurationError(pos_tag_err)
elif not len(vocab._token_to_index['pos_tags']):
raise ConfigurationError(pos_tag_err)
self.label_namespace = label_namespace
self.text_field_embedder = text_field_embedder
self.pos_tag_embedding = pos_tag_embedding
self.num_tags = self.vocab.get_vocab_size(label_namespace)
self.encoder = encoder
self._verbose_metrics = verbose_metrics
embedding_output_dim = self.text_field_embedder.get_output_dim()
if self.pos_tag_embedding is not None:
embedding_output_dim += self.pos_tag_embedding.get_output_dim()
if dropout is not None:
self.dropout = torch.nn.Dropout(dropout)
self.variational_dropout = InputVariationalDropout(dropout)
else:
self.dropout = None
self._feedforward = feedforward
if feedforward is not None:
output_dim = feedforward.get_output_dim()
elif encoder is not None:
output_dim = self.encoder.get_output_dim()
else:
output_dim = embedding_output_dim
self.tag_projection_layer = TimeDistributed(
Linear(output_dim, self.num_tags))
self.pos_tag_loss = pos_tag_loss
if self.pos_tag_loss:
self.num_pos_tags = self.vocab.get_vocab_size("pos_tags")
self.pos_tag_projection_layer = TimeDistributed(
Linear(output_dim, self.num_pos_tags))
self.pos_crf = None
if crf:
self.pos_crf = ConditionalRandomField(self.num_pos_tags, None,
False)
# if constrain_crf_decoding and calculate_span_f1 are not
# provided, (i.e., they're None), set them to True
# if label_encoding is provided and False if it isn't.
if crf:
if constrain_crf_decoding is None:
constrain_crf_decoding = label_encoding is not None
if calculate_span_f1 is None:
calculate_span_f1 = label_encoding is not None
self.label_encoding = label_encoding
if constrain_crf_decoding and crf:
if not label_encoding:
raise ConfigurationError("constrain_crf_decoding is True, but "
"no label_encoding was specified.")
labels = self.vocab.get_index_to_token_vocabulary(label_namespace)
constraints = allowed_transitions(label_encoding, labels)
else:
constraints = None
if crf:
self.include_start_end_transitions = include_start_end_transitions
self.crf = ConditionalRandomField(
self.num_tags,
constraints,
include_start_end_transitions=include_start_end_transitions)
else:
self.crf = None
self.metrics = {
"accuracy": CategoricalAccuracy(),
"accuracy3": CategoricalAccuracy(top_k=3)
}
self.calculate_span_f1 = calculate_span_f1
if calculate_span_f1 is not None:
if not label_encoding:
raise ConfigurationError("calculate_span_f1 is True, but "
"no label_encoding was specified.")
self._f1_metric = SpanBasedF1Measure(vocab,
tag_namespace=label_namespace,
label_encoding=label_encoding)
# If performing POS tagging would be good to keep updated on POS
# accuracy
if self.pos_tag_loss:
self.metrics['POS_accuracy'] = CategoricalAccuracy()
if encoder is not None:
check_dimensions_match(embedding_output_dim,
encoder.get_input_dim(),
"text field embedding dim",
"encoder input dim")
if feedforward is not None and encoder is not None:
check_dimensions_match(encoder.get_output_dim(),
feedforward.get_input_dim(),
"encoder output dim",
"feedforward input dim")
elif feedforward is not None and encoder is None:
check_dimensions_match(embedding_output_dim,
feedforward.get_input_dim(),
"text field output dim",
"feedforward input dim")
initializer(self)
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
span_extractor: SpanExtractor,
entity_embedder: TokenEmbedder,
trigger_embedder: TokenEmbedder,
hidden_dim: int,
loss_weight: float = 1.0,
trigger_gamma: float = None,
role_gamma: float = None,
positive_class_weight: float = 1.0,
triggers_namespace: str = 'trigger_labels',
roles_namespace: str = 'arg_role_labels',
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: RegularizerApplicator = None) -> None:
super().__init__(vocab=vocab, regularizer=regularizer)
self._triggers_namespace = triggers_namespace
self._roles_namespace = roles_namespace
self.num_trigger_classes = self.vocab.get_vocab_size(
triggers_namespace)
self.num_role_classes = self.vocab.get_vocab_size(roles_namespace)
self.hidden_dim = hidden_dim
self.loss_weight = loss_weight
self.trigger_gamma = trigger_gamma
self.role_gamma = role_gamma
self.text_field_embedder = text_field_embedder
self.encoder = encoder
self.entity_embedder = entity_embedder
self.trigger_embedder = trigger_embedder
self.span_extractor = span_extractor
self.trigger_projection = TimeDistributed(
Linear(self.encoder.get_output_dim(), self.num_trigger_classes))
self.trigger_to_hidden = Linear(
self.encoder.get_output_dim() +
self.trigger_embedder.get_output_dim(), self.hidden_dim)
self.entities_to_hidden = Linear(self.encoder.get_output_dim(),
self.hidden_dim)
self.hidden_bias = Parameter(torch.Tensor(self.hidden_dim))
torch.nn.init.normal_(self.hidden_bias)
self.hidden_to_roles = Linear(self.hidden_dim, self.num_role_classes)
self.trigger_accuracy = CategoricalAccuracy()
self.trigger_f1 = SpanBasedF1Measure(
vocab,
tag_namespace=triggers_namespace,
label_encoding="BIO",
ignore_classes=[NEGATIVE_TRIGGER_LABEL])
role_labels_to_idx = self.vocab.get_token_to_index_vocabulary(
namespace=roles_namespace)
evaluated_role_idxs = list(role_labels_to_idx.values())
evaluated_role_idxs.remove(role_labels_to_idx[NEGATIVE_ARGUMENT_LABEL])
self.role_accuracy = CategoricalAccuracy()
self.role_f1 = MicroFBetaMeasure(
average='micro', # Macro averaging in get_metrics
labels=evaluated_role_idxs)
# Trigger class weighting as done in JMEE repo
trigger_labels_to_idx = self.vocab\
.get_token_to_index_vocabulary(namespace=triggers_namespace)
self.trigger_class_weights = torch.ones(
len(trigger_labels_to_idx)) * positive_class_weight
self.trigger_class_weights[
trigger_labels_to_idx[NEGATIVE_TRIGGER_LABEL]] = 1.0
initializer(self)
def __init__(
self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
calculate_span_f1: bool = None,
label_encoding: Optional[str] = None,
label_namespace: str = "labels",
verbose_metrics: bool = False,
initializer: InitializerApplicator = InitializerApplicator(),
parameter_metrics: Dict[str, Metric] = {},
activation_metrics: Dict[str, Metric] = {},
infinity: float=1e3,
**kwargs,
) -> None:
super().__init__(vocab, **kwargs)
self.label_namespace = label_namespace
self.text_field_embedder = text_field_embedder
self.num_classes = self.vocab.get_vocab_size(label_namespace)
self.encoder = encoder
self._verbose_metrics = verbose_metrics
self.tag_projection_layer = TimeDistributed(
Linear(self.encoder.get_output_dim(), self.num_classes)
)
check_dimensions_match(
text_field_embedder.get_output_dim(),
encoder.get_input_dim(),
"text field embedding dim",
"encoder input dim",
)
self.metrics = {
"accuracy": CategoricalAccuracy(),
"accuracy3": CategoricalAccuracy(top_k=3),
}
# We keep calculate_span_f1 as a constructor argument for API consistency with
# the CrfTagger, even it is redundant in this class
# (label_encoding serves the same purpose).
if calculate_span_f1 is None:
calculate_span_f1 = label_encoding is not None
self.calculate_span_f1 = calculate_span_f1
if calculate_span_f1:
if not label_encoding:
raise ConfigurationError(
"calculate_span_f1 is True, but no label_encoding was specified."
)
self._f1_metric = SpanBasedF1Measure(
vocab, tag_namespace=label_namespace, label_encoding=label_encoding
)
else:
self._f1_metric = None
initializer(self)
self.parameter_metrics = parameter_metrics
self.activation_metrics = activation_metrics
self.infinity = infinity
def __init__(self, vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
user_utterance_encoder: Seq2VecEncoder,
prev_user_utterance_encoder: Seq2VecEncoder,
prev_sys_utterance_encoder: Seq2VecEncoder,
classifier_feedforward: FeedForward,
encoder: Seq2SeqEncoder,
calculate_span_f1: bool = None,
tag_encoding: Optional[str] = None,
tag_namespace: str = "tags",
verbose_metrics: bool = False,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super(IntentParamClassifier, self).__init__(vocab, regularizer)
# Intent task
self.text_field_embedder = text_field_embedder
self.label_num_classes = self.vocab.get_vocab_size("labels")
self.user_utterance_encoder = user_utterance_encoder
self.prev_user_utterance_encoder = prev_user_utterance_encoder
self.prev_sys_utterance_encoder = prev_sys_utterance_encoder
self.classifier_feedforward = classifier_feedforward
if text_field_embedder.get_output_dim() != user_utterance_encoder.get_input_dim():
raise ConfigurationError("The output dimension of the text_field_embedder must match the "
"input dimension of the user_utterance_encoder. Found {} and {}, "
"respectively.".format(text_field_embedder.get_output_dim(),
user_utterance_encoder.get_input_dim()))
if text_field_embedder.get_output_dim() != prev_user_utterance_encoder.get_input_dim():
raise ConfigurationError("The output dimension of the text_field_embedder must match the "
"input dimension of the prev_user_utterance_encoder. Found {} and {}, "
"respectively.".format(text_field_embedder.get_output_dim(),
prev_user_utterance_encoder.get_input_dim()))
if text_field_embedder.get_output_dim() != prev_sys_utterance_encoder.get_input_dim():
raise ConfigurationError("The output dimension of the text_field_embedder must match the "
"input dimension of the prev_sys_utterance_encoder. Found {} and {}, "
"respectively.".format(text_field_embedder.get_output_dim(),
prev_sys_utterance_encoder.get_input_dim()))
self.label_accuracy = CategoricalAccuracy()
self.label_f1_metrics = {}
for i in range(self.label_num_classes):
self.label_f1_metrics[vocab.get_token_from_index(index=i, namespace="labels")] = F1Measure(positive_label=i)
self.loss = torch.nn.CrossEntropyLoss()
# Param task
self.tag_namespace = tag_namespace
self.tag_num_classes = self.vocab.get_vocab_size(tag_namespace)
self.encoder = encoder
self._verbose_metrics = verbose_metrics
self.tag_projection_layer = TimeDistributed(Linear(self.encoder.get_output_dim(),
self.tag_num_classes))
check_dimensions_match(text_field_embedder.get_output_dim(), encoder.get_input_dim(),
"text field embedding dim", "encoder input dim")
# We keep calculate_span_f1 as a constructor argument for API consistency with
# the CrfTagger, even it is redundant in this class
# (tag_encoding serves the same purpose).
if calculate_span_f1 and not tag_encoding:
raise ConfigurationError("calculate_span_f1 is True, but "
"no tag_encoding was specified.")
self.tag_accuracy = CategoricalAccuracy()
if calculate_span_f1 or tag_encoding:
self._f1_metric = SpanBasedF1Measure(vocab,
tag_namespace=tag_namespace,
tag_encoding=tag_encoding)
else:
self._f1_metric = None
self.f1 = SpanBasedF1Measure(vocab, tag_namespace=tag_namespace)
self.tag_f1_metrics = {}
for k in range(self.tag_num_classes):
self.tag_f1_metrics[vocab.get_token_from_index(index=k, namespace=tag_namespace)] = F1Measure(
positive_label=k)
initializer(self)
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
label_namespace: str = "labels",
feedforward: Optional[FeedForward] = None,
label_encoding: Optional[str] = None,
include_start_end_transitions: bool = True,
constrain_crf_decoding: bool = None,
calculate_span_f1: bool = None,
dropout: Optional[float] = None,
verbose_metrics: bool = False,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None,
num_virtual_models: int = 0) -> None:
super().__init__(vocab, regularizer)
self.num_virtual_models = num_virtual_models
self.label_namespace = label_namespace
self.text_field_embedder = text_field_embedder
self.num_tags = self.vocab.get_vocab_size(label_namespace)
self.encoder = encoder
self._verbose_metrics = verbose_metrics
if dropout:
self.dropout = torch.nn.Dropout(dropout)
else:
self.dropout = None
self._feedforward = feedforward
if feedforward is not None:
output_dim = feedforward.get_output_dim()
else:
output_dim = self.encoder.get_output_dim()
self.tag_projection_layer = TimeDistributed(
Linear(output_dim, self.num_tags))
# if constrain_crf_decoding and calculate_span_f1 are not
# provided, (i.e., they're None), set them to True
# if label_encoding is provided and False if it isn't.
if constrain_crf_decoding is None:
constrain_crf_decoding = label_encoding is not None
if calculate_span_f1 is None:
calculate_span_f1 = label_encoding is not None
self.label_encoding = label_encoding
if constrain_crf_decoding:
if not label_encoding:
raise ConfigurationError("constrain_crf_decoding is True, but "
"no label_encoding was specified.")
labels = self.vocab.get_index_to_token_vocabulary(label_namespace)
constraints = allowed_transitions(label_encoding, labels)
else:
constraints = None
self.include_start_end_transitions = include_start_end_transitions
self.crf = ConditionalRandomField(
self.num_tags,
constraints,
include_start_end_transitions=include_start_end_transitions)
self.metrics = {
"accuracy": CategoricalAccuracy(),
"accuracy3": CategoricalAccuracy(top_k=3),
}
self.calculate_span_f1 = calculate_span_f1
if calculate_span_f1:
if not label_encoding:
raise ConfigurationError("calculate_span_f1 is True, but "
"no label_encoding was specified.")
self._f1_metric = SpanBasedF1Measure(vocab,
tag_namespace=label_namespace,
label_encoding=label_encoding)
check_dimensions_match(
text_field_embedder.get_output_dim(),
encoder.get_input_dim(),
"text field embedding dim",
"encoder input dim",
)
if feedforward is not None:
check_dimensions_match(
encoder.get_output_dim(),
feedforward.get_input_dim(),
"encoder output dim",
"feedforward input dim",
)
self.index_dict = {
"[pseudo1]": 0,
"[pseudo2]": 1,
"[pseudo3]": 2,
"[pseudo4]": 3,
"[pseudo5]": 4,
"[pseudo6]": 5,
"[pseudo7]": 6,
"[pseudo8]": 7,
"[pseudo9]": 8
}
self.orthogonal_embedding_emb = torch.nn.init.orthogonal_(
torch.empty(self.num_virtual_models,
text_field_embedder.get_output_dim(),
requires_grad=False)).float()
self.orthogonal_embedding_hidden = torch.nn.init.orthogonal_(
torch.empty(self.num_virtual_models,
encoder.get_output_dim(),
requires_grad=False)).float()
self.vocab = vocab
initializer(self)
validation_data_path))
validation_data = validation_and_test_dataset_reader.read(
validation_data_path)
val_generator = validation_iterator(validation_data,
num_epochs=1,
shuffle=False)
#
#ner_coefs, pos_coefs = utils.get_dd_coefs(params['model']['constraints_path'], model.vocab)
ner_coefs, pos_coefs = utils.get_dd_coefs(
params['dd_constraints']['config']
['mtl_implication']['constraints_path'],
model.vocab)
#
ner_metric = [
SpanBasedF1Measure(model.vocab,
tag_namespace="task1_labels",
ignore_classes=['O'])
for _ in range(args.dditer + 1)
]
pos_metric = [
CategoricalAccuracy()
for _ in range(args.dditer + 1)
]
global_num_violations = torch.zeros(args.dditer + 1)
total_words = 0
model.eval()
for (i, batch) in enumerate(val_generator):
if i % 10 == 0:
print('batch#', i)
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
label_namespace: str = "labels",
num_features: Optional[int] = None,
feedforward: Optional[FeedForward] = None,
label_encoding: Optional[str] = None,
include_start_end_transitions: bool = True,
constrain_crf_decoding: bool = None,
calculate_span_f1: bool = None,
dropout: Optional[float] = None,
verbose_metrics: bool = False,
ccm_decoder: Optional[ConstrainedConditionalModule] = None,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super().__init__(vocab, regularizer)
self.label_namespace = label_namespace
self.text_field_embedder = text_field_embedder
self.num_tags = self.vocab.get_vocab_size(label_namespace)
self.encoder = encoder
self._verbose_metrics = verbose_metrics
if dropout:
self.dropout = torch.nn.Dropout(dropout)
else:
self.dropout = None
self._feedforward = feedforward
if feedforward is not None:
output_dim = feedforward.get_output_dim()
else:
output_dim = self.encoder.get_output_dim()
self.tag_projection_layer = TimeDistributed(
Linear(output_dim, self.num_tags))
# Add a ccm decoder if specified
self._ccm_decoder = ccm_decoder
# if constrain_crf_decoding and calculate_span_f1 are not
# provided, (i.e., they're None), set them to True
# if label_encoding is provided and False if it isn't.
if constrain_crf_decoding is None:
constrain_crf_decoding = label_encoding is not None
if calculate_span_f1 is None:
calculate_span_f1 = label_encoding is not None
self.label_encoding = label_encoding
if constrain_crf_decoding:
if not label_encoding:
raise ConfigurationError("constrain_crf_decoding is True, but "
"no label_encoding was specified.")
labels = self.vocab.get_index_to_token_vocabulary(label_namespace)
constraints = allowed_transitions(label_encoding, labels)
else:
constraints = None
self.include_start_end_transitions = include_start_end_transitions
self.crf = ConditionalRandomField(
self.num_tags,
constraints,
include_start_end_transitions=include_start_end_transitions)
self.metrics = {
"accuracy": CategoricalAccuracy(),
"accuracy3": CategoricalAccuracy(top_k=3)
}
self.calculate_span_f1 = calculate_span_f1
if calculate_span_f1:
if not label_encoding:
raise ConfigurationError("calculate_span_f1 is True, but "
"no label_encoding was specified.")
self._f1_metric = SpanBasedF1Measure(vocab,
tag_namespace=label_namespace,
label_encoding=label_encoding)
num_features = num_features or 0
check_dimensions_match(
text_field_embedder.get_output_dim() + num_features,
encoder.get_input_dim(), "text field embedding dim",
"encoder input dim")
if feedforward is not None:
check_dimensions_match(encoder.get_output_dim(),
feedforward.get_input_dim(),
"encoder output dim",
"feedforward input dim")
if self._ccm_decoder:
assert self._ccm_decoder._transition_constraints == constraints
initializer(self)
def __init__(
self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
use_attention: bool = False,
use_positional_encoding: bool = False,
label_namespace: str = "labels",
feedforward: Optional[FeedForward] = None,
label_encoding: Optional[str] = None,
include_start_end_transitions: bool = True,
has_mode: bool = False,
constrain_crf_decoding: bool = None,
calculate_span_f1: bool = None,
calculate_relation_f1: bool = False,
dropout: Optional[float] = None,
verbose_metrics: bool = False,
initializer: InitializerApplicator = InitializerApplicator(),
top_k: int = 1,
max_relation_width:int = 11,
**kwargs,
) -> None:
super().__init__(vocab, **kwargs)
self.label_namespace = label_namespace
self.text_field_embedder = text_field_embedder
self.num_tags = self.vocab.get_vocab_size(label_namespace)
self.encoder = encoder
self.top_k = top_k
self._verbose_metrics = verbose_metrics
self.use_attention = use_attention
self.use_positional_encoding = use_positional_encoding
self._sample_probability = compounding(0.1, 1.0, 0.99)
self.has_mode = has_mode
if dropout:
self.dropout = torch.nn.Dropout(dropout)
else:
self.dropout = None
self._feedforward = feedforward
if feedforward is not None:
output_dim = feedforward.get_output_dim()
else:
output_dim = self.encoder.get_output_dim()
self.tag_projection_layer = TimeDistributed(Linear(output_dim, self.num_tags))
if self.use_attention:
self._attention = SelfAttentionGRU(
output_dim,
embedding_size=encoder.get_output_dim(),
rnn_hidden_size=encoder.get_output_dim(),
bos_index=self.vocab.get_token_index("O", label_namespace)
)
if self.use_positional_encoding:
self.positional_encoding = PositionalEncoding(d_model=encoder.get_output_dim(),dropout=dropout)
# if constrain_crf_decoding and calculate_span_f1 are not
# provided, (i.e., they're None), set them to True
# if label_encoding is provided and False if it isn't.
if constrain_crf_decoding is None:
constrain_crf_decoding = label_encoding is not None
if calculate_span_f1 is None:
calculate_span_f1 = label_encoding is not None
self.label_encoding = label_encoding
if constrain_crf_decoding:
if not label_encoding:
raise ConfigurationError(
"constrain_crf_decoding is True, but no label_encoding was specified."
)
labels = self.vocab.get_index_to_token_vocabulary(label_namespace)
constraints = allowed_transitions(label_encoding, labels)
else:
constraints = None
self.include_start_end_transitions = include_start_end_transitions
self.crf = ConditionalRandomField(
self.num_tags, constraints, include_start_end_transitions=include_start_end_transitions
)
self.metrics = {
"accuracy": CategoricalAccuracy(),
"accuracy3": CategoricalAccuracy(top_k=3),
}
self.calculate_span_f1 = calculate_span_f1
if calculate_span_f1:
if not label_encoding:
raise ConfigurationError(
"calculate_span_f1 is True, but no label_encoding was specified."
)
self._f1_metric = SpanBasedF1Measure(
vocab, tag_namespace=label_namespace, label_encoding=label_encoding
)
self.calculate_relation_f1 = calculate_relation_f1
if calculate_relation_f1:
self._relation_f1_metric = RelationMetric(
vocab, tag_namespace=label_namespace, label_encoding=label_encoding, has_mode=has_mode, max_relation_width=max_relation_width
)
check_dimensions_match(
text_field_embedder.get_output_dim(),
encoder.get_input_dim(),
"text field embedding dim",
"encoder input dim",
)
if feedforward is not None:
check_dimensions_match(
encoder.get_output_dim(),
feedforward.get_input_dim(),
"encoder output dim",
"feedforward input dim",
)
self.j = 0
initializer(self)
def test_span_metrics_are_computed_correctly(self, device: str):
gold_labels = [
"O", "B-ARG1", "I-ARG1", "O", "B-ARG2", "I-ARG2", "O", "O", "O"
]
gold_indices = [
self.vocab.get_token_index(x, "tags") for x in gold_labels
]
gold_tensor = torch.tensor([gold_indices], device=device)
prediction_tensor = torch.rand(
[2, 9, self.vocab.get_vocab_size("tags")], device=device)
# Test that the span measure ignores completely masked sequences by
# passing a mask with a fully masked row.
mask = torch.tensor(
[
[True, True, True, True, True, True, True, True, True],
[
False, False, False, False, False, False, False, False,
False
],
],
device=device,
)
prediction_tensor[:, 0, 0] = 1
prediction_tensor[:, 1, 1] = 1 # (True positive - ARG1
prediction_tensor[:, 2, 2] = 1 # *)
prediction_tensor[:, 3, 0] = 1
prediction_tensor[:, 4, 0] = 1 # (False Negative - ARG2
prediction_tensor[:, 5, 0] = 1 # *)
prediction_tensor[:, 6, 0] = 1
prediction_tensor[:, 7, 1] = 1 # (False Positive - ARG1
prediction_tensor[:, 8, 2] = 1 # *)
metric = SpanBasedF1Measure(self.vocab, "tags")
metric(prediction_tensor, gold_tensor, mask)
assert metric._true_positives["ARG1"] == 1
assert metric._true_positives["ARG2"] == 0
assert "O" not in metric._true_positives.keys()
assert metric._false_negatives["ARG1"] == 0
assert metric._false_negatives["ARG2"] == 1
assert "O" not in metric._false_negatives.keys()
assert metric._false_positives["ARG1"] == 1
assert metric._false_positives["ARG2"] == 0
assert "O" not in metric._false_positives.keys()
# Check things are accumulating correctly.
metric(prediction_tensor, gold_tensor, mask)
assert metric._true_positives["ARG1"] == 2
assert metric._true_positives["ARG2"] == 0
assert "O" not in metric._true_positives.keys()
assert metric._false_negatives["ARG1"] == 0
assert metric._false_negatives["ARG2"] == 2
assert "O" not in metric._false_negatives.keys()
assert metric._false_positives["ARG1"] == 2
assert metric._false_positives["ARG2"] == 0
assert "O" not in metric._false_positives.keys()
metric_dict = metric.get_metric()
assert_allclose(metric_dict["recall-ARG2"], 0.0)
assert_allclose(metric_dict["precision-ARG2"], 0.0)
assert_allclose(metric_dict["f1-measure-ARG2"], 0.0)
assert_allclose(metric_dict["recall-ARG1"], 1.0)
assert_allclose(metric_dict["precision-ARG1"], 0.5)
assert_allclose(metric_dict["f1-measure-ARG1"], 0.666666666)
assert_allclose(metric_dict["recall-overall"], 0.5)
assert_allclose(metric_dict["precision-overall"], 0.5)
assert_allclose(metric_dict["f1-measure-overall"], 0.5)
def __init__(self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
encoder: Seq2SeqEncoder,
intent_encoder: Seq2SeqEncoder = None,
sequence_label_namespace: str = "labels",
intent_label_namespace: str = "intent_labels",
feedforward: Optional[FeedForward] = None,
label_encoding: Optional[str] = None,
include_start_end_transitions: bool = True,
crf_decoding: bool = False,
constrain_crf_decoding: bool = None,
focal_loss_gamma: float = None,
calculate_span_f1: bool = None,
dropout: Optional[float] = None,
verbose_metrics: bool = False,
initializer: InitializerApplicator = InitializerApplicator(),
regularizer: Optional[RegularizerApplicator] = None) -> None:
super().__init__(vocab, regularizer)
self.sequence_label_namespace = sequence_label_namespace
self.intent_label_namespace = intent_label_namespace
self.text_field_embedder = text_field_embedder
self.num_tags = self.vocab.get_vocab_size(sequence_label_namespace)
self.num_intents = self.vocab.get_vocab_size(intent_label_namespace)
self.encoder = encoder
self.intent_encoder = intent_encoder
self._verbose_metrics = verbose_metrics
if dropout:
self.dropout = torch.nn.Dropout(dropout)
else:
self.dropout = None
self._feedforward = feedforward
# if feedforward is not None:
# output_dim = feedforward.get_output_dim()
# else:
# output_dim = self.encoder.get_output_dim()
self.tag_projection_layer = TimeDistributed(
Linear(self.encoder.get_output_dim(), self.num_tags))
if self._feedforward is not None:
self.intent_projection_layer = Linear(feedforward.get_output_dim(),
self.num_intents)
else:
self.intent_projection_layer = Linear(
self.encoder.get_output_dim(), self.num_intents)
if focal_loss_gamma is not None:
self.intent_loss = FocalBCEWithLogitsLoss(gamma=focal_loss_gamma)
# self.intent_loss2 = torch.nn.BCEWithLogitsLoss()
else:
self.intent_loss = torch.nn.BCEWithLogitsLoss()
# if constrain_crf_decoding and calculate_span_f1 are not
# provided, (i.e., they're None), set them to True
# if label_encoding is provided and False if it isn't.
if constrain_crf_decoding is None:
constrain_crf_decoding = label_encoding is not None
if calculate_span_f1 is None:
calculate_span_f1 = label_encoding is not None
self.label_encoding = label_encoding
if constrain_crf_decoding:
if not label_encoding:
raise ConfigurationError("constrain_crf_decoding is True, but "
"no label_encoding was specified.")
labels = self.vocab.get_index_to_token_vocabulary(
sequence_label_namespace)
constraints = allowed_transitions(label_encoding, labels)
else:
constraints = None
self.include_start_end_transitions = include_start_end_transitions
if crf_decoding:
self.crf = ConditionalRandomField(
self.num_tags,
constraints,
include_start_end_transitions=include_start_end_transitions)
else:
self.crf = None
# self.metrics = {
# "int_acc": BinaryAccuracy(),
# "tag_acc": CategoricalAccuracy()
# }
self._intent_f1_metric = MultiLabelF1Measure(
vocab, namespace=intent_label_namespace)
self.calculate_span_f1 = calculate_span_f1
if calculate_span_f1:
if not label_encoding:
raise ConfigurationError("calculate_span_f1 is True, but "
"no label_encoding was specified.")
self._f1_metric = SpanBasedF1Measure(
vocab,
tag_namespace=sequence_label_namespace,
label_encoding=label_encoding)
self._dai_f1_metric = DialogActItemF1Measure()
check_dimensions_match(text_field_embedder.get_output_dim(),
encoder.get_input_dim(),
"text field embedding dim", "encoder input dim")
if feedforward is not None:
check_dimensions_match(encoder.get_output_dim(),
feedforward.get_input_dim(),
"encoder output dim",
"feedforward input dim")
initializer(self)