def __init__(self, tag_dictionary: Dictionary):
"""
:param tag_dictionary: Dictionary of tags for sequence labeling task
"""
self.tag_dictionary = tag_dictionary
self.tagset_size = len(tag_dictionary)
self.start_tag = tag_dictionary.get_idx_for_item(START_TAG)
self.stop_tag = tag_dictionary.get_idx_for_item(STOP_TAG)
def __init__(self, tag_dictionary: Dictionary):
"""
:param tag_dictionary: tag_dictionary of task
"""
super(ViterbiLoss, self).__init__()
self.tag_dictionary = tag_dictionary
self.tagset_size = len(tag_dictionary)
self.start_tag = tag_dictionary.get_idx_for_item(START_TAG)
self.stop_tag = tag_dictionary.get_idx_for_item(STOP_TAG)
def calculate_class_metrics(y_true: List[List[int]], y_pred: List[List[int]],
labels: Dictionary) -> List[Metric]:
"""
Calculates the metrics for the individual classes for the given predictions.
The labels should be converted into a one-hot-list.
:param y_true: list of true labels
:param y_pred: list of predicted labels
:param labels: the label dictionary
:return: the metrics for every class
"""
metrics = []
for label in labels.get_items():
metric = Metric(label)
label_idx = labels.get_idx_for_item(label)
for true, pred in zip(y_true, y_pred):
if true[label_idx] == 1 and pred[label_idx] == 1:
metric.tp()
elif true[label_idx] == 1 and pred[label_idx] == 0:
metric.fn()
elif true[label_idx] == 0 and pred[label_idx] == 1:
metric.fp()
elif true[label_idx] == 0 and pred[label_idx] == 0:
metric.tn()
metrics.append(metric)
return metrics
def test_dictionary_get_idx_for_item():
dictionary = Dictionary(add_unk=False)
dictionary.add_item('class_1')
dictionary.add_item('class_2')
dictionary.add_item('class_3')
idx = dictionary.get_idx_for_item('class_2')
assert (1 == idx)
def convert_labels_to_one_hot(label_list: List[List[str]], label_dict: Dictionary) -> List[List[int]]:
"""
Convert list of labels (strings) to a one hot list.
:param label_list: list of labels
:param label_dict: label dictionary
:return: converted label list
"""
converted_label_list = []
for labels in label_list:
arr = np.empty(len(label_dict))
arr.fill(0)
for label in labels:
arr[label_dict.get_idx_for_item(label)] = 1
converted_label_list.append(arr.tolist())
return converted_label_list
class SequenceTagger(flair.nn.Classifier[Sentence]):
def __init__(
self,
embeddings: TokenEmbeddings,
tag_dictionary: Dictionary,
tag_type: str,
use_rnn: bool = True,
rnn: Optional[torch.nn.RNN] = None,
rnn_type: str = "LSTM",
tag_format: str = "BIOES",
hidden_size: int = 256,
rnn_layers: int = 1,
bidirectional: bool = True,
use_crf: bool = True,
reproject_embeddings: bool = True,
dropout: float = 0.0,
word_dropout: float = 0.05,
locked_dropout: float = 0.5,
train_initial_hidden_state: bool = False,
loss_weights: Dict[str, float] = None,
init_from_state_dict: bool = False,
allow_unk_predictions: bool = False,
):
"""
Sequence Tagger class for predicting labels for single tokens. Can be parameterized by several attributes.
In case of multitask learning, pass shared embeddings or shared rnn into respective attributes.
:param embeddings: Embeddings to use during training and prediction
:param tag_dictionary: Dictionary containing all tags from corpus which can be predicted
:param tag_type: type of tag which is going to be predicted in case a corpus has multiple annotations
:param use_rnn: If true, use a RNN, else Linear layer.
:param rnn: (Optional) Takes a torch.nn.Module as parameter by which you can pass a shared RNN between
different tasks.
:param rnn_type: Specifies the RNN type to use, default is 'LSTM', can choose between 'GRU' and 'RNN' as well.
:param hidden_size: Hidden size of RNN layer
:param rnn_layers: number of RNN layers
:param bidirectional: If True, RNN becomes bidirectional
:param use_crf: If True, use a Conditional Random Field for prediction, else linear map to tag space.
:param reproject_embeddings: If True, add a linear layer on top of embeddings, if you want to imitate
fine tune non-trainable embeddings.
:param dropout: If > 0, then use dropout.
:param word_dropout: If > 0, then use word dropout.
:param locked_dropout: If > 0, then use locked dropout.
:param train_initial_hidden_state: if True, trains initial hidden state of RNN
:param loss_weights: Dictionary of weights for labels for the loss function
(if any label's weight is unspecified it will default to 1.0)
:param init_from_state_dict: Indicator whether we are loading a model from state dict
since we need to transform previous models' weights into CRF instance weights
"""
super(SequenceTagger, self).__init__()
# ----- Create the internal tag dictionary -----
self.tag_type = tag_type
self.tag_format = tag_format.upper()
if init_from_state_dict:
self.label_dictionary = tag_dictionary
else:
# span-labels need special encoding (BIO or BIOES)
if tag_dictionary.span_labels:
# the big question is whether the label dictionary should contain an UNK or not
# without UNK, we cannot evaluate on data that contains labels not seen in test
# with UNK, the model learns less well if there are no UNK examples
self.label_dictionary = Dictionary(
add_unk=allow_unk_predictions)
for label in tag_dictionary.get_items():
if label == "<unk>":
continue
self.label_dictionary.add_item("O")
if tag_format == "BIOES":
self.label_dictionary.add_item("S-" + label)
self.label_dictionary.add_item("B-" + label)
self.label_dictionary.add_item("E-" + label)
self.label_dictionary.add_item("I-" + label)
if tag_format == "BIO":
self.label_dictionary.add_item("B-" + label)
self.label_dictionary.add_item("I-" + label)
else:
self.label_dictionary = tag_dictionary
# is this a span prediction problem?
self.predict_spans = self._determine_if_span_prediction_problem(
self.label_dictionary)
self.tagset_size = len(self.label_dictionary)
log.info(f"SequenceTagger predicts: {self.label_dictionary}")
# ----- Embeddings -----
self.embeddings = embeddings
embedding_dim: int = embeddings.embedding_length
# ----- Initial loss weights parameters -----
self.weight_dict = loss_weights
self.loss_weights = self._init_loss_weights(
loss_weights) if loss_weights else None
# ----- RNN specific parameters -----
self.use_rnn = use_rnn
self.rnn_type = rnn_type if not rnn else rnn._get_name()
self.hidden_size = hidden_size if not rnn else rnn.hidden_size
self.rnn_layers = rnn_layers if not rnn else rnn.num_layers
self.bidirectional = bidirectional if not rnn else rnn.bidirectional
# ----- Conditional Random Field parameters -----
self.use_crf = use_crf
# Previously trained models have been trained without an explicit CRF, thus it is required to check
# whether we are loading a model from state dict in order to skip or add START and STOP token
if use_crf and not init_from_state_dict and not self.label_dictionary.start_stop_tags_are_set(
):
self.label_dictionary.set_start_stop_tags()
self.tagset_size += 2
# ----- Dropout parameters -----
# dropouts
self.use_dropout: float = dropout
self.use_word_dropout: float = word_dropout
self.use_locked_dropout: float = locked_dropout
if dropout > 0.0:
self.dropout = torch.nn.Dropout(dropout)
if word_dropout > 0.0:
self.word_dropout = flair.nn.WordDropout(word_dropout)
if locked_dropout > 0.0:
self.locked_dropout = flair.nn.LockedDropout(locked_dropout)
# ----- Model layers -----
self.reproject_embeddings = reproject_embeddings
if self.reproject_embeddings:
self.embedding2nn = torch.nn.Linear(embedding_dim, embedding_dim)
# ----- RNN layer -----
if use_rnn:
# If shared RNN provided, else create one for model
self.rnn: torch.nn.RNN = (rnn if rnn else self.RNN(
rnn_type,
rnn_layers,
hidden_size,
bidirectional,
rnn_input_dim=embedding_dim,
))
num_directions = 2 if self.bidirectional else 1
hidden_output_dim = self.rnn.hidden_size * num_directions
# Whether to train initial hidden state
self.train_initial_hidden_state = train_initial_hidden_state
if self.train_initial_hidden_state:
(
self.hs_initializer,
self.lstm_init_h,
self.lstm_init_c,
) = self._init_initial_hidden_state(num_directions)
# final linear map to tag space
self.linear = torch.nn.Linear(hidden_output_dim,
len(self.label_dictionary))
else:
self.linear = torch.nn.Linear(embedding_dim,
len(self.label_dictionary))
# the loss function is Viterbi if using CRF, else regular Cross Entropy Loss
self.loss_function = (ViterbiLoss(self.label_dictionary)
if use_crf else torch.nn.CrossEntropyLoss(
weight=self.loss_weights, reduction="sum"))
# if using CRF, we also require a CRF and a Viterbi decoder
if use_crf:
self.crf = CRF(self.label_dictionary, self.tagset_size,
init_from_state_dict)
self.viterbi_decoder = ViterbiDecoder(self.label_dictionary)
self.to(flair.device)
@property
def label_type(self):
return self.tag_type
def _init_loss_weights(self, loss_weights: Dict[str,
float]) -> torch.Tensor:
"""
Intializes the loss weights based on given dictionary:
:param loss_weights: dictionary - contains loss weights
"""
n_classes = len(self.label_dictionary)
weight_list = [1.0 for _ in range(n_classes)]
for i, tag in enumerate(self.label_dictionary.get_items()):
if tag in loss_weights.keys():
weight_list[i] = loss_weights[tag]
return torch.tensor(weight_list).to(flair.device)
def _init_initial_hidden_state(self, num_directions: int):
"""
Intializes hidden states given the number of directions in RNN.
:param num_directions: Number of directions in RNN.
"""
hs_initializer = torch.nn.init.xavier_normal_
lstm_init_h = torch.nn.Parameter(
torch.randn(self.rnn.num_layers * num_directions,
self.hidden_size),
requires_grad=True,
)
lstm_init_c = torch.nn.Parameter(
torch.randn(self.rnn.num_layers * num_directions,
self.hidden_size),
requires_grad=True,
)
return hs_initializer, lstm_init_h, lstm_init_c
@staticmethod
def RNN(
rnn_type: str,
rnn_layers: int,
hidden_size: int,
bidirectional: bool,
rnn_input_dim: int,
) -> torch.nn.RNN:
"""
Static wrapper function returning an RNN instance from PyTorch
:param rnn_type: Type of RNN from torch.nn
:param rnn_layers: number of layers to include
:param hidden_size: hidden size of RNN cell
:param bidirectional: If True, RNN cell is bidirectional
:param rnn_input_dim: Input dimension to RNN cell
"""
if rnn_type in ["LSTM", "GRU", "RNN"]:
RNN = getattr(torch.nn, rnn_type)(
rnn_input_dim,
hidden_size,
num_layers=rnn_layers,
dropout=0.0 if rnn_layers == 1 else 0.5,
bidirectional=bidirectional,
batch_first=True,
)
else:
raise Exception(
f"Unknown RNN type: {rnn_type}. Please use either LSTM, GRU or RNN."
)
return RNN
def forward_loss(
self, sentences: Union[List[Sentence],
Sentence]) -> Tuple[torch.Tensor, int]:
# if there are no sentences, there is no loss
if len(sentences) == 0:
return torch.tensor(0.0,
dtype=torch.float,
device=flair.device,
requires_grad=True), 0
# forward pass to get scores
scores, gold_labels = self.forward(sentences) # type: ignore
# calculate loss given scores and labels
return self._calculate_loss(scores, gold_labels)
def forward(self, sentences: Union[List[Sentence], Sentence]):
"""
Forward propagation through network. Returns gold labels of batch in addition.
:param sentences: Batch of current sentences
"""
if not isinstance(sentences, list):
sentences = [sentences]
self.embeddings.embed(sentences)
# make a zero-padded tensor for the whole sentence
lengths, sentence_tensor = self._make_padded_tensor_for_batch(
sentences)
# sort tensor in decreasing order based on lengths of sentences in batch
sorted_lengths, length_indices = lengths.sort(dim=0, descending=True)
sentences = [sentences[i] for i in length_indices]
sentence_tensor = sentence_tensor[length_indices]
# ----- Forward Propagation -----
if self.use_dropout:
sentence_tensor = self.dropout(sentence_tensor)
if self.use_word_dropout:
sentence_tensor = self.word_dropout(sentence_tensor)
if self.use_locked_dropout:
sentence_tensor = self.locked_dropout(sentence_tensor)
if self.reproject_embeddings:
sentence_tensor = self.embedding2nn(sentence_tensor)
if self.use_rnn:
packed = pack_padded_sequence(sentence_tensor,
sorted_lengths,
batch_first=True,
enforce_sorted=False)
rnn_output, hidden = self.rnn(packed)
sentence_tensor, output_lengths = pad_packed_sequence(
rnn_output, batch_first=True)
if self.use_dropout:
sentence_tensor = self.dropout(sentence_tensor)
if self.use_locked_dropout:
sentence_tensor = self.locked_dropout(sentence_tensor)
# linear map to tag space
features = self.linear(sentence_tensor)
# Depending on whether we are using CRF or a linear layer, scores is either:
# -- A tensor of shape (batch size, sequence length, tagset size, tagset size) for CRF
# -- A tensor of shape (aggregated sequence length for all sentences in batch, tagset size) for linear layer
if self.use_crf:
features = self.crf(features)
scores = (features, sorted_lengths, self.crf.transitions)
else:
scores = self._get_scores_from_features(features, sorted_lengths)
# get the gold labels
gold_labels = self._get_gold_labels(sentences)
return scores, gold_labels
def _calculate_loss(self, scores, labels) -> Tuple[torch.Tensor, int]:
if not any(labels):
return torch.tensor(0.0, requires_grad=True,
device=flair.device), 1
labels = torch.tensor(
[
self.label_dictionary.get_idx_for_item(label[0]) if
len(label) > 0 else self.label_dictionary.get_idx_for_item("O")
for label in labels
],
dtype=torch.long,
device=flair.device,
)
return self.loss_function(scores, labels), len(labels)
def _make_padded_tensor_for_batch(
self,
sentences: List[Sentence]) -> Tuple[torch.Tensor, torch.Tensor]:
names = self.embeddings.get_names()
lengths: List[int] = [len(sentence.tokens) for sentence in sentences]
longest_token_sequence_in_batch: int = max(lengths)
pre_allocated_zero_tensor = torch.zeros(
self.embeddings.embedding_length * longest_token_sequence_in_batch,
dtype=torch.float,
device=flair.device,
)
all_embs = list()
for sentence in sentences:
all_embs += [
emb for token in sentence
for emb in token.get_each_embedding(names)
]
nb_padding_tokens = longest_token_sequence_in_batch - len(sentence)
if nb_padding_tokens > 0:
t = pre_allocated_zero_tensor[:self.embeddings.
embedding_length *
nb_padding_tokens]
all_embs.append(t)
sentence_tensor = torch.cat(all_embs).view([
len(sentences),
longest_token_sequence_in_batch,
self.embeddings.embedding_length,
])
return torch.tensor(lengths, dtype=torch.long), sentence_tensor
@staticmethod
def _get_scores_from_features(features: torch.Tensor,
lengths: torch.Tensor):
"""
Trims current batch tensor in shape (batch size, sequence length, tagset size) in such a way that all
pads are going to be removed.
:param features: torch.tensor containing all features from forward propagation
:param lengths: length from each sentence in batch in order to trim padding tokens
"""
features_formatted = []
for feat, length in zip(features, lengths):
features_formatted.append(feat[:length])
scores = torch.cat(features_formatted)
return scores
def _get_gold_labels(self, sentences: Union[List[Sentence], Sentence]):
"""
Extracts gold labels from each sentence.
:param sentences: List of sentences in batch
"""
# spans need to be encoded as token-level predictions
if self.predict_spans:
all_sentence_labels = []
for sentence in sentences:
sentence_labels = ["O"] * len(sentence)
for label in sentence.get_labels(self.label_type):
span: Span = label.data_point
if len(span) == 1:
sentence_labels[span[0].idx - 1] = "S-" + label.value
else:
sentence_labels[span[0].idx - 1] = "B-" + label.value
sentence_labels[span[-1].idx - 1] = "E-" + label.value
for i in range(span[0].idx, span[-1].idx - 1):
sentence_labels[i] = "I-" + label.value
all_sentence_labels.extend(sentence_labels)
labels = [[label] for label in all_sentence_labels]
# all others are regular labels for each token
else:
labels = [[token.get_label(self.label_type, "O").value]
for sentence in sentences for token in sentence]
return labels
def predict(
self,
sentences: Union[List[Sentence], Sentence],
mini_batch_size: int = 32,
return_probabilities_for_all_classes: bool = False,
verbose: bool = False,
label_name: Optional[str] = None,
return_loss=False,
embedding_storage_mode="none",
):
"""
Predicts labels for current batch with CRF or Softmax.
:param sentences: List of sentences in batch
:param mini_batch_size: batch size for test data
:param return_probabilities_for_all_classes: Whether to return probabilites for all classes
:param verbose: whether to use progress bar
:param label_name: which label to predict
:param return_loss: whether to return loss value
:param embedding_storage_mode: determines where to store embeddings - can be "gpu", "cpu" or None.
"""
if label_name is None:
label_name = self.tag_type
with torch.no_grad():
if not sentences:
return sentences
# make sure its a list
if not isinstance(sentences, list) and not isinstance(
sentences, flair.data.Dataset):
sentences = [sentences]
# filter empty sentences
sentences = [
sentence for sentence in sentences if len(sentence) > 0
]
# reverse sort all sequences by their length
reordered_sentences = sorted(sentences,
key=lambda s: len(s),
reverse=True)
if len(reordered_sentences) == 0:
return sentences
dataloader = DataLoader(
dataset=FlairDatapointDataset(reordered_sentences),
batch_size=mini_batch_size,
)
# progress bar for verbosity
if verbose:
dataloader = tqdm(dataloader, desc="Batch inference")
overall_loss = torch.zeros(1, device=flair.device)
batch_no = 0
label_count = 0
for batch in dataloader:
batch_no += 1
# stop if all sentences are empty
if not batch:
continue
# get features from forward propagation
features, gold_labels = self.forward(batch)
# remove previously predicted labels of this type
for sentence in batch:
sentence.remove_labels(label_name)
# if return_loss, get loss value
if return_loss:
loss = self._calculate_loss(features, gold_labels)
overall_loss += loss[0]
label_count += loss[1]
# Sort batch in same way as forward propagation
lengths = torch.LongTensor(
[len(sentence) for sentence in batch])
_, sort_indices = lengths.sort(dim=0, descending=True)
batch = [batch[i] for i in sort_indices]
# make predictions
if self.use_crf:
predictions, all_tags = self.viterbi_decoder.decode(
features, return_probabilities_for_all_classes)
else:
predictions, all_tags = self._standard_inference(
features, batch, return_probabilities_for_all_classes)
# add predictions to Sentence
for sentence, sentence_predictions in zip(batch, predictions):
# BIOES-labels need to be converted to spans
if self.predict_spans:
sentence_tags = [
label[0] for label in sentence_predictions
]
sentence_scores = [
label[1] for label in sentence_predictions
]
predicted_spans = get_spans_from_bio(
sentence_tags, sentence_scores)
for predicted_span in predicted_spans:
span: Span = sentence[
predicted_span[0][0]:predicted_span[0][-1] + 1]
span.add_label(label_name,
value=predicted_span[2],
score=predicted_span[1])
# token-labels can be added directly
else:
for token, label in zip(sentence.tokens,
sentence_predictions):
token.add_label(typename=label_name,
value=label[0],
score=label[1])
# all_tags will be empty if all_tag_prob is set to False, so the for loop will be avoided
for (sentence, sent_all_tags) in zip(batch, all_tags):
for (token, token_all_tags) in zip(sentence.tokens,
sent_all_tags):
token.add_tags_proba_dist(label_name, token_all_tags)
store_embeddings(sentences,
storage_mode=embedding_storage_mode)
if return_loss:
return overall_loss, label_count
def _standard_inference(self, features: torch.Tensor,
batch: List[Sentence],
probabilities_for_all_classes: bool):
"""
Softmax over emission scores from forward propagation.
:param features: sentence tensor from forward propagation
:param batch: list of sentence
:param probabilities_for_all_classes: whether to return score for each tag in tag dictionary
"""
softmax_batch = F.softmax(features, dim=1).cpu()
scores_batch, prediction_batch = torch.max(softmax_batch, dim=1)
predictions = []
all_tags = []
for sentence in batch:
scores = scores_batch[:len(sentence)]
predictions_for_sentence = prediction_batch[:len(sentence)]
predictions.append([
(self.label_dictionary.get_item_for_index(prediction),
score.item()) for token, score, prediction in zip(
sentence, scores, predictions_for_sentence)
])
scores_batch = scores_batch[len(sentence):]
prediction_batch = prediction_batch[len(sentence):]
if probabilities_for_all_classes:
lengths = [len(sentence) for sentence in batch]
all_tags = self._all_scores_for_token(softmax_batch, lengths)
return predictions, all_tags
def _all_scores_for_token(self, scores: torch.Tensor, lengths: List[int]):
"""
Returns all scores for each tag in tag dictionary.
:param scores: Scores for current sentence.
"""
scores = scores.numpy()
prob_all_tags = [[
Label(self.label_dictionary.get_item_for_index(score_id), score)
for score_id, score in enumerate(score_dist)
] for score_dist in scores]
prob_tags_per_sentence = []
previous = 0
for length in lengths:
prob_tags_per_sentence.append(prob_all_tags[previous:previous +
length])
previous = length
return prob_tags_per_sentence
def _get_state_dict(self):
"""Returns the state dictionary for this model."""
model_state = {
**super()._get_state_dict(),
"embeddings": self.embeddings,
"hidden_size": self.hidden_size,
"tag_dictionary": self.label_dictionary,
"tag_type": self.tag_type,
"use_crf": self.use_crf,
"use_rnn": self.use_rnn,
"rnn_layers": self.rnn_layers,
"use_dropout": self.use_dropout,
"use_word_dropout": self.use_word_dropout,
"use_locked_dropout": self.use_locked_dropout,
"rnn_type": self.rnn_type,
"reproject_embeddings": self.reproject_embeddings,
"weight_dict": self.weight_dict,
}
return model_state
@classmethod
def _init_model_with_state_dict(cls, state, **kwargs):
"""Initialize the model from a state dictionary."""
rnn_type = "LSTM" if "rnn_type" not in state.keys(
) else state["rnn_type"]
use_dropout = 0.0 if "use_dropout" not in state.keys(
) else state["use_dropout"]
use_word_dropout = 0.0 if "use_word_dropout" not in state.keys(
) else state["use_word_dropout"]
use_locked_dropout = 0.0 if "use_locked_dropout" not in state.keys(
) else state["use_locked_dropout"]
reproject_embeddings = True if "reproject_embeddings" not in state.keys(
) else state["reproject_embeddings"]
weights = None if "weight_dict" not in state.keys(
) else state["weight_dict"]
if state["use_crf"]:
if "transitions" in state["state_dict"]:
state["state_dict"]["crf.transitions"] = state["state_dict"][
"transitions"]
del state["state_dict"]["transitions"]
return super()._init_model_with_state_dict(
state,
embeddings=state["embeddings"],
tag_dictionary=state["tag_dictionary"],
tag_type=state["tag_type"],
use_crf=state["use_crf"],
use_rnn=state["use_rnn"],
rnn_layers=state["rnn_layers"],
hidden_size=state["hidden_size"],
dropout=use_dropout,
word_dropout=use_word_dropout,
locked_dropout=use_locked_dropout,
rnn_type=rnn_type,
reproject_embeddings=reproject_embeddings,
loss_weights=weights,
init_from_state_dict=True,
**kwargs,
)
@staticmethod
def _fetch_model(model_name) -> str:
# core Flair models on Huggingface ModelHub
huggingface_model_map = {
"ner": "flair/ner-english",
"ner-fast": "flair/ner-english-fast",
"ner-ontonotes": "flair/ner-english-ontonotes",
"ner-ontonotes-fast": "flair/ner-english-ontonotes-fast",
# Large NER models,
"ner-large": "flair/ner-english-large",
"ner-ontonotes-large": "flair/ner-english-ontonotes-large",
"de-ner-large": "flair/ner-german-large",
"nl-ner-large": "flair/ner-dutch-large",
"es-ner-large": "flair/ner-spanish-large",
# Multilingual NER models
"ner-multi": "flair/ner-multi",
"multi-ner": "flair/ner-multi",
"ner-multi-fast": "flair/ner-multi-fast",
# English POS models
"upos": "flair/upos-english",
"upos-fast": "flair/upos-english-fast",
"pos": "flair/pos-english",
"pos-fast": "flair/pos-english-fast",
# Multilingual POS models
"pos-multi": "flair/upos-multi",
"multi-pos": "flair/upos-multi",
"pos-multi-fast": "flair/upos-multi-fast",
"multi-pos-fast": "flair/upos-multi-fast",
# English SRL models
"frame": "flair/frame-english",
"frame-fast": "flair/frame-english-fast",
# English chunking models
"chunk": "flair/chunk-english",
"chunk-fast": "flair/chunk-english-fast",
# Language-specific NER models
"da-ner": "flair/ner-danish",
"de-ner": "flair/ner-german",
"de-ler": "flair/ner-german-legal",
"de-ner-legal": "flair/ner-german-legal",
"fr-ner": "flair/ner-french",
"nl-ner": "flair/ner-dutch",
}
hu_path: str = "https://nlp.informatik.hu-berlin.de/resources/models"
hu_model_map = {
# English NER models
"ner":
"/".join([hu_path, "ner", "en-ner-conll03-v0.4.pt"]),
"ner-pooled":
"/".join([hu_path, "ner-pooled", "en-ner-conll03-pooled-v0.5.pt"]),
"ner-fast":
"/".join([hu_path, "ner-fast", "en-ner-fast-conll03-v0.4.pt"]),
"ner-ontonotes":
"/".join([hu_path, "ner-ontonotes", "en-ner-ontonotes-v0.4.pt"]),
"ner-ontonotes-fast":
"/".join([
hu_path, "ner-ontonotes-fast", "en-ner-ontonotes-fast-v0.4.pt"
]),
# Multilingual NER models
"ner-multi":
"/".join([hu_path, "multi-ner", "quadner-large.pt"]),
"multi-ner":
"/".join([hu_path, "multi-ner", "quadner-large.pt"]),
"ner-multi-fast":
"/".join([hu_path, "multi-ner-fast", "ner-multi-fast.pt"]),
# English POS models
"upos":
"/".join([hu_path, "upos", "en-pos-ontonotes-v0.4.pt"]),
"upos-fast":
"/".join([hu_path, "upos-fast", "en-upos-ontonotes-fast-v0.4.pt"]),
"pos":
"/".join([hu_path, "pos", "en-pos-ontonotes-v0.5.pt"]),
"pos-fast":
"/".join([hu_path, "pos-fast", "en-pos-ontonotes-fast-v0.5.pt"]),
# Multilingual POS models
"pos-multi":
"/".join([hu_path, "multi-pos", "pos-multi-v0.1.pt"]),
"multi-pos":
"/".join([hu_path, "multi-pos", "pos-multi-v0.1.pt"]),
"pos-multi-fast":
"/".join([hu_path, "multi-pos-fast", "pos-multi-fast.pt"]),
"multi-pos-fast":
"/".join([hu_path, "multi-pos-fast", "pos-multi-fast.pt"]),
# English SRL models
"frame":
"/".join([hu_path, "frame", "en-frame-ontonotes-v0.4.pt"]),
"frame-fast":
"/".join(
[hu_path, "frame-fast", "en-frame-ontonotes-fast-v0.4.pt"]),
# English chunking models
"chunk":
"/".join([hu_path, "chunk", "en-chunk-conll2000-v0.4.pt"]),
"chunk-fast":
"/".join(
[hu_path, "chunk-fast", "en-chunk-conll2000-fast-v0.4.pt"]),
# Danish models
"da-pos":
"/".join([hu_path, "da-pos", "da-pos-v0.1.pt"]),
"da-ner":
"/".join([hu_path, "NER-danish", "da-ner-v0.1.pt"]),
# German models
"de-pos":
"/".join([hu_path, "de-pos", "de-pos-ud-hdt-v0.5.pt"]),
"de-pos-tweets":
"/".join([hu_path, "de-pos-tweets", "de-pos-twitter-v0.1.pt"]),
"de-ner":
"/".join([hu_path, "de-ner", "de-ner-conll03-v0.4.pt"]),
"de-ner-germeval":
"/".join([hu_path, "de-ner-germeval", "de-ner-germeval-0.4.1.pt"]),
"de-ler":
"/".join([hu_path, "de-ner-legal", "de-ner-legal.pt"]),
"de-ner-legal":
"/".join([hu_path, "de-ner-legal", "de-ner-legal.pt"]),
# French models
"fr-ner":
"/".join([hu_path, "fr-ner", "fr-ner-wikiner-0.4.pt"]),
# Dutch models
"nl-ner":
"/".join([hu_path, "nl-ner", "nl-ner-bert-conll02-v0.8.pt"]),
"nl-ner-rnn":
"/".join([hu_path, "nl-ner-rnn", "nl-ner-conll02-v0.5.pt"]),
# Malayalam models
"ml-pos":
"https://raw.githubusercontent.com/qburst/models-repository/master/FlairMalayalamModels/malayalam-xpos-model.pt",
"ml-upos":
"https://raw.githubusercontent.com/qburst/models-repository/master/FlairMalayalamModels/malayalam-upos-model.pt",
# Portuguese models
"pt-pos-clinical":
"/".join([
hu_path,
"pt-pos-clinical",
"pucpr-flair-clinical-pos-tagging-best-model.pt",
]),
# Keyphase models
"keyphrase":
"/".join([hu_path, "keyphrase", "keyphrase-en-scibert.pt"]),
"negation-speculation":
"/".join([
hu_path, "negation-speculation",
"negation-speculation-model.pt"
]),
# Biomedical models
"hunflair-paper-cellline":
"/".join([
hu_path,
"hunflair_smallish_models",
"cellline",
"hunflair-celline-v1.0.pt",
]),
"hunflair-paper-chemical":
"/".join([
hu_path,
"hunflair_smallish_models",
"chemical",
"hunflair-chemical-v1.0.pt",
]),
"hunflair-paper-disease":
"/".join([
hu_path,
"hunflair_smallish_models",
"disease",
"hunflair-disease-v1.0.pt",
]),
"hunflair-paper-gene":
"/".join([
hu_path, "hunflair_smallish_models", "gene",
"hunflair-gene-v1.0.pt"
]),
"hunflair-paper-species":
"/".join([
hu_path,
"hunflair_smallish_models",
"species",
"hunflair-species-v1.0.pt",
]),
"hunflair-cellline":
"/".join([
hu_path,
"hunflair_smallish_models",
"cellline",
"hunflair-celline-v1.0.pt",
]),
"hunflair-chemical":
"/".join([
hu_path,
"hunflair_allcorpus_models",
"huner-chemical",
"hunflair-chemical-full-v1.0.pt",
]),
"hunflair-disease":
"/".join([
hu_path,
"hunflair_allcorpus_models",
"huner-disease",
"hunflair-disease-full-v1.0.pt",
]),
"hunflair-gene":
"/".join([
hu_path,
"hunflair_allcorpus_models",
"huner-gene",
"hunflair-gene-full-v1.0.pt",
]),
"hunflair-species":
"/".join([
hu_path,
"hunflair_allcorpus_models",
"huner-species",
"hunflair-species-full-v1.1.pt",
]),
}
cache_dir = Path("models")
get_from_model_hub = False
# check if model name is a valid local file
if Path(model_name).exists():
model_path = model_name
# check if model key is remapped to HF key - if so, print out information
elif model_name in huggingface_model_map:
# get mapped name
hf_model_name = huggingface_model_map[model_name]
# use mapped name instead
model_name = hf_model_name
get_from_model_hub = True
# if not, check if model key is remapped to direct download location. If so, download model
elif model_name in hu_model_map:
model_path = cached_path(hu_model_map[model_name],
cache_dir=cache_dir)
# special handling for the taggers by the @redewiegergabe project (TODO: move to model hub)
elif model_name == "de-historic-indirect":
model_file = flair.cache_root / cache_dir / "indirect" / "final-model.pt"
if not model_file.exists():
cached_path(
"http://www.redewiedergabe.de/models/indirect.zip",
cache_dir=cache_dir,
)
unzip_file(
flair.cache_root / cache_dir / "indirect.zip",
flair.cache_root / cache_dir,
)
model_path = str(flair.cache_root / cache_dir / "indirect" /
"final-model.pt")
elif model_name == "de-historic-direct":
model_file = flair.cache_root / cache_dir / "direct" / "final-model.pt"
if not model_file.exists():
cached_path(
"http://www.redewiedergabe.de/models/direct.zip",
cache_dir=cache_dir,
)
unzip_file(
flair.cache_root / cache_dir / "direct.zip",
flair.cache_root / cache_dir,
)
model_path = str(flair.cache_root / cache_dir / "direct" /
"final-model.pt")
elif model_name == "de-historic-reported":
model_file = flair.cache_root / cache_dir / "reported" / "final-model.pt"
if not model_file.exists():
cached_path(
"http://www.redewiedergabe.de/models/reported.zip",
cache_dir=cache_dir,
)
unzip_file(
flair.cache_root / cache_dir / "reported.zip",
flair.cache_root / cache_dir,
)
model_path = str(flair.cache_root / cache_dir / "reported" /
"final-model.pt")
elif model_name == "de-historic-free-indirect":
model_file = flair.cache_root / cache_dir / "freeIndirect" / "final-model.pt"
if not model_file.exists():
cached_path(
"http://www.redewiedergabe.de/models/freeIndirect.zip",
cache_dir=cache_dir,
)
unzip_file(
flair.cache_root / cache_dir / "freeIndirect.zip",
flair.cache_root / cache_dir,
)
model_path = str(flair.cache_root / cache_dir / "freeIndirect" /
"final-model.pt")
# for all other cases (not local file or special download location), use HF model hub
else:
get_from_model_hub = True
# if not a local file, get from model hub
if get_from_model_hub:
hf_model_name = "pytorch_model.bin"
revision = "main"
if "@" in model_name:
model_name_split = model_name.split("@")
revision = model_name_split[-1]
model_name = model_name_split[0]
# use model name as subfolder
if "/" in model_name:
model_folder = model_name.split("/", maxsplit=1)[1]
else:
model_folder = model_name
# Lazy import
from huggingface_hub import cached_download, hf_hub_url
url = hf_hub_url(model_name,
revision=revision,
filename=hf_model_name)
try:
model_path = cached_download(
url=url,
library_name="flair",
library_version=flair.__version__,
cache_dir=flair.cache_root / "models" / model_folder,
)
except HTTPError:
# output information
log.error("-" * 80)
log.error(
f"ACHTUNG: The key '{model_name}' was neither found on the ModelHub nor is this a valid path to a file on your system!"
)
# log.error(f" - Error message: {e}")
log.error(
" -> Please check https://huggingface.co/models?filter=flair for all available models."
)
log.error(
" -> Alternatively, point to a model file on your local drive."
)
log.error("-" * 80)
Path(flair.cache_root / "models" /
model_folder).rmdir() # remove folder again if not valid
return model_path
@staticmethod
def _filter_empty_sentences(sentences: List[Sentence]) -> List[Sentence]:
filtered_sentences = [
sentence for sentence in sentences if sentence.tokens
]
if len(sentences) != len(filtered_sentences):
log.warning(
f"Ignore {len(sentences) - len(filtered_sentences)} sentence(s) with no tokens."
)
return filtered_sentences
def _determine_if_span_prediction_problem(self,
dictionary: Dictionary) -> bool:
for item in dictionary.get_items():
if item.startswith("B-") or item.startswith(
"S-") or item.startswith("I-"):
return True
return False
def _print_predictions(self, batch, gold_label_type):
lines = []
if self.predict_spans:
for datapoint in batch:
# all labels default to "O"
for token in datapoint:
token.set_label("gold_bio", "O")
token.set_label("predicted_bio", "O")
# set gold token-level
for gold_label in datapoint.get_labels(gold_label_type):
gold_span: Span = gold_label.data_point
prefix = "B-"
for token in gold_span:
token.set_label("gold_bio", prefix + gold_label.value)
prefix = "I-"
# set predicted token-level
for predicted_label in datapoint.get_labels("predicted"):
predicted_span: Span = predicted_label.data_point
prefix = "B-"
for token in predicted_span:
token.set_label("predicted_bio",
prefix + predicted_label.value)
prefix = "I-"
# now print labels in CoNLL format
for token in datapoint:
eval_line = (f"{token.text} "
f"{token.get_label('gold_bio').value} "
f"{token.get_label('predicted_bio').value}\n")
lines.append(eval_line)
lines.append("\n")
else:
for datapoint in batch:
# print labels in CoNLL format
for token in datapoint:
eval_line = (f"{token.text} "
f"{token.get_label(gold_label_type).value} "
f"{token.get_label('predicted').value}\n")
lines.append(eval_line)
lines.append("\n")
return lines
def evaluate(
self,
data_points: Union[List[DataPoint], Dataset],
gold_label_type: str,
out_path: Union[str, Path] = None,
embedding_storage_mode: str = "none",
mini_batch_size: int = 32,
num_workers: int = 8,
main_evaluation_metric: Tuple[str, str] = ("micro avg", "f1-score"),
exclude_labels: List[str] = [],
gold_label_dictionary: Optional[Dictionary] = None,
) -> Result:
import numpy as np
import sklearn
# read Dataset into data loader (if list of sentences passed, make Dataset first)
if not isinstance(data_points, Dataset):
data_points = SentenceDataset(data_points)
data_loader = DataLoader(data_points,
batch_size=mini_batch_size,
num_workers=num_workers)
with torch.no_grad():
# loss calculation
eval_loss = 0
average_over = 0
# variables for printing
lines: List[str] = []
# variables for computing scores
all_spans: List[str] = []
all_true_values = {}
all_predicted_values = {}
sentence_id = 0
for batch in data_loader:
# remove any previously predicted labels
for datapoint in batch:
datapoint.remove_labels('predicted')
# predict for batch
loss_and_count = self.predict(
batch,
embedding_storage_mode=embedding_storage_mode,
mini_batch_size=mini_batch_size,
label_name='predicted',
return_loss=True)
if isinstance(loss_and_count, Tuple):
average_over += loss_and_count[1]
eval_loss += loss_and_count[0]
else:
eval_loss += loss_and_count
# get the gold labels
for datapoint in batch:
for gold_label in datapoint.get_labels(gold_label_type):
representation = str(
sentence_id) + ': ' + gold_label.identifier
value = gold_label.value
if gold_label_dictionary and gold_label_dictionary.get_idx_for_item(
value) == 0:
value = '<unk>'
if representation not in all_true_values:
all_true_values[representation] = [value]
else:
all_true_values[representation].append(value)
if representation not in all_spans:
all_spans.append(representation)
for predicted_span in datapoint.get_labels("predicted"):
representation = str(
sentence_id) + ': ' + predicted_span.identifier
# add to all_predicted_values
if representation not in all_predicted_values:
all_predicted_values[representation] = [
predicted_span.value
]
else:
all_predicted_values[representation].append(
predicted_span.value)
if representation not in all_spans:
all_spans.append(representation)
sentence_id += 1
store_embeddings(batch, embedding_storage_mode)
# make printout lines
if out_path:
lines.extend(
self._print_predictions(batch, gold_label_type))
# write all_predicted_values to out_file if set
if out_path:
with open(Path(out_path), "w", encoding="utf-8") as outfile:
outfile.write("".join(lines))
# make the evaluation dictionary
evaluation_label_dictionary = Dictionary(add_unk=False)
evaluation_label_dictionary.add_item("O")
for true_values in all_true_values.values():
for label in true_values:
evaluation_label_dictionary.add_item(label)
for predicted_values in all_predicted_values.values():
for label in predicted_values:
evaluation_label_dictionary.add_item(label)
# finally, compute numbers
y_true = []
y_pred = []
for span in all_spans:
true_values = all_true_values[
span] if span in all_true_values else ['O']
predicted_values = all_predicted_values[
span] if span in all_predicted_values else ['O']
y_true_instance = np.zeros(len(evaluation_label_dictionary),
dtype=int)
for true_value in true_values:
y_true_instance[evaluation_label_dictionary.
get_idx_for_item(true_value)] = 1
y_true.append(y_true_instance.tolist())
y_pred_instance = np.zeros(len(evaluation_label_dictionary),
dtype=int)
for predicted_value in predicted_values:
y_pred_instance[evaluation_label_dictionary.
get_idx_for_item(predicted_value)] = 1
y_pred.append(y_pred_instance.tolist())
# now, calculate evaluation numbers
target_names = []
labels = []
counter = Counter()
counter.update(
list(itertools.chain.from_iterable(all_true_values.values())))
counter.update(
list(itertools.chain.from_iterable(all_predicted_values.values())))
for label_name, count in counter.most_common():
if label_name == 'O': continue
if label_name in exclude_labels: continue
target_names.append(label_name)
labels.append(
evaluation_label_dictionary.get_idx_for_item(label_name))
# there is at least one gold label or one prediction (default)
if len(all_true_values) + len(all_predicted_values) > 1:
classification_report = sklearn.metrics.classification_report(
y_true,
y_pred,
digits=4,
target_names=target_names,
zero_division=0,
labels=labels,
)
classification_report_dict = sklearn.metrics.classification_report(
y_true,
y_pred,
target_names=target_names,
zero_division=0,
output_dict=True,
labels=labels,
)
accuracy_score = round(
sklearn.metrics.accuracy_score(y_true, y_pred), 4)
precision_score = round(
classification_report_dict["micro avg"]["precision"], 4)
recall_score = round(
classification_report_dict["micro avg"]["recall"], 4)
micro_f_score = round(
classification_report_dict["micro avg"]["f1-score"], 4)
macro_f_score = round(
classification_report_dict["macro avg"]["f1-score"], 4)
main_score = classification_report_dict[main_evaluation_metric[0]][
main_evaluation_metric[1]]
else:
# issue error and default all evaluation numbers to 0.
log.error(
"ACHTUNG! No gold labels and no all_predicted_values found! Could be an error in your corpus or how you "
"initialize the trainer!")
accuracy_score = precision_score = recall_score = micro_f_score = macro_f_score = main_score = 0.
classification_report = ""
classification_report_dict = {}
detailed_result = ("\nResults:"
f"\n- F-score (micro) {micro_f_score}"
f"\n- F-score (macro) {macro_f_score}"
f"\n- Accuracy {accuracy_score}"
"\n\nBy class:\n" + classification_report)
# line for log file
log_header = "PRECISION\tRECALL\tF1\tACCURACY"
log_line = f"{precision_score}\t" f"{recall_score}\t" f"{micro_f_score}\t" f"{accuracy_score}"
if average_over > 0:
eval_loss /= average_over
result = Result(main_score=main_score,
log_line=log_line,
log_header=log_header,
detailed_results=detailed_result,
classification_report=classification_report_dict,
loss=eval_loss)
return result
def evaluate(
self,
data_points: Union[List[DT], Dataset],
gold_label_type: str,
out_path: Union[str, Path] = None,
embedding_storage_mode: str = "none",
mini_batch_size: int = 32,
num_workers: Optional[int] = 8,
main_evaluation_metric: Tuple[str, str] = ("micro avg", "f1-score"),
exclude_labels: List[str] = [],
gold_label_dictionary: Optional[Dictionary] = None,
**kwargs,
) -> Result:
import numpy as np
import sklearn
# read Dataset into data loader, if list of sentences passed, make Dataset first
if not isinstance(data_points, Dataset):
data_points = FlairDatapointDataset(data_points)
with torch.no_grad():
# loss calculation
eval_loss = torch.zeros(1, device=flair.device)
average_over = 0
# variables for printing
lines: List[str] = []
# variables for computing scores
all_spans: Set[str] = set()
all_true_values = {}
all_predicted_values = {}
loader = DataLoader(data_points,
batch_size=mini_batch_size,
num_workers=0)
sentence_id = 0
for batch in Tqdm.tqdm(loader):
# remove any previously predicted labels
for datapoint in batch:
datapoint.remove_labels("predicted")
# predict for batch
loss_and_count = self.predict(
batch,
embedding_storage_mode=embedding_storage_mode,
mini_batch_size=mini_batch_size,
label_name="predicted",
return_loss=True,
)
if isinstance(loss_and_count, tuple):
average_over += loss_and_count[1]
eval_loss += loss_and_count[0]
else:
eval_loss += loss_and_count
# get the gold labels
for datapoint in batch:
for gold_label in datapoint.get_labels(gold_label_type):
representation = str(
sentence_id) + ": " + gold_label.identifier
value = gold_label.value
if gold_label_dictionary and gold_label_dictionary.get_idx_for_item(
value) == 0:
value = "<unk>"
if representation not in all_true_values:
all_true_values[representation] = [value]
else:
all_true_values[representation].append(value)
if representation not in all_spans:
all_spans.add(representation)
for predicted_span in datapoint.get_labels("predicted"):
representation = str(
sentence_id) + ": " + predicted_span.identifier
# add to all_predicted_values
if representation not in all_predicted_values:
all_predicted_values[representation] = [
predicted_span.value
]
else:
all_predicted_values[representation].append(
predicted_span.value)
if representation not in all_spans:
all_spans.add(representation)
sentence_id += 1
store_embeddings(batch, embedding_storage_mode)
# make printout lines
if out_path:
lines.extend(
self._print_predictions(batch, gold_label_type))
# convert true and predicted values to two span-aligned lists
true_values_span_aligned = []
predicted_values_span_aligned = []
for span in all_spans:
true_values_span_aligned.append(all_true_values[span] if span
in all_true_values else ["O"])
predicted_values_span_aligned.append(
all_predicted_values[span] if span in
all_predicted_values else ["O"])
# write all_predicted_values to out_file if set
if out_path:
with open(Path(out_path), "w", encoding="utf-8") as outfile:
outfile.write("".join(lines))
# make the evaluation dictionary
evaluation_label_dictionary = Dictionary(add_unk=False)
evaluation_label_dictionary.add_item("O")
for true_values in all_true_values.values():
for label in true_values:
evaluation_label_dictionary.add_item(label)
for predicted_values in all_predicted_values.values():
for label in predicted_values:
evaluation_label_dictionary.add_item(label)
# check if this is a multi-label problem
multi_label = False
for true_instance, predicted_instance in zip(
true_values_span_aligned, predicted_values_span_aligned):
if len(true_instance) > 1 or len(predicted_instance) > 1:
multi_label = True
break
log.info(f"Evaluating as a multi-label problem: {multi_label}")
# compute numbers by formatting true and predicted such that Scikit-Learn can use them
y_true = []
y_pred = []
if multi_label:
# multi-label problems require a multi-hot vector for each true and predicted label
for true_instance in true_values_span_aligned:
y_true_instance = np.zeros(len(evaluation_label_dictionary),
dtype=int)
for true_value in true_instance:
y_true_instance[evaluation_label_dictionary.
get_idx_for_item(true_value)] = 1
y_true.append(y_true_instance.tolist())
for predicted_values in predicted_values_span_aligned:
y_pred_instance = np.zeros(len(evaluation_label_dictionary),
dtype=int)
for predicted_value in predicted_values:
y_pred_instance[evaluation_label_dictionary.
get_idx_for_item(predicted_value)] = 1
y_pred.append(y_pred_instance.tolist())
else:
# single-label problems can do with a single index for each true and predicted label
y_true = [
evaluation_label_dictionary.get_idx_for_item(true_instance[0])
for true_instance in true_values_span_aligned
]
y_pred = [
evaluation_label_dictionary.get_idx_for_item(
predicted_instance[0])
for predicted_instance in predicted_values_span_aligned
]
# now, calculate evaluation numbers
target_names = []
labels = []
counter = Counter(
itertools.chain.from_iterable(all_true_values.values()))
counter.update(
list(itertools.chain.from_iterable(all_predicted_values.values())))
for label_name, count in counter.most_common():
if label_name == "O":
continue
if label_name in exclude_labels:
continue
target_names.append(label_name)
labels.append(
evaluation_label_dictionary.get_idx_for_item(label_name))
# there is at least one gold label or one prediction (default)
if len(all_true_values) + len(all_predicted_values) > 1:
classification_report = sklearn.metrics.classification_report(
y_true,
y_pred,
digits=4,
target_names=target_names,
zero_division=0,
labels=labels,
)
classification_report_dict = sklearn.metrics.classification_report(
y_true,
y_pred,
target_names=target_names,
zero_division=0,
output_dict=True,
labels=labels,
)
accuracy_score = round(
sklearn.metrics.accuracy_score(y_true, y_pred), 4)
macro_f_score = round(
classification_report_dict["macro avg"]["f1-score"], 4)
# if there is only one label, then "micro avg" = "macro avg"
if len(target_names) == 1:
classification_report_dict[
"micro avg"] = classification_report_dict["macro avg"]
if "micro avg" in classification_report_dict:
# micro average is only computed if zero-label exists (for instance "O")
precision_score = round(
classification_report_dict["micro avg"]["precision"], 4)
recall_score = round(
classification_report_dict["micro avg"]["recall"], 4)
micro_f_score = round(
classification_report_dict["micro avg"]["f1-score"], 4)
else:
# if no zero-label exists (such as in POS tagging) micro average is equal to accuracy
precision_score = round(classification_report_dict["accuracy"],
4)
recall_score = round(classification_report_dict["accuracy"], 4)
micro_f_score = round(classification_report_dict["accuracy"],
4)
# same for the main score
if "micro avg" not in classification_report_dict and main_evaluation_metric[
0] == "micro avg":
main_score = classification_report_dict["accuracy"]
else:
main_score = classification_report_dict[
main_evaluation_metric[0]][main_evaluation_metric[1]]
else:
# issue error and default all evaluation numbers to 0.
log.error(
"ACHTUNG! No gold labels and no all_predicted_values found! "
"Could be an error in your corpus or how you "
"initialize the trainer!")
accuracy_score = precision_score = recall_score = micro_f_score = macro_f_score = main_score = 0.0
classification_report = ""
classification_report_dict = {}
detailed_result = ("\nResults:"
f"\n- F-score (micro) {micro_f_score}"
f"\n- F-score (macro) {macro_f_score}"
f"\n- Accuracy {accuracy_score}"
"\n\nBy class:\n" + classification_report)
# line for log file
log_header = "PRECISION\tRECALL\tF1\tACCURACY"
log_line = f"{precision_score}\t" f"{recall_score}\t" f"{micro_f_score}\t" f"{accuracy_score}"
if average_over > 0:
eval_loss /= average_over
result = Result(
main_score=main_score,
log_line=log_line,
log_header=log_header,
detailed_results=detailed_result,
classification_report=classification_report_dict,
loss=eval_loss.item(),
)
return result
def __init__(self,
hidden_size: int,
embeddings: TokenEmbeddings,
ce_tag_dictionary: Dictionary,
anon_tag_dictionary: Dictionary,
ce_tag_type: str,
anon_tag_type: str,
use_crf: bool = True,
use_rnn: bool = True,
rnn_layers: int = 1,
dropout: float = 0.0,
word_dropout: float = 0.05,
locked_dropout: float = 0.5,
rnn_type: str = "LSTM",
pickle_module: str = "pickle",
beta: float = 1.0,
loss_weights: Dict[str, float] = None,
hidden_layer_after_lstm: int = 128):
"""
Initializes a MultitaskSequenceTagger
:param hidden_size: number of hidden states in RNN
:param embeddings: word embeddings used in tagger
:param tag_dictionary: dictionary of tags you want to predict
:param tag_type: string identifier for tag type
:param use_crf: if True use CRF decoder, else project directly to tag space
:param use_rnn: if True use RNN layer, otherwise use word embeddings directly
:param rnn_layers: number of RNN layers
:param dropout: dropout probability
:param word_dropout: word dropout probability
:param locked_dropout: locked dropout probability
:param train_initial_hidden_state: if True, trains initial hidden state of RNN
:param beta: Parameter for F-beta score for evaluation and training annealing
:param loss_weights: Dictionary of weights for classes (tags) for the loss function
(if any tag's weight is unspecified it will default to 1.0)
"""
super(flair.models.sequence_tagger_model.SequenceTagger,
self).__init__()
self.use_rnn = use_rnn
self.hidden_size = hidden_size
self.use_crf: bool = use_crf
self.rnn_layers: int = rnn_layers
self.trained_epochs: int = 0
self.embeddings = embeddings
self.embed_model = EmbeddingModule(
embeddings=embeddings,
dropout=dropout,
word_dropout=word_dropout,
locked_dropout=locked_dropout,
)
# set the dictionaries
self.ce_tag_dictionary = ce_tag_dictionary
self.anon_tag_dictionary = anon_tag_dictionary
self.ce_tagset_size: int = len(ce_tag_dictionary)
self.anon_tagset_size: int = len(anon_tag_dictionary)
self.ce_tag_type = ce_tag_type
self.anon_tag_type = anon_tag_type
self.o_label = anon_tag_dictionary.get_idx_for_item("O")
self.beta = beta
self.weight_dict = loss_weights
# Initialize the weight tensor
if loss_weights is not None:
n_classes = len(self.tag_dictionary)
weight_list = [1. for i in range(n_classes)]
for i, tag in enumerate(self.tag_dictionary.get_items()):
if tag in loss_weights.keys():
weight_list[i] = loss_weights[tag]
self.loss_weights = torch.FloatTensor(weight_list).to(flair.device)
else:
self.loss_weights = None
# initialize the network architecture
self.nlayers: int = rnn_layers
self.hidden_word = None
self.pickle_module = pickle_module
rnn_input_dim: int = self.embeddings.embedding_length
self.bidirectional = True
self.rnn_type = "LSTM"
self.hidden_layer_after_lstm = hidden_layer_after_lstm
num_directions = 2 if self.bidirectional else 1
self.shared_rnn = RecurrentModule(
rnn_input_dim=rnn_input_dim,
hidden_size=hidden_size,
rnn_layers=rnn_layers,
)
if self.hidden_layer_after_lstm > 0:
self.ce_hidden_layer = torch.nn.Linear(
hidden_size * num_directions, hidden_layer_after_lstm)
self.ce_relu = torch.nn.ReLU()
self.ce_linear = torch.nn.Linear(hidden_layer_after_lstm,
len(self.ce_tag_dictionary))
self.anon_hidden_layer = torch.nn.Linear(
hidden_size * num_directions, hidden_layer_after_lstm)
self.anon_relu = torch.nn.ReLU()
self.anon_linear = torch.nn.Linear(hidden_layer_after_lstm,
len(self.anon_tag_dictionary))
else:
# final linear map to tag space
self.anon_linear = torch.nn.Linear(hidden_size * num_directions,
len(anon_tag_dictionary))
self.ce_linear = torch.nn.Linear(hidden_size * num_directions,
len(ce_tag_dictionary))
if self.use_crf:
self.ce_transitions = torch.nn.Parameter(
torch.randn(self.ce_tagset_size, self.ce_tagset_size))
self.ce_transitions.detach()[
self.ce_tag_dictionary.get_idx_for_item(START_TAG), :] = -10000
self.ce_transitions.detach()[:,
self.ce_tag_dictionary.
get_idx_for_item(STOP_TAG)] = -10000
self.anon_transitions = torch.nn.Parameter(
torch.randn(self.anon_tagset_size, self.anon_tagset_size))
self.anon_transitions.detach(
)[self.anon_tag_dictionary.get_idx_for_item(START_TAG), :] = -10000
self.anon_transitions.detach()[:,
self.anon_tag_dictionary.
get_idx_for_item(STOP_TAG)] = -10000
self.set_output_to_ce()
self.to(flair.device)
