def benchmark_polyglot_mdl(): """ Running ployglot requires these packages: # Morfessor==2.0.6 # PyICU==2.4.2 # pycld2==0.41 # polyglot """ from polyglot.tag import NEChunker from polyglot.text import WordList start = time.time() predictions = [] for tokens in sentences_tokens: word_list = WordList(tokens, language='da') ne_chunker = NEChunker(lang='da') word_ent_tuples = list(ne_chunker.annotate(word_list)) predictions.append([entity for word, entity in word_ent_tuples]) print('polyglot:') print_speed_performance(start, num_sentences, num_tokens) assert len(predictions) == len(sentences_entities) print(f1_report(sentences_entities, remove_miscs(predictions), bio=True))
def benchmark_dacy_mdl(dacy_model="da_dacy_large_tft-0.0.0"): """ an adaption of benchmark spacy model which is compatible with spacy v. 3 running this requires: spacy >= 3.0.0 spacy-transformers """ from spacy.tokens import Doc import dacy nlp = dacy.load(dacy_model) trf = nlp.get_pipe('transformer') ner = nlp.get_pipe('ner') predictions = [] start = time.time() for token in sentences_tokens: doc = Doc(nlp.vocab, words=token) doc = trf(doc) doc = ner(doc) ents = [] for t in doc: if t.ent_iob_ == 'O': ents.append(t.ent_iob_) else: ents.append(t.ent_iob_ + "-" + t.ent_type_) predictions.append(ents) print('DaCy ({}):'.format(dacy_model)) print_speed_performance(start, num_sentences, num_tokens) assert len(predictions) == num_sentences print(f1_report(sentences_entities, remove_miscs(predictions), bio=True))
def benchmark_dacy_mdl(dacy_model="da_dacy_large_tft-0.0.0"): """ an adaption of benchmark spacy model which is compatible with spacy v. 3 running this requires: spacy >= 3.0.0 spacy-transformers """ import dacy from spacy.tokens import Doc nlp = dacy.load(dacy_model) trf = nlp.get_pipe('transformer') tagger = nlp.get_pipe('tagger') start = time.time() tags_pred = [] for sent in sentences_tokens: doc = Doc(nlp.vocab, words=sent) doc = trf(doc) doc = tagger(doc) tags = [] for tok in doc: tags.append(tok.tag_) tags_pred.append(tags) print('**Spacy model**') print_speed_performance(start, num_sentences, num_tokens) assert len(tags_pred) == num_sentences assert sum([len(s) for s in tags_pred]) == num_tokens print(accuracy_report(tags_true, tags_pred), end="\n\n")
def benchmark_polyglot_mdl(corrected_output=False): """ Running polyglot requires these packages: # Morfessor==2.0.6 # PyICU==2.4.2 # pycld2==0.41 # polyglot """ def udify_tag(tag, word): if tag == "CONJ": return "CCONJ" if tag == "VERB" and word in auxiliary_verbs: return "AUX" return tag start = time.time() tags_pred = [] for tokens in sentences_tokens: word_list = WordList(tokens, language='da') tagger = POSTagger(lang='da') word_tag_tuples = list(tagger.annotate(word_list)) tags_pred.append([ udify_tag(tag, word) if corrected_output else tag for word, tag in word_tag_tuples ]) print('**Polyglot model' + (' (corrected output) ' if corrected_output else '') + '**') print_speed_performance(start, num_sentences, num_tokens) assert len(tags_pred) == num_sentences assert sum([len(s) for s in tags_pred]) == num_tokens print(accuracy_report(tags_true, tags_pred), end="\n\n")
def benchmark_spacy_mdl(): def normalize_spacy_head(i, hd): return 0 if i == hd else hd + 1 nlp = load_spacy_model() parser = nlp.parser start = time.time() deps_pred = [] for sent in sentences_tokens: doc = nlp.tokenizer.tokens_from_list(sent) doc = parser(doc) deprels = [] depheads = [] for i, tok in enumerate(doc): deprels.append(tok.dep_.lower()) depheads.append(normalize_spacy_head(i, tok.head.i)) deps_pred.append([(r, h) for r, h in zip(deprels, depheads)]) print('**Spacy model**') print_speed_performance(start, num_sentences, num_tokens) assert len(deps_pred) == num_sentences assert sum([len(s) for s in deps_pred]) == num_tokens print(dependency_report(deps_true, deps_pred))
def benchmark_stanza_mdl(): nlp = stanza.Pipeline('da', processors='tokenize,pos,lemma,depparse', tokenize_pretokenized=True) start = time.time() deps_pred = [] for sent in sentences_tokens: doc = nlp(" ".join(sent)) deprels = [] depheads = [] for tok in doc.iter_tokens(): deprels.append(tok.words[0].deprel) depheads.append(tok.words[0].head) deps_pred.append([(r, h) for r, h in zip(deprels, depheads)]) print('**Stanza model**') print_speed_performance(start, num_sentences, num_tokens) assert len(deps_pred) == num_sentences assert sum([len(s) for s in deps_pred]) == num_tokens print(dependency_report(deps_true, deps_pred))
def benchmark_daluke_mdl(): from daluke import AutoNERDaLUKE, predict_ner model = AutoNERDaLUKE() sents = [" ".join(s) for s in sentences_tokens] start = time.time() predictions = predict_ner(sents, model) print('DaLUKE:') print_speed_performance(start, num_sentences, num_tokens) assert len(predictions) == num_sentences print(f1_report(sentences_entities, remove_miscs(predictions), bio=True))
def benchmark_flair_mdl(): tagger = load_flair_pos_model() start = time.time() tagger.predict(corpus_flair.test) tags_pred = [[tok.tags['upos'].value for tok in fs] for fs in corpus_flair.test] print('**Flair model** ') print_speed_performance(start, num_sentences, num_tokens) assert len(tags_pred)==num_sentences assert sum([len(s) for s in tags_pred])==num_tokens print(accuracy_report(tags_true, tags_pred), end="\n\n")
def benchmark_bert_mdl(): bert = load_bert_ner_model() start = time.time() predictions = [] for i, sentence in enumerate(sentences_tokens): _, pred_ents = bert.predict(sentence) predictions.append(pred_ents) print('BERT:') print_speed_performance(start, num_sentences, num_tokens) assert len(predictions) == num_sentences print(f1_report(sentences_entities, remove_miscs(predictions), bio=True))
def benchmark_xlmr_mdl(): xlmr = load_xlmr_ned_model() start = time.time() predictions = [] for sent, kg in zip(sentences, kgs): pred = xlmr.predict(sent, kg) predictions.append(pred) print('XLMR:') print_speed_performance(start, num_sentences) assert len(predictions) == num_sentences print(f1_report(gold_tags, predictions, 'XLM-R', 'DaNED'))
def benchmark_spacy_mdl(): start = time.time() chks_pred = [] for sent in sentences_tokens: bio_chunks = chunker.predict(sent) chks_pred.append(bio_chunks) print('**Spacy model**') print_speed_performance(start, num_sentences, num_tokens) assert len(chks_pred)==num_sentences assert sum([len(s) for s in chks_pred])==num_tokens print(f1_report(chks_true, chks_pred, bio=True))
def benchmark_nerda_electra_mdl(): nerda = DA_ELECTRA_DA() nerda.download_network() nerda.load_network() start = time.time() predictions = nerda.predict(sentences_tokens) print('NERDA DA electra:') print_speed_performance(start, num_sentences, num_tokens) assert len(predictions) == num_sentences print(f1_report(sentences_entities, remove_miscs(predictions), bio=True))
def benchmark_nerda_multi_mdl(): nerda = DA_BERT_ML() nerda.download_network() nerda.load_network() start = time.time() predictions = nerda.predict(sentences_tokens) print('NERDA multilingual:') print_speed_performance(start, num_sentences, num_tokens) assert len(predictions) == num_sentences print(f1_report(sentences_entities, remove_miscs(predictions), bio=True))
def benchmark_xlmr_mdl(): from allennlp.data import DataLoader from allennlp.training.util import evaluate xlmr = load_xlmr_coref_model() instances = xlmr.dataset_reader.load_dataset(testset) data_loader = SimpleDataLoader(instances, 1) data_loader.index_with(xlmr.model.vocab) start = time.time() metrics = evaluate(xlmr.model, data_loader) print('**XLM-R model**') print_speed_performance(start, num_sentences, num_tokens) print('Precision : ', metrics['coref_precision']) print('Recall : ', metrics['coref_recall']) print('F1 : ', metrics['coref_f1']) print('Mention Recall : ', metrics['mention_recall'])
def benchmark_dacy_mdl(dacy_model="da_dacy_large_tft-0.0.0"): """ an adaption of benchmark spacy model which is compatible with spacy v. 3 running this requires: spacy >= 3.0.0 spacy-transformers """ def normalize_spacy_head(i, hd): return 0 if i == hd else hd+1 from spacy.tokens import Doc import dacy nlp = dacy.load(dacy_model) trf = nlp.get_pipe('transformer') parser = nlp.get_pipe('parser') start = time.time() deps_pred = [] for sent in sentences_tokens: doc = Doc(nlp.vocab, words=sent) doc = trf(doc) doc = parser(doc) deprels = [] depheads = [] for i, tok in enumerate(doc): deprels.append(tok.dep_.lower()) depheads.append(normalize_spacy_head(i, tok.head.i)) deps_pred.append([(r,h) for r,h in zip(deprels, depheads)]) print('**Spacy model**') print_speed_performance(start, num_sentences, num_tokens) assert len(deps_pred)==num_sentences assert sum([len(s) for s in deps_pred])==num_tokens print(dependency_report(deps_true, deps_pred))
def benchmark_scandiner_mdl(): from transformers import AutoModelForTokenClassification from transformers import AutoTokenizer import torch model_name = "saattrupdan/nbailab-base-ner-scandi" model = AutoModelForTokenClassification.from_pretrained(model_name) tokenizer = AutoTokenizer.from_pretrained(model_name) id2labels = model.config.id2label tokenized_sents = [] for sentence in sentences_tokens: tok_mask = [] for tok in sentence: t = tokenizer.tokenize(tok) tok_mask += [1] + [0] * (len(t) - 1) inputs = tokenizer.encode(sentence, return_tensors="pt", is_split_into_words=True) assert (len(inputs[0]) == len(tok_mask) + 2) tokenized_sents.append((inputs, tok_mask)) start = time.time() predictions = [] for (input, mask) in tokenized_sents: preds = model(input)[0] preds = torch.argmax(preds, dim=2) preds = preds[0].tolist()[1:-1] assert (len(preds) == len(mask)) pred_ents = [id2labels[p] for p, m in zip(preds, mask) if m] assert (len(pred_ents) == sum(mask)) predictions.append(pred_ents) print('ScandiNER:') print_speed_performance(start, num_sentences, num_tokens) assert len(predictions) == num_sentences print(f1_report(sentences_entities, remove_miscs(predictions), bio=True))
def benchmark_flair_mdl(): tagger = load_flair_ner_model() start = time.time() flair_sentences = [] for i, sentence in enumerate(sentences_tokens): flair_sentence = Sentence() for token_txt in sentence: flair_sentence.add_token(Token(token_txt)) flair_sentences.append(flair_sentence) tagger.predict(flair_sentences, verbose=True) predictions = [[tok.get_tag('ner').value for tok in fs] for fs in flair_sentences] print('Flair:') print_speed_performance(start, num_sentences, num_tokens) assert len(predictions) == num_sentences print(f1_report(sentences_entities, remove_miscs(predictions), bio=True))
def benchmark_spacy_mdl(): nlp = load_spacy_model() ner = nlp.entity predictions = [] start = time.time() for token in sentences_tokens: doc = nlp.tokenizer.tokens_from_list(token) ner(doc) ents = [] for t in doc: if t.ent_iob_ == 'O': ents.append(t.ent_iob_) else: ents.append(t.ent_iob_ + "-" + t.ent_type_) predictions.append(ents) print('spaCy:') print_speed_performance(start, num_sentences, num_tokens) assert len(predictions) == num_sentences print(f1_report(sentences_entities, remove_miscs(predictions), bio=True))
def benchmark_stanza_mdl(): nlp = stanza.Pipeline('da', processors='tokenize,pos', tokenize_pretokenized=True) start = time.time() tags_pred = [] for sent in sentences_tokens: doc = nlp(" ".join(sent)) tags = [] for tok in doc.iter_tokens(): tags.append(tok.words[0].upos) tags_pred.append(tags) print('**Stanza model**') print_speed_performance(start, num_sentences, num_tokens) assert len(tags_pred) == num_sentences assert sum([len(s) for s in tags_pred]) == num_tokens print(accuracy_report(tags_true, tags_pred), end="\n\n")
def benchmark_spacy_mdl(): nlp = load_spacy_model() tagger = nlp.tagger start = time.time() tags_pred = [] for sent in sentences_tokens: doc = nlp.tokenizer.tokens_from_list(sent) doc = tagger(doc) tags = [] for tok in doc: tags.append(tok.pos_) tags_pred.append(tags) print('**Spacy model**') print_speed_performance(start, num_sentences, num_tokens) assert len(tags_pred) == num_sentences assert sum([len(s) for s in tags_pred]) == num_tokens print(accuracy_report(tags_true, tags_pred), end="\n\n")