from flair.models import SequenceTagger, TARSSequenceTagger, TARSSequenceTagger2
from flair.trainers import ModelTrainer
from flair.embeddings import WordEmbeddings, TransformerWordEmbeddings, TransformerDocumentEmbeddings
from flair.data import Sentence
from flair.data import MultiCorpus
from flair.datasets import MIT_MOVIE_NER_COMPLEX
flair.set_seed(1)
label_name_map = {
#"LOC":"Location","PER":"Person","ORG":"Organization","MISC":"Miscellaneous"
"Character_Name":"Character Name"
}
print(label_name_map)
corpus = MIT_MOVIE_NER_COMPLEX(tag_to_bioes=None, tag_to_bio2="ner", label_name_map=label_name_map)
corpus = corpus.downsample(0.002)
tag_type = "ner"
label_dictionary = corpus.make_label_dictionary(tag_type)
print(label_dictionary)
#embeddings = WordEmbeddings("glove")
#embeddings = TransformerWordEmbeddings()
tagger = TARSSequenceTagger2.load("resources/testfaelle-studproj/conll_3/final-model.pt")
#tagger = TARSSequenceTagger2(tag_dictionary=label_dictionary, tag_type=tag_type, task_name="TEST_NER")
#tagger = TARSSequenceTagger(tag_dictionary=label_dictionary, tag_type=tag_type, task_name="TEST_NER")
#tagger = SequenceTagger(tag_dictionary=corpus.make_tag_dictionary(tag_type), tag_type=tag_type, hidden_size=256, embeddings=embeddings)
tagger.add_and_switch_to_new_task("TEST_NER_CONLL3+MIT10", tag_dictionary=label_dictionary, tag_type=tag_type)
import flair
import torch
from flair.models import TARSSequenceTagger2
from flair.data import Sentence
from flair.datasets import MIT_MOVIE_NER_COMPLEX
flair.set_seed(3)
tagger = TARSSequenceTagger2.load("resources/v3/conll_3-cryptic/final-model.pt")
label_name_map = {
"Character_Name":"Character Name"
}
print(label_name_map)
corpus = MIT_MOVIE_NER_COMPLEX(tag_to_bioes=None, tag_to_bio2="ner", label_name_map=label_name_map)
corpus = corpus.downsample(0.1)
tag_type = "ner"
tag_dictionary = corpus.make_label_dictionary(tag_type)
tagger.add_and_switch_to_new_task("zeroshot-conll_3-cryptic-to-moviecomplex", tag_dictionary=tag_dictionary, tag_type=tag_type)
result, eval_loss = tagger.evaluate(corpus.test)
print(result.main_score)
print(result.log_header)
print(result.log_line)
print(result.detailed_results)
print(eval_loss)
# evaluation
sentences = [
Sentence("The Parlament of the United Kingdom is discussing a variety of topics."),
Sentence("A man fell in love with a woman. This takes place in the last century. The film received the Golden Love Film Award."),
####
flair.set_seed(3)
tagger = TARSSequenceTagger2.load(
"resources/v3/conll_3-synonyms/final-model.pt")
### train k sentences for each tag in new corpus
k = 1
label_name_map = {"Character_Name": "Character Name"}
print(label_name_map)
corpus = MIT_MOVIE_NER_COMPLEX(tag_to_bioes=None,
tag_to_bio2="ner",
label_name_map=label_name_map)
corpus_small = corpus.downsample(0.1)
tag_type = "ner"
tag_dictionary = corpus.make_label_dictionary(tag_type)
corpus_sents = []
tag_dictionary_no_prefix = _get_tag_dictionary_no_prefix(tag_dictionary)
tag_countdown = [k for i in range(len(tag_dictionary_no_prefix.idx2item))]
for idx in range(len(corpus.train)):
sent = corpus.train[idx]
sent_picked = False
for tkn in sent:
if sent_picked:
break
tag = tkn.get_tag("ner").value
pref, tag_no_pref = _split_tag(tag)
if tag_no_pref is None: