import os
#sys.path.insert(0, os.path.join('C:/', 'Users', 'pharn', 'flair'))
#sys.path.insert(0, os.path.join('C:/', 'Users', 'pharn', 'AppData', 'Local', 'Packages', 'PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0', 'LocalCache', 'local-packages', 'Python38', 'site-packages'))
sys.path.insert(0, "/vol/fob-vol7/mi19/harnisph/flair")
sys.path.insert(0, os.path.join("vol", "fob-vol7", "mi19", "harnisph",
"flair"))
import flair
from flair.models import TARSSequenceTagger2
from flair.data import Sentence
#from flair.datasets import CONLL_3, MIT_MOVIE_NER_COMPLEX
from flair.datasets import WNUT_2020_NER
flair.set_seed(1)
tagger = TARSSequenceTagger2.load(
"resources/testfaelle-studproj/conll_3/final-model.pt")
sentences = [
Sentence(
"The Parlament of the United Kingdom is discussing a variety of topics."
),
Sentence(
"A man fell in love with a woman. This takes place in the last century. The film received the Golden Love Film Award."
),
Sentence("The Company of Coca Cola was invented in 1901."),
Sentence("This is very frustrating! I was smiling since I saw you."),
Sentence("The Green Party received only a small percentage of the vote."),
Sentence(
"Bayern Munich won the german soccer series the sixth time in a row.")
]
import sys
import os
sys.path.insert(0, "/vol/fob-vol7/mi19/harnisph/flair")
import flair
import torch
from flair.models import TARSSequenceTagger2
from flair.data import Sentence
from flair.datasets import CONLL_03
tagger = TARSSequenceTagger2.load(
"resources/v3/moviecomplex-long/final-model.pt")
flair.set_seed(3)
label_name_map = {
"LOC": "Location",
"PER": "Person",
"ORG": "Organization",
"MISC": "Miscellaneous"
}
print(label_name_map)
corpus = CONLL_03(
tag_to_bioes=None,
tag_to_bio2="ner",
label_name_map=label_name_map,
base_path="/vol/fob-vol7/mi19/harnisph/studienprojekt-dokumentation")
corpus = corpus.downsample(0.1)
tag_type = "ner"
tag_dictionary = corpus.make_label_dictionary(tag_type)
tagger.add_and_switch_to_new_task("zeroshot-moviecomplex-long-to-conll3",
import sys
import os
sys.path.insert(0, "/vol/fob-vol7/mi19/harnisph/flair")
import flair
import torch
from flair.models import TARSSequenceTagger2
from flair.data import Sentence
from flair.datasets import MIT_MOVIE_NER_COMPLEX
flair.set_seed(3)
tagger = TARSSequenceTagger2.load("resources/v3/conll_3-cryptic/final-model.pt")
label_name_map = {
"Character_Name":"Character Name"
}
print(label_name_map)
corpus = MIT_MOVIE_NER_COMPLEX(tag_to_bioes=None, tag_to_bio2="ner", label_name_map=label_name_map)
corpus = corpus.downsample(0.1)
tag_type = "ner"
tag_dictionary = corpus.make_label_dictionary(tag_type)
tagger.add_and_switch_to_new_task("zeroshot-conll_3-cryptic-to-moviecomplex", tag_dictionary=tag_dictionary, tag_type=tag_type)
result, eval_loss = tagger.evaluate(corpus.test)
print(result.main_score)
print(result.log_header)
print(result.log_line)
print(result.detailed_results)
print(eval_loss)
# evaluation
import sys
import os
sys.path.insert(0, "/vol/fob-vol7/mi19/harnisph/flair")
import flair
import torch
from flair.models import TARSSequenceTagger2
from flair.data import Sentence
from flair.datasets import MIT_MOVIE_NER_COMPLEX, CONLL_03, WNUT_17, WNUT_2020_NER, BIOSCOPE
from flair.trainers import ModelTrainer
from torch.optim.lr_scheduler import OneCycleLR
flair.set_seed(3)
tagger = TARSSequenceTagger2.load("resources/v3/sequence-2/final-model.pt")
label_name_map = {
"person": "Person",
"location": "Location",
"creative-work": "Creative Work",
"product": "Product",
"corporation": "Corporation",
"group": "Group"
}
print(label_name_map)
corpus = WNUT_17(tag_to_bioes=None,
tag_to_bio2="ner",
label_name_map=label_name_map)
corpus = corpus.downsample(0.1)
tag_type = "ner"
}
print(label_name_map)
corpus = CONLL_03(
tag_to_bioes=None,
tag_to_bio2="ner",
label_name_map=label_name_map,
base_path="/vol/fob-vol7/mi19/harnisph/studienprojekt-dokumentation")
corpus = corpus.downsample(0.1)
tag_type = "ner"
label_dictionary = corpus.make_label_dictionary(tag_type)
print(label_dictionary)
tagger = TARSSequenceTagger2(tag_dictionary=label_dictionary,
tag_type=tag_type,
task_name="TEST_NER")
trainer = ModelTrainer(tagger, corpus, optimizer=torch.optim.AdamW)
from torch.optim.lr_scheduler import OneCycleLR
trainer.train(
base_path='resources/v3/conll_3-synonyms',
learning_rate=5.0e-5,
mini_batch_size=32,
mini_batch_chunk_size=None,
max_epochs=10,
weight_decay=0.,
embeddings_storage_mode="none",
scheduler=OneCycleLR,
)
import sys
import os
sys.path.insert(0, "/vol/fob-vol7/mi19/harnisph/flair")
import flair
import torch
from flair.models import TARSSequenceTagger2
from flair.data import Sentence
from flair.datasets import MIT_MOVIE_NER_COMPLEX, CONLL_03, WNUT_17, WNUT_2020_NER, BIOSCOPE
from flair.trainers import ModelTrainer
from torch.optim.lr_scheduler import OneCycleLR
flair.set_seed(3)
tagger = TARSSequenceTagger2.load("resources/v3/conll_3-simple/final-model.pt")
label_name_map = {"Character_Name": "Character Name"}
print(label_name_map)
corpus = MIT_MOVIE_NER_COMPLEX(tag_to_bioes=None,
tag_to_bio2="ner",
label_name_map=label_name_map)
corpus = corpus.downsample(0.1)
tag_type = "ner"
tag_dictionary = corpus.make_label_dictionary(tag_type)
tagger.add_and_switch_to_new_task("sequence-2-train",
tag_dictionary=tag_dictionary,
tag_type=tag_type)
trainer = ModelTrainer(tagger, corpus, optimizer=torch.optim.AdamW)
trainer.train(
from flair.trainers import ModelTrainer
from flair.embeddings import WordEmbeddings, TransformerWordEmbeddings, TransformerDocumentEmbeddings
from flair.data import Sentence
from flair.data import MultiCorpus
from flair.datasets import CONLL_03
flair.set_seed(1)
label_name_map = {
"LOC": "Location",
"PER": "Person",
"ORG": "Organization",
"MISC": "Miscellaneous"
}
tagger = TARSSequenceTagger2.load(
"resources/taggers/tars/test-bio/final-model.pt")
sentence1 = Sentence("I like Berlin and Germany")
sentence2 = Sentence("I like Berlin and Germany")
print()
print()
print("NEW SENTENCES FOR PRELEARNED CLASSES:\n")
tagger.predict(sentence2)
print(sentence2.to_tagged_string)
sent3 = Sentence(
"Donald Trump is the former president of the United States of America.")
tagger.predict(sent3)
print(sent3.to_tagged_string)