Exemplos de MIT_MOVIE_NER_COMPLEX.downsample em Python

Exemplo n.º 1

Arquivo: conll_3+mit10sent.py Projeto: pharnisch/sequence-tars-evaluation

from flair.models import SequenceTagger, TARSSequenceTagger, TARSSequenceTagger2
from flair.trainers import ModelTrainer
from flair.embeddings import WordEmbeddings, TransformerWordEmbeddings, TransformerDocumentEmbeddings
from flair.data import Sentence
from flair.data import MultiCorpus
from flair.datasets import MIT_MOVIE_NER_COMPLEX
flair.set_seed(1)

label_name_map = {
#"LOC":"Location","PER":"Person","ORG":"Organization","MISC":"Miscellaneous"
"Character_Name":"Character Name"
}

print(label_name_map)
corpus = MIT_MOVIE_NER_COMPLEX(tag_to_bioes=None, tag_to_bio2="ner", label_name_map=label_name_map)
corpus = corpus.downsample(0.002)

tag_type = "ner"
label_dictionary = corpus.make_label_dictionary(tag_type)
print(label_dictionary)

#embeddings = WordEmbeddings("glove")
#embeddings = TransformerWordEmbeddings()

tagger = TARSSequenceTagger2.load("resources/testfaelle-studproj/conll_3/final-model.pt")
#tagger = TARSSequenceTagger2(tag_dictionary=label_dictionary, tag_type=tag_type, task_name="TEST_NER")
#tagger = TARSSequenceTagger(tag_dictionary=label_dictionary, tag_type=tag_type, task_name="TEST_NER")
#tagger = SequenceTagger(tag_dictionary=corpus.make_tag_dictionary(tag_type), tag_type=tag_type, hidden_size=256, embeddings=embeddings)

tagger.add_and_switch_to_new_task("TEST_NER_CONLL3+MIT10", tag_dictionary=label_dictionary, tag_type=tag_type)

Exemplo n.º 2

Arquivo: zeroshot-conll3-cryptic-to-moviecomplex.py Projeto: pharnisch/sequence-tars-evaluation

import flair
import torch
from flair.models import TARSSequenceTagger2
from flair.data import Sentence
from flair.datasets import MIT_MOVIE_NER_COMPLEX

flair.set_seed(3)

tagger = TARSSequenceTagger2.load("resources/v3/conll_3-cryptic/final-model.pt")

label_name_map = {
"Character_Name":"Character Name"
}
print(label_name_map)
corpus = MIT_MOVIE_NER_COMPLEX(tag_to_bioes=None, tag_to_bio2="ner", label_name_map=label_name_map)
corpus = corpus.downsample(0.1)
tag_type = "ner"
tag_dictionary = corpus.make_label_dictionary(tag_type)
tagger.add_and_switch_to_new_task("zeroshot-conll_3-cryptic-to-moviecomplex", tag_dictionary=tag_dictionary, tag_type=tag_type)
result, eval_loss = tagger.evaluate(corpus.test)
print(result.main_score)
print(result.log_header)
print(result.log_line)
print(result.detailed_results)
print(eval_loss)

# evaluation

sentences = [
Sentence("The Parlament of the United Kingdom is discussing a variety of topics."),
Sentence("A man fell in love with a woman. This takes place in the last century. The film received the Golden Love Film Award."),

Exemplo n.º 3

####

flair.set_seed(3)

tagger = TARSSequenceTagger2.load(
    "resources/v3/conll_3-synonyms/final-model.pt")

### train k sentences for each tag in new corpus
k = 1
label_name_map = {"Character_Name": "Character Name"}
print(label_name_map)
corpus = MIT_MOVIE_NER_COMPLEX(tag_to_bioes=None,
                               tag_to_bio2="ner",
                               label_name_map=label_name_map)
corpus_small = corpus.downsample(0.1)
tag_type = "ner"
tag_dictionary = corpus.make_label_dictionary(tag_type)
corpus_sents = []
tag_dictionary_no_prefix = _get_tag_dictionary_no_prefix(tag_dictionary)
tag_countdown = [k for i in range(len(tag_dictionary_no_prefix.idx2item))]

for idx in range(len(corpus.train)):
    sent = corpus.train[idx]
    sent_picked = False
    for tkn in sent:
        if sent_picked:
            break
        tag = tkn.get_tag("ner").value
        pref, tag_no_pref = _split_tag(tag)
        if tag_no_pref is None: