def _get_embedding_model( model_name_or_path: Union[str, HFModelResult, FlairModelResult] ) -> Union[FlairEmbeddings, WordEmbeddings, TransformerWordEmbeddings, Sentence]: "Load the proper `Embeddings` model from `model_name_or_path`" if isinstance(model_name_or_path, FlairModelResult): nm = model_name_or_path.name try: return WordEmbeddings(nm.strip('flairNLP/')) except: return FlairEmbeddings(nm.strip('flairNLP/')) elif isinstance(model_name_or_path, HFModelResult): return TransformerWordEmbeddings(model_name_or_path.name) else: res = _flair_hub.search_model_by_name(model_name_or_path, user_uploaded=True) if len(res) < 1: # No models found res = _hf_hub.search_model_by_name(model_name_or_path, user_uploaded=True) if len(res) < 1: raise ValueError( f'Embeddings not found for the model key: {model_name_or_path}, check documentation or custom model path to verify specified model' ) else: return TransformerWordEmbeddings( res[0].name ) # Returning the first should always be the non-fast option else: nm = res[0].name try: return WordEmbeddings(nm.strip('flairNLP/')) except: return FlairEmbeddings(nm.strip('flairNLP/'))
def create_embeddings(self) -> StackedEmbeddings: embedding_types: List[FlairEmbeddings] = [] if self.config['use_word_embeddings']: embedding_types.append(W2vWordEmbeddings(self.config['word_embeddings_path'])) if self.config['use_char_embeddings']: embedding_types.append(CharacterEmbeddings()) if self.config['use_flair_embeddings']: embedding_types.append(FlairEmbeddings('es-clinical-forward')) embedding_types.append(FlairEmbeddings('es-clinical-backward')) if self.config['use_beto_embeddings']: embedding_types.append( TransformerWordEmbeddings( 'dccuchile/bert-base-spanish-wwm-cased', layers = self.config['layers'], layer_mean = self.config['layer_mean'], subtoken_pooling = self.config['subtoken_pooling'])) embeddings: StackedEmbeddings = StackedEmbeddings(embeddings = embedding_types) return embeddings
def get_scibert_flair_embeddings(): return [ TransformerWordEmbeddings(model="allenai/scibert_scivocab_uncased", fine_tune=True), FlairEmbeddings("pubmed-forward"), FlairEmbeddings("pubmed-backward") ]
def __init__(self, *embeddings: str): print("May need a couple moments to instantiate...") self.embedding_stack = [] # Load correct Embeddings module for model_name_or_path in embeddings: if ("flair" in model_name_or_path or model_name_or_path in FLAIR_PRETRAINED_MODEL_NAMES): self.embedding_stack.append( FlairEmbeddings(model_name_or_path)) else: try: self.embedding_stack.append( WordEmbeddings(model_name_or_path)) except ValueError: try: self.embedding_stack.append( TransformerWordEmbeddings(model_name_or_path)) except ValueError: raise ValueError( f"Embeddings not found for the model key: {model_name_or_path}, check documentation or custom model path to verify specified model" ) assert len(self.embedding_stack) != 0 self.stacked_embeddings = StackedEmbeddings( embeddings=self.embedding_stack)
def train(): generate_datasets() DATA_FOLDER = '../content/data' # MAX_TOKENS = 500 columns = {0: 'text', 1: 'pos', 2: 'tag'} data_folder = DATA_FOLDER corpus: Corpus = ColumnCorpus(data_folder, columns, train_file='train-labelled.txt', test_file='dev-labelled.txt', in_memory=False) # corpus._train = [x for x in corpus.train if len(x) < MAX_TOKENS] # corpus._test = [x for x in corpus.test if len(x) < MAX_TOKENS] tag_type = 'tag' tag_dictionary = corpus.make_tag_dictionary(tag_type=tag_type) print(tag_dictionary) embeddings = TransformerWordEmbeddings('roberta-base', layers='-4', fine_tune=True) tagger: SequenceTagger = SequenceTagger(hidden_size=128, embeddings=embeddings, tag_dictionary=tag_dictionary, tag_type=tag_type, # dropout=0.3334816033039888, use_crf=True) trainer: ModelTrainer = ModelTrainer(tagger, corpus) trainer.train('resources/taggers/task-TC', learning_rate=0.2, mini_batch_size=64, max_epochs=100, embeddings_storage_mode='gpu'),
def build_embedding(self, lang, embedding_codes: List[str]) -> None: self.tic = time.time() self.embedding_name: str = "-".join(embedding_codes) self.lang = lang embedding_types: List[TokenEmbeddings] = [] for code in embedding_codes: code = code.lower() assert code in [ "bpe", "bert", "flair", "ft", "char", "ohe", "elmo", ], f"{code} - Invalid embedding code" if code == "ohe": embedding_types.append(OneHotEmbeddings(corpus=self.corpus)) elif code == "ft": embedding_types.append(WordEmbeddings(self.lang)) elif code == "bpe": embedding_types.append(BytePairEmbeddings(self.lang)) elif code == "bert": embedding_types.append( TransformerWordEmbeddings( model=self.huggingface_ref[self.lang], pooling_operation="first", layers="-1", fine_tune=False, ) ) elif code == "char": embedding_types.append(CharacterEmbeddings()) elif code == "flair": embedding_types.append(FlairEmbeddings(f"{self.lang}-forward")) embedding_types.append(FlairEmbeddings(f"{self.lang}-backward")) elif code == "elmo": embedding_types.append( ELMoEmbeddings(model="large", embedding_mode="all") ) self.embedding: StackedEmbeddings = StackedEmbeddings( embeddings=embedding_types ) self.tagger: SequenceTagger = SequenceTagger( hidden_size=256, embeddings=self.embedding, tag_dictionary=self.tag_dictionary, tag_type=self.tag_type, use_crf=True, ) self.trainer: ModelTrainer = ModelTrainer(self.tagger, self.corpus)
def train(self, training_dir=None): from flair.trainers import ModelTrainer if training_dir is None: training_dir = flair_splitter_dep_dir # define columns columns = {0: "text", 1: "ner"} # this is the folder in which train, test and dev files reside data_folder = flair_splitter_dep_dir + "data" # init a corpus using column format, data folder and the names of the train, dev and test files # note that training data should be unescaped, i.e. tokens like "&", not "&" corpus: Corpus = ColumnCorpus( data_folder, columns, train_file="sent_train.txt", test_file="sent_test.txt", dev_file="sent_dev.txt", document_separator_token="-DOCSTART-", ) print(corpus) tag_type = "ner" tag_dictionary = corpus.make_tag_dictionary(tag_type=tag_type) print(tag_dictionary) # initialize embeddings embedding_types = [ # WordEmbeddings('glove'), # comment in this line to use character embeddings CharacterEmbeddings(), # comment in these lines to use flair embeddings #FlairEmbeddings("news-forward"), #FlairEmbeddings("news-backward"), # BertEmbeddings('distilbert-base-cased') TransformerWordEmbeddings('google/electra-base-discriminator') ] embeddings: StackedEmbeddings = StackedEmbeddings( embeddings=embedding_types) tagger: SequenceTagger = SequenceTagger( hidden_size=128, embeddings=embeddings, tag_dictionary=tag_dictionary, tag_type=tag_type, use_crf=True, ) trainer: ModelTrainer = ModelTrainer(tagger, corpus) trainer.train(training_dir, learning_rate=0.1, mini_batch_size=16, max_epochs=50) self.model = tagger
def test_train_load_use_classifier(results_base_path, tasks_base_path): corpus = ColumnCorpus( data_folder=tasks_base_path / "conllu", train_file="train.conllup", dev_file="train.conllup", test_file="train.conllup", column_format={ 1: "text", 2: "pos", 3: "ner" }, ) relation_label_dict = corpus.make_label_dictionary(label_type="relation") embeddings = TransformerWordEmbeddings() model: RelationExtractor = RelationExtractor( embeddings=embeddings, label_dictionary=relation_label_dict, label_type="relation", entity_label_type="ner", train_on_gold_pairs_only=True, ) # initialize trainer trainer: ModelTrainer = ModelTrainer(model, corpus) trainer.train( results_base_path, learning_rate=0.1, mini_batch_size=2, max_epochs=3, shuffle=False, ) del trainer, model, relation_label_dict, corpus loaded_model: RelationExtractor = RelationExtractor.load( results_base_path / "final-model.pt") loaded_model.train_on_gold_pairs_only = False sentence = Sentence( ["Apple", "was", "founded", "by", "Steve", "Jobs", "."]) for token, tag in zip(sentence.tokens, ["B-ORG", "O", "O", "O", "B-PER", "I-PER", "O"]): token.set_label("ner", tag) loaded_model.predict(sentence) assert "founded_by" == sentence.get_labels("relation")[0].value # loaded_model.predict([sentence, sentence_empty]) # loaded_model.predict([sentence_empty]) del loaded_model
def train_model(directory='Data', use_BERT=True): # define columns columns = { 0: 'ID', 1: 'text', 2: 'empty_0', 3: 'pos', 4: 'empty_1', 5: 'empty_2', 6: 'empty_3', 7: 'empty_4', 8: 'empty_5', 9: 'tox' } # this is the folder in which train, test and dev files reside data_folder = directory # init a corpus using column format, data folder and the names of the train, dev and test files corpus: Corpus = ColumnCorpus(data_folder, columns, train_file='converted_data_train.conll', test_file='converted_data_test.conll', dev_file='converted_data_dev.conll') # tag to predict tag_type = 'tox' tag_dictionary = corpus.make_tag_dictionary(tag_type=tag_type) # embeddings if use_BERT: bert_embeddings = [ TransformerWordEmbeddings('bert-large-uncased', fine_tune=True) ] embeddings: StackedEmbeddings = StackedEmbeddings( embeddings=bert_embeddings) else: embedding_types = [WordEmbeddings('glove')] embeddings: StackedEmbeddings = StackedEmbeddings( embeddings=embedding_types) # initialize sequence tagger tagger: SequenceTagger = SequenceTagger(hidden_size=256, embeddings=embeddings, tag_dictionary=tag_dictionary, tag_type=tag_type, use_crf=True) # initialize trainer trainer: ModelTrainer = ModelTrainer(tagger, corpus) # start training trainer.train('resources/taggers/toxic_classifier_bert', learning_rate=0.1, mini_batch_size=32, max_epochs=5)
def test_transformer_word_embeddings(): embeddings = TransformerWordEmbeddings('distilbert-base-uncased', layers='-1,-2,-3,-4', layer_mean=False) sentence: Sentence = Sentence("I love Berlin") embeddings.embed(sentence) for token in sentence.tokens: assert len(token.get_embedding()) == 3072 token.clear_embeddings() assert len(token.get_embedding()) == 0 embeddings = TransformerWordEmbeddings('distilbert-base-uncased', layers='all', layer_mean=False) embeddings.embed(sentence) for token in sentence.tokens: assert len(token.get_embedding()) == 5376 token.clear_embeddings() assert len(token.get_embedding()) == 0 del embeddings embeddings = TransformerWordEmbeddings('distilbert-base-uncased', layers='all', layer_mean=True) embeddings.embed(sentence) for token in sentence.tokens: assert len(token.get_embedding()) == 768 token.clear_embeddings() assert len(token.get_embedding()) == 0 del embeddings
def test_sequence_tagger_transformer_finetune(results_base_path, tasks_base_path): flair.set_seed(123) # load dataset corpus: Corpus = ColumnCorpus( data_folder=tasks_base_path / "trivial" / "trivial_bioes", column_format={ 0: "text", 1: "ner" }, ) tag_dictionary = corpus.make_label_dictionary("ner") # tagger without CRF tagger: SequenceTagger = SequenceTagger( hidden_size=64, embeddings=TransformerWordEmbeddings("distilbert-base-uncased", fine_tune=True), tag_dictionary=tag_dictionary, tag_type="ner", use_crf=False, use_rnn=False, reproject_embeddings=False, ) # train trainer = ModelTrainer(tagger, corpus) trainer.fine_tune( results_base_path, mini_batch_size=2, max_epochs=10, shuffle=True, learning_rate=0.5e-4, ) loaded_model: SequenceTagger = SequenceTagger.load(results_base_path / "final-model.pt") sentence = Sentence("this is New York") sentence_empty = Sentence(" ") loaded_model.predict(sentence) loaded_model.predict([sentence, sentence_empty]) loaded_model.predict([sentence_empty]) # check if loaded model can predict entities = [span.text for span in sentence.get_spans("ner")] assert "New York" in entities # check if loaded model successfully fit the training data result: Result = loaded_model.evaluate(corpus.test, gold_label_type="ner") assert result.classification_report["micro avg"]["f1-score"] == 1.0 del loaded_model
def trainNER(data_dir, model_dir): parser = argparse.ArgumentParser() parser.add_argument("--model", default='bert-base-cased', type=str, required=True, help="The pretrained model to produce embeddings") args = parser.parse_args() model = args.model # pdb.set_trace() try: corpus: Corpus = CONLL_03(base_path=data_dir + '/') except FileNotFoundError: columns = {0: 'text', 1: 'ner'} corpus: Corpus = ColumnCorpus(data_dir, columns) corpus.filter_empty_sentences() tag_type = 'ner' # tag_dictionary = corpus.make_tag_dictionary(tag_type=tag_type) tag_dictionary = corpus.make_label_dictionary('ner') print(tag_dictionary.get_items()) stats = corpus.obtain_statistics() print(stats) # ['<unk>', 'O', 'B-DEVICE', 'I-DEVICE', 'B-TREE', 'I-TREE', 'B-APPLICATION', 'I-APPLICATION', 'B-LOCATION', 'I-LOCATION', '<START>', '<STOP>'] # pdb.set_trace() embedding_types: List[TokenEmbeddings] = [ WordEmbeddings('glove'), TransformerWordEmbeddings( model=model, layers='0', # dtype: str pooling_operation='first_last', use_scalar_mix=False, batch_size=16, fine_tune=False, allow_long_sentences=False) ] embeddings: StackedEmbeddings = StackedEmbeddings( embeddings=embedding_types) # biLSTM + CRF # tagger: SequenceTagger = SequenceTagger(hidden_size=256, # embeddings=embeddings, # tag_dictionary=tag_dictionary, # tag_type=tag_type) model_path = '/home/carolyn/Projects/mygit/Flair-NER/exprmt-20201120/conll_frac/10ptdata/models-5e-20201124/final-model.pt' tagger: SequenceTagger = SequenceTagger.load(model_path) trainer: ModelTrainer = ModelTrainer(tagger, corpus) trainer.train(model_dir, train_with_dev=False, max_epochs=10) # 150
def test_transformers_keep_tokenizer_when_saving(results_base_path): embeddings = TransformerWordEmbeddings( "sentence-transformers/paraphrase-albert-small-v2") results_base_path.mkdir(exist_ok=True, parents=True) initial_tagger_path = results_base_path / "initial_tokenizer.pk" reloaded_tagger_path = results_base_path / "reloaded_tokenizer.pk" initial_tagger = SequenceTagger(embeddings, Dictionary(), "ner") initial_tagger.save(initial_tagger_path) reloaded_tagger = SequenceTagger.load(initial_tagger_path) reloaded_tagger.save(reloaded_tagger_path)
def dump_bert_vecs(df, dump_dir): print("Getting BERT vectors...") embedding = TransformerWordEmbeddings('bert-base-uncased') word_counter = defaultdict(int) stop_words = set(stopwords.words('english')) stop_words.add("would") except_counter = 0 for index, row in df.iterrows(): if index % 100 == 0: print("Finished sentences: " + str(index) + " out of " + str(len(df))) line = row["sentence"] sentences = sent_tokenize(line) for sentence_ind, sent in enumerate(sentences): flag = 0 i = 0 sentence = None while flag == 0: sentence = Sentence(sent[:(len(sent) - i * 100)], use_tokenizer=True) try: embedding.embed(sentence) flag = 1 except Exception as e: except_counter += 1 print("Length of sentence: ", len(sent) - i * 100) print("Exception Counter while getting BERT: ", except_counter, sentence_ind, index, e) i += 1 if sentence is None or len(sentence) == 0: print("Length of sentence is 0: ", index) for token_ind, token in enumerate(sentence): word = token.text word = word.translate( str.maketrans('', '', string.punctuation)) if word in stop_words or "/" in word or len(word) == 0: continue word_dump_dir = dump_dir + word os.makedirs(word_dump_dir, exist_ok=True) fname = word_dump_dir + "/" + str( word_counter[word]) + ".pkl" word_counter[word] += 1 vec = token.embedding.cpu().numpy() try: with open(fname, "wb") as handler: pickle.dump(vec, handler) except Exception as e: except_counter += 1 print("Exception Counter while dumping BERT: ", except_counter, sentence_ind, index, word, e)
def dump_bert_vecs(df, dump_dir): print("Getting BERT vectors...") embedding = TransformerWordEmbeddings('roberta-base', layers='-1') word_counter = defaultdict(int) stop_words = set(stopwords.words('english')) stop_words.add("would") except_counter = 0 key = list(word_cnt.keys()) for index, row in df.iterrows(): file1 = open("progress.txt", "w+") file1.write(str(index)) print(index) if index % 100 == 0: print("Finished sentences: " + str(index) + " out of " + str(len(df))) line = row["news"] sentences = sent_tokenize(line) for sentence_ind, sent in enumerate(sentences): sentence = Sentence(sent, use_tokenizer=True) try: embedding.embed(sentence) except Exception as e: except_counter += 1 print("Exception Counter while getting BERT: ", except_counter, sentence_ind, index, e) continue for token_ind, token in enumerate(sentence): word = token.text word = word.translate( str.maketrans('', '', string.punctuation)) if word in stop_words or "/" in word or len(word) == 0 or ( word not in key) or word_cnt[word] < 10: #print("word") continue word_dump_dir = dump_dir + word os.makedirs(word_dump_dir, exist_ok=True) fname = word_dump_dir + "/" + str( word_counter[word]) + ".pkl" word_counter[word] += 1 vec = token.embedding.cpu().numpy() try: with open(fname, "wb") as handler: pickle.dump(vec, handler) except Exception as e: except_counter += 1 print("Exception Counter while dumping BERT: ", except_counter, sentence_ind, index, word, e)
def __init__(self, args, name, asp_word2idx, selected_idx=None, need_neg_senti=False): self.asp_word2idx = asp_word2idx self.need_neg_senti = need_neg_senti self.args = args self.embedding = TransformerWordEmbeddings('bert-base-uncased',layers='-1') if name == 'train': self.load_corpus_with_NULL_ITEM(os.path.join(args.data_dir,args.train), 'train', selected_idx, filter_null=args.unsupervised) elif name == 'dev': self.load_corpus_with_NULL_ITEM(os.path.join(args.data_dir,args.dev), 'dev', selected_idx) elif name == 'test': self.load_corpus_with_NULL_ITEM(os.path.join(args.data_dir,args.test), 'test', selected_idx) else: raise NotImplementedError self.len = len(self.corpus_y)
def test_transformer_weird_sentences(): embeddings = TransformerWordEmbeddings('distilbert-base-uncased', layers='all', use_scalar_mix=True) sentence = Sentence("Hybrid mesons , qq ̄ states with an admixture") embeddings.embed(sentence) for token in sentence: assert len(token.get_embedding()) == 768 sentence = Sentence("typical proportionalities of ∼ 1nmV − 1 [ 3,4 ] .") embeddings.embed(sentence) for token in sentence: assert len(token.get_embedding()) == 768
def main(directory, embeddings, strategy): # 1. find corpora in data directory corpora = {"train": None, "dev": None, "test": None} for labelset in corpora: for file in sorted(os.listdir(directory)): if infer_split(file) == labelset: corpora[labelset] = pd.read_csv( os.path.join(directory, file), sep="\t", names=["text", "pos", "lemma", "label"], engine="python", error_bad_lines=False, quoting=csv.QUOTE_NONE).fillna("") break if embeddings == "elmo": embedder = ELMoEmbeddings("original") elif embeddings == "flair": embedder = FlairEmbeddings("news-forward") elif embeddings == "bert": embedder = TransformerWordEmbeddings('bert-base-cased') embeddings_dir = os.path.join(directory, embeddings + "_embeddings") if not os.path.exists(embeddings_dir): os.makedirs(embeddings_dir, exist_ok=True) strategy = np.mean if strategy == "mean" else np.max if strategy == "max" else np.sum if strategy == "sum" else None for labelset, corpus in corpora.items(): if corpus is None: print(f"empty corpus: {labelset}") continue voc = sorted(corpus["text"].unique()) print(f"Unique tokens: {len(voc)}") with open(os.path.join(embeddings_dir, labelset + ".w2v"), "w") as f: for word in voc: sentence = Sentence(word) if len(sentence) == 0: continue embedder.embed(sentence) token_embedding = strategy( [token.embedding.cpu().numpy() for token in sentence], axis=0) f.write( word + " " + " ".join([str(num) for num in token_embedding.tolist()]) + '\n')
def __init__(self, config): """ Load pretrained language model """ super(LanguageModel, self).__init__() embeddings_stack = [] transformers = config.get("language_model", "transformers") if transformers is not "": transformers = transformers.split(";") for model in transformers: embeddings_stack.append( TransformerWordEmbeddings( model, layers="-1", pooling_operation='mean', # use_scalar_mix=True, fine_tune=True)) word_embeddings = config.get("language_model", "word_embeddings") if word_embeddings is not "": word_embeddings = word_embeddings.split(";") for model in word_embeddings: embeddings_stack.append(WordEmbeddings(model)) flair_embeddings = config.get("language_model", "flair_embeddings") if flair_embeddings is not "": flair_embeddings = flair_embeddings.split(";") for model in flair_embeddings: embeddings_stack.append(FlairEmbeddings(model, fine_tune=True)) character_embeddings = config.get("language_model", "character_embeddigs") if character_embeddings.lower() is "yes": embeddings_stack.append(CharacterEmbeddings(character_embeddings)) bytepair_embeddings = config.get("language_model", "bytepair_embeddings") if bytepair_embeddings.lower() is "yes": embeddings_stack.append(BytePairEmbeddings()) custom_embeddings = config.get("language_model", "custom_embeddings") if custom_embeddings is not "": custom_embeddings = custom_embeddings.split(";") for path in custom_embeddings: embeddings_stack.append(WordEmbeddings(path)) self.lm = StackedEmbeddings(embeddings_stack) self.embedding_dim = self.lm.embedding_length self.dropout = torch.nn.Dropout( float(config.get("language_model", "dropout"))) self.classify = torch.nn.Linear(self.embedding_dim, 2) if config.get("language_model", "relu") == "yes": self.relu = torch.nn.ReLU()
def trainNER(data_dir, model_dir): parser = argparse.ArgumentParser() parser.add_argument("--model", default='bert-base-cased', type=str, required=True, help="The pretrained model to produce embeddings") args = parser.parse_args() model = args.model columns = {0: 'text', 1: 'ner'} # pdb.set_trace() # print(data_dir + '/eng.train') corpus: Corpus = ColumnCorpus(data_dir, columns) corpus.filter_empty_sentences() tag_type = 'ner' # tag_dictionary = corpus.make_tag_dictionary(tag_type=tag_type) tag_dictionary = corpus.make_label_dictionary('ner') print(tag_dictionary.get_items()) stats = corpus.obtain_statistics() print(stats) # ['<unk>', 'O', 'B-DEVICE', 'I-DEVICE', 'B-TREE', 'I-TREE', 'B-APPLICATION', 'I-APPLICATION', 'B-LOCATION', 'I-LOCATION', '<START>', '<STOP>'] # pdb.set_trace() embedding_types: List[TokenEmbeddings] = [ WordEmbeddings('glove'), TransformerWordEmbeddings( model=model, layers='0', # dtype: str pooling_operation='first_last', use_scalar_mix=False, batch_size=16, fine_tune=False, allow_long_sentences=False) ] embeddings: StackedEmbeddings = StackedEmbeddings( embeddings=embedding_types) # biLSTM + CRF tagger: SequenceTagger = SequenceTagger(hidden_size=256, embeddings=embeddings, tag_dictionary=tag_dictionary, tag_type=tag_type) trainer: ModelTrainer = ModelTrainer(tagger, corpus) trainer.train(model_dir, train_with_dev=True, max_epochs=10) # 150
def _get_embedding_model( model_name_or_path: str ) -> Union[FlairEmbeddings, WordEmbeddings, TransformerWordEmbeddings, Sentence]: "Load the proper `Embeddings` model from `model_name_or_path`" if ("flair" in model_name_or_path or model_name_or_path in FLAIR_PRETRAINED_MODEL_NAMES): return FlairEmbeddings(model_name_or_path) else: try: return WordEmbeddings(model_name_or_path) except ValueError: try: return TransformerWordEmbeddings(model_name_or_path) except ValueError: raise ValueError( f"Embeddings not found for the model key: {model_name_or_path}, check documentation or custom model path to verify specified model" )
def test_transformer_weird_sentences(): embeddings = TransformerWordEmbeddings('distilbert-base-uncased', layers='all', layer_mean=True) sentence = Sentence("Hybrid mesons , qq ̄ states with an admixture") embeddings.embed(sentence) for token in sentence: assert len(token.get_embedding()) == 768 sentence = Sentence( "typical proportionalities of ∼ 1nmV − 1 [ 3,4 ] .") embeddings.embed(sentence) for token in sentence: assert len(token.get_embedding()) == 768 sentence = Sentence("🤟 🤟 🤟 hüllo") embeddings.embed(sentence) for token in sentence: assert len(token.get_embedding()) == 768 sentence = Sentence("🤟hallo 🤟 🤟 🤟 🤟") embeddings.embed(sentence) for token in sentence: assert len(token.get_embedding()) == 768 sentence = Sentence("🤟hallo 🤟 🤟 🤟 🤟") embeddings.embed(sentence) for token in sentence: assert len(token.get_embedding()) == 768 sentence = Sentence("🤟") embeddings.embed(sentence) for token in sentence: assert len(token.get_embedding()) == 768 sentence = Sentence("🤟") sentence_2 = Sentence("second sentence") embeddings.embed([sentence, sentence_2]) for token in sentence: assert len(token.get_embedding()) == 768 for token in sentence_2: assert len(token.get_embedding()) == 768
def test_transformer_word_embeddings_forward_language_ids(): cos = torch.nn.CosineSimilarity(dim=0, eps=1e-10) sent_en = Sentence(["This", "is", "a", "sentence"], language_code="en") sent_de = Sentence(["Das", "ist", "ein", "Satz"], language_code="de") embeddings = TransformerWordEmbeddings("xlm-mlm-ende-1024", allow_long_sentences=False) embeddings.embed([sent_de, sent_en]) expected_similarities = [ 0.7102344036102295, 0.7598986625671387, 0.7437312602996826, 0.5584433674812317 ] for (token_de, token_en, exp_sim) in zip(sent_de, sent_en, expected_similarities): sim = cos(token_de.embedding, token_en.embedding).item() assert abs(exp_sim - sim) < 1e-5
def handle(self, *args, **options): file = options.get('file') or 'annotated_sentences' model_folder = options.get('model_folder') or 'model-var' columns = {0: 'text', 1: 'var'} data_folder = 'data/txt' corpus = ColumnCorpus(data_folder, columns, train_file=f'{file}.txt') tag_type = 'var' tag_dictionary = corpus.make_tag_dictionary(tag_type=tag_type) embedding_types = [ WordEmbeddings('glove'), # comment in this line to use character embeddings # CharacterEmbeddings(), # comment in these lines to use flair embeddings # FlairEmbeddings('news-forward'), # FlairEmbeddings('news-backward'), TransformerWordEmbeddings('bert-base-uncased'), ] embeddings = StackedEmbeddings(embeddings=embedding_types) tagger = SequenceTagger(hidden_size=256, embeddings=embeddings, tag_dictionary=tag_dictionary, tag_type=tag_type, use_crf=True) trainer = ModelTrainer(tagger, corpus) trainer.train(f'data/models/taggers/{model_folder}', learning_rate=0.1, mini_batch_size=32, max_epochs=150) self.stdout.write(self.style.SUCCESS(f'Successfully trained model on dataset file.'))
def embed_text( self, text: Union[List[Sentence], Sentence, List[str], str], model_name_or_path: str = "bert-base-cased", ) -> List[Sentence]: """Produces embeddings for text * **text** - Text input, it can be a string or any of Flair's `Sentence` input formats * **model_name_or_path** - The hosted model name key or model path **return** - A list of Flair's `Sentence`s """ # Convert into sentences if isinstance(text, str): sentences = Sentence(text) elif isinstance(text, list) and all(isinstance(t, str) for t in text): sentences = [Sentence(t) for t in text] else: sentences = text # Load correct Embeddings module if not self.models[model_name_or_path]: if ("flair" in model_name_or_path or model_name_or_path in FLAIR_PRETRAINED_MODEL_NAMES): self.models[model_name_or_path] = FlairEmbeddings( model_name_or_path) else: try: self.models[model_name_or_path] = WordEmbeddings( model_name_or_path) except ValueError: try: self.models[ model_name_or_path] = TransformerWordEmbeddings( model_name_or_path) except ValueError: raise ValueError( f"Embeddings not found for the model key: {model_name_or_path}, check documentation or custom model path to verify specified model" ) return Sentence("") embedding = self.models[model_name_or_path] return embedding.embed(sentences)
def EmbeddingFactory(parameters, corpus): from flair.embeddings import FlairEmbeddings, StackedEmbeddings, \ WordEmbeddings, OneHotEmbeddings, CharacterEmbeddings, TransformerWordEmbeddings stack = [] for emb in parameters.embedding.split(): if any((spec in emb) for spec in ("bert", "gpt", "xlnet")): stack.append( TransformerWordEmbeddings(model=pretrainedstr( emb, parameters.language), fine_tune=parameters.tune_embedding)) elif emb == "flair": stack += [ FlairEmbeddings(f"{parameters.language}-forward", fine_tune=parameters.tune_embedding), FlairEmbeddings(f"{parameters.language}-backward", fine_tune=parameters.tune_embedding) ] elif emb == "pos": stack.append( OneHotEmbeddings(corpus, field="pos", embedding_length=parameters.pos_embedding_dim, min_freq=1)) elif emb == "fasttext": stack.append(WordEmbeddings(parameters.language)) elif emb == "word": stack.append( OneHotEmbeddings( corpus, field="text", embedding_length=parameters.word_embedding_dim, min_freq=parameters.word_minfreq)) elif emb == "char": stack.append( CharacterEmbeddings( char_embedding_dim=parameters.char_embedding_dim, hidden_size_char=parameters.char_bilstm_dim)) else: raise NotImplementedError() return StackedEmbeddings(stack)
def load_model(bert=None, document=False, flair=False): """Load word embeddings model.""" if bert == 'bio': # https://github.com/flairNLP/flair/issues/1085 # also see readme for instructions bertpath = './bert/bert-base-biobert-cased' elif bert == 'sci': # https://github.com/flairNLP/flair/issues/744 # https://github.com/flairNLP/flair/issues/1239 bertpath = './bert/scibert_scivocab_uncased' else: bertpath = 'bert-base-uncased' if document and not flair: bert_embedding = TransformerDocumentEmbeddings(model=bertpath, batch_size=4) return bert_embedding bert_embedding = TransformerWordEmbeddings(model=bertpath, pooling_operation='first', batch_size=4) if flair: flair_embedding_forward = FlairEmbeddings('en-forward') flair_embedding_backward = FlairEmbeddings('en-backward') embed_arr = [ bert_embedding, flair_embedding_backward, flair_embedding_forward, ] else: embed_arr = [bert_embedding] if document: document_embeddings = DocumentPoolEmbeddings( embed_arr, fine_tune_mode='nonlinear') else: document_embeddings = StackedEmbeddings(embed_arr) return document_embeddings
def train(self, training_dir=None): from flair.trainers import ModelTrainer if training_dir is None: training_dir = script_dir + "flair" + os.sep # define columns columns = {0: "text", 1: "ner"} # this is the folder in which train, test and dev files reside data_folder = training_dir + "data" # init a corpus using column format, data folder and the names of the train, dev and test files # note that training data should be unescaped, i.e. tokens like "&", not "&" corpus: Corpus = ColumnCorpus( data_folder, columns, train_file="sent_train.txt", test_file="sent_test.txt", dev_file="sent_dev.txt", ) print(corpus) tag_type = "ner" tag_dictionary = corpus.make_tag_dictionary(tag_type=tag_type) print(tag_dictionary) # initialize embeddings embeddings: TransformerWordEmbeddings = TransformerWordEmbeddings('onlplab/alephbert-base') tagger: SequenceTagger = SequenceTagger( hidden_size=128, embeddings=embeddings, tag_dictionary=tag_dictionary, tag_type=tag_type, use_crf=True, ) trainer: ModelTrainer = ModelTrainer(tagger, corpus) trainer.train(training_dir, learning_rate=0.1, mini_batch_size=32, max_epochs=50) self.model = tagger
data_path = args[2] model_path = args[3] # CKGE Graph embeddings ckge_dataset = Dataset(os.path.join(data_path, "CKGE"), use_cpu=True) ckge_model = CP(ckge_dataset.get_shape(), 50) ckge_model.load_state_dict( torch.load(os.path.join(model_path, "CKGE.pickle"), map_location=torch.device('cpu'))) ckge_graph_vectorizer = GraphEmbeddingTransformer(ckge_dataset, ckge_model) # Distil RoBERTa (DR) flair_vectorizer_DR = FlairTransformer([ TransformerWordEmbeddings(model="distilroberta-base", use_scalar_mix=True) ], batch_size=1) # GPT2 flair_vectorizer_GPT2 = FlairTransformer([ TransformerWordEmbeddings(model="gpt2-large", use_scalar_mix=True) ], batch_size=1) ckge_dr_union = FeatureUnion([('TE', flair_vectorizer_DR), ('CP', ckge_graph_vectorizer)]) ckge_gpt2_union = FeatureUnion([('TE', flair_vectorizer_GPT2), ('CP', ckge_graph_vectorizer)]) # parametres_grid_ridge = { # "estimator__alpha": [0.01, 0.1, 0.5, 1, 1.5, 3, 6], # "estimator__normalize": [True, False], # "estimator__tol": [1e-5, 1e-3, 1e-1]
from preprocessing.normalize import normalize from utility.frequency_loader import load_frequencies, load_doc_frequencies from utility.run_experiment import run_experiment import os if not os.path.exists(IMAGE_PATH): os.makedirs(IMAGE_PATH) sick_all, sick_train, sick_test, sick_dev = download_and_load_sick_dataset() print('Downloaded data') frequency = load_frequencies("data/frequencies/frequencies.tsv") doc_frequency = load_doc_frequencies("data/frequencies/doc_frequencies.tsv") word2vec = load_word2vec(w2v_path) elmo = ELMoEmbeddings('large') bert = TransformerWordEmbeddings('bert-large-cased') flair = StackedEmbeddings([ WordEmbeddings('glove'), FlairEmbeddings('news-forward'), FlairEmbeddings('news-backward') ]) elmo_bert = StackedEmbeddings([elmo, bert]) print("Loaded Resources") benchmarks = [("AVG-W2V", ft.partial(run_avg_benchmark, model=word2vec, use_stoplist=False)), ("AVG-ELMO", ft.partial(run_context_avg_benchmark,