def main(config, name, args):
from flair.trainers import ModelTrainer
from flair.visual.training_curves import Plotter
from math import ceil
from torch.optim import Adam
from torch import manual_seed
from pickle import load
from discodop.lexgrammar import SupertagGrammar
cp = corpusparam(**config["Corpus"], **config["Grammar"])
corpus = SupertagParseCorpus(cp.filename)
grammar = load(open(f"{cp.filename}.grammar", "rb"))
tc = FindlrParameters(**config["Training"],
**config["Eval-common"],
**config["Eval-Development"],
language=cp.language)
model = Supertagger.from_corpus(corpus, grammar, tc)
model.set_eval_param(tc)
if args.downsample:
corpus = corpus.downsample(args.downsample)
if args.iterations is None:
epoch = ceil(len(corpus.train) / tc.batchsize)
args.iterations = epoch * 5
trainer = ModelTrainer(model, corpus)
learning_rate_tsv = trainer.find_learning_rate(
name,
start_learning_rate=args.min_lr,
end_learning_rate=args.max_lr,
iterations=args.iterations)
plotter = Plotter()
plotter.plot_learning_rate(learning_rate_tsv)
def optimize_lr():
corpus, label_dictionary = load_corpus()
embeddings = [
WordEmbeddings('glove'),
FlairEmbeddings('news-forward'),
FlairEmbeddings('news-backward')
]
document_embeddings = DocumentRNNEmbeddings(embeddings,
hidden_size=512,
reproject_words=True,
reproject_words_dimension=256,
bidirectional=True)
classifier = TextClassifier(document_embeddings,
label_dictionary=label_dictionary,
multi_label=False)
trainer = ModelTrainer(classifier, corpus)
# 7. find learning rate
learning_rate_tsv = trainer.find_learning_rate('resources/classifiers/',
'learning_rate.tsv')
# 8. plot the learning rate finder curve
from flair.visual.training_curves import Plotter
plotter = Plotter()
plotter.plot_learning_rate(learning_rate_tsv)
def find_learning_rate(
self,
output_dir: Union[Path, str],
file_name: str = "learning_rate.tsv",
start_learning_rate: float = 1e-8,
end_learning_rate: float = 10,
iterations: int = 100,
mini_batch_size: int = 32,
stop_early: bool = True,
smoothing_factor: float = 0.7,
plot_learning_rate: bool = True,
**kwargs,
) -> float:
"""
Uses Leslie's cyclical learning rate finding method to generate and save the loss x learning rate plot
This method returns a suggested learning rate using the static method `LMFineTuner.suggest_learning_rate()`
which is implicitly run in this method.
* **output_dir** - Path to dir for learning rate file to be saved
* **file_name** - Name of learning rate .tsv file
* **start_learning_rate** - Initial learning rate to start cyclical learning rate finder method
* **end_learning_rate** - End learning rate to stop exponential increase of the learning rate
* **iterations** - Number of optimizer iterations for the ExpAnnealLR scheduler
* **mini_batch_size** - Batch size for dataloader
* **stop_early** - Bool for stopping early once loss diverges
* **smoothing_factor** - Smoothing factor on moving average of losses
* **adam_epsilon** - Epsilon for Adam optimizer.
* **weight_decay** - Weight decay if we apply some.
* **kwargs** - Additional keyword arguments for the Adam optimizer
**return** - Learning rate as a float
"""
# 7. find learning rate
learning_rate_tsv = self.trainer.find_learning_rate(
base_path=output_dir,
file_name=file_name,
start_learning_rate=start_learning_rate,
end_learning_rate=end_learning_rate,
iterations=iterations,
mini_batch_size=mini_batch_size,
stop_early=stop_early,
smoothing_factor=smoothing_factor,
)
# Reinitialize optimizer and parameters by reinitializing trainer
self._initial_setup(self.label_dict, **self.trainer_kwargs)
if plot_learning_rate:
plotter = Plotter()
plotter.plot_learning_rate(learning_rate_tsv)
# Use the automated learning rate finder
with open(learning_rate_tsv) as lr_f:
lr_tsv = list(csv.reader(lr_f, delimiter="\t"))
losses = np.array([float(row[-1]) for row in lr_tsv[1:]])
lrs = np.array([float(row[-2]) for row in lr_tsv[1:]])
lr_to_use = self.suggested_learning_rate(losses, lrs, **kwargs)
print(f"Recommended Learning Rate {lr_to_use}")
return lr_to_use
def find_learning_rate(trainer, params):
learning_rate_tsv = trainer.find_learning_rate(
os.path.join("hyperparameter_search", params['model_output_dirpath']),
'learning_rate_search_log.tsv',
iterations=400,
stop_early=False,
mini_batch_size=16)
from flair.visual.training_curves import Plotter
plotter = Plotter()
plotter.plot_learning_rate(learning_rate_tsv)
def train_sequence_labeling_model(data_folder, proposed_tags_vocabulary_size):
# define columns
columns = {0: 'text', 1: 'pos', 2: 'is_separator', 3: 'proposed_tags'}
# init a corpus using column format, data folder and the names of the train and test files
# 1. get the corpus
corpus: Corpus = ColumnCorpus(data_folder, columns,
train_file='train',
test_file='test',
dev_file=None)
log.info(corpus)
# 2. what tag do we want to predict
tag_type = 'pos'
# 3. make the tag dictionary from the corpus
tag_dictionary = corpus.make_tag_dictionary(tag_type=tag_type)
log.info(tag_dictionary)
# 4. initialize embeddings
local_model_path = use_scratch_dir_if_available('resources/polish_FastText_embeddings')
embedding_types: List[TokenEmbeddings] = [
FlairEmbeddings('pl-forward', chars_per_chunk=64),
FlairEmbeddings('pl-backward', chars_per_chunk=64),
OneHotEmbeddings(corpus=corpus, field='is_separator', embedding_length=3, min_freq=3),
OneHotEmbeddings(corpus=corpus, field='proposed_tags',
embedding_length=math.ceil((proposed_tags_vocabulary_size + 1)**0.25),
min_freq=3),
WordEmbeddings(local_model_path) if os.path.exists(local_model_path) else WordEmbeddings('pl')
]
embeddings: StackedEmbeddings = StackedEmbeddings(embeddings=embedding_types)
# 5. initialize sequence tagger
tagger: SequenceTagger = SequenceTagger(hidden_size=256,
embeddings=embeddings,
tag_dictionary=tag_dictionary,
tag_type=tag_type,
use_crf=False,
rnn_layers=2)
# 6. initialize trainer
trainer: ModelTrainer = ModelTrainer(tagger, corpus)
# 7. start training
trainer.train(use_scratch_dir_if_available('resources_pol_eval/taggers/example-pos/'),
learning_rate=0.1,
mini_batch_size=32,
embeddings_storage_mode='gpu',
max_epochs=sys.maxsize,
monitor_test=True)
# 8. plot weight traces (optional)
plotter = Plotter()
plotter.plot_weights(use_scratch_dir_if_available('resources_pol_eval/taggers/example-pos/weights.txt'))
def __init__(self, path: Union[Path, str], model: str = 'final-model.pt'):
if type(path) == str:
path = Path(path)
assert path.exists()
self.path = path
self.model = SequenceTagger.load(path / model)
self.cv_results = {}
for file in ['summary', 'details']:
try:
self.cv_results[file] = pickle.load(
(path / (file + '.pkl')).open(mode='rb'))
except FileNotFoundError:
print(
f"{file+'.pkl'} not found. Setting cv_results['{file}'] to None"
)
self.plotter = Plotter()
def train(
self,
output_dir: Union[Path, str],
learning_rate: float = 0.07,
mini_batch_size: int = 32,
anneal_factor: float = 0.5,
patience: int = 5,
max_epochs: int = 150,
plot_weights: bool = False,
**kwargs,
) -> None:
"""
Train the Sequence Classifier
* **output_dir** - The output directory where the model predictions and checkpoints will be written.
* **learning_rate** - The initial learning rate
* **mini_batch_size** - Batch size for the dataloader
* **anneal_factor** - The factor by which the learning rate is annealed
* **patience** - Patience is the number of epochs with no improvement the Trainer waits until annealing the learning rate
* **max_epochs** - Maximum number of epochs to train. Terminates training if this number is surpassed.
* **plot_weights** - Bool to plot weights or not
* **kwargs** - Keyword arguments for the rest of Flair's `Trainer.train()` hyperparameters
"""
if isinstance(output_dir, str):
output_dir = Path(output_dir)
# Start the training
self.trainer.train(
output_dir,
learning_rate=learning_rate,
mini_batch_size=mini_batch_size,
anneal_factor=anneal_factor,
patience=patience,
max_epochs=max_epochs,
**kwargs,
)
# Plot weight traces
if plot_weights:
plotter = Plotter()
plotter.plot_weights(output_dir / "weights.txt")
def test_plotting_training_curves_and_weights():
plotter = Plotter()
plotter.plot_training_curves('./resources/visual/loss.tsv')
plotter.plot_weights('./resources/visual/weights.txt')
# clean up directory
os.remove('./resources/visual/weights.png')
os.remove('./resources/visual/training.png')
def test_plotting_training_curves_and_weights(resources_path):
plotter = Plotter()
plotter.plot_training_curves(resources_path / "visual/loss.tsv")
plotter.plot_weights(resources_path / "visual/weights.txt")
# clean up directory
(resources_path / "visual/weights.png").unlink()
(resources_path / "visual/training.png").unlink()
def plot_curve(self,
traing_curve_path=os.path.normpath(
r'./resources/taggers/slow_bert/loss.tsv'),
weights_path=os.path.normpath(
r'./resources/taggers/slow_bert/loss.tsv')):
from flair.visual.training_curves import Plotter
plotter = Plotter()
plotter.plot_training_curves(traing_curve_path)
plotter.plot_weights(weights_path)
def main(args):
args = parser.parse_args()
# 1. get the corpus
corpus: TaggedCorpus = NLPTaskDataFetcher.load_classification_corpus(
args.data_dir[0],
train_file='train.txt',
dev_file='dev.txt',
test_file='test.txt')
# 2. create the label dictionary
label_dict = corpus.make_label_dictionary()
# 3. make a list of word embeddings
word_embeddings = [
WordEmbeddings('glove'),
# comment in flair embeddings for state-of-the-art results
# FlairEmbeddings('news-forward'),
# FlairEmbeddings('news-backward'),
# ELMoEmbeddings()
]
# 4. init document embedding by passing list of word embeddings
document_embeddings: DocumentLSTMEmbeddings = DocumentLSTMEmbeddings(
word_embeddings,
hidden_size=128,
reproject_words=True,
reproject_words_dimension=64,
)
# 5. create the text classifier
classifier = TextClassifier(document_embeddings,
label_dictionary=label_dict,
multi_label=False)
# 6. initialize the text classifier trainer
trainer = ModelTrainer(classifier, corpus)
# 7. start the training
model_out = 'resources/classifiers/sentence-classification/glove'
trainer.train(model_out,
learning_rate=0.1,
mini_batch_size=32,
anneal_factor=0.5,
patience=5,
max_epochs=100)
# 8. plot training curves (optional)
from flair.visual.training_curves import Plotter
plotter = Plotter()
plotter.plot_training_curves(join(model_out, 'loss.tsv'))
plotter.plot_weights(join(model_out, 'weights.txt'))
def __init__(self, corpus_name: str):
corpus = NLPTaskDataFetcher.load_column_corpus(
loc.abs_path([loc.ASSETS, loc.MODELS, loc.DIRKSON]), {
0: 'text',
1: 'ner'
},
train_file=corpus_name + loc.DIRKSON_VALIDATION_TXT,
test_file=corpus_name + loc.DIRKSON_TEST_TXT)
embedding_types = [
BertEmbeddings('bert-base-uncased'),
FlairEmbeddings('mix-forward'),
FlairEmbeddings('mix-backward')
]
tag_type = 'ner'
embeddings = StackedEmbeddings(embeddings=embedding_types)
tag_dictionary = corpus.make_tag_dictionary(tag_type=tag_type)
tagger: SequenceTagger = SequenceTagger(hidden_size=256,
embeddings=embeddings,
tag_dictionary=tag_dictionary,
tag_type=tag_type,
use_crf=True)
trainer: ModelTrainer = ModelTrainer(tagger, corpus)
if not path.exists:
os.mkdir(
loc.abs_path(
[loc.ASSETS, loc.MODELS, loc.DIRKSON, corpus_name]))
trainer.train(loc.abs_path(
[loc.ASSETS, loc.MODELS, loc.DIRKSON, corpus_name]),
learning_rate=0.1,
mini_batch_size=32,
max_epochs=150)
plotter = Plotter()
plotter.plot_training_curves(
loc.abs_path([
loc.ASSETS, loc.MODELS, loc.DIRKSON, corpus_name, loc.LOSS_TSV
]))
plotter.plot_weights(
loc.abs_path([
loc.ASSETS, loc.MODELS, loc.DIRKSON, corpus_name,
loc.WEIGHTS_TXT
]))
def train(self,
trainfile,
devfile,
testfile,
resfolder,
embtype="bert",
chunk_len=100,
batch_len=8):
"""
*** This method can be used to train new models with the settings used in project Redewiedergabe
It is not accessible from rwtagger_script and not documented in detail. Use at your own risk. ;-)
***
:param trainfile:
:param devfile:
:param testfile:
:param resfolder:
:param embtype:
:param chunk_len:
:param batch_len:
:return:
"""
emb_name, embeddings = self._get_embeddings(embtype)
corpus: Corpus = self.create_corpus(trainfile, devfile, testfile,
chunk_len)
tag_dictionary = corpus.make_tag_dictionary(tag_type="cat")
if not os.path.exists(resfolder):
os.makedirs(resfolder)
tagger: SequenceTagger = SequenceTagger(hidden_size=256,
embeddings=embeddings,
tag_dictionary=tag_dictionary,
tag_type="cat",
use_crf=True,
rnn_layers=2)
trainer: ModelTrainer = ModelTrainer(tagger, corpus)
trainer.train(resfolder,
learning_rate=0.1,
mini_batch_size=batch_len,
max_epochs=150,
checkpoint=True)
# plot training curves
plotter = Plotter()
plotter.plot_training_curves(os.path.join(resfolder, 'loss.tsv'))
plotter.plot_weights(os.path.join(resfolder, 'weights.txt'))
def train(data_folder, model_output_folder):
corpus: TaggedCorpus = NLPTaskDataFetcher.load_corpus(
NLPTask.CONLL_03, base_path=data_folder)
# 2. what tag do we want to predict?
tag_type = 'ner'
# 3. make the tag dictionary from the corpus
tag_dictionary = corpus.make_tag_dictionary(tag_type=tag_type)
print(tag_dictionary.idx2item)
# init Flair embeddings
flair_forward_embedding = FlairEmbeddings('multi-forward')
flair_backward_embedding = FlairEmbeddings('multi-backward')
# init multilingual BERT
bert_embedding = BertEmbeddings('bert-base-multilingual-cased')
# 4. initialize embeddings
embedding_types: List[TokenEmbeddings] = [
flair_forward_embedding, flair_backward_embedding, bert_embedding
]
embeddings: StackedEmbeddings = StackedEmbeddings(
embeddings=embedding_types)
# 5. initialize sequence tagger
from flair.models import SequenceTagger
tagger: SequenceTagger = SequenceTagger(hidden_size=256,
embeddings=embeddings,
tag_dictionary=tag_dictionary,
tag_type=tag_type)
# 6. initialize trainer
from flair.trainers import ModelTrainer
trainer: ModelTrainer = ModelTrainer(tagger, corpus)
# 7. start training
trainer.train(model_output_folder, mini_batch_size=256, max_epochs=150)
# 8. plot training curves (optional)
from flair.visual.training_curves import Plotter
plotter = Plotter()
plotter.plot_training_curves(model_output_folder + '/loss.tsv')
plotter.plot_weights(model_output_folder + '/weights.txt')
def main():
train_dev_corpus = NLPTaskDataFetcher.load_classification_corpus(
Path(DATA_PATH),
train_file='flair_train.csv',
test_file='flair_test.csv',
dev_file='flair_dev.csv')
label_dict = train_dev_corpus.make_label_dictionary()
word_embeddings = [
WordEmbeddings('crawl'),
FlairEmbeddings('news-forward-fast', chars_per_chunk=128),
FlairEmbeddings('news-backward-fast', chars_per_chunk=128)
]
document_embeddings = DocumentRNNEmbeddings(word_embeddings,
rnn_type='LSTM',
hidden_size=128,
reproject_words=True,
reproject_words_dimension=64)
classifier = TextClassifier(document_embeddings,
label_dictionary=label_dict,
multi_label=False)
trainer = ModelTrainer(classifier, train_dev_corpus)
trainer.train(PRETRAINED_FLAIR,
max_epochs=40,
learning_rate=0.2,
mini_batch_size=32,
embeddings_in_memory=False,
checkpoint=True)
plotter = Plotter()
plotter.plot_training_curves(FLAIR_LOSS)
plotter.plot_weights(FLAIR_WEIGHTS)
def train_sequence_labeling_model(data_folder, proposed_tags_vocabulary_size,
skf_split_no):
"""
Trains the sequence labeling model (by default model uses one RNN layer).
Model is trained to predict part of speech tag and takes into account information about:
- text (plain text made of tokens that together form a sentence),
- occurrence of separator before token,
- proposed tags for given token.
It is trained with use of Stacked Embeddings used to combine different embeddings together. Words are embedded
using a concatenation of two vector embeddings:
- Flair Embeddings - contextual string embeddings that capture latent syntactic-semantic
information that goes beyond standard word embeddings. Key differences are: (1) they are trained without any
explicit notion of words and thus fundamentally model words as sequences of characters. And (2) they are
contextualized by their surrounding text, meaning that the same word will have different embeddings depending on
its contextual use.
There are forward (that goes through the given on input plain text form left to right) and backward model (that
goes through the given on input plain text form right to left) used for part of speech (pos) tag training.
- One Hot Embeddings - embeddings that encode each word in a vocabulary as a one-hot vector, followed by an
embedding layer. These embeddings thus do not encode any prior knowledge as do most other embeddings. They also
differ in that they require to see a Corpus during instantiation, so they can build up a vocabulary consisting of
the most common words seen in the corpus, plus an UNK token for all rare words.
There are two One Hot Embeddings used in training:
- first to embed information about occurrence of separator before token,
- second to embed information about concatenated with a ';' proposed tags.
Model and training logs are saved in resources/taggers/example-pos directory.
This is the method where internal states of forward and backward Flair models are taken at the end of each token
and, supplemented by information about occurrence of separator before token and proposed tags for given token used
to train model for one of stratified 10 fold cross validation splits.
:param data_folder: folder where files with column corpus split are stored. Those columns are used to initialize
ColumnCorpus object
:param proposed_tags_vocabulary_size: number of proposed tags
:param skf_split_no: number that indicates one of stratified 10 fold cross validation splits (from range 1 to 10)
used to train the model
"""
# define columns
columns = {0: 'text', 1: 'pos', 2: 'is_separator', 3: 'proposed_tags'}
# init a corpus using column format, data folder and the names of the train and test files
# 1. get the corpus
corpus: Corpus = ColumnCorpus(data_folder,
columns,
train_file='train_' + str(skf_split_no),
test_file='test_' + str(skf_split_no),
dev_file=None)
log.info(corpus)
# 2. what tag do we want to predict
tag_type = 'pos'
# 3. make the tag dictionary from the corpus
tag_dictionary = corpus.make_tag_dictionary(tag_type=tag_type)
log.info(tag_dictionary)
# 4. initialize embeddings
embedding_types: List[TokenEmbeddings] = [
FlairEmbeddings('pl-forward', chars_per_chunk=64),
FlairEmbeddings('pl-backward', chars_per_chunk=64),
OneHotEmbeddings(corpus=corpus,
field='is_separator',
embedding_length=3,
min_freq=3),
OneHotEmbeddings(corpus=corpus,
field='proposed_tags',
embedding_length=math.ceil(
(proposed_tags_vocabulary_size + 1)**0.25),
min_freq=3)
]
embeddings: StackedEmbeddings = StackedEmbeddings(
embeddings=embedding_types)
# 5. initialize sequence tagger
tagger: SequenceTagger = SequenceTagger(hidden_size=256,
embeddings=embeddings,
tag_dictionary=tag_dictionary,
tag_type=tag_type,
use_crf=False,
rnn_layers=1)
# 6. initialize trainer
trainer: ModelTrainer = ModelTrainer(tagger, corpus)
# 7. start training
trainer.train(
use_scratch_dir_if_available('resources/taggers/example-pos/it-' +
str(skf_split_no)),
learning_rate=0.1,
mini_batch_size=32,
embeddings_storage_mode='gpu',
max_epochs=sys.maxsize,
monitor_test=True)
# 8. plot weight traces (optional)
plotter = Plotter()
plotter.plot_weights(
use_scratch_dir_if_available('resources/taggers/example-pos/it-' +
str(skf_split_no) + '/weights.txt'))
# comment in this line to use character embeddings
# CharacterEmbeddings(),
# comment in these lines to use flair embeddings
# FlairEmbeddings('news-forward'),
# FlairEmbeddings('news-backward'),
]
embeddings: StackedEmbeddings = StackedEmbeddings(embeddings=embedding_types)
# 5. initialize sequence tagger
tagger: SequenceTagger = SequenceTagger(hidden_size=256,
embeddings=embeddings,
tag_dictionary=tag_dictionary,
tag_type=tag_type,
use_crf=True)
# 6. initialize trainer
trainer: ModelTrainer = ModelTrainer(tagger, corpus)
# 7. start training
trainer.train('resources/taggers/example-ner',
learning_rate=0.1,
mini_batch_size=32,
max_epochs=150)
# 8. plot weight traces (optional)
plotter = Plotter()
plotter.plot_weights('resources/taggers/example-ner/weights.txt')
embeddings: StackedEmbeddings = StackedEmbeddings(embeddings=embedding_types)
# initialize sequence tagger
from flair.models import SequenceTagger
tagger: SequenceTagger = SequenceTagger(
hidden_size=256,
embeddings=embeddings,
tag_dictionary=tag_dictionary,
tag_type=tag_type,
use_crf=True,
)
# initialize trainer
from flair.trainers import ModelTrainer
trainer: ModelTrainer = ModelTrainer(tagger, corpus)
trainer.train(
"resources/taggers/example-ner",
learning_rate=0.1,
mini_batch_size=32,
max_epochs=1,
shuffle=False,
)
plotter = Plotter()
plotter.plot_training_curves("resources/taggers/example-ner/loss.tsv")
plotter.plot_weights("resources/taggers/example-ner/weights.txt")
embeddings: StackedEmbeddings = StackedEmbeddings(embeddings=embedding_types)
# initialize sequence tagger
from flair.models import SequenceTagger
tagger: SequenceTagger = SequenceTagger(
hidden_size=nb_cells,
embeddings=embeddings,
tag_dictionary=tag_dictionary,
tag_type=tag_type,
use_crf=True,
)
# initialize trainer
from flair.trainers import ModelTrainer
trainer: ModelTrainer = ModelTrainer(tagger, corpus)
trainer.train(
"resources/taggers/" + exp_name,
learning_rate=0.1,
embeddings_storage_mode="cpu",
mini_batch_size=32,
max_epochs=150,
shuffle=False,
)
plotter = Plotter()
plotter.plot_training_curves("resources/taggers/" + exp_name + "/loss.tsv")
plotter.plot_weights("resources/taggers/" + exp_name + "/weights.txt")
# Just replace the names of model and print the details of training and weights
from flair.visual.training_curves import Plotter
plotter = Plotter()
plotter.plot_training_curves('FLAIR/resources/taggers/flairpos1/loss.tsv')
plotter.plot_weights('FLAIR/resources/taggers/flairpos1/weights.txt')
# comment in these lines to use contextual string embeddings
# CharLMEmbeddings('news-forward'),
# CharLMEmbeddings('news-backward'),
]
embeddings = WordEmbeddings("tmp/glove.bin")
# 5. initialize sequence tagger
from flair.models import SequenceTagger
tagger: SequenceTagger = SequenceTagger(hidden_size=1024,
embeddings=embeddings,
tag_dictionary=tag_dictionary,
tag_type=tag_type,
use_crf=True)
# 6. initialize trainer
from flair.trainers import SequenceTaggerTrainer
trainer: SequenceTaggerTrainer = SequenceTaggerTrainer(tagger, corpus)
# 7. start training
trainer.train('resources/taggers/example-ner',
learning_rate=0.1,
mini_batch_size=8,
max_epochs=150)
# 8. plot training curves (optional)
from flair.visual.training_curves import Plotter
plotter = Plotter()
plotter.plot_training_curves('resources/taggers1/example-ner/loss.tsv')
plotter.plot_weights('resources/taggers1/example-ner/weights.txt')
from flair.visual.training_curves import Plotter
clf_dir = 'resources/binary_unbiased_031219/'
plotter = Plotter()
plotter.plot_training_curves('./resources/loss.tsv')
plotter.plot_weights(clf_dir + 'weights.txt')
def trainer(file_path: Path, filenames: Tuple[str, str, str], checkpoint: str,
stack: str, n_epochs: int) -> None:
"""Train sentiment model using Flair NLP library:
https://github.com/zalandoresearch/flair/blob/master/resources/docs/TUTORIAL_7_TRAINING_A_MODEL.md
To help provide added context, we can stack Glove, Bert or ELMo embeddings along with Flair embeddings.
"""
# pip install flair allennlp
from flair.datasets import ClassificationCorpus
from flair.embeddings import FlairEmbeddings, DocumentRNNEmbeddings, DocumentPoolEmbeddings
from flair.models import TextClassifier
from flair.trainers import ModelTrainer
from flair.training_utils import EvaluationMetric
from flair.visual.training_curves import Plotter
if stack == "glove":
from flair.embeddings import WordEmbeddings
stacked_embedding = WordEmbeddings('glove')
elif stack == "fasttext":
from flair.embeddings import WordEmbeddings
stacked_embedding = WordEmbeddings('it')
elif stack == "elmo":
from flair.embeddings import ELMoEmbeddings
stacked_embedding = ELMoEmbeddings('original')
elif stack == "bert":
from flair.embeddings import BertEmbeddings
stacked_embedding = BertEmbeddings('bert-base-uncased')
elif stack == "bert-multi":
from flair.embeddings import BertEmbeddings
stacked_embedding = BertEmbeddings('bert-base-multilingual-uncased')
elif stack == 'bpe':
from flair.embeddings import BytePairEmbeddings
stacked_embedding = BytePairEmbeddings('it')
else:
stacked_embedding = None
# Define and Load corpus from the provided dataset
train, dev, test = filenames
corpus = ClassificationCorpus(
file_path,
train_file=train,
dev_file=dev,
test_file=test,
)
# Create label dictionary from provided labels in data
label_dict = corpus.make_label_dictionary()
# Stack Flair string-embeddings with optional embeddings
word_embeddings = list(
filter(None, [
stacked_embedding,
FlairEmbeddings('it-forward'),
FlairEmbeddings('it-backward'),
]))
# Initialize document embedding by passing list of word embeddings
document_embeddings = DocumentRNNEmbeddings(
word_embeddings,
hidden_size=256,
reproject_words=True,
dropout=0.5,
reproject_words_dimension=256,
)
#document_embeddings = DocumentPoolEmbeddings([
# stacked_embedding,
# FlairEmbeddings('it-forward'),
# FlairEmbeddings('it-backward')],pooling='mean')
# Define classifier
classifier = TextClassifier(document_embeddings,
label_dictionary=label_dict,
multi_label=True)
if not checkpoint:
trainer = ModelTrainer(classifier, corpus)
else:
# If checkpoint file is defined, resume training
#checkpoint = classifier.load_checkpoint(Path(checkpoint))
trainer = ModelTrainer.load_checkpoint(checkpoint, corpus)
# Begin training (enable checkpointing to continue training at a later time, if desired)
trainer.train(
file_path,
max_epochs=n_epochs,
checkpoint=True,
)
# Plot curves and store weights and losses
plotter = Plotter()
plotter.plot_training_curves(file_path + '/loss.tsv')
plotter.plot_weights(file_path + '/weights.txt')
)
# 5. create the text classifier
classifier = TextClassifier(document_embeddings, label_dictionary=label_dict)
# 6. initialize the text classifier trainer
trainer = ModelTrainer(classifier, corpus)
# 7. start the training
trainer.train('resources/taggers/ag_news',
learning_rate=0.1,
mini_batch_size=32,
anneal_factor=0.5,
patience=5,
max_epochs=150)
# 8. plot training curves (optional)
from flair.visual.training_curves import Plotter
plotter = Plotter()
plotter.plot_training_curves('resources/taggers/ag_news/loss.tsv')
plotter.plot_weights('resources/taggers/ag_news/weights.txt')
classifier = TextClassifier.load('resources/taggers/ag_news/final-model.pt')
# create example sentence
sentence = Sentence('France is the current world cup winner.')
# predict tags and print
classifier.predict(sentence)
print(sentence.labels)
tagger: SequenceTagger = SequenceTagger(hidden_size=256,
embeddings=embeddings,
tag_dictionary=tag_dictionary,
tag_type=tag_type,
use_crf=True)
# 6. initialize trainer
trainer: ModelTrainer = ModelTrainer(tagger, corpus)
# 7. find learning rate
learning_rate_tsv = trainer.find_learning_rate('resources/taggers/example-ner',
'learning_rate.tsv')
# 8. plot the learning rate finder curve
from flair.visual.training_curves import Plotter
plotter = Plotter()
plotter.plot_learning_rate(learning_rate_tsv)
from torch.optim.adam import Adam
trainer: ModelTrainer = ModelTrainer(tagger,
corpus,
optimizer=Adam,
weight_decay=1e-4)
from flair.optim import SGDW
trainer: ModelTrainer = ModelTrainer(tagger,
corpus,
optimizer=SGDW,
momentum=0.9)
embedding_types: List[TokenEmbeddings] = [
CharacterEmbeddings(),
WordEmbeddings("tmp/glove.1.8G.bin")
]
embeddings: StackedEmbeddings = StackedEmbeddings(embeddings=embedding_types)
# 5. initialize sequence tagger
from flair.models import SequenceTagger
tagger: SequenceTagger = SequenceTagger(hidden_size=1024,
embeddings=embeddings,
tag_dictionary=tag_dictionary,
tag_type=tag_type,
use_crf=True)
# 6. initialize trainer
from flair.trainers import SequenceTaggerTrainer
trainer: SequenceTaggerTrainer = SequenceTaggerTrainer(tagger, corpus)
# 7. start training
model_path = "tmp/model2"
trainer.train(model_path, learning_rate=0.1, mini_batch_size=8, max_epochs=150)
# 8. plot training curves (optional)
from flair.visual.training_curves import Plotter
plotter = Plotter()
plotter.plot_training_curves(f'{model_path}/loss.tsv')
plotter.plot_weights(f'{model_path}/weights.txt')
embeddings: StackedEmbeddings = StackedEmbeddings(embeddings=embedding_types)
# initialize sequence tagger
from flair.models import SequenceTagger
tagger: SequenceTagger = SequenceTagger(hidden_size=256,
embeddings=embeddings,
tag_dictionaries=tag_dictionaries,
tag_types=tag_types,
use_crf=True)
# initialize trainer
from flair.trainers import ModelTrainer
trainer: ModelTrainer = ModelTrainer(tagger, corpus)
# trainer: ModelTrainer = ModelTrainer(tagger, corpus, optimizer=Adam)
trainer.train('resources/taggers/famulus_eda_test_n_bert_long2',
EvaluationMetric.MICRO_F1_SCORE,
learning_rate=0.1,
mini_batch_size=32,
max_epochs=1000,
test_mode=True)
# trainer.train('resources/taggers/famulus_test', EvaluationMetric.MICRO_F1_SCORE, learning_rate=0.001, mini_batch_size=32,
# max_epochs=100, test_mode=True)
plotter = Plotter()
plotter.plot_training_curves(
'resources/taggers/famulus_eda_test_n_bert_long2/loss.tsv')
plotter.plot_weights(
'resources/taggers/famulus_eda_test_n_bert_long2/weights.txt')
# 4. initialize embeddings
embedding_types = WordEmbeddings('ru-wiki')
# 5. initialize sequence tagger
from flair.models import SequenceTagger
tagger: SequenceTagger = SequenceTagger(hidden_size=256,
embeddings=embedding_types,
tag_dictionary=tag_dictionary,
tag_type=tag_type,
use_crf=True)
# 6. initialize trainer
from flair.trainers import ModelTrainer
trainer: ModelTrainer = ModelTrainer(tagger, corpus)
# 7. start training
trainer.train('/home/anna/Desktop/markup/brat_data/without_out/model',
learning_rate=0.1,
mini_batch_size=32,
max_epochs=150)
# 8. plot weight traces (optional)
from flair.visual.training_curves import Plotter
plotter = Plotter()
plotter.plot_weights(
'/home/anna/Desktop/markup/brat_data/without_out/model/weights.txt')
reproject_words_dimension=256,
)
# 5. create the text classifier
classifier = TextClassifier(document_embeddings, label_dictionary=label_dict)
# 6. initialize the text classifier trainer
trainer = ModelTrainer(classifier, corpus)
# 7. start the training
trainer.train('/home/anna/Desktop/markup/8',
learning_rate=0.1,
mini_batch_size=32,
anneal_factor=0.5,
patience=5,
max_epochs=150)
# 8. plot weight traces (optional)
from flair.visual.training_curves import Plotter
plotter = Plotter()
plotter.plot_weights('/home/anna/Desktop/markup/8/weights.txt')
classifier = TextClassifier.load('/home/anna/Desktop/markup/8/final-model.pt')
# create example sentences
sentence = Sentence('France is the current world cup winner.')
# predict class and print
classifier.predict(sentence)
print(sentence.labels)
dropout=0.3169,
rnn_layers=1,
rnn_type='RNN_RELU')
classifier = TextClassifier(document_embeddings,
label_dictionary=corpus.make_label_dictionary(),
multi_label=False)
trainer = ModelTrainer(classifier, corpus)
trainer.train('./',
learning_rate=0.05,
mini_batch_size=32,
max_epochs=10,
evaluation_metric=EvaluationMetric.MACRO_F1_SCORE)
#plot training curves
from flair.visual.training_curves import Plotter
plotter = Plotter()
plotter.plot_training_curves('loss.tsv')
plotter.plot_weights('weights.txt')
#make a table with predictions
test_pred = pd.read_csv('test.tsv',
sep='\t',
encoding="utf-8",
names=['text', 'label', 'prediction', 'confidence'])
test_full = pd.concat([data_test_filtered, test_pred], axis=1, sort=False)
#make a table with wrong predicions
df = test_full.loc[test_full['prediction'] != test_full['label']]
wrong_prediction = df[[
'attribute::id', 'text', 'label', 'prediction', 'confidence'
]]
wrong_prediction.to_csv("cat_without_lemma_flair_wrong_pred.csv",
