def main(config, name, args):
from flair.trainers import ModelTrainer
from flair.visual.training_curves import Plotter
from math import ceil
from torch.optim import Adam
from torch import manual_seed
from pickle import load
from discodop.lexgrammar import SupertagGrammar
cp = corpusparam(**config["Corpus"], **config["Grammar"])
corpus = SupertagParseCorpus(cp.filename)
grammar = load(open(f"{cp.filename}.grammar", "rb"))
tc = FindlrParameters(**config["Training"],
**config["Eval-common"],
**config["Eval-Development"],
language=cp.language)
model = Supertagger.from_corpus(corpus, grammar, tc)
model.set_eval_param(tc)
if args.downsample:
corpus = corpus.downsample(args.downsample)
if args.iterations is None:
epoch = ceil(len(corpus.train) / tc.batchsize)
args.iterations = epoch * 5
trainer = ModelTrainer(model, corpus)
learning_rate_tsv = trainer.find_learning_rate(
name,
start_learning_rate=args.min_lr,
end_learning_rate=args.max_lr,
iterations=args.iterations)
plotter = Plotter()
plotter.plot_learning_rate(learning_rate_tsv)
def optimize_lr():
corpus, label_dictionary = load_corpus()
embeddings = [
WordEmbeddings('glove'),
FlairEmbeddings('news-forward'),
FlairEmbeddings('news-backward')
]
document_embeddings = DocumentRNNEmbeddings(embeddings,
hidden_size=512,
reproject_words=True,
reproject_words_dimension=256,
bidirectional=True)
classifier = TextClassifier(document_embeddings,
label_dictionary=label_dictionary,
multi_label=False)
trainer = ModelTrainer(classifier, corpus)
# 7. find learning rate
learning_rate_tsv = trainer.find_learning_rate('resources/classifiers/',
'learning_rate.tsv')
# 8. plot the learning rate finder curve
from flair.visual.training_curves import Plotter
plotter = Plotter()
plotter.plot_learning_rate(learning_rate_tsv)
def find_learning_rate(
self,
output_dir: Union[Path, str],
file_name: str = "learning_rate.tsv",
start_learning_rate: float = 1e-8,
end_learning_rate: float = 10,
iterations: int = 100,
mini_batch_size: int = 32,
stop_early: bool = True,
smoothing_factor: float = 0.7,
plot_learning_rate: bool = True,
**kwargs,
) -> float:
"""
Uses Leslie's cyclical learning rate finding method to generate and save the loss x learning rate plot
This method returns a suggested learning rate using the static method `LMFineTuner.suggest_learning_rate()`
which is implicitly run in this method.
* **output_dir** - Path to dir for learning rate file to be saved
* **file_name** - Name of learning rate .tsv file
* **start_learning_rate** - Initial learning rate to start cyclical learning rate finder method
* **end_learning_rate** - End learning rate to stop exponential increase of the learning rate
* **iterations** - Number of optimizer iterations for the ExpAnnealLR scheduler
* **mini_batch_size** - Batch size for dataloader
* **stop_early** - Bool for stopping early once loss diverges
* **smoothing_factor** - Smoothing factor on moving average of losses
* **adam_epsilon** - Epsilon for Adam optimizer.
* **weight_decay** - Weight decay if we apply some.
* **kwargs** - Additional keyword arguments for the Adam optimizer
**return** - Learning rate as a float
"""
# 7. find learning rate
learning_rate_tsv = self.trainer.find_learning_rate(
base_path=output_dir,
file_name=file_name,
start_learning_rate=start_learning_rate,
end_learning_rate=end_learning_rate,
iterations=iterations,
mini_batch_size=mini_batch_size,
stop_early=stop_early,
smoothing_factor=smoothing_factor,
)
# Reinitialize optimizer and parameters by reinitializing trainer
self._initial_setup(self.label_dict, **self.trainer_kwargs)
if plot_learning_rate:
plotter = Plotter()
plotter.plot_learning_rate(learning_rate_tsv)
# Use the automated learning rate finder
with open(learning_rate_tsv) as lr_f:
lr_tsv = list(csv.reader(lr_f, delimiter="\t"))
losses = np.array([float(row[-1]) for row in lr_tsv[1:]])
lrs = np.array([float(row[-2]) for row in lr_tsv[1:]])
lr_to_use = self.suggested_learning_rate(losses, lrs, **kwargs)
print(f"Recommended Learning Rate {lr_to_use}")
return lr_to_use
def find_learning_rate(trainer, params):
learning_rate_tsv = trainer.find_learning_rate(
os.path.join("hyperparameter_search", params['model_output_dirpath']),
'learning_rate_search_log.tsv',
iterations=400,
stop_early=False,
mini_batch_size=16)
from flair.visual.training_curves import Plotter
plotter = Plotter()
plotter.plot_learning_rate(learning_rate_tsv)
embeddings=embeddings,
tag_dictionary=tag_dictionary,
tag_type=tag_type,
use_crf=True)
# 6. initialize trainer
trainer: ModelTrainer = ModelTrainer(tagger, corpus)
# 7. find learning rate
learning_rate_tsv = trainer.find_learning_rate('resources/taggers/example-ner',
'learning_rate.tsv')
# 8. plot the learning rate finder curve
from flair.visual.training_curves import Plotter
plotter = Plotter()
plotter.plot_learning_rate(learning_rate_tsv)
from torch.optim.adam import Adam
trainer: ModelTrainer = ModelTrainer(tagger,
corpus,
optimizer=Adam,
weight_decay=1e-4)
from flair.optim import SGDW
trainer: ModelTrainer = ModelTrainer(tagger,
corpus,
optimizer=SGDW,
momentum=0.9)
class SequenceTaggerEvaluation():
def __init__(self, path: Union[Path, str], model: str = 'final-model.pt'):
if type(path) == str:
path = Path(path)
assert path.exists()
self.path = path
self.model = SequenceTagger.load(path / model)
self.cv_results = {}
for file in ['summary', 'details']:
try:
self.cv_results[file] = pickle.load(
(path / (file + '.pkl')).open(mode='rb'))
except FileNotFoundError:
print(
f"{file+'.pkl'} not found. Setting cv_results['{file}'] to None"
)
self.plotter = Plotter()
def result_tables(self, save_as_html: bool = True):
html_0 = self.cv_results['summary'].to_frame('value').to_html()
html_1 = self.cv_results['details'].to_html()
display(HTML(html_0))
print('\n')
display(HTML(html_1))
if save_as_html:
(self.path / 'summary.html').write_text(html_0)
(self.path / 'details.html').write_text(html_1)
def plot_tag_stats(self, mode: str, savefig: bool = False, **kwargs):
"""
mode
tp-fn: stacked barplot - true-positives and false-negatives
tp-fp: bar plot - true-positives and false-positives
"""
details = self.cv_results['details']
if mode == 'tp_fn':
details[['true-positive', 'false-negative']].plot.bar(stacked=True,
**kwargs)
elif mode == 'tp_fp':
details[['true-positive',
'false-positive']].plot.bar(stacked=False, **kwargs)
else:
details[mode.split('_')].plot.bar(stacked=False, **kwargs)
plt.gca().yaxis.grid(True, linestyle='--')
plt.tight_layout()
if savefig:
plt.savefig(self.path / (mode + '.png'))
def confusion_matrix(self, ):
# confusion matrix tags
pass
def predict(self,
sentences: Union[str, Sentence, List[Sentence], List[str]],
display_html: bool = True,
html_file: str = None,
display_str: bool = False,
**kwargs):
if type(sentences) == Sentence:
sentences = [sentences]
elif type(sentences) == str:
sentences = split_single(sentences)
if type(sentences[0]) == str:
sentences = [Sentence(s, use_tokenizer=True) for s in sentences]
self.model.predict(sentences)
if display_html or html_file:
html = render_ner_html(sentences, **kwargs)
if display_html:
display(HTML(html))
if html_file:
(self.path / html_file).write_text(html)
if display_str:
for sentence in sentences:
print(sentence.to_tagged_string())
def plot_training_curves(self, plot_values: List[str] = ["loss", "F1"]):
self.plotter.plot_training_curves(self.path / 'loss.tsv', plot_values)
def plot_weights(self):
self.plotter.plot_weights(self.path / 'weights.txt')
def plot_learning_rate(self, skip_first: int = 10, skip_last: int = 5):
self.plotter.plot_learning_rate(self.path / 'loss.tsv', skip_first,
skip_last)
@staticmethod
def _preprocess(text, mode=None):
'''helper function to preprocess text. returns List of Sentences'''
sentences = split_single(text)
if mode:
nlp = spacy.load('de_core_news_sm')
if mode == 'lemmatize':
sentences = [
Sentence((' ').join([token.lemma_ for token in nlp(s)]))
for s in sentences
]
elif mode == 'stem':
stemmer = GermanStemmer()
sentences = [
Sentence((' ').join(
[stemmer.stem(token.text) for token in nlp(s)]))
for s in sentences
]
else:
sentences = [Sentence(s, use_tokenizer=True) for s in sentences]
return sentences
from flair.visual.training_curves import Plotter
plotter = Plotter()
# plotter.plot_weights('flair_outputs_glove/weights.txt')
# plotter.plot_training_curves('flair_outputs_glove/loss.tsv')
# plotter.plot_learning_rate('flair_outputs_glove/loss.tsv')
plotter.plot_weights("flair_outputs_fastText/weights.txt")
plotter.plot_training_curves("flair_outputs_fastText/loss.tsv")
plotter.plot_learning_rate("flair_outputs_fastText/loss.tsv")