def main(config, name, args): from flair.trainers import ModelTrainer from flair.visual.training_curves import Plotter from math import ceil from torch.optim import Adam from torch import manual_seed from pickle import load from discodop.lexgrammar import SupertagGrammar cp = corpusparam(**config["Corpus"], **config["Grammar"]) corpus = SupertagParseCorpus(cp.filename) grammar = load(open(f"{cp.filename}.grammar", "rb")) tc = FindlrParameters(**config["Training"], **config["Eval-common"], **config["Eval-Development"], language=cp.language) model = Supertagger.from_corpus(corpus, grammar, tc) model.set_eval_param(tc) if args.downsample: corpus = corpus.downsample(args.downsample) if args.iterations is None: epoch = ceil(len(corpus.train) / tc.batchsize) args.iterations = epoch * 5 trainer = ModelTrainer(model, corpus) learning_rate_tsv = trainer.find_learning_rate( name, start_learning_rate=args.min_lr, end_learning_rate=args.max_lr, iterations=args.iterations) plotter = Plotter() plotter.plot_learning_rate(learning_rate_tsv)
def optimize_lr(): corpus, label_dictionary = load_corpus() embeddings = [ WordEmbeddings('glove'), FlairEmbeddings('news-forward'), FlairEmbeddings('news-backward') ] document_embeddings = DocumentRNNEmbeddings(embeddings, hidden_size=512, reproject_words=True, reproject_words_dimension=256, bidirectional=True) classifier = TextClassifier(document_embeddings, label_dictionary=label_dictionary, multi_label=False) trainer = ModelTrainer(classifier, corpus) # 7. find learning rate learning_rate_tsv = trainer.find_learning_rate('resources/classifiers/', 'learning_rate.tsv') # 8. plot the learning rate finder curve from flair.visual.training_curves import Plotter plotter = Plotter() plotter.plot_learning_rate(learning_rate_tsv)
def find_learning_rate( self, output_dir: Union[Path, str], file_name: str = "learning_rate.tsv", start_learning_rate: float = 1e-8, end_learning_rate: float = 10, iterations: int = 100, mini_batch_size: int = 32, stop_early: bool = True, smoothing_factor: float = 0.7, plot_learning_rate: bool = True, **kwargs, ) -> float: """ Uses Leslie's cyclical learning rate finding method to generate and save the loss x learning rate plot This method returns a suggested learning rate using the static method `LMFineTuner.suggest_learning_rate()` which is implicitly run in this method. * **output_dir** - Path to dir for learning rate file to be saved * **file_name** - Name of learning rate .tsv file * **start_learning_rate** - Initial learning rate to start cyclical learning rate finder method * **end_learning_rate** - End learning rate to stop exponential increase of the learning rate * **iterations** - Number of optimizer iterations for the ExpAnnealLR scheduler * **mini_batch_size** - Batch size for dataloader * **stop_early** - Bool for stopping early once loss diverges * **smoothing_factor** - Smoothing factor on moving average of losses * **adam_epsilon** - Epsilon for Adam optimizer. * **weight_decay** - Weight decay if we apply some. * **kwargs** - Additional keyword arguments for the Adam optimizer **return** - Learning rate as a float """ # 7. find learning rate learning_rate_tsv = self.trainer.find_learning_rate( base_path=output_dir, file_name=file_name, start_learning_rate=start_learning_rate, end_learning_rate=end_learning_rate, iterations=iterations, mini_batch_size=mini_batch_size, stop_early=stop_early, smoothing_factor=smoothing_factor, ) # Reinitialize optimizer and parameters by reinitializing trainer self._initial_setup(self.label_dict, **self.trainer_kwargs) if plot_learning_rate: plotter = Plotter() plotter.plot_learning_rate(learning_rate_tsv) # Use the automated learning rate finder with open(learning_rate_tsv) as lr_f: lr_tsv = list(csv.reader(lr_f, delimiter="\t")) losses = np.array([float(row[-1]) for row in lr_tsv[1:]]) lrs = np.array([float(row[-2]) for row in lr_tsv[1:]]) lr_to_use = self.suggested_learning_rate(losses, lrs, **kwargs) print(f"Recommended Learning Rate {lr_to_use}") return lr_to_use
def find_learning_rate(trainer, params): learning_rate_tsv = trainer.find_learning_rate( os.path.join("hyperparameter_search", params['model_output_dirpath']), 'learning_rate_search_log.tsv', iterations=400, stop_early=False, mini_batch_size=16) from flair.visual.training_curves import Plotter plotter = Plotter() plotter.plot_learning_rate(learning_rate_tsv)
embeddings=embeddings, tag_dictionary=tag_dictionary, tag_type=tag_type, use_crf=True) # 6. initialize trainer trainer: ModelTrainer = ModelTrainer(tagger, corpus) # 7. find learning rate learning_rate_tsv = trainer.find_learning_rate('resources/taggers/example-ner', 'learning_rate.tsv') # 8. plot the learning rate finder curve from flair.visual.training_curves import Plotter plotter = Plotter() plotter.plot_learning_rate(learning_rate_tsv) from torch.optim.adam import Adam trainer: ModelTrainer = ModelTrainer(tagger, corpus, optimizer=Adam, weight_decay=1e-4) from flair.optim import SGDW trainer: ModelTrainer = ModelTrainer(tagger, corpus, optimizer=SGDW, momentum=0.9)
class SequenceTaggerEvaluation(): def __init__(self, path: Union[Path, str], model: str = 'final-model.pt'): if type(path) == str: path = Path(path) assert path.exists() self.path = path self.model = SequenceTagger.load(path / model) self.cv_results = {} for file in ['summary', 'details']: try: self.cv_results[file] = pickle.load( (path / (file + '.pkl')).open(mode='rb')) except FileNotFoundError: print( f"{file+'.pkl'} not found. Setting cv_results['{file}'] to None" ) self.plotter = Plotter() def result_tables(self, save_as_html: bool = True): html_0 = self.cv_results['summary'].to_frame('value').to_html() html_1 = self.cv_results['details'].to_html() display(HTML(html_0)) print('\n') display(HTML(html_1)) if save_as_html: (self.path / 'summary.html').write_text(html_0) (self.path / 'details.html').write_text(html_1) def plot_tag_stats(self, mode: str, savefig: bool = False, **kwargs): """ mode tp-fn: stacked barplot - true-positives and false-negatives tp-fp: bar plot - true-positives and false-positives """ details = self.cv_results['details'] if mode == 'tp_fn': details[['true-positive', 'false-negative']].plot.bar(stacked=True, **kwargs) elif mode == 'tp_fp': details[['true-positive', 'false-positive']].plot.bar(stacked=False, **kwargs) else: details[mode.split('_')].plot.bar(stacked=False, **kwargs) plt.gca().yaxis.grid(True, linestyle='--') plt.tight_layout() if savefig: plt.savefig(self.path / (mode + '.png')) def confusion_matrix(self, ): # confusion matrix tags pass def predict(self, sentences: Union[str, Sentence, List[Sentence], List[str]], display_html: bool = True, html_file: str = None, display_str: bool = False, **kwargs): if type(sentences) == Sentence: sentences = [sentences] elif type(sentences) == str: sentences = split_single(sentences) if type(sentences[0]) == str: sentences = [Sentence(s, use_tokenizer=True) for s in sentences] self.model.predict(sentences) if display_html or html_file: html = render_ner_html(sentences, **kwargs) if display_html: display(HTML(html)) if html_file: (self.path / html_file).write_text(html) if display_str: for sentence in sentences: print(sentence.to_tagged_string()) def plot_training_curves(self, plot_values: List[str] = ["loss", "F1"]): self.plotter.plot_training_curves(self.path / 'loss.tsv', plot_values) def plot_weights(self): self.plotter.plot_weights(self.path / 'weights.txt') def plot_learning_rate(self, skip_first: int = 10, skip_last: int = 5): self.plotter.plot_learning_rate(self.path / 'loss.tsv', skip_first, skip_last) @staticmethod def _preprocess(text, mode=None): '''helper function to preprocess text. returns List of Sentences''' sentences = split_single(text) if mode: nlp = spacy.load('de_core_news_sm') if mode == 'lemmatize': sentences = [ Sentence((' ').join([token.lemma_ for token in nlp(s)])) for s in sentences ] elif mode == 'stem': stemmer = GermanStemmer() sentences = [ Sentence((' ').join( [stemmer.stem(token.text) for token in nlp(s)])) for s in sentences ] else: sentences = [Sentence(s, use_tokenizer=True) for s in sentences] return sentences
from flair.visual.training_curves import Plotter plotter = Plotter() # plotter.plot_weights('flair_outputs_glove/weights.txt') # plotter.plot_training_curves('flair_outputs_glove/loss.tsv') # plotter.plot_learning_rate('flair_outputs_glove/loss.tsv') plotter.plot_weights("flair_outputs_fastText/weights.txt") plotter.plot_training_curves("flair_outputs_fastText/loss.tsv") plotter.plot_learning_rate("flair_outputs_fastText/loss.tsv")