def fine_tune(base_model, corpus_dir, output_dir):
# print stats
print(f'Fine tuning base model: {base_model}')
print(f'Corpus dir: {corpus_dir}')
print(f'Output dir: {output_dir}')
# instantiate an existing LM, such as one from the FlairEmbeddings
language_model = FlairEmbeddings(base_model).lm
# are you fine-tuning a forward or backward LM?
is_forward_lm = language_model.is_forward_lm
# get the dictionary from the existing language model
dictionary: Dictionary = language_model.dictionary
# get your corpus, process forward and at the character level
corpus = TextCorpus(corpus_dir,
dictionary,
is_forward_lm,
character_level=True)
# use the model trainer to fine-tune this model on your corpus
trainer = LanguageModelTrainer(language_model, corpus)
trainer.train(output_dir,
sequence_length=100,
mini_batch_size=100,
learning_rate=20,
patience=10,
checkpoint=True)
def train_LM(file_path, model_path, is_forward_lm=True):
from flair.data import Dictionary
from flair.models import LanguageModel
from flair.trainers.language_model_trainer import LanguageModelTrainer, TextCorpus
dictionary = Dictionary.load_from_file(file_path + 'mappings')
# get your corpus, process forward and at the character level
corpus = TextCorpus(file_path,
dictionary,
is_forward_lm,
character_level=True)
# instantiate your language model, set hidden size and number of layers
language_model = LanguageModel(dictionary,
is_forward_lm,
hidden_size=128,
nlayers=1)
# train your language model
trainer = LanguageModelTrainer(language_model, corpus)
trainer.train(model_path,
sequence_length=100,
mini_batch_size=32,
max_epochs=10)
def test_train_resume_language_model_training(resources_path,
results_base_path,
tasks_base_path):
dictionary = Dictionary.load(u'chars')
language_model = LanguageModel(dictionary,
is_forward_lm=True,
hidden_size=128,
nlayers=1)
corpus = TextCorpus((resources_path / u'corpora/lorem_ipsum'),
dictionary,
language_model.is_forward_lm,
character_level=True)
trainer = LanguageModelTrainer(language_model, corpus, test_mode=True)
trainer.train(results_base_path,
sequence_length=10,
mini_batch_size=10,
max_epochs=2,
checkpoint=True)
trainer = LanguageModelTrainer.load_from_checkpoint(
(results_base_path / u'checkpoint.pt'), corpus)
trainer.train(results_base_path,
sequence_length=10,
mini_batch_size=10,
max_epochs=2)
shutil.rmtree(results_base_path)
def test_training():
# get default dictionary
dictionary: Dictionary = Dictionary.load('chars')
# init forward LM with 128 hidden states and 1 layer
language_model: LanguageModel = LanguageModel(dictionary,
is_forward_lm=True,
hidden_size=128,
nlayers=1)
# get the example corpus and process at character level in forward direction
corpus: TextCorpus = TextCorpus('resources/corpora/lorem_ipsum',
dictionary,
language_model.is_forward_lm,
character_level=True)
# train the language model
trainer: LanguageModelTrainer = LanguageModelTrainer(
language_model, corpus)
trainer.train('./results',
sequence_length=10,
mini_batch_size=10,
max_epochs=5)
# use the character LM as embeddings to embed the example sentence 'I love Berlin'
char_lm_embeddings = CharLMEmbeddings('./results/best-lm.pt')
sentence = Sentence('I love Berlin')
char_lm_embeddings.embed(sentence)
print(sentence[1].embedding.size())
# clean up results directory
shutil.rmtree('./results', ignore_errors=True)
def test_train_language_model(results_base_path, resources_path):
# get default dictionary
dictionary: Dictionary = Dictionary.load('chars')
# init forward LM with 128 hidden states and 1 layer
language_model: LanguageModel = LanguageModel(dictionary, is_forward_lm=True, hidden_size=128, nlayers=1)
# get the example corpus and process at character level in forward direction
corpus: TextCorpus = TextCorpus(resources_path / 'corpora/lorem_ipsum',
dictionary,
language_model.is_forward_lm,
character_level=True)
# train the language model
trainer: LanguageModelTrainer = LanguageModelTrainer(language_model, corpus, test_mode=True)
trainer.train(results_base_path, sequence_length=10, mini_batch_size=10, max_epochs=2)
# use the character LM as embeddings to embed the example sentence 'I love Berlin'
char_lm_embeddings = FlairEmbeddings(str(results_base_path / 'best-lm.pt'))
sentence = Sentence('I love Berlin')
char_lm_embeddings.embed(sentence)
text, likelihood = language_model.generate_text(number_of_characters=100)
assert (text is not None)
assert (len(text) >= 100)
# clean up results directory
shutil.rmtree(results_base_path, ignore_errors=True)
def ft(self):
if isinstance(self.document_embedding, LanguageModel):
trainer = LanguageModelTrainer(self.document_embedding, corpus)
trainer.train('resources/taggers/language_model',
sequence_length=100,
mini_batch_size=100,
learning_rate=20,
patience=10,
checkpoint=True)
def train_elmo(args):
if args.finetune and args.checkpoint_path == '':
print("finetune")
from flair.embeddings import FlairEmbeddings
language_model = FlairEmbeddings('he-forward').lm
corpus: TextCorpus = TextCorpus(args.corpus_path,
language_model.dictionary,
language_model.is_forward_lm,
character_level=True)
trainer = LanguageModelTrainer(language_model, corpus)
elif args.checkpoint_path == '' and not args.finetune:
# Training from scrach
print('Training from scarch')
#Downloading data
if not os.path.exists(args.corpus_path):
print('Corpus _path', args.corpus_path)
download_corpus(args)
language_model, corpus = create_corpus(args)
trainer = LanguageModelTrainer(language_model, corpus)
else:
print("Training from checpoint")
from pathlib import Path
checkpoint = Path(args.checkpoint_path)
if args.finetune:
load_dict_from_lm = True
else:
load_dict_from_lm = False
trainer = LanguageModelTrainer.load_from_checkpoint(
checkpoint,
create_corpus(args, load_dict_from_lm, return_back='corpus'))
trainer.train(args.save_model,
sequence_length=args.seq_length,
mini_batch_size=args.mini_batch,
max_epochs=args.epochs,
checkpoint=args.checkpoint)
def train(self) -> None:
trainer = LanguageModelTrainer(self.lm, self.corpus)
trainer.train(self.save_dir,
sequence_length=self.sequence_length,
mini_batch_size=self.mini_batch_size,
learning_rate=self.learning_rate,
patience=self.patience,
checkpoint=self.checkpoint,
write_weights=True,
use_tensorboard=True)
def trainLanguage(self, corpusPath):
self.corpus = TextCorpus(Path(corpusPath),
self.dictionary,
self.is_forward_lm,
character_level=True)
self.language_model = LanguageModel(self.dictionary,
self.is_forward_lm,
hidden_size=128,
nlayers=10)
self.trainer = LanguageModelTrainer(self.language_model, self.corpus)
self.trainer.train('resources/taggers/language_model',
sequence_length=10,
mini_batch_size=10,
max_epochs=10)
def test_train_resume_language_model_training(resources_path,
results_base_path,
tasks_base_path):
# get default dictionary
dictionary: Dictionary = Dictionary.load("chars")
# init forward LM with 128 hidden states and 1 layer
language_model: LanguageModel = LanguageModel(dictionary,
is_forward_lm=True,
hidden_size=128,
nlayers=1)
# get the example corpus and process at character level in forward direction
corpus: TextCorpus = TextCorpus(
resources_path / "corpora/lorem_ipsum",
dictionary,
language_model.is_forward_lm,
character_level=True,
)
# train the language model
trainer: LanguageModelTrainer = LanguageModelTrainer(language_model,
corpus,
test_mode=True)
trainer.train(
results_base_path,
sequence_length=10,
mini_batch_size=10,
max_epochs=2,
checkpoint=True,
)
del trainer, language_model
trainer = LanguageModelTrainer.load_from_checkpoint(
results_base_path / "checkpoint.pt", corpus)
trainer.train(results_base_path,
sequence_length=10,
mini_batch_size=10,
max_epochs=2)
# clean up results directory
shutil.rmtree(results_base_path)
del trainer
def retrain_flair(cls,
corpus_path: str,
model_path_dest: str,
flair_algorithm: str = 'de-forward',
epochs: int = 10):
use_embedding, algorithm = cls.determine_algorithm_from_string(
flair_algorithm_string=flair_algorithm)
# instantiate an existing LM, such as one from the FlairEmbeddings
model = use_embedding(flair_algorithm)
if algorithm == 'bert':
language_model = model.model
else:
language_model = model.lm
# are you fine-tuning a forward or backward LM?
try:
is_forward_lm = language_model.is_forward_lm
except AttributeError:
is_forward_lm = True
# todo: no support for finetuning BERT with Flair Library for now
# get the dictionary from the existing language model
dictionary: Dictionary = language_model.dictionary
# get your corpus, process forward and at the character level
corpus = TextCorpus(corpus_path,
dictionary,
is_forward_lm,
character_level=True)
# use the model trainer to fine-tune this model on your corpus
trainer = LanguageModelTrainer(language_model, corpus)
trainer.train(model_path_dest,
sequence_length=10,
mini_batch_size=10,
learning_rate=20,
max_epochs=epochs,
patience=10,
checkpoint=True)
def process(options):
"""
Do the processing
"""
# are you training a forward or backward LM?
is_forward_lm = not options.is_backward_lm
# load the default character dictionary
dictionary: Dictionary = Dictionary.load('chars')
# get your corpus, process forward and at the character level
corpus = TextCorpus(options.corpus_dir,
dictionary,
is_forward_lm,
character_level=True)
# instantiate your language model, set hidden size and number of layers
language_model = LanguageModel(
dictionary,
is_forward_lm,
hidden_size=2048,
nlayers=1,
embedding_size=100, # recommendations?
dropout=0) # dropout probs?
# train your language model
trainer = LanguageModelTrainer(language_model, corpus)
trainer.train(
options.
model_dir, # embeddings_in_memory=False: effect on 'RuntimeError: CUDA out of memory'?
sequence_length=250,
learning_rate=20,
mini_batch_size=100,
anneal_factor=0.25,
patience=
22, # 'patience' value of the learning rate scheduler: 1/2 training splits
clip=0.25, # clipping gradients?
max_epochs=75)
def test_train_language_model(results_base_path, resources_path):
dictionary = Dictionary.load(u'chars')
language_model = LanguageModel(dictionary,
is_forward_lm=True,
hidden_size=128,
nlayers=1)
corpus = TextCorpus((resources_path / u'corpora/lorem_ipsum'),
dictionary,
language_model.is_forward_lm,
character_level=True)
trainer = LanguageModelTrainer(language_model, corpus, test_mode=True)
trainer.train(results_base_path,
sequence_length=10,
mini_batch_size=10,
max_epochs=2)
char_lm_embeddings = FlairEmbeddings(
unicode((results_base_path / u'best-lm.pt')))
sentence = Sentence(u'I love Berlin')
char_lm_embeddings.embed(sentence)
(text, likelihood) = language_model.generate_text(number_of_characters=100)
assert (text is not None)
assert (len(text) >= 100)
shutil.rmtree(results_base_path, ignore_errors=True)
with open(os.path.join(tmp_path.name, "test.txt"), 'w') as f:
f.writelines("\n".join(dev_set))
print("load original model")
language_model = FlairEmbeddings('fr-backward').lm
is_forward_lm = language_model.is_forward_lm
dictionary: Dictionary = language_model.dictionary
print("load corpus")
corpus = TextCorpus(tmp_path.name,
dictionary,
is_forward_lm,
character_level=True)
print("start training")
trainer = LanguageModelTrainer(language_model, corpus)
trainer.train('resources/flair_ner/lm/ca_backward',
sequence_length=100,
mini_batch_size=100,
learning_rate=20,
patience=10,
max_epochs=5,
checkpoint=True)
print("load original model")
language_model = FlairEmbeddings('fr-forward').lm
is_forward_lm = language_model.is_forward_lm
dictionary: Dictionary = language_model.dictionary
print("load corpus")
item2idx = model.dictionary.item2idx
print(item2idx["\n".encode()])
inputs = open('corpus/train/train.txt', 'r').read().splitlines()[-1]
inputs = [item2idx.get(char.encode(), 0) for char in inputs]
inputs = torch.LongTensor(inputs).unsqueeze(-1) # (seqlen, 1)
inputs = inputs.to(device)
print("# load corpus")
corpus = TextCorpus(Path('corpus/'),
model.dictionary,
model.is_forward_lm,
character_level=True)
print("# trainer")
trainer = LanguageModelTrainer(model, corpus)
print("# Generating characters with pretraned model")
generate(model, inputs, hp.n_chars, f"{hp.output_dir}/0.out", device)
print("# continue training the model on the new corpus")
for epoch in range(1, hp.n_epochs):
print(f"# epoch: {epoch}")
print("training ..")
trainer.train(f'{hp.ckpt_dir}', sequence_length=hp.seqlen, max_epochs=1)
print("Generating ..")
generate(model, inputs, hp.n_chars, f"{hp.output_dir}/{epoch}.out", device)
print("Loading saved model")
model = LanguageModel.load_language_model(f'{hp.ckpt_dir}/best-lm.pt')