def train_model(self,
learn,
test,
output_dim,
lr=0.001,
batch_size=512,
n_epochs=5,
enable_checkpoint_ensemble=True):
all_test_preds = []
checkpoint_weights = [1, 2, 4, 8, 8]
test_loader = torch.utils.data.DataLoader(test,
batch_size=batch_size,
shuffle=False)
n = len(learn.data.train_dl)
phases = [(TrainingPhase(n).schedule_hp('lr', lr * (0.6**(i))))
for i in range(n_epochs)]
sched = GeneralScheduler(learn, phases)
learn.callbacks.append(sched)
for epoch in range(n_epochs):
learn.fit(1)
test_preds = np.zeros((len(test), output_dim))
for i, x_batch in enumerate(test_loader):
X = x_batch[0].cuda()
y_pred = self.sigmoid(learn.model(X).detach().cpu().numpy())
test_preds[i * batch_size:(i + 1) * batch_size, :] = y_pred
all_test_preds.append(test_preds)
if enable_checkpoint_ensemble:
test_preds = np.average(all_test_preds,
weights=checkpoint_weights,
axis=0)
else:
test_preds = all_test_preds[-1]
return test_preds
def flattenAnneal(learn:Learner, lr:float, n_epochs:int, start_pct:float):
n = len(learn.data.train_dl)
anneal_start = int(n*n_epochs*start_pct)
anneal_end = int(n*n_epochs) - anneal_start
phases = [TrainingPhase(anneal_start).schedule_hp('lr', lr),
TrainingPhase(anneal_end).schedule_hp('lr', lr, anneal=annealing_cos)]
sched = GeneralScheduler(learn, phases)
learn.callbacks.append(sched)
learn.fit(n_epochs)
def train_model(learn, lr=0.001, lr_decay=0.8, batch_size=512, n_epochs=20, model_name='fastai_'):
n = len(learn.data.train_dl)
phases = [(TrainingPhase(n).schedule_hp('lr', lr * (lr_decay ** (i)))) for i in range(n_epochs)]
sched = GeneralScheduler(learn, phases)
learn.callbacks.append(sched)
learn.fit(n_epochs,
callbacks=[SaveModelCallback(learn, name=model_name),
EarlyStoppingCallback(learn, min_delta=0.001, patience=5)])
def fit_with_annealing(learn:Learner, num_epoch:int, lr:float=1e-3, annealing_start:float=0.7,
callbacks:list=None)->None:
n = len(learn.data.train_dl)
anneal_start = int(n*num_epoch*annealing_start)
phase0 = TrainingPhase(anneal_start).schedule_hp('lr', lr)
phase1 = TrainingPhase(n*num_epoch - anneal_start).schedule_hp('lr', lr, anneal=annealing_cos)
phases = [phase0, phase1]
sched = GeneralScheduler(learn, phases)
learn.callbacks.append(sched)
learn.fit(num_epoch, callbacks=callbacks)
def train_lm(n_cycles: int = 3,
cycle_len: int = 1,
cycle_mult: int = 2,
momentum: float = 0.8,
bptt: int = 40,
lr: float = 1e-3,
wd: float = 1e-7):
datasets = create_or_restore(DATA_ROOT)
lm_data = [
fastai_patch(ds)
for ds in (datasets['train_unsup'], datasets['test_unsup'])
]
bunch = TextLMDataBunch.create(lm_data, path=LM_PATH, bptt=bptt)
n = sum(len(ds) for ds in lm_data)
phases = [
TrainingPhase(n * (cycle_len * cycle_mult**i),
lrs=lr,
moms=momentum,
lr_anneal=annealing_cos) for i in range(n_cycles)
]
learner = RNNLearner.language_model(bunch, bptt)
cbs = [
EarlyStopping(learner, patience=2),
GeneralScheduler(learner, phases),
SaveModel(learner)
]
if cycle_mult == 1:
total_epochs = n_cycles * cycle_len
else:
total_epochs = int(cycle_len * (1 - cycle_mult**n_cycles) /
(1 - cycle_mult))
print(f'Total number of epochs: {total_epochs:d}')
try:
learner.fit(total_epochs, wd=wd, callbacks=cbs)
except RuntimeError as e:
print(f'Model training error: {e}')
finally:
folder = learner.path / learner.model_dir
print(f'Saving latest model state into {folder}')
learner.save('lm_final')
learner.save_encoder('lm_final_enc')
def main():
train_ds = TextDataset.from_folder(IMDB_PATH, name='train', shuffle=True)
valid_ds = TextDataset.from_folder(IMDB_PATH, name='test')
lm_data = [train_ds, valid_ds]
lm_bunch = TextLMDataBunch.create(lm_data, path=LM_PATH)
learner = RNNLearner.language_model(lm_bunch)
n = sum(len(ds) for ds in lm_data)
num_epochs, phases = create_phases(3, n)
callbacks = [
EarlyStopping(learner, patience=2),
SaveModel(learner),
GeneralScheduler(learner, phases)
]
learner.fit(num_epochs, )