os.makedirs(checkpoint_dir, exist_ok=True)
# early stopper
if config['early_stopping_epochs'] > 1:
early_stopper = EarlyStopper(config['early_stopping_epochs'])
else:
early_stopper = None
scheduler_metric = None
for epoch in range(config['epochs']):
try:
# epoch start
epoch += trainer.init_epoch
epoch_info(epoch, config['epochs'] + trainer.init_epoch)
# train an epoch
loss, accu = trainer.run(data_gen['train'])
# epoch summary
print('Avg Loss: %.5f, Avg Accu: %.5f' % (loss, accu))
if logger is not None:
logger.add_scalar('metrics/epoch_loss', loss, epoch + 1)
logger.add_scalar('metrics/epoch_accu', accu, epoch + 1)
if epoch % config['validation_frequency'] == 0:
# get validation score
val_score = validator.run(data_gen['valid'])
init_epoch = 1
# main running loop
terminated = False
scheduler_metric = None
stage_info = {'train': 'Training', 'valid': 'Validating'}
best = 0.
os.makedirs(args.output_dir, exist_ok=True)
for epoch in range(init_epoch, init_epoch + config['epochs']):
if terminated:
break
# epoch start
epoch_info(epoch - 1, init_epoch + config['epochs'] - 1)
outputs = []
for stage in stages:
training = True if stage == 'train' else False
# skip validation stage by validation_frequency
if stage != 'train' and epoch % config['validation_frequency'] != 0:
break
# run on an epoch
try:
n_steps = len(data_gen[stage])
progress_bar = tqdm(data_gen[stage],
total=n_steps,