def train_and_decode(self, train_loader, dev_loader, test_loader, max_epoch, reduction='mean'):
eval_iter = 0
for epoch in range(max_epoch):
start_time = time.time()
epoch_loss = 0.0
for i, data_batch in enumerate(train_loader):
self.model.train()
self.optimizer['all'].zero_grad()
word_ids, segment_ids, masks, labels = to_device(data_batch, self.device)
loss = self.model(word_ids, segment_ids, masks, labels)
if reduction == 'mean':
loss = torch.mean(loss)
else:
loss = torch.sum(loss)
loss.backward()
self.optimizer['all'].step()
epoch_loss += loss.item()
torch.cuda.empty_cache()
gc.collect()
eval_iter += 1
if eval_iter % 100 == 0:
start_time = time.time()
dev_acc, dev_loss = self.decode(dev_loader, labeled=True, add_loss=True)
self.logger.info('Dev Evaluation:\tIter : %d\tTime : %.4fs\tLoss : %.5f\tAcc : %.4f' \
% (eval_iter, time.time() - start_time, dev_loss, dev_acc))
if dev_acc > self.history['dev_acc'] or (dev_acc == self.history['dev_acc'] and dev_loss < self.history['dev_loss']):
self.history['dev_acc'] = dev_acc
self.history['dev_loss'] = dev_loss
self.history['best_iter'] = eval_iter
self.model.save_model(os.path.join(self.exp_path, 'model.pkl'))
self.logger.warn('NEW BEST:\tIter : %d\tBest Valid Loss/Acc : %.4f/%.4f' % (eval_iter, dev_loss, dev_acc))
self.history['train_losses'].append(epoch_loss)
self.logger.info('Training:\tEpoch : %d\tTime : %.4fs\t Loss: %.5f' \
% (epoch, time.time() - start_time, epoch_loss))
start_time = time.time()
dev_acc, dev_loss = self.decode(dev_loader, labeled=True, add_loss=True)
self.logger.info('Dev Evaluation:\tIter : %d\tTime : %.4fs\tLoss : %.5f\tAcc : %.4f' \
% (eval_iter, time.time() - start_time, dev_loss, dev_acc))
if dev_acc > self.history['dev_acc'] or (dev_acc == self.history['dev_acc'] and dev_loss < self.history['dev_loss']):
self.history['dev_acc'] = dev_acc
self.history['dev_loss'] = dev_loss
self.history['best_iter'] = eval_iter
self.model.save_model(os.path.join(self.exp_path, 'model.pkl'))
self.logger.warn('NEW BEST:\tIter : %d\tBest Valid Loss/Acc : %.4f/%.4f' % (eval_iter, dev_loss, dev_acc))
torch.cuda.empty_cache()
gc.collect()
self.model.load_model(os.path.join(self.exp_path, 'model.pkl'))
self.logger.error('FINAL BEST RESULT: \tIter : %d\tBest Valid (Loss: %.5f Acc : %.4f)'
% (self.history['best_iter'], self.history['dev_loss'], self.history['dev_acc']))
test_file = os.path.join(self.exp_path, 'QNLI.tsv')
test_results = self.decode(test_loader, labeled=False, add_loss=False)
self.logger.info('Start writing test predictions to file %s ...' % (test_file))
write_results(test_results, test_file)
def train_and_decode(self, train_loader, dev_loader, test_loader,
max_epoch):
for epoch in range(max_epoch):
start_time = time.time()
epoch_loss = 0.0
self.model.train()
for data_batch in train_loader:
self.optimizer['all'].zero_grad()
word_ids, segment_ids, masks, labels = to_device(
data_batch, self.device)
loss = self.model(word_ids, segment_ids, masks, labels)
loss.backward()
self.optimizer['all'].step()
epoch_loss += loss.item()
self.history['train_losses'].append(epoch_loss)
self.logger.info('Training:\tEpoch : %d\tTime : %.4fs\t Loss: %.5f' \
% (epoch, time.time() - start_time, epoch_loss))
if epoch < 5: # start to evaluate after some epochs
continue
start_time = time.time()
dev_acc, dev_loss = self.decode(dev_loader,
labeled=True,
add_loss=True)
self.logger.info('Dev Evaluation:\tEpoch : %d\tTime : %.4fs\tLoss : %.5f\tAcc : %.4f' \
% (epoch, time.time() - start_time, dev_loss, dev_acc))
if dev_acc > self.history['dev_acc'] or (
dev_acc == self.history['dev_acc']
and dev_loss < self.history['dev_loss']):
self.history['dev_acc'] = dev_acc
self.history['dev_loss'] = dev_loss
self.history['best_epoch'] = epoch
self.model.save_model(os.path.join(self.exp_path, 'model.pkl'))
self.logger.warn(
'NEW BEST:\tEpoch : %d\tBest Valid Loss/Acc : %.4f/%.4f' %
(epoch, dev_loss, dev_acc))
self.model.load_model(os.path.join(self.exp_path, 'model.pkl'))
self.logger.error(
'FINAL BEST RESULT: \tEpoch : %d\tBest Valid (Loss: %.5f Acc : %.4f)'
% (self.history['best_epoch'], self.history['dev_loss'],
self.history['dev_acc']))
test_file = os.path.join(self.exp_path, 'QNLI.tsv')
test_results = self.decode(test_loader, labeled=False, add_loss=False)
self.logger.info('Start writing test predictions to file %s ...' %
(test_file))
write_results(test_results, test_file)
from utils.writer import write_results
directory = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'backup')
files = os.listdir(directory)
outfile = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'QNLI.tsv')
results_list = []
def read_results(file_path):
results = []
with open(file_path, 'r') as infile:
infile.readline()
for line in infile:
line = line.strip()
idx, label = line.split('\t')
label = 1 if label == 'entailment' else 0
results.append(label)
return np.array(results, dtype=np.int)
for each in files:
file_path = os.path.join(directory, each)
results = read_results(file_path)
results_list.append(results.reshape(-1))
results = np.mean(np.stack(results_list, axis=0), axis=0, dtype=np.float)
write_results(results, outfile)
reduction=args.reduction,
label_smoothing=args.label_smoothing,
device=device)
model = model.to(device)
if args.testing:
model.load_model(os.path.join(args.read_model_path, 'model.pkl'))
test_solver = QNLISolver(model, None, exp_path, logger, device)
logger.info("*" * 50)
logger.info("Testing starts at %s" %
(time.asctime(time.localtime(time.time()))))
predictions = test_solver.decode(test_loader,
labeled=False,
add_loss=False)
test_file = os.path.join(args.read_model_path, 'QNLI.tsv')
logger.info("Start to write predictions to file %s" % (test_file))
write_results(predictions, test_file)
else:
model.init_weights(args.init_weights)
if args.optim == 'bertadam':
t_total = int((len(train_loader.dataset) + args.batch_size - 1) /
args.batch_size) * args.max_epoch
bert_optimizer = set_bertadam_optimizer(model.bert_parameters(),
args.lr,
t_total,
warmup=args.warmup,
schedule=args.schedule,
weight_decay=args.weight_decay)
clsfy_optimizer = set_bertadam_optimizer(
model.clsfy_parameters(),
args.lr,
t_total,