def main(): logger.info('Launching the SAN') opt = vars(args) logger.info('Loading data') embedding, opt = load_meta(opt, os.path.join(args.data_dir, args.meta)) train_data = BatchGen(os.path.join(args.data_dir, args.train_data), batch_size=args.batch_size, gpu=args.cuda, elmo_on=args.elmo_on) dev_data = BatchGen(os.path.join(args.data_dir, args.dev_data), batch_size=args.batch_size_eval, gpu=args.cuda, is_train=False, elmo_on=args.elmo_on) logger.info('#' * 20) logger.info(opt) logger.info('#' * 20) model = DocReaderModel(opt, embedding) model.setup_eval_embed(embedding) logger.info("Total number of params: {}".format(model.total_param)) if args.cuda: model.cuda() best_em_score, best_f1_score = 0.0, 0.0 print("PROGRESS: 00.00%") for epoch in range(0, args.epoches): logger.warning('At epoch {}'.format(epoch)) train_data.reset() start = datetime.now() for i, batch in enumerate(train_data): model.update(batch) if (i + 1) % args.log_per_updates == 0 or i == 0: logger.info('updates[{0:6}] train loss[{1:.5f}] remaining[{2}]'.format( model.updates, model.train_loss.avg, str((datetime.now() - start) / (i + 1) * (len(train_data) - i - 1)).split('.')[0])) # dev eval em, f1, results = check(model, dev_data, os.path.join(args.data_dir, args.dev_gold)) output_path = os.path.join(model_dir, 'dev_output_{}.json'.format(epoch)) with open(output_path, 'w') as f: json.dump(results, f) # setting up scheduler if model.scheduler is not None: logger.info('scheduler_type {}'.format(opt['scheduler_type'])) if opt['scheduler_type'] == 'rop': model.scheduler.step(f1, epoch=epoch) else: model.scheduler.step() # save model_file = os.path.join(model_dir, 'checkpoint_epoch_{}.pt'.format(epoch)) if not args.philly_on: model.save(model_file) if em + f1 > best_em_score + best_f1_score: model.save(os.path.join(model_dir, 'best_checkpoint.pt')) # copyfile(model_file, os.path.join(model_dir, 'best_checkpoint.pt')) best_em_score, best_f1_score = em, f1 logger.info('Saved the new best model and prediction') logger.warning("Epoch {0} - dev EM: {1:.3f} F1: {2:.3f} (best EM: {3:.3f} F1: {4:.3f})".format(epoch, em, f1, best_em_score, best_f1_score)) print("PROGRESS: {0:.2f}%".format(100.0 * (epoch + 1) / args.epoches))
def main(): logger.info('Launching the SAN') opt = vars(args) logger.info(opt) embedding, opt, vocab = load_meta(opt, args.meta) max_doc = opt['max_doc'] smooth = opt['smooth'] is_rep = opt['is_rep'] eval_step = opt['eval_step'] curve_file = opt['curve_file'] training_step = 0 cur_eval_step = 1 checkpoint_path = args.resume if checkpoint_path == '': if not args.if_train: print('checkpoint path can not be empty during testing...') exit() model = DocReaderModel(opt, embedding) else: state_dict = torch.load(checkpoint_path)["state_dict"] model = DocReaderModel(opt, embedding, state_dict) model.setup_eval_embed(embedding) logger.info("Total number of params: {}".format(model.total_param)) if args.cuda: model.cuda() pred_output_path = os.path.join(model_dir, 'pred_output') if not os.path.exists(pred_output_path): os.makedirs(pred_output_path) full_output_path = os.path.join(model_dir, 'full_output_path') if not os.path.exists(full_output_path): os.makedirs(full_output_path) if args.if_train: logger.info('Loading training data') train_data = BatchGen(os.path.join(args.data_dir, args.train_data), batch_size=args.batch_size, gpu=args.cuda, doc_maxlen=max_doc) logger.info('Loading dev data') dev_data = BatchGen(os.path.join(args.data_dir, args.dev_data), batch_size=8, gpu=args.cuda, is_train=False, doc_maxlen=max_doc) curve_file = os.path.join(model_dir, curve_file) full_path = os.path.join(args.data_dir, args.dev_full) pred_output = os.path.join(pred_output_path, str( model.updates)) + '.txt' full_output = os.path.join(full_output_path, str( model.updates)) + '_full.txt' for epoch in range(0, args.epoches): logger.warning('At epoch {}'.format(epoch)) train_data.reset() start = datetime.now() for i, batch in enumerate(train_data): training_step += 1 model.update(batch, smooth, is_rep) if (i + 1) % args.log_per_updates == 0: logger.info( 'updates[{0:6}] train: loss[{1:.5f}]' ' ppl[{2:.5f}] remaining[{3}]'.format( model.updates, model.train_loss.avg, np.exp(model.train_loss.avg), str((datetime.now() - start) / (i + 1) * (len(train_data) - i - 1)).split('.')[0])) # setting up scheduler if model.scheduler is not None: if opt['scheduler_type'] == 'rop': model.scheduler.step(model.train_loss.avg, epoch=epoch) else: model.scheduler.step() dev_loss = 0.0 if (training_step) == cur_eval_step: print('evaluating_step is {} ....'.format(training_step)) bleu, bleu_fact, diver_uni, diver_bi = check( model, dev_data, vocab, full_path, pred_output, full_output) dev_loss = eval_test_loss(model, dev_data).data.cpu().numpy()[0] # dev_loss = dev_loss.data.cpu().numpy()[0] logger.info( 'updates[{0:6}] train: loss[{1:.5f}] ppl[{2:.5f}]\n' 'dev: loss[{3:.5f}] ppl[{4:.5f}]'.format( model.updates, model.train_loss.avg, np.exp(model.train_loss.avg), dev_loss, np.exp(dev_loss))) print('{0},{1:.5f},{2:.5f},{3:.5f},{4:.5f},' '{5:.5f},{6:.5f},{7:.5f},{8:.5f}\n'.format( model.updates, model.train_loss.avg, np.exp(model.train_loss.avg), dev_loss, np.exp(dev_loss), float(bleu), float(diver_uni), float(diver_bi), float(bleu_fact))) with open(curve_file, 'a+') as fout_dev: fout_dev.write( '{0},{1:.5f},{2:.5f},{3:.5f},{4:.5f},' '{5:.5f},{6:.5f},{7:.5f},{8:.5f}\n'.format( model.updates, model.train_loss.avg, np.exp(model.train_loss.avg), dev_loss, np.exp(dev_loss), float(bleu), float(diver_uni), float(diver_bi), float(bleu_fact))) if cur_eval_step == 1: cur_eval_step = cur_eval_step - 1 cur_eval_step += eval_step if (i + 1) % (args.log_per_updates * 50) == 0: logger.info( 'have saved model as checkpoint_step_{0}_{1:.5f}.pt'. format(model.updates, np.exp(dev_loss))) model_file = os.path.join( model_dir, 'checkpoint_step_{0}_{1:.5f}.pt'.format( model.updates, np.exp(dev_loss))) model.save(model_file, epoch) #save dev_loss = eval_test_loss(model, dev_data) dev_loss = dev_loss.data.cpu().numpy()[0] logger.info( 'have saved model as checkpoint_epoch_{0}_{1}_{2:.5f}.pt'. format(epoch, args.learning_rate, np.exp(dev_loss))) model_file = os.path.join( model_dir, 'checkpoint_epoch_{0}_{1}_{2:.5f}.pt'.format( epoch, args.learning_rate, np.exp(dev_loss))) model.save(model_file, epoch) else: logger.info('Loading evaluation data') checkpoint_path = args.resume state_dict = torch.load(checkpoint_path)["state_dict"] model = DocReaderModel(opt, embedding, state_dict) model.setup_eval_embed(embedding) logger.info("Total number of params: {}".format(model.total_param)) if args.cuda: model.cuda() def _eval_output(file_path=args.dev_data, full_path=args.dev_full, test_type='dev'): data = BatchGen(os.path.join(args.data_dir, file_path), batch_size=args.batch_size, gpu=args.cuda, is_train=False) print(len(data)) full_path = os.path.join(args.data_dir, full_path) pred_output_path = os.path.join('./output/', test_type) + '/' full_output_path = os.path.join('./full_output/', test_type) + '/' if not os.path.exists(pred_output_path): os.makedirs(pred_output_path) if not os.path.exists(full_output_path): os.makedirs(full_output_path) t = args.test_output pred_output = pred_output_path + t + '.txt' full_output = full_output_path + t + '_full.txt' bleu, bleu_fact, diver_uni, diver_bi = \ check(model, data, vocab, full_path, pred_output, full_output) _loss = eval_test_loss(model, data) _loss = _loss.data.cpu().numpy()[0] logger.info('dev loss[{0:.5f}] ppl[{1:.5f}]'.format( _loss, np.exp(_loss))) print('{0},{1:.5f},{2:.5f},{3:.5f},{4:.5f},{5:.5f},' '{6:.5f},{7:.5f},{8:.5f}\n'.format( model.updates, model.train_loss.avg, np.exp(model.train_loss.avg), _loss, np.exp(_loss), float(bleu), float(diver_uni), float(diver_bi), float(bleu_fact))) print('test result is:') _eval_output(args.test_data, args.test_full, 'test')
def main(): logger.info('Launching the SAN') opt = vars(args) logger.info('Loading data') version = 'v1' gold_version = 'v1.1' dev_path = gen_name(args.data_dir, args.dev_data, version) dev_gold_path = gen_gold_name(args.data_dir, args.dev_gold, gold_version) test_path = gen_name(args.data_dir, args.test_data, version) test_gold_path = gen_gold_name(args.data_dir, args.test_gold, gold_version) if args.v2_on: version = 'v2' gold_version = 'v2.0' dev_labels = load_squad_v2_label(args.dev_gold) embedding, opt = load_meta( opt, gen_name(args.data_dir, args.meta, version, suffix='pick')) # train_data = BatchGen(gen_name(args.data_dir, args.train_data, version), # batch_size=args.batch_size, # gpu=args.cuda, # with_label=args.v2_on, # elmo_on=args.elmo_on) # import pdb; pdb.set_trace() dev_data = BatchGen(gen_name(args.data_dir, args.dev_data, version), batch_size=args.batch_size, gpu=args.cuda, is_train=False, elmo_on=args.elmo_on) test_data = None test_gold = None # if os.path.exists(test_path): # test_data = BatchGen(test_path, # batch_size=args.batch_size, # gpu=args.cuda, is_train=False, elmo_on=args.elmo_on) # load golden standard dev_gold = load_squad(dev_gold_path) if os.path.exists(test_gold_path): test_gold = load_squad(test_gold_path) model = DocReaderModel(opt, embedding) ### model = your_model() # model meta str headline = '############# Model Arch of SAN #############' # print network logger.info('\n{}\n{}\n'.format(headline, model.network)) model.setup_eval_embed(embedding) logger.info("Total number of params: {}".format(model.total_param)) if args.cuda: model.cuda() best_em_score, best_f1_score = 0.0, 0.0 # test epoch value.. epoch = 2 # for epoch in range(0, args.epoches): # logger.warning('At epoch {}'.format(epoch)) # train_data.reset() # start = datetime.now() # for i, batch in enumerate(train_data): # model.update(batch) # if (model.updates) % args.log_per_updates == 0 or i == 0: # logger.info('#updates[{0:6}] train loss[{1:.5f}] remaining[{2}]'.format( # model.updates, model.train_loss.avg, # str((datetime.now() - start) / (i + 1) * (len(train_data) - i - 1)).split('.')[0])) # dev eval # load the best model from disk... # import pdb;pdb.set_trace() f'loading the model from disk........' path1 = '/demo-mount/san_mrc/checkpoint/checkpoint_v1_epoch_0_full_model.pt' # model = torch.load('/home/ofsdms/san_mrc/checkpoint/best_v1_checkpoint.pt', map_location='cpu') # checkpoint_test = torch.load('/home/ofsdms/san_mrc/checkpoint/best_v1_checkpoint.pt', map_location='cpu') model = torch.load(path1) results, labels = predict_squad(model, dev_data, v2_on=args.v2_on) if args.v2_on: metric = evaluate_v2(dev_gold, results, na_prob_thresh=args.classifier_threshold) em, f1 = metric['exact'], metric['f1'] acc = compute_acc(labels, dev_labels) else: metric = evaluate(dev_gold, results) em, f1 = metric['exact_match'], metric['f1'] output_path = os.path.join(model_dir, 'dev_output_{}.json'.format(epoch)) with open(output_path, 'w') as f: json.dump(results, f) if test_data is not None: test_results, test_labels = predict_squad(model, test_data, v2_on=args.v2_on) test_output_path = os.path.join( model_dir, 'test_output_{}.json'.format(epoch)) with open(test_output_path, 'w') as f: json.dump(test_results, f) if (test_gold is not None): if args.v2_on: test_metric = evaluate_v2( test_gold, test_results, na_prob_thresh=args.classifier_threshold) test_em, test_f1 = test_metric['exact'], test_metric['f1'] test_acc = compute_acc(labels, test_labels) else: test_metric = evaluate(test_gold, test_results) test_em, test_f1 = test_metric['exact_match'], test_metric[ 'f1'] # setting up scheduler if model.scheduler is not None: logger.info('scheduler_type {}'.format(opt['scheduler_type'])) if opt['scheduler_type'] == 'rop': model.scheduler.step(f1, epoch=epoch) else: model.scheduler.step() # save model_file = os.path.join( model_dir, 'checkpoint_{}_epoch_{}.pt'.format(version, epoch)) model.save(model_file, epoch) if em + f1 > best_em_score + best_f1_score: copyfile( os.path.join(model_dir, model_file), os.path.join(model_dir, 'best_{}_checkpoint.pt'.format(version))) best_em_score, best_f1_score = em, f1 logger.info('Saved the new best model and prediction') logger.warning( "Epoch {0} - dev EM: {1:.3f} F1: {2:.3f} (best EM: {3:.3f} F1: {4:.3f})" .format(epoch, em, f1, best_em_score, best_f1_score)) if args.v2_on: logger.warning("Epoch {0} - ACC: {1:.4f}".format(epoch, acc)) if metric is not None: logger.warning("Detailed Metric at Epoch {0}: {1}".format( epoch, metric)) if (test_data is not None) and (test_gold is not None): logger.warning("Epoch {0} - test EM: {1:.3f} F1: {2:.3f}".format( epoch, test_em, test_f1)) if args.v2_on: logger.warning("Epoch {0} - test ACC: {1:.4f}".format( epoch, test_acc))
def main(): logger.info('Launching the SAN') opt = vars(args) logger.info('Loading data') version = 'v1' if args.v2_on: version = 'v2' dev_labels = load_squad_v2_label(args.dev_gold) embedding, opt = load_meta(opt, gen_name(args.data_dir, args.meta, version, suffix='pick')) train_data = BatchGen(gen_name(args.data_dir, args.train_data, version), batch_size=args.batch_size, gpu=args.cuda, with_label=args.v2_on) dev_data = BatchGen(gen_name(args.data_dir, args.dev_data, version), batch_size=args.batch_size, gpu=args.cuda, is_train=False) # load golden standard dev_gold = load_squad(args.dev_gold) model = DocReaderModel(opt, embedding) # model meta str headline = '############# Model Arch of SAN #############' # print network logger.info('\n{}\n{}\n'.format(headline, model.network)) model.setup_eval_embed(embedding) logger.info("Total number of params: {}".format(model.total_param)) if args.cuda: model.cuda() best_em_score, best_f1_score = 0.0, 0.0 for epoch in range(0, args.epoches): logger.warning('At epoch {}'.format(epoch)) train_data.reset() start = datetime.now() for i, batch in enumerate(train_data): #pdb.set_trace() model.update(batch) if (model.updates) % args.log_per_updates == 0 or i == 0: logger.info('#updates[{0:6}] train loss[{1:.5f}] remaining[{2}]'.format( model.updates, model.train_loss.avg, str((datetime.now() - start) / (i + 1) * (len(train_data) - i - 1)).split('.')[0])) # dev eval results, labels = predict_squad(model, dev_data, v2_on=args.v2_on) if args.v2_on: metric = evaluate_v2(dev_gold, results, na_prob_thresh=args.classifier_threshold) em, f1 = metric['exact'], metric['f1'] acc = compute_acc(labels, dev_labels) cls_pr, cls_rec, cls_f1 = compute_classifier_pr_rec(labels, dev_labels) else: metric = evaluate(dev_gold, results) em, f1 = metric['exact_match'], metric['f1'] output_path = os.path.join(model_dir, 'dev_output_{}.json'.format(epoch)) with open(output_path, 'w') as f: json.dump(results, f) # setting up scheduler if model.scheduler is not None: logger.info('scheduler_type {}'.format(opt['scheduler_type'])) if opt['scheduler_type'] == 'rop': model.scheduler.step(f1, epoch=epoch) else: model.scheduler.step() # save model_file = os.path.join(model_dir, 'checkpoint_{}_epoch_{}.pt'.format(version, epoch)) model.save(model_file, epoch) if em + f1 > best_em_score + best_f1_score: copyfile(os.path.join(model_dir, model_file), os.path.join(model_dir, 'best_{}_checkpoint.pt'.format(version))) best_em_score, best_f1_score = em, f1 logger.info('Saved the new best model and prediction') logger.warning("Epoch {0} - dev EM: {1:.3f} F1: {2:.3f} (best EM: {3:.3f} F1: {4:.3f})".format(epoch, em, f1, best_em_score, best_f1_score)) if args.v2_on: logger.warning("Epoch {0} - Precision: {1:.4f}, Recall: {2:.4f}, F1: {3:.4f}, Accuracy: {4:.4f}".format(epoch, cls_pr, cls_rec, cls_f1, acc)) if metric is not None: logger.warning("Detailed Metric at Epoch {0}: {1}".format(epoch, metric))
def main(): opt = vars(args) logger.info('Loading Squad') version = 'v1' gold_version = 'v1.1' if args.v2_on: version = 'v2' gold_version = 'v2.0' dev_labels = load_squad_v2_label(args.dev_gold) logger.info('Loading Meta') embedding, opt = load_meta( opt, gen_name(args.data_dir, args.meta, version, suffix='pick')) logger.info('Loading Train Batcher') if args.elmo_on: logger.info('ELMO ON') dev_path = gen_name(args.data_dir, args.dev_data, version) dev_gold_path = gen_gold_name(args.data_dir, args.dev_gold, gold_version) train_data = BatchGen(gen_name(args.data_dir, args.train_data, version), batch_size=args.batch_size, gpu=args.cuda, with_label=args.v2_on, elmo_on=args.elmo_on) logger.info('Loading Dev Batcher') dev_data = BatchGen(dev_path, batch_size=args.batch_size, gpu=args.cuda, is_train=False, elmo_on=args.elmo_on) logger.info('Loading Golden Standards') # load golden standard dev_gold = load_squad(args.dev_gold) if len(args.resume) > 0: logger.info('Loading resumed model') model = DocReaderModel.load(args.resume, embedding, gpu=args.cuda) resumeSplit = args.resume.split('_') best_f1_score = float(resumeSplit[6].replace('.pt', '')) best_em_score = float(resumeSplit[4]) resumed_epoch = int(resumeSplit[2]) + 1 #step scheduler for i in range(resumed_epoch): model.scheduler.step() logger.info( "RESUMING MODEL TRAINING. BEST epoch {} EM {} F1 {} ".format( str(resumed_epoch), str(best_em_score), str(best_f1_score))) else: model = DocReaderModel(opt, embedding) best_em_score, best_f1_score = 0.0, 0.0 resumed_epoch = 0 # model meta str # headline = '############# Model Arch of SAN #############' # print network # logger.info('\n{}\n{}\n'.format(headline, model.network)) model.setup_eval_embed(embedding) logger.info("Total number of params: {}".format(model.total_param)) logger.info("Batch Size {}".format(args.batch_size)) if args.cuda: model.cuda() else: model.cpu() for epoch in range(resumed_epoch, args.epoches): logger.warning('At epoch {}'.format(epoch)) #shuffle training batch train_data.reset() start = datetime.now() for i, batch in enumerate(train_data): model.update(batch) if (model.updates) % args.log_per_updates == 0 or i == 0: logger.info( '#updates[{0:6}] train loss[{1:.5f}] remaining[{2}]'. format( model.updates, model.train_loss.avg, str((datetime.now() - start) / (i + 1) * (len(train_data) - i - 1)).split('.')[0])) # dev eval results, labels = predict_squad(model, dev_data, v2_on=args.v2_on) if args.v2_on: metric = evaluate_v2(dev_gold, results, na_prob_thresh=args.classifier_threshold) em, f1 = metric['exact'], metric['f1'] acc = compute_acc(labels, dev_labels) else: metric = evaluate(dev_gold, results) em, f1 = metric['exact_match'], metric['f1'] output_path = os.path.join(model_dir, 'dev_output_{}.json'.format(epoch)) with open(output_path, 'w') as f: json.dump(results, f) # setting up scheduler if model.scheduler is not None: logger.info('scheduler_type {}'.format(opt['scheduler_type'])) if opt['scheduler_type'] == 'rop': model.scheduler.step(f1, epoch=epoch) else: model.scheduler.step() # save model_file = os.path.join( model_dir, 'cp_epoch_{}_em_{}_f1_{}.pt'.format(epoch, int(em), int(f1))) model.save(model_file, epoch) if em + f1 > best_em_score + best_f1_score: copyfile( os.path.join(model_dir, model_file), os.path.join( model_dir, 'best_epoch_{}_em_{}_f1_{}.pt'.format( epoch, int(em), int(f1)))) best_em_score, best_f1_score = em, f1 logger.info('Saved the new best model and prediction') logger.warning( "Epoch {0} - dev EM: {1:.3f} F1: {2:.3f} (best EM: {3:.3f} F1: {4:.3f})" .format(epoch, em, f1, best_em_score, best_f1_score)) if args.v2_on: logger.warning("Epoch {0} - ACC: {1:.4f}".format(epoch, acc)) if metric is not None: logger.warning("Detailed Metric at Epoch {0}: {1}".format( epoch, metric))
def main(): logger.info('Launching the SAN') start_test = False dev_name = 'dev' opt = vars(args) logger.info('Loading data') embedding, opt = load_meta( opt, os.path.join(args.multitask_data_path, args.meta)) gold_data = load_gold(args.dev_datasets, args.data_dir, dev_name=dev_name) best_em_score, best_f1_score = 0.0, 0.0 all_train_batchgen = [] all_dev_batchgen = [] all_train_iters = [] for dataset_name in args.train_datasets: path = os.path.join(args.multitask_data_path, dataset_name + '_train.json') this_extra_score = extra_score.get(dataset_name, None) all_train_batchgen.append( BatchGen(path, batch_size=args.batch_size, gpu=args.cuda, dataset_name=dataset_name, doc_maxlen=args.doc_maxlen, drop_less=args.drop_less, num_gpu=args.num_gpu, dropout_w=args.dropout_w, dw_type=args.dw_type, extra_score=this_extra_score, extra_score_cap=args.extra_score_cap)) all_train_iters = [iter(item) for item in all_train_batchgen] for dataset_name in args.dev_datasets: path = os.path.join(args.multitask_data_path, dataset_name + '_%s.json' % dev_name) all_dev_batchgen.append( BatchGen(path, batch_size=args.valid_batch_size, gpu=args.cuda, is_train=False, dataset_name=dataset_name, doc_maxlen=args.doc_maxlen, num_gpu=args.num_gpu)) if 'marco' in dataset_name: rank_path = os.path.join(args.data_dir, dataset_name) dev_rank_path = os.path.join(rank_path, 'dev_rank_scores.json') dev_rank_scores = load_rank_score(dev_rank_path) dev_yn = json.load( open(os.path.join(rank_path, 'dev_yn_dict.json'))) dev_gold_path = os.path.join(args.data_dir, dataset_name, 'dev_original.json') dev_gold_data_marco = load_jsonl(dev_gold_path) if args.resume_last_epoch: latest_time = 0 for o in os.listdir(model_dir): if o.startswith('checkpoint_') and 'trim' not in o: edit_time = os.path.getmtime(os.path.join(model_dir, o)) if edit_time > latest_time: latest_time = edit_time args.resume_dir = model_dir args.resume = o if args.resume_dir is not None: print('resuming model in ', os.path.join(args.resume_dir, args.resume)) checkpoint = torch.load(os.path.join(args.resume_dir, args.resume)) model_opt = checkpoint['config'] if args.resume_options else opt model_opt['multitask_data_path'] = opt['multitask_data_path'] model_opt['covec_path'] = opt['covec_path'] model_opt['data_dir'] = opt['data_dir'] if args.resume_options: logger.info('resume old options') else: logger.info('use new options.') model_opt['train_datasets'] = checkpoint['config']['train_datasets'] state_dict = checkpoint['state_dict'] model = DocReaderModel(model_opt, embedding, state_dict) if not args.new_random_state: logger.info('use old random state.') random.setstate(checkpoint['random_state']) torch.random.set_rng_state(checkpoint['torch_state']) if args.cuda: torch.cuda.set_rng_state(checkpoint['torch_cuda_state']) if model.scheduler: if args.new_scheduler: model.scheduler = torch.optim.lr_scheduler.MultiStepLR( model.optimizer, milestones=[2, 5, 8], gamma=args.lr_gamma) elif 'scheduler_state' in checkpoint: model.scheduler.load_state_dict(checkpoint['scheduler_state']) else: print( 'warning: not loading scheduler state because didn\'t save.' ) start_epoch = checkpoint['epoch'] + 1 else: model = DocReaderModel(opt, embedding) start_epoch = 0 logger.info('using {} GPUs'.format(torch.cuda.device_count())) headline = '############# Model Arch of SAN #############' # print network logger.info('\n{}\n{}\n'.format(headline, model.network)) model.setup_eval_embed(embedding) logger.info("Total number of params: {}".format(model.total_param)) if args.cuda: model.cuda() all_lens = [len(bg) for bg in all_train_batchgen] if args.continue_epoches is not None: args.epoches = start_epoch + args.continue_epoches num_all_batches = args.epoches * sum(all_lens) best_performance = {name: 0.0 for name in args.dev_datasets} best_performance['total'] = 0.0 for epoch in range(start_epoch, args.epoches): logger.warning('At epoch {}'.format(epoch)) # batch indices all_call_indices = [] for train_data in all_train_batchgen: train_data.reset() if args.dataset_include_ratio >= 0: other_indices = [] for i in range(1, len(all_train_batchgen)): other_indices += [i] * len(all_train_batchgen[i]) if args.dataset_include_ratio > 1: inverse_ratio = 1 / args.dataset_include_ratio batch0_indices = [0] * (int( len(other_indices) * inverse_ratio)) else: batch0_indices = [0] * len(all_train_batchgen[0]) other_picks = int( len(other_indices) * args.dataset_include_ratio) other_indices = random.sample(other_indices, other_picks) all_call_indices = batch0_indices + other_indices else: for i in range(len(all_train_batchgen)): all_call_indices += [i] * len(all_train_batchgen[i]) random.shuffle(all_call_indices) all_call_indices = all_call_indices[:10] start = datetime.now() for i in range(len(all_call_indices)): batch_list, name_map = next(all_train_iters[all_call_indices[i]]) dataset_name = args.train_datasets[all_call_indices[i]] model.update(batch_list, name_map, dataset_name) if (model.updates) % args.log_per_updates == 0 or i == 0: logger.info( 'o(*^~^*) Task [{0:2}] #updates[{1:6}] train loss[{2:.5f}] remaining[{3}]' .format( all_call_indices[i], model.updates, model.train_loss.avg, str((datetime.now() - start) / (i + 1) * (len(all_call_indices) - i - 1)).split('.')[0])) em_sum = 0 f1_sum = 0 model.eval() this_performance = {} for i in range(len(all_dev_batchgen)): dataset_name = args.dev_datasets[i] if dataset_name in ['squad', 'newsqa']: em, f1, results, scores = check(model, all_dev_batchgen[i], gold_data[dataset_name]) output_path = os.path.join( model_dir, 'dev_output_{}_{}.json'.format(dataset_name, epoch)) output_scores_path = os.path.join( model_dir, 'dev_scores_{}_{}.pt'.format(dataset_name, epoch)) for repeat_times in range(10): try: with open(output_path, 'w') as f: json.dump(results, f) with open(output_scores_path, 'wb') as f: pickle.dump(scores, f) break except Exception as e: print('save predict failed. error:', e) em_sum += em f1_sum += f1 this_performance[dataset_name] = em + f1 logger.warning( "Epoch {0} - Task {1:6} dev EM: {2:.3f} F1: {3:.3f}". format(epoch, dataset_name, em, f1)) elif dataset_name == 'wdw': acc, results, scores = check_wdw(model, all_dev_batchgen[i]) output_path = os.path.join( model_dir, 'dev_output_{}_{}.json'.format(dataset_name, epoch)) output_scores_path = os.path.join( model_dir, 'dev_scores_{}_{}.pt'.format(dataset_name, epoch)) for repeat_times in range(10): try: with open(output_path, 'w') as f: json.dump(results, f) with open(output_scores_path, 'wb') as f: pickle.dump(scores, f) break except Exception as e: print('save predict failed. error:', e) em_sum += acc f1_sum += acc logger.warning( "Epoch {0} - Task {1:6} dev ACC: {2:.3f}".format( epoch, dataset_name, acc)) this_performance[dataset_name] = acc elif 'marco' in dataset_name: # dev eval output = os.path.join(model_dir, 'dev_pred_{}.json'.format(epoch)) output_yn = os.path.join(model_dir, 'dev_pred_yn_{}.json'.format(epoch)) span_output = os.path.join( model_dir, 'dev_pred_span_{}.json'.format(epoch)) dev_predictions, dev_best_scores, dev_ids_list = eval_model_marco( model, all_dev_batchgen[i]) answer_list, rank_answer_list, yesno_answer_list = generate_submit( dev_ids_list, dev_best_scores, dev_predictions, dev_rank_scores, dev_yn) dev_gold_path = os.path.join(args.data_dir, dataset_name, 'dev_original.json') metrics = compute_metrics_from_files(dev_gold_data_marco, \ rank_answer_list, \ MAX_BLEU_ORDER) rouge_score = metrics['rouge_l'] blue_score = metrics['bleu_1'] logger.warning( "Epoch {0} - dev ROUGE-L: {1:.4f} BLEU-1: {2:.4f}".format( epoch, rouge_score, blue_score)) for metric in sorted(metrics): logger.info('%s: %s' % (metric, metrics[metric])) this_performance[dataset_name] = rouge_score + blue_score this_performance['total'] = sum([v for v in this_performance.values()]) model.train() # setting up scheduler if model.scheduler is not None: logger.info('scheduler_type {}'.format(opt['scheduler_type'])) if opt['scheduler_type'] == 'rop': model.scheduler.step(f1, epoch=epoch) else: model.scheduler.step() # save for try_id in range(10): try: model_file = os.path.join( model_dir, 'checkpoint_epoch_{}.pt'.format(epoch)) model.save(model_file, epoch, best_em_score, best_f1_score) if em_sum + f1_sum > best_em_score + best_f1_score: copyfile(os.path.join(model_dir, model_file), os.path.join(model_dir, 'best_checkpoint.pt')) best_em_score, best_f1_score = em_sum, f1_sum logger.info('Saved the new best model and prediction') break except Exception as e: print('save model failed: outer step. error=', e)
def main(): logger.info('Launching the SAN') opt = vars(args) logger.info('Loading data') version = 'v2' if args.v2_on else 'v1' gold_version = 'v2.0' if args.v2_on else 'v1.1' dev_path = gen_name(args.data_dir, args.dev_data, version) dev_gold_path = gen_gold_name(args.data_dir, args.dev_gold, gold_version) dev_labels = load_squad_v2_label(dev_gold_path) embedding, opt = load_meta(opt, gen_name(args.data_dir, args.meta, version, suffix='pick')) dev_data = BatchGen(dev_path, batch_size=args.batch_size, gpu=args.cuda, is_train=False, elmo_on=args.elmo_on) dev_gold = load_squad(dev_gold_path) checkpoint_path = args.checkpoint_path logger.info(f'path to given checkpoint is {checkpoint_path}') checkpoint = torch.load(checkpoint_path) if args.cuda else torch.load(checkpoint_path, map_location='cpu') state_dict = checkpoint['state_dict'] # Set up the model logger.info('Loading model ...') model = DocReaderModel(opt, embedding,state_dict) model.setup_eval_embed(embedding) logger.info('done') if args.cuda: model.cuda() # dev eval logger.info('Predicting ...') results, labels = predict_squad(model, dev_data, v2_on=args.v2_on) logger.info('done') # get actual and predicted labels (as lists) actual_labels = [] predicted_labels = [] dropped = 0 for key in dev_labels.keys(): # convert from dictionaries to lists try: actual_labels.append(dev_labels[key]) predicted_labels.append(labels[key]) except: dropped += 1 print(f'dropped: {dropped}') actual_labels = np.array(actual_labels) predicted_labels = np.array(predicted_labels) # convert from continuous to discrete actual_labels = (actual_labels > args.classifier_threshold).astype(np.int32) predicted_labels = (predicted_labels > args.classifier_threshold).astype(np.int32) # Print all metrics print('accuracy', 100 - 100 * sum(abs(predicted_labels-actual_labels)) / len(actual_labels), '%') print('confusion matrix', confusion_matrix(predicted_labels, actual_labels)) precision, recall, f1, _ = precision_recall_fscore_support(actual_labels, predicted_labels, average='binary') print(f'Precision: {precision} recall: {recall} f1-score: {f1}')
def main(): logger.info('Launching the SAN') opt = vars(args) logger.info('Loading data') version = 'v2' if args.v2_on else 'v1' gold_version = 'v2.0' if args.v2_on else 'v1.1' train_path = gen_name(args.data_dir, args.train_data, version) train_gold_path = gen_gold_name(args.data_dir, 'train', gold_version) dev_path = gen_name(args.data_dir, args.dev_data, version) dev_gold_path = gen_gold_name(args.data_dir, args.dev_gold, gold_version) test_path = gen_name(args.data_dir, args.test_data, version) test_gold_path = gen_gold_name(args.data_dir, args.test_gold, gold_version) train_labels = load_squad_v2_label(train_gold_path) dev_labels = load_squad_v2_label(dev_gold_path) #train_labels = load_squad_v2_label(train_gold_path) embedding, opt = load_meta( opt, gen_name(args.data_dir, args.meta, version, suffix='pick')) train_data = BatchGen(train_path, batch_size=args.batch_size, gpu=args.cuda, with_label=args.v2_on, elmo_on=args.elmo_on) dev_data = BatchGen(dev_path, batch_size=args.batch_size, gpu=args.cuda, is_train=False, elmo_on=args.elmo_on) test_data = None test_gold = None if os.path.exists(test_path): test_data = BatchGen(test_path, batch_size=args.batch_size, gpu=args.cuda, is_train=False, elmo_on=args.elmo_on) # load golden standard train_gold = load_squad(train_gold_path) dev_gold = load_squad(dev_gold_path) #train_gold = load_squad(train_gold_path) if os.path.exists(test_gold_path): test_gold = load_squad(test_gold_path) #define csv path csv_head = [ 'epoch', 'train_loss', 'train_loss_san', 'train_loss_class', 'dev_em', 'dev_f1', 'dev_acc', 'train_em', 'train_f1', 'train_acc' ] csvfile = 'results_{}.csv'.format(args.classifier_gamma) csv_path = os.path.join(args.data_dir, csvfile) result_params = [] #load previous checkpoint start_epoch = 0 state_dict = None if (args.load_checkpoint != 0): start_epoch = args.load_checkpoint + 1 checkpoint_file = 'checkpoint_{}_epoch_{}.pt'.format( version, args.load_checkpoint) checkpoint_path = os.path.join(args.model_dir, checkpoint_file) logger.info('path to prev checkpoint is {}'.format(checkpoint_path)) checkpoint = torch.load(checkpoint_path) state_dict = checkpoint['state_dict'] opt = checkpoint['config'] #logger.warning('the checkpoint is {}'.format(checkpoint)) #load previous metrics with open(csv_path, 'r') as csvfile: csvreader = csv.reader(csvfile) dummy = next(csvreader) for row in csvreader: result_params.append(row) logger.info('Previous metrics loaded') model = DocReaderModel(opt, embedding, state_dict) # model meta str #headline = '############# Model Arch of SAN #############' # print network #logger.info('\n{}\n{}\n'.format(headline, model.network)) model.setup_eval_embed(embedding) logger.info("Total number of params: {}".format(model.total_param)) if args.cuda: model.cuda() best_em_score, best_f1_score = 0.0, 0.0 for epoch in range(start_epoch, args.epoches): logger.warning('At epoch {}'.format(epoch)) loss, loss_san, loss_class = 0.0, 0.0, 0.0 train_data.reset() start = datetime.now() for i, batch in enumerate(train_data): losses = model.update(batch) loss += losses[0].item() loss_san += losses[1].item() if losses[2]: loss_class += losses[2].item() if (model.updates) % args.log_per_updates == 0 or i == 0: logger.info( '#updates[{0:6}] train loss[{1:.5f}] remaining[{2}]'. format( model.updates, model.train_loss.avg, str((datetime.now() - start) / (i + 1) * (len(train_data) - i - 1)).split('.')[0])) # train eval tr_results, tr_labels = predict_squad(model, train_data, v2_on=args.v2_on) if args.v2_on and args.classifier_on: train_metric = evaluate_v2( train_gold, tr_results, na_prob_thresh=args.classifier_threshold) train_em, train_f1 = train_metric['exact'], train_metric['f1'] train_acc = compute_acc(tr_labels, train_labels) else: train_metric = evaluate(train_gold, tr_results) train_em, train_f1 = train_metric['exact_match'], train_metric[ 'f1'] train_acc = -1 # dev eval results, labels = predict_squad(model, dev_data, v2_on=args.v2_on) if args.v2_on and args.classifier_on: metric = evaluate_v2(dev_gold, results, na_prob_thresh=args.classifier_threshold) em, f1 = metric['exact'], metric['f1'] acc = compute_acc(labels, dev_labels) else: metric = evaluate(dev_gold, results) em, f1 = metric['exact_match'], metric['f1'] acc = -1 output_path = os.path.join(model_dir, 'dev_output_{}.json'.format(epoch)) with open(output_path, 'w') as f: json.dump(results, f) if test_data is not None: test_results, test_labels = predict_squad(model, test_data, v2_on=args.v2_on) test_output_path = os.path.join( model_dir, 'test_output_{}.json'.format(epoch)) with open(test_output_path, 'w') as f: json.dump(test_results, f) if (test_gold is not None): if args.v2_on: test_metric = evaluate_v2( test_gold, test_results, na_prob_thresh=args.classifier_threshold) test_em, test_f1 = test_metric['exact'], test_metric['f1'] test_acc = compute_acc( labels, test_labels ) #?? should be test_labels,test_gold_labels else: test_metric = evaluate(test_gold, test_results) test_em, test_f1 = test_metric['exact_match'], test_metric[ 'f1'] # setting up scheduler # halves learning rate every 10 epochs if model.scheduler is not None: logger.info('scheduler_type {}'.format(opt['scheduler_type'])) if opt['scheduler_type'] == 'rop': model.scheduler.step(f1, epoch=epoch) else: model.scheduler.step() # save model_file = os.path.join( model_dir, 'checkpoint_{}_epoch_{}.pt'.format(version, epoch)) model.save(model_file, epoch) if em + f1 > best_em_score + best_f1_score: copyfile( os.path.join(model_dir, model_file), os.path.join(model_dir, 'best_{}_checkpoint.pt'.format(version))) best_em_score, best_f1_score = em, f1 logger.info('Saved the new best model and prediction') approx = lambda x: round(x, 3) logger.warning(f""" Epoch {str(epoch).zfill(2)} --- Train | acc: {approx(train_acc)} EM: {approx(train_em)} F1: {approx(train_f1)} loss ({approx(loss)}) = {approx(loss_san)} + {approx(loss_class)} Dev | acc: {approx(acc)} EM: {approx(em)} F1: {approx(f1)} -------------------------------- """) #writing in CSV result_params.append([ epoch, loss, loss_san, loss_class, em, f1, acc, train_em, train_f1, train_acc ]) logger.info('Writing in {} the values {}'.format( csv_path, result_params)) with open(csv_path, 'w') as csvfile: csvwriter = csv.writer(csvfile) csvwriter.writerow(csv_head) csvwriter.writerows(result_params)
def main(): logger.info('Launching the SAN') opt = vars(args) logger.info('Loading data') version = 'v1' if args.v2_on: version = 'v2' dev_labels = load_squad_v2_label(args.dev_gold) dev_labels_adv = load_squad_v2_label('data/adv-dev-v2.0.json') embedding, opt = load_meta( opt, gen_name(args.data_dir, args.meta, version, suffix='pick')) dev_data = BatchGen(gen_name(args.data_dir, args.dev_data, version), batch_size=args.batch_size, gpu=args.cuda, is_train=False) dev_data_adv = BatchGen(gen_name(args.data_dir, 'adv_' + args.dev_data, version), batch_size=args.batch_size, gpu=args.cuda, is_train=False) # load golden standard dev_gold = load_squad(args.dev_gold) dev_gold_adv = load_squad('data/adv-dev-v2.0.json') # TODO best_checkpoint_path = os.path.join( model_dir, 'best_{}_checkpoint.pt'.format(version)) check = torch.load(best_checkpoint_path) model = DocReaderModel(check['config'], embedding, state_dict=check['state_dict']) model.setup_eval_embed(embedding) if args.cuda: model.cuda() # dev eval results, labels = predict_squad(model, dev_data, v2_on=args.v2_on) if args.v2_on: metric = evaluate_v2(dev_gold, results, labels, na_prob_thresh=args.classifier_threshold) em, f1 = metric['exact'], metric['f1'] acc = compute_acc(labels, dev_labels) print("Original validation EM {}, F1 {}, Acc {}".format(em, f1, acc)) else: metric = evaluate(dev_gold, results) em, f1 = metric['exact_match'], metric['f1'] results, labels = predict_squad(model, dev_data_adv, v2_on=args.v2_on) if args.v2_on: metric = evaluate_v2(dev_gold_adv, results, labels, na_prob_thresh=args.classifier_threshold) em, f1 = metric['exact'], metric['f1'] acc = compute_acc(labels, dev_labels_adv) print("Adversarial EM {}, F1 {}, Acc {}".format(em, f1, acc)) else: metric = evaluate(dev_gold, results) em, f1 = metric['exact_match'], metric['f1']
batch_size, have_gpu, is_train=False, with_label=True) #batches.reset() #batches = list(batches) model_path = model_root + 'best_checkpoint.pt' checkpoint = torch.load(model_path) opt = checkpoint['config'] set_environment(opt['seed'], have_gpu) opt['covec_path'] = mtlstm_path opt['cuda'] = have_gpu opt['multi_gpu'] = False opt['max_len'] = max_len state_dict = checkpoint['state_dict'] model = DocReaderModel(opt, state_dict=state_dict) model.setup_eval_embed(torch.Tensor(test_embedding)) logger.info('Loaded model!') if have_gpu: model.cuda() results, score_list = evaluate_squad_v2(model, dev_data) dev_gold = load_squad_v2(test_file) results = my_evaluation(dev_gold, results, score_list, 0.4) logger.info('{}'.format(results))
class InteractiveModel: def __init__(self, args): self.is_cuda = args.cuda self.embedding, self.opt, self.vocab = load_meta(vars(args), args.meta) self.opt['skip_tokens'] = self.get_skip_tokens( self.opt["skip_tokens_file"]) self.opt['skip_tokens_first'] = self.get_skip_tokens( self.opt["skip_tokens_first_file"]) self.state_dict = th.load(args.model_dir)["state_dict"] self.model = DocReaderModel(self.opt, self.embedding, self.state_dict) self.model.setup_eval_embed(self.embedding) if self.is_cuda: self.model.cuda() def get_skip_tokens(self, path): skip_tokens = None if path and os.path.isfile(path): skip_tokens = [] with open(path, 'r') as f: for word in f: word = word.strip().rstrip('\n') try: skip_tokens.append(self.vocab[word]) except: print("Token %s not present in dictionary" % word) return skip_tokens def predict(self, data, top_k=2): processed_data = prepare_batch_data( [self.preprocess_data(x) for x in data], ground_truth=False) prediction, prediction_topks = self.model.predict(processed_data, top_k=top_k) pred_word = pred2words(prediction, self.vocab) prediction = [np.asarray(x, dtype=np.str).tolist() for x in pred_word] return (prediction, prediction_topks) def preprocess_data(self, sample, q_cutoff=30, doc_cutoff=500): def tok_func(toks): return [self.vocab[w] for w in toks] fea_dict = {} query_tokend = filter_query(sample['query'].strip(), max_len=q_cutoff).split() doc_tokend = filter_fact(sample['fact'].strip()).split() if len(doc_tokend) > doc_cutoff: doc_tokend = doc_tokend[:doc_cutoff] + ['<TRNC>'] # TODO fea_dict['query_tok'] = tok_func(query_tokend) fea_dict['query_pos'] = [] fea_dict['query_ner'] = [] fea_dict['doc_tok'] = tok_func(doc_tokend) fea_dict['doc_pos'] = [] fea_dict['doc_ner'] = [] fea_dict['doc_fea'] = '' if len(fea_dict['query_tok']) == 0: fea_dict['query_tok'] = [0] if len(fea_dict['doc_tok']) == 0: fea_dict['doc_tok'] = [0] return fea_dict
embedding, opt = load_meta(opt, gen_name(args.data_dir, args.meta, version, suffix='pick')) data = BatchGen('./proc_data.json', batch_size=args.batch_size, gpu=args.cuda, is_train=False, elmo_on=args.elmo_on) checkpoint_path = args.checkpoint_path logger.info(f'path to given checkpoint is {checkpoint_path}') checkpoint = torch.load(checkpoint_path) if args.cuda else torch.load(checkpoint_path, map_location='cpu') state_dict = checkpoint['state_dict'] # Set up the model logger.info('Loading model ...') model = DocReaderModel(opt, embedding,state_dict) model.setup_eval_embed(embedding) logger.info('done') if args.cuda: model.cuda() # dev eval logger.info('Predicting ...') results, labels = predict_squad(model, data, v2_on=args.v2_on) logger.info('done') print('\n\n\n------------ Results ------------\n') print(para, '\n') for q in ques: print(f'Ques: {q["question"]}') id = q["id"]