def main(config_path, out_path): logger.info('------------Match-LSTM Evaluate--------------') logger.info('loading config file...') global_config = read_config(config_path) # set random seed seed = global_config['model']['global']['random_seed'] torch.manual_seed(seed) enable_cuda = global_config['test']['enable_cuda'] device = torch.device("cuda" if enable_cuda else "cpu") if torch.cuda.is_available() and not enable_cuda: logger.warning("CUDA is avaliable, you can enable CUDA in config file") elif not torch.cuda.is_available() and enable_cuda: raise ValueError("CUDA is not abaliable, please unable CUDA in config file") torch.no_grad() # make sure all tensors below have require_grad=False logger.info('reading squad dataset...') dataset = SquadDataset(global_config) logger.info('constructing model...') model = MatchLSTMModel(global_config).to(device) model.eval() # let training = False, make sure right dropout # load model weight logger.info('loading model weight...') model_weight_path = global_config['data']['model_path'] assert os.path.exists(model_weight_path), "not found model weight file on '%s'" % model_weight_path weight = torch.load(model_weight_path, map_location=lambda storage, loc: storage) if enable_cuda: weight = torch.load(model_weight_path, map_location=lambda storage, loc: storage.cuda()) model.load_state_dict(weight, strict=False) # forward logger.info('forwarding...') enable_char = global_config['model']['encoder']['enable_char'] batch_size = global_config['test']['batch_size'] # batch_dev_data = dataset.get_dataloader_dev(batch_size) batch_dev_data = list(dataset.get_batch_dev(batch_size)) # to just evaluate score or write answer to file if out_path is None: criterion = MyNLLLoss() score_em, score_f1, sum_loss = eval_on_model(model=model, criterion=criterion, batch_data=batch_dev_data, epoch=None, device=device, enable_char=enable_char, batch_char_func=dataset.gen_batch_with_char) logger.info("test: ave_score_em=%.2f, ave_score_f1=%.2f, sum_loss=%.5f" % (score_em, score_f1, sum_loss)) else: predict_ans = predict_on_model(model=model, batch_data=batch_dev_data, device=device, enable_char=enable_char, batch_char_func=dataset.gen_batch_with_char, id_to_word_func=dataset.sentence_id2word) samples_id = dataset.get_all_samples_id_dev() ans_with_id = dict(zip(samples_id, predict_ans)) logging.info('writing predict answer to file %s' % out_path) with open(out_path, 'w') as f: json.dump(ans_with_id, f) logging.info('finished.')
def test(config_path, out_path): logger.info('------------MODEL PREDICT--------------') logger.info('loading config file...') global_config = read_config(config_path) # set random seed seed = global_config['global']['random_seed'] torch.manual_seed(seed) enable_cuda = global_config['test']['enable_cuda'] device = torch.device("cuda" if enable_cuda else "cpu") if torch.cuda.is_available() and not enable_cuda: logger.warning("CUDA is avaliable, you can enable CUDA in config file") elif not torch.cuda.is_available() and enable_cuda: raise ValueError( "CUDA is not abaliable, please unable CUDA in config file") torch.set_grad_enabled( False) # make sure all tensors below have require_grad=False, logger.info('reading squad dataset...') dataset = SquadDataset(global_config) logger.info('constructing model...') model_choose = global_config['global']['model'] dataset_h5_path = global_config['data']['dataset_h5'] if model_choose == 'base': model_config = read_config('config/base_model.yaml') model = BaseModel(dataset_h5_path, model_config) elif model_choose == 'match-lstm': model = MatchLSTM(dataset_h5_path) elif model_choose == 'match-lstm+': model = MatchLSTMPlus(dataset_h5_path) elif model_choose == 'r-net': model = RNet(dataset_h5_path) elif model_choose == 'm-reader': model = MReader(dataset_h5_path) else: raise ValueError('model "%s" in config file not recoginized' % model_choose) model = model.to(device) model.eval() # let training = False, make sure right dropout # load model weight logger.info('loading model weight...') model_weight_path = global_config['data']['model_path'] assert os.path.exists( model_weight_path ), "not found model weight file on '%s'" % model_weight_path weight = torch.load(model_weight_path, map_location=lambda storage, loc: storage) if enable_cuda: weight = torch.load(model_weight_path, map_location=lambda storage, loc: storage.cuda()) model.load_state_dict(weight, strict=False) # forward logger.info('forwarding...') batch_size = global_config['test']['batch_size'] num_workers = global_config['global']['num_data_workers'] batch_dev_data = dataset.get_dataloader_dev(batch_size, num_workers) # to just evaluate score or write answer to file if out_path is None: criterion = MyNLLLoss() score_em, score_f1, sum_loss = eval_on_model(model=model, criterion=criterion, batch_data=batch_dev_data, epoch=None, device=device) logger.info( "test: ave_score_em=%.2f, ave_score_f1=%.2f, sum_loss=%.5f" % (score_em, score_f1, sum_loss)) else: context_right_space = dataset.get_all_ct_right_space_dev() predict_ans = predict_on_model( model=model, batch_data=batch_dev_data, device=device, id_to_word_func=dataset.sentence_id2word, right_space=context_right_space) samples_id = dataset.get_all_samples_id_dev() ans_with_id = dict(zip(samples_id, predict_ans)) logging.info('writing predict answer to file %s' % out_path) with open(out_path, 'w') as f: json.dump(ans_with_id, f) logging.info('finished.')
def test(config_path, out_path): logger.info('------------MODEL PREDICT--------------') logger.info('loading config file...') global_config = read_config(config_path) # set random seed seed = global_config['global']['random_seed'] torch.manual_seed(seed) enable_cuda = global_config['test']['enable_cuda'] device = torch.device("cuda" if enable_cuda else "cpu") if torch.cuda.is_available() and not enable_cuda: logger.warning("CUDA is avaliable, you can enable CUDA in config file") elif not torch.cuda.is_available() and enable_cuda: raise ValueError("CUDA is not abaliable, please unable CUDA in config file") torch.set_grad_enabled(False) # make sure all tensors below have require_grad=False, logger.info('reading squad dataset...') dataset = SquadDataset(global_config) logger.info('constructing model...') model_choose = global_config['global']['model'] dataset_h5_path = global_config['data']['dataset_h5'] if model_choose == 'base': model_config = read_config('config/base_model.yaml') model = BaseModel(dataset_h5_path, model_config) elif model_choose == 'match-lstm': model = MatchLSTM(dataset_h5_path) elif model_choose == 'match-lstm+': model = MatchLSTMPlus(dataset_h5_path) elif model_choose == 'r-net': model = RNet(dataset_h5_path) elif model_choose == 'm-reader': model = MReader(dataset_h5_path) else: raise ValueError('model "%s" in config file not recoginized' % model_choose) model = model.to(device) model.eval() # let training = False, make sure right dropout # load model weight logger.info('loading model weight...') model_weight_path = global_config['data']['model_path'] assert os.path.exists(model_weight_path), "not found model weight file on '%s'" % model_weight_path weight = torch.load(model_weight_path, map_location=lambda storage, loc: storage) if enable_cuda: weight = torch.load(model_weight_path, map_location=lambda storage, loc: storage.cuda()) model.load_state_dict(weight, strict=False) # forward logger.info('forwarding...') batch_size = global_config['test']['batch_size'] num_workers = global_config['global']['num_data_workers'] batch_dev_data = dataset.get_dataloader_dev(batch_size, num_workers) # to just evaluate score or write answer to file if out_path is None: criterion = MyNLLLoss() score_em, score_f1, sum_loss = eval_on_model(model=model, criterion=criterion, batch_data=batch_dev_data, epoch=None, device=device) logger.info("test: ave_score_em=%.2f, ave_score_f1=%.2f, sum_loss=%.5f" % (score_em, score_f1, sum_loss)) else: context_right_space = dataset.get_all_ct_right_space_dev() predict_ans = predict_on_model(model=model, batch_data=batch_dev_data, device=device, id_to_word_func=dataset.sentence_id2word, right_space=context_right_space) samples_id = dataset.get_all_samples_id_dev() ans_with_id = dict(zip(samples_id, predict_ans)) logging.info('writing predict answer to file %s' % out_path) with open(out_path, 'w') as f: json.dump(ans_with_id, f) logging.info('finished.')