def main(): args, a_config, r_config = parse_args() if args.cudnn_off: torch.backends.cudnn.enabled = False with jsonlines.open(args.annot) as reader: gts = [(obj['answer_label'], obj['rationale_label']) for obj in reader] a_gt = np.array([gt[0] for gt in gts], dtype=np.int64) r_gt = np.array([gt[1] for gt in gts], dtype=np.int64) # cache a_cache_fn = os.path.join(args.result_path, '{}_a_pred.npy'.format(args.result_name)) r_cache_fn = os.path.join(args.result_path, '{}_r_pred.npy'.format(args.result_name)) a_pred = r_pred = None if not os.path.exists(args.result_path): os.makedirs(args.result_path) if args.use_cache: if os.path.exists(a_cache_fn): print("Load cached predictions from {}...".format(a_cache_fn)) a_pred = np.load(a_cache_fn) if os.path.exists(r_cache_fn): print("Load cached predictions from {}...".format(r_cache_fn)) r_pred = np.load(r_cache_fn) else: if a_config is not None and args.a_ckpt is not None: print("Build model and dataloader for Q->A...") # get model device_ids = args.gpus # os.environ['CUDA_VISIBLE_DEVICES'] = ','.join([str(k) for k in args.gpus]) a_config.GPUS = ','.join([str(k) for k in args.gpus]) answer_model = eval(a_config.MODULE)(a_config) if len(device_ids) > 1: answer_model = torch.nn.DataParallel( answer_model, device_ids=device_ids).cuda() else: torch.cuda.set_device(device_ids[0]) answer_model = answer_model.cuda() if args.fp16: [answer_model] = amp.initialize([answer_model], opt_level='O2', keep_batchnorm_fp32=False) a_ckpt = torch.load(args.a_ckpt, map_location=lambda storage, loc: storage) smart_load_model_state_dict(answer_model, a_ckpt['state_dict']) answer_model.eval() # get data loader a_config.DATASET.TASK = 'Q2A' a_config.VAL.SHUFFLE = False answer_loader = make_dataloader(a_config, mode='val', distributed=False) label_index_in_batch = a_config.DATASET.LABEL_INDEX_IN_BATCH print("Inference Q->A...") # inference n_batch = len(answer_loader) a_pred = np.zeros((len(gts), 4), dtype=np.float) i_sample = 0 for nbatch, a_batch in zip(trange(len(answer_loader)), answer_loader): # for a_batch in answer_loader: a_batch = to_cuda(a_batch) a_batch = [ a_batch[i] for i in range(len(a_batch)) if i != label_index_in_batch % len(a_batch) ] a_out = answer_model(*a_batch) a_batch_pred = a_out['label_logits'] batch_size = a_batch_pred.shape[0] if a_batch_pred.dim() == 2: a_pred[i_sample:( i_sample + batch_size )] = a_batch_pred.detach().cpu().numpy().astype(np.float, copy=False) elif a_batch_pred.dim() == 1: assert a_batch_pred.shape[0] % 4 == 0 a_batch_pred = a_batch_pred.view((-1, 4)) a_pred[int(i_sample / 4):int((i_sample + batch_size) / 4)] \ = a_batch_pred.float().detach().cpu().numpy().astype(np.float, copy=False) else: raise ValueError("Invalid") i_sample += batch_size # print("inference {}/{}".format(i_sample, len(answer_loader.dataset))) np.save(a_cache_fn, a_pred) if r_config is not None and args.r_ckpt is not None: print("Build model and dataloader for QA->R...") # get model device_ids = args.gpus # os.environ['CUDA_VISIBLE_DEVICES'] = ','.join([str(k) for k in args.gpus]) r_config.GPUS = ','.join([str(k) for k in args.gpus]) rationale_model = eval(r_config.MODULE)(r_config) if len(device_ids) > 1: rationale_model = torch.nn.DataParallel( rationale_model, device_ids=device_ids).cuda() else: torch.cuda.set_device(device_ids[0]) rationale_model = rationale_model.cuda() if args.fp16: [rationale_model] = amp.initialize([rationale_model], opt_level='O2', keep_batchnorm_fp32=False) r_ckpt = torch.load(args.r_ckpt, map_location=lambda storage, loc: storage) smart_load_model_state_dict(rationale_model, r_ckpt['state_dict']) rationale_model.eval() # get data loader r_config.DATASET.TASK = 'QA2R' r_config.VAL.SHUFFLE = False rationale_loader = make_dataloader(r_config, mode='val', distributed=False) label_index_in_batch = r_config.DATASET.LABEL_INDEX_IN_BATCH print("Inference QA->R...") # inference n_batch = len(rationale_loader) r_pred = np.zeros((len(rationale_loader.dataset), 4), dtype=np.float) i_sample = 0 for nbatch, r_batch in zip(trange(len(rationale_loader)), rationale_loader): # for r_batch in rationale_loader: r_batch = to_cuda(r_batch) r_batch = [ r_batch[i] for i in range(len(r_batch)) if i != label_index_in_batch % len(r_batch) ] r_out = rationale_model(*r_batch) r_batch_pred = r_out['label_logits'] batch_size = r_batch_pred.shape[0] r_pred[i_sample:(i_sample + batch_size)] =\ r_batch_pred.float().detach().cpu().numpy().astype(np.float, copy=False) i_sample += batch_size # print("inference {}/{}".format(i_sample, len(rationale_loader.dataset))) np.save(r_cache_fn, r_pred) # evaluate print("Evaluate...") if a_pred is not None: acc_a = (a_pred.argmax(1) == a_gt).sum() * 1.0 / a_gt.size print("Q->A\t{:.1f}".format(acc_a * 100.0)) if r_pred is not None: acc_r = (r_pred.argmax(1) == r_gt).sum() * 1.0 / r_gt.size print("QA->R\t{:.1f}".format(acc_r * 100.0)) if a_pred is not None and r_pred is not None: acc_joint = ((a_pred.argmax(1) == a_gt) * (r_pred.argmax(1) == r_gt)).sum() * 1.0 / a_gt.size print("Q->AR\t{:.1f}".format(acc_joint * 100.0))
def test_net(args, config, ckpt_path=None, save_path=None, save_name=None): if save_path is None: logger, test_output_path = create_logger(config.OUTPUT_PATH, args.cfg, config.DATASET.IMAGE_SET, split='test') save_path = test_output_path if ckpt_path is None: _, train_output_path = create_logger(config.OUTPUT_PATH, args.cfg, config.DATASET.IMAGE_SET, split='train') model_prefix = os.path.join(train_output_path, config.MODEL_PREFIX) ckpt_path = '{}-best.model'.format(model_prefix) print('Use best checkpoint {}...'.format(ckpt_path)) if save_name is None: save_name = os.path.split(ckpt_path)[-1] if not os.path.exists(save_path): os.makedirs(save_path) result_csv_path = os.path.join(save_path, '{}_test_result.csv'.format(save_name)) if args.repredict or not os.path.isfile(result_csv_path): print('test net...') pprint.pprint(args) pprint.pprint(config) device_ids = [int(d) for d in config.GPUS.split(',')] # os.environ['CUDA_VISIBLE_DEVICES'] = config.GPUS if ckpt_path is None: _, train_output_path = create_logger(config.OUTPUT_PATH, args.cfg, config.DATASET.IMAGE_SET, split='train') model_prefix = os.path.join(train_output_path, config.MODEL_PREFIX) ckpt_path = '{}-best.model'.format(model_prefix) print('Use best checkpoint {}...'.format(ckpt_path)) shutil.copy2( ckpt_path, os.path.join(save_path, '{}_test_ckpt.model'.format(config.MODEL_PREFIX))) # torch.backends.cudnn.enabled = False # torch.backends.cudnn.deterministic = True # torch.backends.cudnn.benchmark = False # get network model = eval(config.MODULE)(config) if len(device_ids) > 1: model = torch.nn.DataParallel(model, device_ids=device_ids).cuda() else: model = model.cuda() if args.fp16: [model] = amp.initialize([model], opt_level='O2', keep_batchnorm_fp32=False) checkpoint = torch.load(ckpt_path, map_location=lambda storage, loc: storage) smart_load_model_state_dict(model, checkpoint['state_dict']) # loader test_loader = make_dataloader(config, mode='test', distributed=False) test_dataset = test_loader.dataset test_database = test_dataset.database # test sentence_logits = [] test_ids = [] sentence_labels = [] cur_id = 0 model.eval() for batch in test_loader: batch = to_cuda(batch) output = model(*batch) sentence_logits.append( output['sentence_label_logits'].float().detach().cpu().numpy()) batch_size = batch[0].shape[0] sentence_labels.append([ test_database[cur_id + k]['label'] for k in range(batch_size) ]) test_ids.append([ test_database[cur_id + k]['pair_id'] for k in range(batch_size) ]) cur_id += batch_size sentence_logits = np.concatenate(sentence_logits, axis=0) test_ids = np.concatenate(test_ids, axis=0) sentence_labels = np.concatenate(sentence_labels, axis=0) if config.DATASET.ALIGN_CAPTION_IMG: sentence_prediction = np.argmax(sentence_logits, axis=1).reshape(-1) else: sentence_prediction = (sentence_logits > 0.).astype(int).reshape(-1) # generate final result csv dataframe = pd.DataFrame(data=sentence_prediction, columns=["sentence_pred_label"]) dataframe['pair_id'] = test_ids dataframe['sentence_labels'] = sentence_labels # Save predictions dataframe = dataframe.set_index('pair_id', drop=True) dataframe.to_csv(result_csv_path) print('result csv saved to {}.'.format(result_csv_path)) else: print( "Cache found in {}, skip test prediction!".format(result_csv_path)) dataframe = pd.read_csv(result_csv_path) sentence_prediction = np.array(dataframe["sentence_pred_label"].values) sentence_labels = np.array(dataframe["sentence_labels"].values) # Evaluate predictions for metric in ["overall_accuracy", "easy_accuracy", "alignment_accuracy"]: accuracy = compute_metrics_sentence_level(metric, sentence_prediction, sentence_labels) print("{} on test set is: {}".format(metric, str(accuracy)))
def test_net(args, config, ckpt_path=None, save_path=None, save_name=None): if save_path is None: logger, test_output_path = create_logger(config.OUTPUT_PATH, args.cfg, config.DATASET.TEST_IMAGE_SET, split='test') save_path = test_output_path if save_name is None: save_name = config.MODEL_PREFIX if not os.path.exists(save_path): os.makedirs(save_path) result_csv_path = os.path.join(save_path, '{}_test_result_{}.csv'.format(save_name, config.DATASET.TASK)) if args.use_cache and os.path.isfile(result_csv_path): print("Cache found in {}, skip test!".format(result_csv_path)) return result_csv_path print('test net...') pprint.pprint(args) pprint.pprint(config) device_ids = [int(d) for d in config.GPUS.split(',')] # os.environ['CUDA_VISIBLE_DEVICES'] = config.GPUS if ckpt_path is None: _, train_output_path = create_logger(config.OUTPUT_PATH, args.cfg, config.DATASET.TRAIN_IMAGE_SET, split='train') model_prefix = os.path.join(train_output_path, config.MODEL_PREFIX) ckpt_path = '{}-best.model'.format(model_prefix) print('Use best checkpoint {}...'.format(ckpt_path)) shutil.copy2(ckpt_path, os.path.join(save_path, '{}_test_ckpt_{}.model'.format(config.MODEL_PREFIX, config.DATASET.TASK))) # torch.backends.cudnn.enabled = False # torch.backends.cudnn.deterministic = True # torch.backends.cudnn.benchmark = False # get network model = eval(config.MODULE)(config) if len(device_ids) > 1: model = torch.nn.DataParallel(model, device_ids=device_ids).cuda() else: model = model.cuda() if args.fp16: [model] = amp.initialize([model], opt_level='O2', keep_batchnorm_fp32=False) checkpoint = torch.load(ckpt_path, map_location=lambda storage, loc: storage) smart_load_model_state_dict(model, checkpoint['state_dict']) # loader test_loader = make_dataloader(config, mode='test', distributed=False) test_dataset = test_loader.dataset test_database = test_dataset.database # test test_probs = [] test_ids = [] cur_id = 0 model.eval() for nbatch, batch in zip(trange(len(test_loader)), test_loader): # for nbatch, batch in tqdm(enumerate(test_loader)): batch = to_cuda(batch) if config.DATASET.TASK == 'Q2A': output = model(*batch) probs = F.softmax(output['label_logits'].float(), dim=1) batch_size = probs.shape[0] test_probs.append(probs.float().detach().cpu().numpy()) test_ids.append([test_database[cur_id + k]['annot_id'] for k in range(batch_size)]) cur_id += batch_size elif config.DATASET.TASK == 'QA2R': conditioned_probs = [] for a_id in range(4): q_index_in_batch = test_loader.dataset.data_names.index('question') q_align_mat_index_in_batch = test_loader.dataset.data_names.index('question_align_matrix') batch_ = [*batch] batch_[q_index_in_batch] = batch[q_index_in_batch][:, a_id, :, :] batch_[q_align_mat_index_in_batch] = batch[q_align_mat_index_in_batch][:, a_id, :, :] output = model(*batch_) probs = F.softmax(output['label_logits'].float(), dim=1) conditioned_probs.append(probs.float().detach().cpu().numpy()) conditioned_probs = np.concatenate(conditioned_probs, axis=1) test_probs.append(conditioned_probs) test_ids.append([test_database[cur_id + k]['annot_id'] for k in range(conditioned_probs.shape[0])]) cur_id += conditioned_probs.shape[0] else: raise ValueError('Not Support Task {}'.format(config.DATASET.TASK)) test_probs = np.concatenate(test_probs, axis=0) test_ids = np.concatenate(test_ids, axis=0) result_npy_path = os.path.join(save_path, '{}_test_result_{}.npy'.format(save_name, config.DATASET.TASK)) np.save(result_npy_path, test_probs) print('result npy saved to {}.'.format(result_npy_path)) # generate final result csv if config.DATASET.TASK == 'Q2A': columns = ['answer_{}'.format(i) for i in range(4)] else: columns = ['rationale_conditioned_on_a{}_{}'.format(i, j) for i in range(4) for j in range(4)] dataframe = pd.DataFrame(data=test_probs, columns=columns) dataframe['annot_id'] = test_ids dataframe = dataframe.set_index('annot_id', drop=True) dataframe.to_csv(result_csv_path) print('result csv saved to {}.'.format(result_csv_path)) return result_csv_path
def test_net2018(args, config, ckpt_path=None, save_path=None, save_name=None): print('test net...') pprint.pprint(args) pprint.pprint(config) device_ids = [int(d) for d in config.GPUS.split(',')] # os.environ['CUDA_VISIBLE_DEVICES'] = config.GPUS torch.backends.cudnn.enabled = False torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False if ckpt_path is None: _, train_output_path = create_logger(config.OUTPUT_PATH, args.cfg, config.DATASET.TRAIN_IMAGE_SET, split='train') model_prefix = os.path.join(train_output_path, config.MODEL_PREFIX) ckpt_path = '{}-best.model'.format(model_prefix) print('Use best checkpoint {}...'.format(ckpt_path)) if save_path is None: logger, test_output_path = create_logger(config.OUTPUT_PATH, args.cfg, config.DATASET.TEST_IMAGE_SET, split='test') save_path = test_output_path if not os.path.exists(save_path): os.makedirs(save_path) shutil.copy2( ckpt_path, os.path.join( save_path, '{}_test_ckpt_{}.model'.format(config.MODEL_PREFIX, config.DATASET.TASK))) # ************ # Step 1: Select model architecture and preload trained model model = eval(config.MODULE)(config) if len(device_ids) > 1: model = torch.nn.DataParallel(model, device_ids=device_ids).cuda() else: torch.cuda.set_device(device_ids[0]) model = model.cuda() checkpoint = torch.load(ckpt_path, map_location=lambda storage, loc: storage) smart_load_model_state_dict(model, checkpoint['state_dict']) # ************ # Step 2: Create dataloader to include all caption-image pairs test_loader = make_dataloader(config, mode='test', distributed=False) test_dataset = test_loader.dataset test_database = test_dataset.database vocab = test_dataset.MLT_vocab # ************ # Step 3: Run all pairs through model for inference word_de_ids = [] words_de = [] words_en = [] captions_en = [] captions_de = [] logit_words = [] logits = [] model.eval() cur_id = 0 for nbatch, batch in zip(trange(len(test_loader)), test_loader): bs = test_loader.batch_sampler.batch_size if test_loader.batch_sampler is not None else test_loader.batch_size # for id in range(cur_id, min(cur_id + bs, len(test_database))): # print(test_database[id]) words_de.extend([ test_database[id]['word_de'] for id in range(cur_id, min(cur_id + bs, len(test_database))) ]) words_en.extend([ test_database[id]['word_en'] for id in range(cur_id, min(cur_id + bs, len(test_database))) ]) captions_en.extend([ test_database[id]['caption_en'] for id in range(cur_id, min(cur_id + bs, len(test_database))) ]) captions_de.extend([ test_database[id]['caption_de'] for id in range(cur_id, min(cur_id + bs, len(test_database))) ]) batch = to_cuda(batch) output = model(*batch) # FM note: output is tuple (outputs, loss) probs = F.softmax(output[0]['MLT_logits'].float(), dim=1) batch_size = probs.shape[0] logits.extend(probs.argmax(dim=1).detach().cpu().tolist()) # word_de_ids.extend(output[0]['MLT_label'].detach().cpu().tolist()) logit_words.extend([ vocab[id] for id in logits[cur_id:min(cur_id + bs, len(test_database))] ]) cur_id += bs # output = model(*batch) # probs = F.softmax(output['label_logits'].float(), dim=1) # batch_size = probs.shape[0] # test_probs.append(probs.float().detach().cpu().numpy()) # test_ids.append([test_database[cur_id + k]['annot_id'] for k in range(batch_size)]) # logits.extend(F.sigmoid(output[0]['relationship_logits']).detach().cpu().tolist()) # ************ # Step 3: Store all logit results in file for later evalution result = [{ 'logit': l_id, 'word_en': word_en, 'word_de': word_de, 'word_pred': logit_word, 'caption_en': caption_en, 'caption_de': caption_de } for l_id, word_en, word_de, logit_word, caption_en, caption_de in zip( logits, words_en, words_de, logit_words, captions_en, captions_de)] cfg_name = os.path.splitext(os.path.basename(args.cfg))[0] result_json_path = os.path.join( save_path, '{}_MLT_{}.json'.format(cfg_name if save_name is None else save_name, config.DATASET.TEST_IMAGE_SET)) with open(result_json_path, 'w') as f: json.dump(result, f) print('result json saved to {}.'.format(result_json_path)) return result_json_path
def test_net(args, config, ckpt_path=None, save_path=None, save_name=None): print('test net...') pprint.pprint(args) pprint.pprint(config) device_ids = [int(d) for d in config.GPUS.split(',')] obj_cats = config.OBJECT_CATEGORIES pred_cats = config.PREDICATE_CATEGORIES torch.backends.cudnn.enabled = False torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False if ckpt_path is None: _, train_output_path = create_logger(config.OUTPUT_PATH, args.cfg, config.DATASET.TRAIN_IMAGE_SET, split='train') model_prefix = os.path.join(train_output_path, config.MODEL_PREFIX) ckpt_path = '{}-best.model'.format(model_prefix) print('Use best checkpoint {}...'.format(ckpt_path)) if save_path is None: logger, test_output_path = create_logger(config.OUTPUT_PATH, args.cfg, config.DATASET.TEST_IMAGE_SET, split='test') save_path = test_output_path if not os.path.exists(save_path): os.makedirs(save_path) shutil.copy2(ckpt_path, os.path.join(save_path, '{}_test_ckpt_{}.model'.format( config.MODEL_PREFIX, config.DATASET.TASK ))) # get network model = eval(config.MODULE)(config) if len(device_ids) > 1: model = torch.nn.DataParallel(model, device_ids=device_ids).cuda() else: torch.cuda.set_device(device_ids[0]) model = model.cuda() checkpoint = torch.load(ckpt_path, map_location=lambda storage, loc: storage) smart_load_model_state_dict(model, checkpoint['state_dict']) split = args.split loader = make_dataloader(config, mode=split, distributed=False) nb_of_correct_50 = nb_of_sample = nb_of_correct_top100 = 0 model.eval() save_dir = '' if args.visualize_mask: # For mask visualization purpose save_dir = 'heatmap/vrd' if not os.path.isdir(save_dir): os.makedirs(save_dir) for nbatch, batch in zip(trange(len(loader)), loader): batch = to_cuda(batch) output = model(*batch) n_correct, n_sample, n_correct_top100 = compute_recall(output, obj_cats, pred_cats, remove_bg=config.TRAIN.SAMPLE_RELS != -1, visualize_mask=args.visualize_mask, save_dir=save_dir) nb_of_correct_50 += n_correct nb_of_correct_top100 += n_correct_top100 nb_of_sample += n_sample recall_50 = nb_of_correct_50 / nb_of_sample recall_100 = nb_of_correct_top100 / nb_of_sample return recall_50, recall_100
def val_net(args, config, ckpt_path=None, save_path=None, save_name=None): print('test net...') pprint.pprint(args) pprint.pprint(config) device_ids = [int(d) for d in config.GPUS.split(',')] # os.environ['CUDA_VISIBLE_DEVICES'] = config.GPUS torch.backends.cudnn.enabled = False torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False if ckpt_path is None: _, train_output_path = create_logger(config.OUTPUT_PATH, args.cfg, config.DATASET.TRAIN_IMAGE_SET, split='train') model_prefix = os.path.join(train_output_path, config.MODEL_PREFIX) ckpt_path = '{}-best.model'.format(model_prefix) print('Use best checkpoint {}...'.format(ckpt_path)) # if save_path is None: # logger, test_output_path = create_logger(config.OUTPUT_PATH, args.cfg, config.DATASET.TEST_IMAGE_SET, # split='test') # save_path = test_output_path # if not os.path.exists(save_path): # os.makedirs(save_path) # shutil.copy2(ckpt_path, # os.path.join(save_path, '{}_test_ckpt_{}.model'.format(config.MODEL_PREFIX, config.DATASET.TASK))) # get network model = eval(config.MODULE)(config) if hasattr(model, 'setup_adapter'): model.setup_adapter() if len(device_ids) > 1: model = torch.nn.DataParallel(model, device_ids=device_ids).cuda() else: torch.cuda.set_device(device_ids[0]) model = model.cuda() checkpoint = torch.load(ckpt_path, map_location=lambda storage, loc: storage) smart_load_model_state_dict(model, checkpoint['state_dict']) # loader test_loader = make_dataloader(config, mode='val', distributed=False) test_dataset = test_loader.dataset test_database = test_dataset.database # test predicts = [] model.eval() cur_id = 0 for nbatch, batch in zip(trange(len(test_loader)), test_loader): # for nbatch, batch in tqdm(enumerate(test_loader)): bs = test_loader.batch_sampler.batch_size if test_loader.batch_sampler is not None else test_loader.batch_size batch = to_cuda(batch) outputs = model(*batch[:-1]) if outputs['label_logits'].shape[-1] == 1: prob = torch.sigmoid(outputs['label_logits'][:, 0]).detach().cpu().tolist() else: prob = torch.softmax(outputs['label_logits'], dim=-1)[:, 1].detach().cpu().tolist() sample_ids = batch[-1].cpu().tolist() targets = batch[config.DATASET.LABEL_INDEX_IN_BATCH] for pb, id, tg in zip(prob, sample_ids, targets): predicts.append({ 'id': int(id), 'proba': float(pb), 'label': int(pb > 0.5), 'target': float(tg) }) pred_probs = [p['proba'] for p in predicts] pred_labels = [p['label'] for p in predicts] targets = [p['target'] for p in predicts] roc_auc = roc_auc_score(targets, pred_probs) print(f"roc_auc: {roc_auc}") max_accuracy = 0.0 best_threshold = 1e-2 for th in range(1, 100): targets_idx = [int(p['target'] > 1e-2 * th) for p in predicts] accuracy = accuracy_score(targets_idx, pred_labels) if accuracy > max_accuracy: max_accuracy = accuracy best_threshold = th * 1e-2 print(f"max accuracy: {max_accuracy}, best_threshold: {best_threshold}")
def test_net(args, config, ckpt_path=None, save_path=None, save_name=None): print('test net...') pprint.pprint(args) pprint.pprint(config) device_ids = [int(d) for d in config.GPUS.split(',')] # os.environ['CUDA_VISIBLE_DEVICES'] = config.GPUS torch.backends.cudnn.enabled = False torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False if ckpt_path is None: _, train_output_path = create_logger(config.OUTPUT_PATH, args.cfg, config.DATASET.TRAIN_IMAGE_SET, split='train') model_prefix = os.path.join(train_output_path, config.MODEL_PREFIX) ckpt_path = '{}-best.model'.format(model_prefix) print('Use best checkpoint {}...'.format(ckpt_path)) if save_path is None: logger, test_output_path = create_logger(config.OUTPUT_PATH, args.cfg, config.DATASET.TEST_IMAGE_SET, split='test') save_path = test_output_path if not os.path.exists(save_path): os.makedirs(save_path) test_ckpt_path = '{}_test_ckpt_{}.model'.format(config.MODEL_PREFIX, config.DATASET.TASK) try: shutil.copy2(ckpt_path, os.path.join(save_path, test_ckpt_path)) except shutil.SameFileError: print(f'Test checkpoints is alredy exist: {test_ckpt_path}') # get network model = eval(config.MODULE)(config) if hasattr(model, 'setup_adapter'): model.setup_adapter() # if len(device_ids) > 1: # model = torch.nn.DataParallel(model, device_ids=device_ids).cuda() # else: torch.cuda.set_device(min(device_ids)) model = model.cuda() checkpoint = torch.load(ckpt_path, map_location=lambda storage, loc: storage) smart_load_model_state_dict(model, checkpoint['state_dict']) # loader test_loader = make_dataloader(config, mode='test', distributed=False) test_dataset = test_loader.dataset test_database = test_dataset.database # test predicts = [] model.eval() cur_id = 0 for nbatch, batch in zip(trange(len(test_loader)), test_loader): # for nbatch, batch in tqdm(enumerate(test_loader)): bs = test_loader.batch_sampler.batch_size if test_loader.batch_sampler is not None else test_loader.batch_size batch = to_cuda(batch) outputs = model(*batch[:-1]) if outputs['label_logits'].shape[-1] == 1: prob = torch.sigmoid(outputs['label_logits'][:, 0]).detach().cpu().tolist() else: prob = torch.softmax(outputs['label_logits'], dim=-1)[:, 1].detach().cpu().tolist() sample_ids = batch[-1].cpu().tolist() for pb, id in zip(prob, sample_ids): predicts.append({ 'id': int(id), 'proba': float(pb), 'label': int(pb > 0.5) }) cfg_name = os.path.splitext(os.path.basename(args.cfg))[0] output_name = cfg_name if save_name is None else save_name result_json_path = os.path.join(save_path, f'{output_name}_cls_{config.DATASET.TEST_IMAGE_SET}.json') result_csv_path = os.path.join(save_path, f'{output_name}_cls_{config.DATASET.TEST_IMAGE_SET}.csv') with open(result_json_path, 'w') as f: json.dump(predicts, f) print('result json saved to {}.'.format(result_json_path)) pd.DataFrame.from_dict(predicts).to_csv(result_csv_path, index=False) return result_json_path
def test_net(args, config, ckpt_path=None, save_path=None, save_name=None): print('test net...') pprint.pprint(args) pprint.pprint(config) device_ids = [int(d) for d in config.GPUS.split(',')] # os.environ['CUDA_VISIBLE_DEVICES'] = config.GPUS torch.backends.cudnn.enabled = False torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False if ckpt_path is None: _, train_output_path = create_logger(config.OUTPUT_PATH, args.cfg, config.DATASET.TRAIN_IMAGE_SET, split='train') model_prefix = os.path.join(train_output_path, config.MODEL_PREFIX) ckpt_path = '{}-best.model'.format(model_prefix) print('Use best checkpoint {}...'.format(ckpt_path)) if save_path is None: logger, test_output_path = create_logger(config.OUTPUT_PATH, args.cfg, config.DATASET.TEST_IMAGE_SET, split='test') save_path = test_output_path if not os.path.exists(save_path): os.makedirs(save_path) shutil.copy2( ckpt_path, os.path.join( save_path, '{}_test_ckpt_{}.model'.format(config.MODEL_PREFIX, config.DATASET.TASK))) # get network model = eval(config.MODULE)(config) if len(device_ids) > 1: model = torch.nn.DataParallel(model, device_ids=device_ids).cuda() else: torch.cuda.set_device(device_ids[0]) model = model.cuda() checkpoint = torch.load(ckpt_path, map_location=lambda storage, loc: storage) smart_load_model_state_dict(model, checkpoint['state_dict']) # loader test_loader = make_dataloader(config, mode='test', distributed=False) test_dataset = test_loader.dataset test_database = test_dataset.database # test q_ids = [] answer_ids = [] model.eval() cur_id = 0 for nbatch, batch in zip(trange(len(test_loader)), test_loader): # for nbatch, batch in tqdm(enumerate(test_loader)): bs = test_loader.batch_sampler.batch_size if test_loader.batch_sampler is not None else test_loader.batch_size q_ids.extend([ str(test_database[id]['annot_id']) for id in range(cur_id, min(cur_id + bs, len(test_database))) ]) batch = to_cuda(batch) output = model(*batch) answer_ids.extend(output['label_logits'].cpu().numpy().tolist()) cur_id += bs result = [q_ids, answer_ids] cfg_name = os.path.splitext(os.path.basename(args.cfg))[0] result_json_path = os.path.join( save_path, '{}_vqa2_{}.json'.format(cfg_name if save_name is None else save_name, config.DATASET.TEST_IMAGE_SET)) with open(result_json_path, 'w') as f: json.dump(result, f) print('result json saved to {}.'.format(result_json_path)) return result_json_path
def test_translation_net(args, config, ckpt_path=None, save_path=None, save_name=None): print('test net...') pprint.pprint(args) pprint.pprint(config) device_ids = [int(d) for d in config.GPUS.split(',')] # os.environ['CUDA_VISIBLE_DEVICES'] = config.GPUS torch.backends.cudnn.enabled = False torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False if ckpt_path is None: _, train_output_path = create_logger(config.OUTPUT_PATH, args.cfg, config.DATASET.TRAIN_IMAGE_SET, split='train') model_prefix = os.path.join(train_output_path, config.MODEL_PREFIX) ckpt_path = '{}-best.model'.format(model_prefix) print('Use best checkpoint {}...'.format(ckpt_path)) if save_path is None: logger, test_output_path = create_logger(config.OUTPUT_PATH, args.cfg, config.DATASET.TEST_IMAGE_SET, split='test') save_path = test_output_path if not os.path.exists(save_path): os.makedirs(save_path) shutil.copy2(ckpt_path, os.path.join(save_path, '{}_test_ckpt_{}.model'.format(config.MODEL_PREFIX, config.DATASET.TASK))) # ************ # Step 1: Select model architecture and preload trained model model = eval(config.MODULE)(config) if len(device_ids) > 1: model = torch.nn.DataParallel(model, device_ids=device_ids).cuda() else: torch.cuda.set_device(device_ids[0]) model = model.cuda() checkpoint = torch.load(ckpt_path, map_location=lambda storage, loc: storage) smart_load_model_state_dict(model, checkpoint['state_dict']) # ************ # Step 2: Create dataloader to include all caption-image pairs test_loader = make_dataloader(config, mode='test', distributed=False) test_dataset = test_loader.dataset test_database = test_dataset.database # ************ # Step 3: Run all pairs through model for inference caption_ids = [] image_ids = [] logits = [] model.eval() cur_id = 0 for nbatch, batch in zip(trange(len(test_loader)), test_loader): bs = test_loader.batch_sampler.batch_size if test_loader.batch_sampler is not None else test_loader.batch_size caption_ids.extend([test_database[id]['caption_en_index'] for id in range(cur_id, min(cur_id + bs, len(test_database)))]) image_ids.extend([test_database[id]['caption_de_index'] for id in range(cur_id, min(cur_id + bs, len(test_database)))]) batch = to_cuda(batch) output = model(*batch) logits.extend(F.sigmoid(output[0]['relationship_logits']).detach().cpu().tolist()) cur_id += bs #TODO: remove this is just for checking # if nbatch>900: # break # ************ # Step 3: Store all logit results in file for later evalution result = [{'caption_en_index': c_id, 'caption_de_index': i_id, 'logit': l_id} for c_id, i_id, l_id in zip(caption_ids, image_ids, logits)] cfg_name = os.path.splitext(os.path.basename(args.cfg))[0] result_json_path = os.path.join(save_path, '{}_retrieval_translation_{}.json'.format(cfg_name if save_name is None else save_name, config.DATASET.TEST_IMAGE_SET)) with open(result_json_path, 'w') as f: json.dump(result, f) print('result json saved to {}.'.format(result_json_path)) return result_json_path
def test_net(args, config, ckpt_path=None, save_path=None, save_name=None): print('test net...') pprint.pprint(args) pprint.pprint(config) device_ids = [int(d) for d in config.GPUS.split(',')] # os.environ['CUDA_VISIBLE_DEVICES'] = config.GPUS torch.backends.cudnn.enabled = False torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False if ckpt_path is None: _, train_output_path = create_logger(config.OUTPUT_PATH, args.cfg, config.DATASET.TRAIN_IMAGE_SET, split='train') model_prefix = os.path.join(train_output_path, config.MODEL_PREFIX) ckpt_path = '{}-best.model'.format(model_prefix) print('Use best checkpoint {}...'.format(ckpt_path)) if save_path is None: logger, test_output_path = create_logger(config.OUTPUT_PATH, args.cfg, config.DATASET.TEST_IMAGE_SET, split='test') save_path = test_output_path if not os.path.exists(save_path): os.makedirs(save_path) shutil.copy2(ckpt_path, os.path.join(save_path, '{}_test_ckpt_{}.model'.format(config.MODEL_PREFIX, config.DATASET.TASK))) # get network model = eval(config.MODULE)(config) if len(device_ids) > 1: model = torch.nn.DataParallel(model, device_ids=device_ids).cuda() else: torch.cuda.set_device(device_ids[0]) model = model.cuda() checkpoint = torch.load(ckpt_path, map_location=lambda storage, loc: storage) smart_load_model_state_dict(model, checkpoint['state_dict']) # loader test_loader = make_dataloader(config, mode='test', distributed=False) split = args.split + 'id' if args.split == 'val' else args.split # 'val' -> 'valid' # test if config.TEST.EXCL_LEFT_RIGHT: precompute_test_cache = f'{args.log_dir}/pred_{split}_{ckpt_path[-10:-6]}_excl-left-right.pickle' else: precompute_test_cache = f'{args.log_dir}/pred_{split}_{ckpt_path[-10:-6]}.pickle' if not os.path.isdir(args.log_dir): os.makedirs(args.log_dir) pred_file = precompute_test_cache if not os.path.exists(precompute_test_cache): _ids = [] losses = [] predictions = [] model.eval() if args.visualize_mask: # For mask visualization purpose save_dir = 'heatmap/spasen' if not os.path.isdir(save_dir): # os.mkdir(save_dir) os.makedirs(save_dir) for nbatch, batch in zip(trange(len(test_loader)), test_loader): _ids.extend(batch[0]) # the first input element is _id batch = to_cuda(batch) output = model(*batch) predictions.append(output['prediction']) losses.append(output['ans_loss'].item()) if args.visualize_mask: # For mask visualization purpose mask = output['spo_fused_masks'].cpu() # torch.Size([8, 3, 14, 14]) subj_name = output['subj_name'] # list of 8 strs obj_name = output['obj_name'] # list of 8 strs pred_name = output['pred_name'] # list of 8 strs im_path = output['im_path'] # list of 8 img urls for i in range(mask.shape[0]): img, dataset = read_img(im_path[i], config.IMAGEPATH) img_name = dataset + '-' + im_path[i].split('/')[-1] show_cam_on_image(img, mask[i], img_name, subj_name[i], obj_name[i], pred_name[i], save_dir) predictions = [v.item() for v in torch.cat(predictions)] loss = sum(losses) / len(losses) pickle.dump((_ids, predictions, loss), open(pred_file, 'wb')) accs, loss = accuracies(pred_file, 'data/spasen/annotations.json', split) return accs, loss
def test_net(args, config): print('test net...') pprint.pprint(args) pprint.pprint(config) device_ids = [int(d) for d in config.GPUS.split(',')] #os.environ['CUDA_VISIBLE_DEVICES'] = config.GPUS config.DATASET.TEST_IMAGE_SET = args.split ckpt_path = args.ckpt save_path = args.result_path if not os.path.exists(save_path): os.makedirs(save_path) shutil.copy2(ckpt_path, os.path.join(save_path, '{}_test_ckpt_{}.model'.format(config.MODEL_PREFIX, config.DATASET.TASK))) torch.backends.cudnn.enabled = False torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False # get network model = eval(config.MODULE)(config) if len(device_ids) > 1: model = torch.nn.DataParallel(model, device_ids=device_ids).cuda() else: torch.cuda.set_device(device_ids[0]) model = model.cuda() checkpoint = torch.load(ckpt_path, map_location=lambda storage, loc: storage) print(ckpt_path) smart_load_model_state_dict(model, checkpoint['state_dict']) # loader test_loader = make_dataloader(config, mode='test', distributed=False) test_dataset = test_loader.dataset test_database = test_dataset.database # test ref_ids = [] pred_boxes = [] model.eval() cur_id = 0 test_loader.dataset.get_image_features(model.image_feature_extractor) score_matrix = torch.zeros((len(test_loader), test_loader.dataset.image_nums)).cuda()#test_loader.dataset.image_nums)) # lihui target_matrix = np.zeros((len(test_loader), test_loader.dataset.image_nums))#test_loader.dataset.image_nums)) # lihui jump_val = 666 for nbatch, batch in zip(trange(len(test_loader)), test_loader): batch[0] = batch[0].cuda() # for nbatch, batch in tqdm(enumerate(test_loader)): # bs = test_loader.batch_sampler.batch_size if test_loader.batch_sampler is not None else test_loader.batch_size for i in range(int(test_loader.dataset.image_nums / jump_val)+1): start_i = i*jump_val stop_i = (i+1)*jump_val if (i+1)*jump_val < test_loader.dataset.image_nums else test_loader.dataset.image_nums-1 batch_b = [] for i in range(stop_i - start_i): batch_b.append(batch[0].squeeze(0)) batch_b = torch.stack(batch_b, dim=0) #image_b = test_loader.dataset.val_images[start_i: stop_i] boxes_b = test_loader.dataset.val_boxes[start_i: stop_i] im_info_b = test_loader.dataset.val_im_info[start_i: stop_i] ids_b = test_loader.dataset.val_ids[start_i: stop_i] feats = test_loader.dataset.val_feat[start_i: stop_i] #image_b = torch.stack(image_b, dim=0) boxes_b = torch.stack(boxes_b, dim=0) im_info_b = torch.stack(im_info_b, dim=0) feats = torch.stack(feats, dim=0).view(-1,feats[0].size(1), feats[0].size(2)) batch_test = [None, boxes_b, im_info_b, batch_b, None, feats] score_none, sim = model(*batch_test) score_matrix[nbatch, start_i: stop_i] = sim[:, 0] target_matrix[nbatch, start_i: stop_i][np.array(ids_b) == batch[1].numpy()] = 1 #target_matrix[nbatch, start_i: stop_i][ids_b == batch[1]] = 1 if nbatch % 10 == 0: r1, r5, r10, r50, mAP = compute_topk(score_matrix[:(nbatch + 1)].cpu().numpy(), target_matrix[:(nbatch + 1)]) print(" r1, r5, r10, r50, mAP: ", r1, r5, r10, r50, mAP) #r1, r5, r10 = compute_topk(score_matrix[:(nbatch + 1)].cpu().numpy(), target_matrix[:(nbatch + 1)]) r1, r5, r10, r50, mAP = compute_topk(score_matrix.cpu().numpy(), target_matrix) print(" r1, r5, r10, r50, mAP: ", r1, r5, r10, r50, mAP) r1, r5, r10, r50, mAP = compute_topk(score_matrix.cpu().numpy().T, target_matrix.T) print(" r1, r5, r10, r50, mAP: ", r1, r5, r10, r50, mAP) return None
def test_net(args, config): print('test net...') pprint.pprint(args) pprint.pprint(config) task = config.FOIL_TASK device_ids = [int(d) for d in config.GPUS.split(',')] #os.environ['CUDA_VISIBLE_DEVICES'] = config.GPUS config.DATASET.TEST_IMAGE_SET = args.split ckpt_path = args.ckpt save_path = args.result_path if not os.path.exists(save_path): os.makedirs(save_path) shutil.copy2( ckpt_path, os.path.join( save_path, '{}_test_ckpt_{}.model'.format(config.MODEL_PREFIX, config.DATASET.TASK))) torch.backends.cudnn.enabled = False torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False # get network model = eval(config.MODULE)(config) if len(device_ids) > 1: model = torch.nn.DataParallel(model, device_ids=device_ids).cuda() else: torch.cuda.set_device(device_ids[0]) model = model.cuda() checkpoint = torch.load(ckpt_path, map_location=lambda storage, loc: storage) smart_load_model_state_dict(model, checkpoint['state_dict']) # loader test_loader = make_dataloader(config, mode='test', distributed=False) test_dataset = test_loader.dataset test_database = test_dataset.database # test ref_ids = [] pred_boxes = [] model.eval() cur_id = 0 for nbatch, batch in zip(trange(len(test_loader)), test_loader): # for nbatch, batch in tqdm(enumerate(test_loader)): bs = test_loader.batch_sampler.batch_size if test_loader.batch_sampler is not None else test_loader.batch_size ref_ids.extend([ test_database[id]['ref_id'] for id in range(cur_id, min(cur_id + bs, len(test_database))) ]) batch = to_cuda(batch) output = model(*batch) pred_boxes.extend(output['pred_boxes'].detach().cpu().tolist()) cur_id += bs result = [{ 'ref_id': ref_id, 'box': box } for ref_id, box in zip(ref_ids, pred_boxes)] result_json_path = os.path.join( save_path, '{}_refcoco+_{}.json'.format( config.MODEL_PREFIX if args.result_name is None else args.result_name, config.DATASET.TEST_IMAGE_SET)) with open(result_json_path, 'w') as f: json.dump(result, f) print('result json saved to {}.'.format(result_json_path)) # evaluate (test label of refcoco+ has been released) print("Evaluate on split: {}...".format(config.DATASET.TEST_IMAGE_SET)) pred_boxes_arr = np.array(pred_boxes) gt_boxes_arr = np.array( [test_dataset.refer.getRefBox(ref_id=ref_id) for ref_id in ref_ids]) gt_boxes_arr[:, [2, 3]] += gt_boxes_arr[:, [0, 1]] iou = cacluate_iou(pred_boxes_arr, gt_boxes_arr) acc = float((iou >= POSITIVE_THRESHOLD).sum() * 1.0 / iou.shape[0]) print("Accuracy: {}.".format(acc * 100.0)) return result_json_path