def test(**kwargs): opt.parse(kwargs) train_L, query_L, retrieval_L, train_x, query_x, retrieval_x, train_y, query_y, retrieval_y = load_data( opt.data_path) y_dim = query_y.shape[1] print('...loading and splitting data finish') img_model = ImgModule(opt.bit) txt_model = TxtModule(y_dim, opt.bit) if opt.load_img_path: img_model.load(opt.load_img_path) if opt.load_txt_path: txt_model.load(opt.load_txt_path) if opt.use_gpu: img_model = img_model.cuda() txt_model = txt_model.cuda() qBX = generate_image_code(img_model, query_x, opt.bit) qBY = generate_text_code(txt_model, query_y, opt.bit) rBX = generate_image_code(img_model, retrieval_x, opt.bit) rBY = generate_text_code(txt_model, retrieval_y, opt.bit) if opt.use_gpu: query_L = query_L.cuda() retrieval_L = retrieval_L.cuda() mapi2t = calc_map_k(qBX, rBY, query_L, retrieval_L) mapt2i = calc_map_k(qBY, rBX, query_L, retrieval_L) print('...test MAP: MAP(i->t): %3.3f, MAP(t->i): %3.3f' % (mapi2t, mapt2i))
def calc_maps_k(self, qBX, qBY, rBX, rBY, qLX, qLY, rLX, rLY, k): """ Calculate MAPs, in regards to K :param: qBX: query hashes, modality X :param: qBY: query hashes, modality Y :param: rBX: response hashes, modality X :param: rBY: response hashes, modality Y :param: qLX: query labels, modality X :param: qLY: query labels, modality Y :param: rLX: response labels, modality X :param: rLY: response labels, modality Y :param: k: k :returns: MAPs """ mapi2t = calc_map_k(qBX, rBY, qLX, rLY, k) mapt2i = calc_map_k(qBY, rBX, qLY, rLX, k) mapi2i = calc_map_k(qBX, rBX, qLX, rLX, k) mapt2t = calc_map_k(qBY, rBY, qLY, rLY, k) avg = (mapi2t.item() + mapt2i.item() + mapi2i.item() + mapt2t.item()) * 0.25 mapi2t, mapt2i, mapi2i, mapt2t, mapavg = mapi2t.item(), mapt2i.item( ), mapi2i.item(), mapt2t.item(), avg s = 'Valid: mAP@{}, avg: {:3.3f}, i->t: {:3.3f}, t->i: {:3.3f}, i->i: {:3.3f}, t->t: {:3.3f}' self.logger.info(s.format(k, mapavg, mapi2t, mapt2i, mapi2i, mapt2t)) return mapi2t, mapt2i, mapi2i, mapt2t, mapavg
def valid(img_model, txt_model, query_x, retrieval_x, query_y, retrieval_y, query_L, retrieval_L): qBX = generate_image_code(img_model, query_x, opt.bit) qBY = generate_text_code(txt_model, query_y, opt.bit) rBX = generate_image_code(img_model, retrieval_x, opt.bit) rBY = generate_text_code(txt_model, retrieval_y, opt.bit) mapi2t = calc_map_k(qBX, rBY, query_L, retrieval_L) mapt2i = calc_map_k(qBY, rBX, query_L, retrieval_L) return mapi2t, mapt2i
def valid(img_model, txt_model, img_memory_net, text_memory_net, img_centroids, text_centoids, query_x, retrieval_x, query_y, retrieval_y, query_L, retrieval_L, save=False): qBX = generate_image_code(img_model, img_memory_net, query_x, img_centroids, opt.bit) qBY = generate_text_code(txt_model, text_memory_net, query_y, text_centoids, opt.bit) rBX = generate_image_code(img_model, img_memory_net, retrieval_x, img_centroids, opt.bit) rBY = generate_text_code(txt_model, text_memory_net, retrieval_y, text_centoids, opt.bit) dir_name = 'hashcodes' + time.strftime("%Y_%m_%d_%H%M", time.localtime()) os.makedirs(dir_name) if save: np.save(os.path.join(dir_name, 'qBX.npy'), qBX.cpu().numpy()) np.save(os.path.join(dir_name, 'qBY.npy'), qBY.cpu().numpy()) np.save(os.path.join(dir_name, 'rBX.npy'), rBX.cpu().numpy()) np.save(os.path.join(dir_name, 'rBY.npy'), rBY.cpu().numpy()) print("Hash codes saved.\n") num_class = query_L.shape[1] num_test = query_L.shape[0] index = np.arange(num_test) mlist_i2t = [] mlist_t2i = [] for i in range(num_class): ind = index[query_L[:, i] == 1] class_mapi2t = calc_map_k(qBX[ind, :], rBY, query_L[ind, :], retrieval_L) class_mapt2i = calc_map_k(qBY[ind, :], rBX, query_L[ind, :], retrieval_L) mlist_i2t.append(class_mapi2t) mlist_t2i.append(class_mapt2i) f = open(os.path.join(dir_name, 'maplist.txt'), "w") f.write("i2t: " + str(mlist_i2t)) f.write("\n") f.write("t2i: " + str(mlist_t2i)) mapi2t = calc_map_k(qBX, rBY, query_L, retrieval_L) mapt2i = calc_map_k(qBY, rBX, query_L, retrieval_L) return mapi2t, mapt2i
def test(**kwargs): opt.parse(kwargs) if opt.device is not None: opt.device = torch.device(opt.device) elif opt.gpus: opt.device = torch.device(0) else: opt.device = torch.device('cpu') pretrain_model = load_pretrain_model(opt.pretrain_model_path) generator = GEN(opt.dropout, opt.image_dim, opt.text_dim, opt.hidden_dim, opt.bit, pretrain_model=pretrain_model).to(opt.device) path = 'checkpoints/' + opt.dataset + '_' + str(opt.bit) load_model(generator, path) generator.eval() images, tags, labels = load_data(opt.data_path, opt.dataset) i_query_data = Dataset(opt, images, tags, labels, test='image.query') i_db_data = Dataset(opt, images, tags, labels, test='image.db') t_query_data = Dataset(opt, images, tags, labels, test='text.query') t_db_data = Dataset(opt, images, tags, labels, test='text.db') i_query_dataloader = DataLoader(i_query_data, opt.batch_size, shuffle=False) i_db_dataloader = DataLoader(i_db_data, opt.batch_size, shuffle=False) t_query_dataloader = DataLoader(t_query_data, opt.batch_size, shuffle=False) t_db_dataloader = DataLoader(t_db_data, opt.batch_size, shuffle=False) qBX = generate_img_code(generator, i_query_dataloader, opt.query_size) qBY = generate_txt_code(generator, t_query_dataloader, opt.query_size) rBX = generate_img_code(generator, i_db_dataloader, opt.db_size) rBY = generate_txt_code(generator, t_db_dataloader, opt.db_size) query_labels, db_labels = i_query_data.get_labels() query_labels = query_labels.to(opt.device) db_labels = db_labels.to(opt.device) mapi2t = calc_map_k(qBX, rBY, query_labels, db_labels) mapt2i = calc_map_k(qBY, rBX, query_labels, db_labels) print('...test MAP: MAP(i->t): %3.4f, MAP(t->i): %3.4f' % (mapi2t, mapt2i))
def valid(model, x_query_dataloader, x_db_dataloader, y_query_dataloader, y_db_dataloader, query_labels, db_labels): model.eval() qBX = generate_img_code(model, x_query_dataloader, opt.query_size) qBY = generate_txt_code(model, y_query_dataloader, opt.query_size) rBX = generate_img_code(model, x_db_dataloader, opt.db_size) rBY = generate_txt_code(model, y_db_dataloader, opt.db_size) mapi2t = calc_map_k(qBX, rBY, query_labels, db_labels) mapt2i = calc_map_k(qBY, rBX, query_labels, db_labels) model.train() return mapi2t.item(), mapt2i.item()
def test(**kwargs): opt.parse(kwargs) images, tags, labels = load_data(opt.data_path) y_dim = tags.shape[1] X, Y, L = split_data(images, tags, labels) print('...loading and splitting data finish') img_model = ImgModule(opt.bit) txt_model = TxtModule(y_dim, opt.bit) if opt.load_img_path: img_model.load(opt.load_img_path) if opt.load_txt_path: txt_model.load(opt.load_txt_path) if opt.use_gpu: img_model = img_model.cuda() txt_model = txt_model.cuda() print('-----------------------') query_L = torch.from_numpy(L['query']) query_x = torch.from_numpy(X['query']) query_y = torch.from_numpy(Y['query']) retrieval_L = torch.from_numpy(L['retrieval']) retrieval_x = torch.from_numpy(X['retrieval']) retrieval_y = torch.from_numpy(Y['retrieval']) qBX = generate_image_code(img_model, query_x, opt.bit) qBY = generate_text_code(txt_model, query_y, opt.bit) rBX = generate_image_code(img_model, retrieval_x, opt.bit) rBY = generate_text_code(txt_model, retrieval_y, opt.bit) if opt.use_gpu: query_L = query_L.cuda() retrieval_L = retrieval_L.cuda() mapi2t = calc_map_k(qBX, rBY, query_L, retrieval_L) mapt2i = calc_map_k(qBY, rBX, query_L, retrieval_L) print('...test MAP: MAP(i->t): %3.3f, MAP(t->i): %3.3f' % (mapi2t, mapt2i))
def train_model(model, dataloader, criterion, criterion_hash, optimizer, scheduler, num_epochs, bits, classes, log_file): train_codes = calc_train_codes(dataloader, bits, classes) for epoch in range(num_epochs): model.train() ce_loss = 0.0 for batch_cnt, (inputs, labels, item) in enumerate(dataloader['train']): codes = torch.tensor(train_codes[item, :]).float().cuda() inputs = inputs.cuda() labels = labels.cuda() optimizer.zero_grad() feature_map, outputs_class, outputs_codes = model(inputs) # ------------------------------------------------------------ attention = torch.sum(feature_map.detach(), dim=1, keepdim=True) attention = nn.functional.interpolate(attention, size=(224, 224), mode='bilinear', align_corners=True) masks = [] for i in range(labels.size()[0]): threshold = random.uniform(0.9, 1.0) mask = (attention[i] < threshold * attention[i].max()).float() masks.append(mask) masks = torch.stack(masks) hide_imgs = inputs * masks _, outputs_hide, _ = model(hide_imgs) # ------------------------------------------------------------ loss_class = criterion(outputs_class, labels) loss_class_hide = criterion(outputs_hide, labels) loss_codes = criterion_hash(outputs_codes, codes) loss = loss_class + loss_codes + loss_class_hide # 0.1* loss.backward() optimizer.step() ce_loss += loss.item() * inputs.size(0) epoch_loss = ce_loss / dataloader['train'].total_item_len scheduler.step() if (epoch+1)%1 == 0: ground_q, code_q = eval_turn(model, dataloader['val']) ground_d, code_d = eval_turn(model, dataloader['base']) labels_onehot_q = label2onehot(ground_q.cpu(), classes) labels_onehot_d = label2onehot(ground_d.cpu(), classes) map_1 = calc_map_k(torch.sign(code_q), torch.tensor(train_codes).float().cuda(), labels_onehot_q, labels_onehot_d) print('epoch:{}: loss:{:.4f}, MAP:{:.4f}'.format(epoch+1, epoch_loss, map_1)) log_file.write('epoch:{}: loss:{:.4f}, MAP:{:.4f}'.format(epoch+1, epoch_loss, map_1) + '\n')
def valid(opt, img_model: nn.Module, txt_model: nn.Module, dataset: DatasetMirflckr25KValid, return_hash=False): # get query img and txt binary code dataset.query() qB_img = get_img_code(opt, img_model, dataset) qB_txt = get_txt_code(opt, txt_model, dataset) query_label = dataset.get_all_label() # get retrieval img and txt binary code dataset.retrieval() rB_img = get_img_code(opt, img_model, dataset) rB_txt = get_txt_code(opt, txt_model, dataset) retrieval_label = dataset.get_all_label() mAPi2t = calc_map_k(qB_img, rB_txt, query_label, retrieval_label) mAPt2i = calc_map_k(qB_txt, rB_img, query_label, retrieval_label) if return_hash: return mAPi2t, mAPt2i, qB_img.cpu(), qB_txt.cpu(), rB_img.cpu( ), rB_txt.cpu(), query_label, retrieval_label return mAPi2t, mAPt2i
def test(**kwargs): opt.parse(kwargs) if opt.device is not None: opt.device = torch.device(opt.device) elif opt.gpus: opt.device = torch.device(0) else: opt.device = torch.device('cpu') pretrain_model = load_pretrain_model(opt.pretrain_model_path) model = AGAH(opt.bit, opt.tag_dim, opt.num_label, opt.emb_dim, lambd=opt.lambd, pretrain_model=pretrain_model).to(opt.device) path = 'checkpoints/' + opt.dataset + '_' + str(opt.bit) load_model(model, path) FEATURE_MAP = torch.load(os.path.join(path, 'feature_map.pth')).to(opt.device) model.eval() images, tags, labels = load_data(opt.data_path, opt.dataset) x_query_data = Dataset(opt, images, tags, labels, test='image.query') x_db_data = Dataset(opt, images, tags, labels, test='image.db') y_query_data = Dataset(opt, images, tags, labels, test='text.query') y_db_data = Dataset(opt, images, tags, labels, test='text.db') x_query_dataloader = DataLoader(x_query_data, opt.batch_size, shuffle=False) x_db_dataloader = DataLoader(x_db_data, opt.batch_size, shuffle=False) y_query_dataloader = DataLoader(y_query_data, opt.batch_size, shuffle=False) y_db_dataloader = DataLoader(y_db_data, opt.batch_size, shuffle=False) qBX = generate_img_code(model, x_query_dataloader, opt.query_size, FEATURE_MAP) qBY = generate_txt_code(model, y_query_dataloader, opt.query_size, FEATURE_MAP) rBX = generate_img_code(model, x_db_dataloader, opt.db_size, FEATURE_MAP) rBY = generate_txt_code(model, y_db_dataloader, opt.db_size, FEATURE_MAP) query_labels, db_labels = x_query_data.get_labels() query_labels = query_labels.to(opt.device) db_labels = db_labels.to(opt.device) p_i2t, r_i2t = pr_curve(qBX, rBY, query_labels, db_labels) p_t2i, r_t2i = pr_curve(qBY, rBX, query_labels, db_labels) K = [1, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000] pk_i2t = p_topK(qBX, rBY, query_labels, db_labels, K) pk_t2i = p_topK(qBY, rBX, query_labels, db_labels, K) path = 'checkpoints/' + opt.dataset + '_' + str(opt.bit) np.save(os.path.join(path, 'P_i2t.npy'), p_i2t.numpy()) np.save(os.path.join(path, 'R_i2t.npy'), r_i2t.numpy()) np.save(os.path.join(path, 'P_t2i.npy'), p_t2i.numpy()) np.save(os.path.join(path, 'R_t2i.npy'), r_t2i.numpy()) np.save(os.path.join(path, 'P_at_K_i2t.npy'), pk_i2t.numpy()) np.save(os.path.join(path, 'P_at_K_t2i.npy'), pk_t2i.numpy()) mapi2t = calc_map_k(qBX, rBY, query_labels, db_labels) mapt2i = calc_map_k(qBY, rBX, query_labels, db_labels) print('...test MAP: MAP(i->t): %3.4f, MAP(t->i): %3.4f' % (mapi2t, mapt2i))
def test(**kwargs): opt.parse(kwargs) if opt.device is not None: opt.device = torch.device(opt.device) elif opt.gpus: opt.device = torch.device(0) else: opt.device = torch.device('cpu') with torch.no_grad(): model = CPAH(opt.image_dim, opt.text_dim, opt.hidden_dim, opt.bit, opt.num_label).to(opt.device) path = 'checkpoints/' + opt.dataset + '_' + str(opt.bit) + str(opt.proc) load_model(model, path) model.eval() images, tags, labels = load_data(opt.data_path, opt.dataset) i_query_data = Dataset(opt, images, tags, labels, test='image.query') i_db_data = Dataset(opt, images, tags, labels, test='image.db') t_query_data = Dataset(opt, images, tags, labels, test='text.query') t_db_data = Dataset(opt, images, tags, labels, test='text.db') i_query_dataloader = DataLoader(i_query_data, opt.batch_size, shuffle=False) i_db_dataloader = DataLoader(i_db_data, opt.batch_size, shuffle=False) t_query_dataloader = DataLoader(t_query_data, opt.batch_size, shuffle=False) t_db_dataloader = DataLoader(t_db_data, opt.batch_size, shuffle=False) qBX = generate_img_code(model, i_query_dataloader, opt.query_size) qBY = generate_txt_code(model, t_query_dataloader, opt.query_size) rBX = generate_img_code(model, i_db_dataloader, opt.db_size) rBY = generate_txt_code(model, t_db_dataloader, opt.db_size) query_labels, db_labels = i_query_data.get_labels() query_labels = query_labels.to(opt.device) db_labels = db_labels.to(opt.device) #K = [1, 10, 100, 1000] #p_top_k(qBX, rBY, query_labels, db_labels, K, tqdm_label='I2T') # pr_curve2(qBY, rBX, query_labels, db_labels) p_i2t, r_i2t = pr_curve(qBX, rBY, query_labels, db_labels, tqdm_label='I2T') p_t2i, r_t2i = pr_curve(qBY, rBX, query_labels, db_labels, tqdm_label='T2I') p_i2i, r_i2i = pr_curve(qBX, rBX, query_labels, db_labels, tqdm_label='I2I') p_t2t, r_t2t = pr_curve(qBY, rBY, query_labels, db_labels, tqdm_label='T2T') K = [1, 10, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000] pk_i2t = p_top_k(qBX, rBY, query_labels, db_labels, K, tqdm_label='I2T') pk_t2i = p_top_k(qBY, rBX, query_labels, db_labels, K, tqdm_label='T2I') pk_i2i = p_top_k(qBX, rBX, query_labels, db_labels, K, tqdm_label='I2I') pk_t2t = p_top_k(qBY, rBY, query_labels, db_labels, K, tqdm_label='T2T') mapi2t = calc_map_k(qBX, rBY, query_labels, db_labels) mapt2i = calc_map_k(qBY, rBX, query_labels, db_labels) mapi2i = calc_map_k(qBX, rBX, query_labels, db_labels) mapt2t = calc_map_k(qBY, rBY, query_labels, db_labels) pr_dict = {'pi2t': p_i2t.cpu().numpy(), 'ri2t': r_i2t.cpu().numpy(), 'pt2i': p_t2i.cpu().numpy(), 'rt2i': r_t2i.cpu().numpy(), 'pi2i': p_i2i.cpu().numpy(), 'ri2i': r_i2i.cpu().numpy(), 'pt2t': p_t2t.cpu().numpy(), 'rt2t': r_t2t.cpu().numpy()} pk_dict = {'k': K, 'pki2t': pk_i2t.cpu().numpy(), 'pkt2i': pk_t2i.cpu().numpy(), 'pki2i': pk_i2i.cpu().numpy(), 'pkt2t': pk_t2t.cpu().numpy()} map_dict = {'mapi2t': float(mapi2t.cpu().numpy()), 'mapt2i': float(mapt2i.cpu().numpy()), 'mapi2i': float(mapi2i.cpu().numpy()), 'mapt2t': float(mapt2t.cpu().numpy())} print(' Test MAP: MAP(i->t) = {:3.4f}, MAP(t->i) = {:3.4f}, MAP(i->i) = {:3.4f}, MAP(t->t) = {:3.4f}'.format(mapi2t, mapt2i, mapi2i, mapt2t)) path = 'checkpoints/' + opt.dataset + '_' + str(opt.bit) + str(opt.proc) write_pickle(os.path.join(path, 'pr_dict.pkl'), pr_dict) write_pickle(os.path.join(path, 'pk_dict.pkl'), pk_dict) write_pickle(os.path.join(path, 'map_dict.pkl'), map_dict)
def test(self): self.ImgNet.eval().cuda() self.TxtNet.eval().cuda() re_BI, re_BT, re_LT, qu_BI, qu_BT, qu_LT = generate_hashes_from_dataloader( self.database_loader, self.test_loader, self.ImgNet, self.TxtNet, self.cfg.LABEL_DIM) qu_BI = self.get_each_5th_element(qu_BI) re_BI = self.get_each_5th_element(re_BI) qu_LI = self.get_each_5th_element(qu_LT) re_LI = self.get_each_5th_element(re_LT) p_i2t, r_i2t = pr_curve(qu_BI, re_BT, qu_LI, re_LT, tqdm_label='I2T') p_t2i, r_t2i = pr_curve(qu_BT, re_BI, qu_LT, re_LI, tqdm_label='T2I') p_i2i, r_i2i = pr_curve(qu_BI, re_BI, qu_LI, re_LI, tqdm_label='I2I') p_t2t, r_t2t = pr_curve(qu_BT, re_BT, qu_LT, re_LT, tqdm_label='T2T') K = [1, 10, 50] + list(range(100, 1000, 100)) + list( range(1000, 10001, 1000)) pk_i2t = p_top_k(qu_BI, re_BT, qu_LI, re_LT, K, tqdm_label='I2T') pk_t2i = p_top_k(qu_BT, re_BI, qu_LT, re_LI, K, tqdm_label='T2I') pk_i2i = p_top_k(qu_BI, re_BI, qu_LI, re_LI, K, tqdm_label='I2I') pk_t2t = p_top_k(qu_BT, re_BT, qu_LT, re_LT, K, tqdm_label='T2T') MAP_I2T = calc_map_k(qu_BI, re_BT, qu_LI, re_LT, self.cfg.MAP_K) MAP_T2I = calc_map_k(qu_BT, re_BI, qu_LT, re_LI, self.cfg.MAP_K) MAP_I2I = calc_map_k(qu_BI, re_BI, qu_LI, re_LI, self.cfg.MAP_K) MAP_T2T = calc_map_k(qu_BT, re_BT, qu_LT, re_LT, self.cfg.MAP_K) MAPS = (MAP_I2T, MAP_T2I, MAP_I2I, MAP_T2T) pr_dict = { 'pi2t': p_i2t.cpu().numpy(), 'ri2t': r_i2t.cpu().numpy(), 'pt2i': p_t2i.cpu().numpy(), 'rt2i': r_t2i.cpu().numpy(), 'pi2i': p_i2i.cpu().numpy(), 'ri2i': r_i2i.cpu().numpy(), 'pt2t': p_t2t.cpu().numpy(), 'rt2t': r_t2t.cpu().numpy() } pk_dict = { 'k': K, 'pki2t': pk_i2t.cpu().numpy(), 'pkt2i': pk_t2i.cpu().numpy(), 'pki2i': pk_i2i.cpu().numpy(), 'pkt2t': pk_t2t.cpu().numpy() } map_dict = { 'mapi2t': float(MAP_I2T.cpu().numpy()), 'mapt2i': float(MAP_T2I.cpu().numpy()), 'mapi2i': float(MAP_I2I.cpu().numpy()), 'mapt2t': float(MAP_T2T.cpu().numpy()) } self.logger.info( 'mAP I->T: %.3f, mAP T->I: %.3f, mAP I->I: %.3f, mAP T->T: %.3f' % MAPS) write_pickle(osp.join(self.cfg.MODEL_DIR, self.path, 'pr_dict.pkl'), pr_dict) write_pickle(osp.join(self.cfg.MODEL_DIR, self.path, 'pk_dict.pkl'), pk_dict) write_pickle(osp.join(self.cfg.MODEL_DIR, self.path, 'map_dict.pkl'), map_dict)