def evaluate(qf, ql, qc, gf, gl, gc): """ Evaluation """ query = qf.view(-1, 1) score = torch.mm(gf, query) score = score.squeeze(1).cpu() score = score.numpy() # prediction index index = np.argsort(score) index = index[::-1] query_index = np.argwhere(gl == ql) camera_index = np.argwhere(gc == qc) good_index = np.setdiff1d(query_index, camera_index, assume_unique=True) junk_index1 = np.argwhere(gl == -1) junk_index2 = np.intersect1d(query_index, camera_index) junk_index = np.append(junk_index2, junk_index1) #.flatten()) ap_tmp, CMC_tmp = compute_map(index, good_index, junk_index) return ap_tmp, CMC_tmp
fine_val_loader = load_dataloader(bs, fine_val_dataset) fine_nb = len(fine_val_loader) for i, fine_val in tqdm.tqdm(enumerate(fine_val_loader), total=fine_nb): fine_results += make_results(fine_model, fine_val, device) if len(coarse_dataset.tolist()) > 0: coarse_val_dataset = load_dataset(coarse_dataset, fine_tr, bs) coarse_val_loader = load_dataloader(bs, coarse_val_dataset) coarse_nb = len(coarse_train_loader) for i, coarse_val in tqdm.tqdm(enumerate(coarse_val_loader), total=coarse_nb): coarse_results += make_results(coarse_model, coarse_val, device) map50 = compute_map(fine_results, coarse_results) print('Validation MAP: \n', map50) # save if e % opt.save_freq == 0: torch.save(fine_model, os.path.join(opt.save, 'fine_model')) torch.save(coarse_model, os.path.join(opt.save, 'coarse_model')) # Testing fine_dataset, coarse_dataset, policies = rl_agent.eval(split_test_path, original_img_path_test) fine_results, coarse_results = [], [] if len(fine_dataset.tolist()) > 0: fine_test_dataset = load_dataset(fine_dataset, fine_tr, bs) fine_test_loader = load_dataloader(bs, fine_test_dataset)
fine_val_loader = load_dataloader(bs, fine_val_dataset) fine_nb = len(fine_val_loader) for i, fine_val in tqdm.tqdm(enumerate(fine_val_loader), total=fine_nb): for j in fine_detector.test(fine_val): fine_results.append(j) print('len(coarse_dataset.tolist()): \n', len(coarse_dataset.tolist())) if len(coarse_dataset.tolist()) > 0: coarse_val_dataset = load_dataset(coarse_dataset, coarse_tr, bs) coarse_val_loader = load_dataloader(bs, coarse_val_dataset) coarse_nb = len(coarse_val_loader) for i, coarse_val in tqdm.tqdm(enumerate(coarse_val_loader), total=coarse_nb): for j in coarse_detector.test(coarse_val): coarse_results.append(j) map50 = compute_map(fine_results, coarse_results) print('Validation mAP: \n', map50) print('Validation find mAP: \n', compute_map(fine_results, [])) print('Validation coarse mAP: \n', compute_map([], coarse_results)) print('Time for validation: \n', time.time() - s_time) with open(opt.save_path + '/val_result.txt', 'a') as f: f.write(str(map50) + '\n') eff = 0 for i in policies: eff += int(i) with open(opt.save_path + '/val_policies.txt', 'a') as f: f.write(str(eff / len(policies)) + '\n') # Testing
# # Build recommender system # item = Item(tracks_info, 0.075, 0.075) slim = Slim(lambda_i=0.001, lambda_j=0.0001, epochs=3, lr=0.1) h1 = HybridSimilarity(item, 0.7, slim, 0.3) user = User(knn=64) h2 = Hybrid(h1, 0.85, user, 0.15) w = Warp(NUM_TRACKS=NUM_TRACKS, no_components=300, epochs=50) a = ALS(factors=1024, iterations=5) h3 = Hybrid(w, 0.7, a, 0.3) recsys = Hybrid(h2, 0.85, h3, 0.15) # # Run recommender system # recs = recsys.run(train_data, targets) # # Compute MAP or export recommendations # if TEST: utils.compute_map(recs, test_data, targets) else: utils.save_recommendations(recs)
sim = torch.mm(train_features, query_features.T) ranks = torch.argsort(-sim, dim=0).cpu().numpy() ############################################################################ # Step 3: evaluate gnd = dataset_train.cfg['gnd'] # evaluate ranks ks = [1, 5, 10] # search for easy & hard gnd_t = [] for i in range(len(gnd)): g = {} g['ok'] = np.concatenate([gnd[i]['easy'], gnd[i]['hard']]) g['junk'] = np.concatenate([gnd[i]['junk']]) gnd_t.append(g) mapM, apsM, mprM, prsM = utils.compute_map(ranks, gnd_t, ks) # search for hard gnd_t = [] for i in range(len(gnd)): g = {} g['ok'] = np.concatenate([gnd[i]['hard']]) g['junk'] = np.concatenate([gnd[i]['junk'], gnd[i]['easy']]) gnd_t.append(g) mapH, apsH, mprH, prsH = utils.compute_map(ranks, gnd_t, ks) print('>> {}: mAP M: {}, H: {}'.format( args.dataset, np.around(mapM * 100, decimals=2), np.around(mapH * 100, decimals=2))) print('>> {}: mP@k{} M: {}, H: {}'.format( args.dataset, np.array(ks), np.around(mprM * 100, decimals=2), np.around(mprH * 100, decimals=2))) dist.barrier()