def get_knn_salience(labeled_imidx, ny, nx): ''' as name ''' knnsal_path = '../../data_viper/salience_all.mat' tmp = loadfile(knnsal_path) knn_gal = tmp['salience_all_gal'] # view a knn_prb = tmp['salience_all_prb'] # view b salmap_knn_sub = knn_gal[:, :, labeled_imidx].transpose((2, 0, 1)) salmap_knn_query = [mapresize(im, size=(ny, nx)) for im in salmap_knn_sub] salmap_knn_sub = knn_prb[:, :, labeled_imidx].transpose((2, 0, 1)) salmap_knn_gallery = [mapresize(im, size=(ny, nx)) for im in salmap_knn_sub] return np.asarray(salmap_knn_query), np.asarray(salmap_knn_gallery)
def get_calibrated_gtsal(): ''' get calibrated groundtruth salience maps ''' # load viper dataset viper = loadfile('../../data_viper/viper.pkl') ## load knn salience knnsal_path = '../../data_viper/salience_all.mat' tmp = loadfile(knnsal_path) knn_gal = tmp['salience_all_gal'] # view a knn_prb = tmp['salience_all_gal'] # view b mapsz = viper.imgs[0].shape[:2] labeled_imidx_path = '../../data_viper/labeled_imidx.mat' tmp = loadfile(labeled_imidx_path) labeled_imidx = tmp['labeled_imidx'].flatten() salmap_knn_small = knn_gal[:, :, labeled_imidx].transpose((2, 0, 1)) salmap_knn_all = [mapresize(im, size=mapsz) for im in salmap_knn_small] knn_arr = [] gt_arr = [] for seg, sal, msk in zip(viper.segmsks, viper.salmsks, salmap_knn_all): idx = seg != 0 gt_arr = np.hstack((gt_arr, sal[idx])) knn_arr = np.hstack((knn_arr, msk[idx])) nBin_gt = 100 nBin_knn = 100 gt_hist = np.histogram(gt_arr, bins=nBin_gt, range=(0, 1))[0]/float(len(gt_arr)) knn_hist = np.histogram(knn_arr, bins=nBin_knn, range=(0, 1))[0]/float(len(knn_arr)) mapping = np.asarray(histmapping(gt_hist, knn_hist)) pl.figure(1) pl.plot(mapping) pl.savefig('mapping.jpg') # mapping from knn to gt maps # salmap_knn_mapped = [mapping[np.floor(sal*nBin_knn).astype(int)] for sal in salmap_knn_small] salmap_gt_mapped = [mapping[np.floor(sal*nBin_gt).astype(int)] for sal in viper.salmsks] return salmap_gt_mapped
def main(): ''' pipeline for evaluating salience ''' # three types: # 1) unsupervised, knnsal # 2) groundtruth, gtsal # 3) prediction, predsal ## load cnn salience with groundtruth supsal_path = '../data_viper/model_feat/salmaps_comparison.pkl' test_imids, salmap_gt, salmap_pred = loadfile(supsal_path) mapsz = salmap_gt[0].shape ## load knn salience knnsal_path = '../data_viper/salience_all.mat' tmp = loadfile(knnsal_path) knn_gal = tmp['salience_all_gal'] # view a knn_prb = tmp['salience_all_gal'] # view b labeled_imidx_path = '../data_viper/labeled_imidx.mat' tmp = loadfile(labeled_imidx_path) labeled_imidx = tmp['labeled_imidx'].flatten() salmap_knn_small = knn_gal[:, :, labeled_imidx].transpose((2, 0, 1)) salmap_knn_all = [mapresize(im, size=mapsz) for im in salmap_knn_small] # get rid of background for better illustration datafile_viper = '../data_viper/viper.pkl' viper = loadfile(datafile_viper) salmap_knn = [] for seg, msk in zip(viper.segmsks, salmap_knn_all): idx = seg == 0 msk[idx] = 0 salmap_knn.append(msk) salmap_knn = np.asarray(salmap_knn) # qualitative evaluation save_path = '../data_viper/model_feat/' for i in test_imids: pl.figure(1) pl.subplot(1, 4, 1) # show image pl.imshow(viper.imgs[i]) pl.title('image') pl.subplot(1, 4, 2) # show groundtruth salience pl.imshow(salmap_gt[i]*255., cmap='hot', vmin=0, vmax=255) pl.title('groundtruth') pl.subplot(1, 4, 3) # show knn salience pl.imshow(salmap_knn[i]*255., cmap='hot', vmin=0, vmax=255) pl.title('KNN salience') pl.xlabel('abserr={0:.2f}'.format(abs_error(salmap_knn[i].flatten(), salmap_gt[i].flatten()))) pl.subplot(1, 4, 4) # show CNN prediction salience pl.imshow(salmap_pred[i]*255., cmap='hot', vmin=0, vmax=255) pl.title('CNN salience') pl.xlabel('abserr={0:.2f}'.format(abs_error(salmap_pred[i].flatten(), salmap_gt[i].flatten()))) pl.savefig(save_path + '{0:03d}.jpg'.format(i)) print save_path +'{0:03d}.jpg'.format(i) + ' saved!' # quantitative evaluation test_idx = np.unique(test_imids) print 'mean abs error - KNN vs Gt: {0:.2f}'.format(abs_error(salmap_knn[test_idx], salmap_gt[test_idx])) print 'mean abs error - CNN vs Gt: {0:.2f}'.format(abs_error(salmap_pred[test_idx], salmap_gt[test_idx])) # pl.figure(2) # test_idx = np.unique(test_imids) # recall_knn, precision_knn = get_roc_curve(salmap_gt[test_idx], salmap_knn[test_idx]) # pl.plot(recall_knn, precision_knn, 'b', linewidth=2, label='knn vs. gt') # recall_cnn, precision_cnn = get_roc_curve(salmap_gt[test_idx], salmap_pred[test_idx]) # pl.plot(recall_cnn, precision_cnn, 'r', linewidth=2, label='cnn vs. gt') # pl.xlabel('recall') # pl.ylabel('precision') # pl.legend() # pl.savefig(save_path+'roc.jpg') # print 'ROC curve saved!' os.system('xdg-open '+save_path)
def get_gt_salience(ny, nx): ''' get groundtruth salience ''' gtsal_path = '../../data_viper/salience_gt.pkl' salmap_gt_query = loadfile(gtsal_path) return np.asarray([mapresize(im, size=(ny, nx)) for im in salmap_gt_query])
def get_mask(segmsks, ny, nx): ''' get foreground masks ''' return np.asarray([mapresize(msk, size=(ny, nx))>0 for msk in segmsks])
def main(): ''' pipeline for supervised salience training ''' if os.path.isdir('../data_viper/'): datapath = '../data_viper/' else: datapath = '../data/' save_path = '../data_viper/model_feat/' DATA_OPT = 'feat' # feature type TRAIN_OPT = 'SVR' # training model option TRAIN = True # wheather re-train the model # prepare training data for supervised salience training #======================================================= datafile_viper = datapath + 'viper.pkl' if not os.path.isfile(datafile_viper): viper = DataMan_viper_small() viper.make_data() savefile(datafile_viper, viper) else: viper = loadfile(datafile_viper) viper = preprocess_data(viper, DATA_OPT) # training # ============== modelfile = datapath + 'model_feat/model.pkl' if TRAIN: tic = time.clock() model = train_model(viper, TRAIN_OPT) toc = time.clock() print 'Elapsed training time: {0:.2f} min'.format((toc-tic)/60.) savefile(modelfile, model) os.system('ls -lh ' + modelfile) else: model = loadfile(modelfile) ## validation #========================================= print 'validating trained model' nValid = 5000 valididx = np.random.permutation(viper.valid_feat.shape[0])[:nValid] # valid_ypred = model.predict(viper.valid_feat[valididx]) valid_ypred = predict(model, viper.valid_feat[valididx], viper.yy[viper.valid_ctrids][valididx], viper.imH) #- quantize patches based on testing scores kmeans = KMeans(init='k-means++', n_clusters=10, n_init=10, verbose=1) kmeans.fit(valid_ypred.reshape(nValid, 1)) #- crop patches for testing image valid_patset = np.asarray(viper.get_patchset('valid'))[valididx] #- save to result folder os.system('rm '+save_path+'*.jpg') for i in range(10): idx = kmeans.labels_== i if any(idx): pats = immontage(list(valid_patset[idx])) imsave(save_path+'{}.jpg'.format(kmeans.cluster_centers_[i]), pats) print 'patchset {} saved'.format(i) ### testing #=============== print 'testing' # test_ypred = model.predict(viper.test_feat) test_ypred = predict(model, viper.test_feat, viper.yy_test[viper.test_ctrids], viper.imH) ## assign predicted scores to images h, w = viper.imgs[0].shape[:2] mh, mw = len(np.unique(viper.yy_test)), len(np.unique(viper.xx_test)) msk0 = np.zeros(mh*mw, dtype=np.float32) msks = [msk0.copy() for im in viper.imgs] showlist = [] n_test = len(test_ypred) for i in range(n_test): imgid = viper.test_imgids[i] patid = viper.test_ctrids[i] score = test_ypred[i] msks[imgid][patid] = score # resize predicted salience map to match image size msks_rs = [mapresize(msk.reshape((mw, mh)).T, size=(h, w)) for msk in msks] # msks_rs = msks # save salience map for comparison test_imids = np.asarray(np.unique(viper.test_imgids)) salmap_gt = np.asarray(viper.salmsks) #np.asarray([viper.salmsks[imid] for imid in test_imids]) salmap_pred = np.asarray(msks_rs) #np.asarray([msks_rs[imid]/255. for imid in test_imids]) savefile(save_path+'salmaps_comparison.pkl', [test_imids, salmap_gt, salmap_pred])