def main():
''' pipeline for testing and evaluation '''
# preset parameters
save_path = '../data_viper/model_feat/'
# load data
# imL = 48
bs = 100
datafile_viper = '../data_viper/viper.pkl'
viper = loadfile(datafile_viper)
# load model
modelfile_viper = '../data_viper/model_feat/model.pkl'
model = loadfile(modelfile_viper)
# evaluation and testing
# test_x = viper.test_x.get_value(borrow=True)
test_x = np.asarray(viper.test_feat)
test_y = viper.test_y
n_test = test_x.shape[0]
test_ypred = model.predict(viper.test_feat)
test_ypred = np.asarray(test_ypred).flatten()
# test_ims = test_x.reshape((n_test, imL, imL, -1))
# assign predicted scores to images
h, w = viper.imgs[0].shape[:2]
mh, mw = len(np.unique(viper.yy)), len(np.unique(viper.xx))
msk0 = np.zeros(mh*mw).astype(np.uint8)
msks = [msk0.copy() for im in viper.imgs]
showlist = []
for i in range(n_test):
imgid = viper.test_imgids[i]
patid = viper.test_ctrids[i]
score = test_ypred[i]
msks[imgid][patid] = score*255
# resize predicted salience map to match image size
msks_rs = [imresize(msk.reshape((mw, mh)).T, size=(h, w))/255. for msk in msks]
# save salience map for comparison
test_imids = np.asarray(np.unique(viper.test_imgids))
salmap_gt = np.asarray(viper.salmsks) #np.asarray([viper.salmsks[imid] for imid in test_imids])
salmap_pred = np.asarray(msks_rs) #np.asarray([msks_rs[imid]/255. for imid in test_imids])
savefile(save_path+'salmaps_comparison.pkl', [test_imids, salmap_gt, salmap_pred])
# quantize to show different test patches
# kmeans = KMeans(init='k-means++', n_clusters=10, n_init=10)
# kmeans.fit(test_ypred.reshape(n_test, 1))
# # save to result folder
# for i in range(10):
# idx = kmeans.labels_== i
# if any(idx):
# im = immontage(list(test_ims[idx]))
# imsave(save_path+'{}.jpg'.format(kmeans.cluster_centers_[i]), im)
print 'testing finished'
def test_SVR():
datafile_viper = '../data_viper/viper.pkl'
viper = loadfile(datafile_viper)
from sklearn.svm import SVR
model = SVR(C=10, kernel='rbf', shrinking=False, verbose=True)
model.fit(viper.train_feat, viper.train_y)
y_pred = model.predict(viper.test_feat)
print 'testing error {}'.format(abs_error(y_pred, viper.test_y))
def test_lasso_regression():
datafile_viper = '../data_viper/viper.pkl'
viper = loadfile(datafile_viper)
from sklearn.linear_model import Lasso
model = Lasso(alpha=1e-3)
model.fit(viper.train_feat, viper.train_y)
y_pred = model.predict(viper.test_feat)
print 'testing error {}'.format(abs_error(y_pred, viper.test_y))
def test_linear_regression():
datafile_viper = '../data_viper/viper.pkl'
viper = loadfile(datafile_viper)
from sklearn.linear_model import LinearRegression
model = LinearRegression(normalize=True)
model.fit(viper.train_feat, viper.train_y)
y_pred = model.predict(viper.test_feat)
print 'testing error {}'.format(abs_error(y_pred, viper.test_y))
def test_knn_regression():
datafile_viper = '../data_viper/viper.pkl'
viper = loadfile(datafile_viper)
from sklearn.neighbors import KNeighborsRegressor
model = KNeighborsRegressor(n_neighbors=5, weights='uniform', metric='euclidean')
model.fit(viper.train_feat, viper.train_y)
n_test = len(viper.test_feat)
y_pred = np.zeros(n_test)
for i, feat in zip(np.arange(n_test), viper.test_feat):
dist, ind = model.kneighbors(feat)
y_pred[i] = (viper.train_y[ind]*np.exp(-dist**2)).sum()/(np.exp(-dist**2)).sum()
# y_pred = model.predict(viper.test_feat)
print 'testing error {}'.format(abs_error(y_pred, viper.test_y))
def main():
''' pipeline for evaluating salience '''
# three types:
# 1) unsupervised, knnsal
# 2) groundtruth, gtsal
# 3) prediction, predsal
## load cnn salience with groundtruth
supsal_path = '../data_viper/model_feat/salmaps_comparison.pkl'
test_imids, salmap_gt, salmap_pred = loadfile(supsal_path)
mapsz = salmap_gt[0].shape
## load knn salience
knnsal_path = '../data_viper/salience_all.mat'
tmp = loadfile(knnsal_path)
knn_gal = tmp['salience_all_gal'] # view a
knn_prb = tmp['salience_all_gal'] # view b
labeled_imidx_path = '../data_viper/labeled_imidx.mat'
tmp = loadfile(labeled_imidx_path)
labeled_imidx = tmp['labeled_imidx'].flatten()
salmap_knn_small = knn_gal[:, :, labeled_imidx].transpose((2, 0, 1))
salmap_knn_all = [mapresize(im, size=mapsz) for im in salmap_knn_small]
# get rid of background for better illustration
datafile_viper = '../data_viper/viper.pkl'
viper = loadfile(datafile_viper)
salmap_knn = []
for seg, msk in zip(viper.segmsks, salmap_knn_all):
idx = seg == 0
msk[idx] = 0
salmap_knn.append(msk)
salmap_knn = np.asarray(salmap_knn)
# qualitative evaluation
save_path = '../data_viper/model_feat/'
for i in test_imids:
pl.figure(1)
pl.subplot(1, 4, 1) # show image
pl.imshow(viper.imgs[i])
pl.title('image')
pl.subplot(1, 4, 2) # show groundtruth salience
pl.imshow(salmap_gt[i]*255., cmap='hot', vmin=0, vmax=255)
pl.title('groundtruth')
pl.subplot(1, 4, 3) # show knn salience
pl.imshow(salmap_knn[i]*255., cmap='hot', vmin=0, vmax=255)
pl.title('KNN salience')
pl.xlabel('abserr={0:.2f}'.format(abs_error(salmap_knn[i].flatten(), salmap_gt[i].flatten())))
pl.subplot(1, 4, 4) # show CNN prediction salience
pl.imshow(salmap_pred[i]*255., cmap='hot', vmin=0, vmax=255)
pl.title('CNN salience')
pl.xlabel('abserr={0:.2f}'.format(abs_error(salmap_pred[i].flatten(), salmap_gt[i].flatten())))
pl.savefig(save_path + '{0:03d}.jpg'.format(i))
print save_path +'{0:03d}.jpg'.format(i) + ' saved!'
# quantitative evaluation
test_idx = np.unique(test_imids)
print 'mean abs error - KNN vs Gt: {0:.2f}'.format(abs_error(salmap_knn[test_idx], salmap_gt[test_idx]))
print 'mean abs error - CNN vs Gt: {0:.2f}'.format(abs_error(salmap_pred[test_idx], salmap_gt[test_idx]))
# pl.figure(2)
# test_idx = np.unique(test_imids)
# recall_knn, precision_knn = get_roc_curve(salmap_gt[test_idx], salmap_knn[test_idx])
# pl.plot(recall_knn, precision_knn, 'b', linewidth=2, label='knn vs. gt')
# recall_cnn, precision_cnn = get_roc_curve(salmap_gt[test_idx], salmap_pred[test_idx])
# pl.plot(recall_cnn, precision_cnn, 'r', linewidth=2, label='cnn vs. gt')
# pl.xlabel('recall')
# pl.ylabel('precision')
# pl.legend()
# pl.savefig(save_path+'roc.jpg')
# print 'ROC curve saved!'
os.system('xdg-open '+save_path)
def main():
''' pipeline for supervised salience training '''
if os.path.isdir('../data_viper/'):
datapath = '../data_viper/'
else:
datapath = '../data/'
save_path = '../data_viper/model_feat/'
DATA_OPT = 'feat' # feature type
TRAIN_OPT = 'SVR' # training model option
TRAIN = True # wheather re-train the model
# prepare training data for supervised salience training
#=======================================================
datafile_viper = datapath + 'viper.pkl'
if not os.path.isfile(datafile_viper):
viper = DataMan_viper_small()
viper.make_data()
savefile(datafile_viper, viper)
else:
viper = loadfile(datafile_viper)
viper = preprocess_data(viper, DATA_OPT)
# training
# ==============
modelfile = datapath + 'model_feat/model.pkl'
if TRAIN:
tic = time.clock()
model = train_model(viper, TRAIN_OPT)
toc = time.clock()
print 'Elapsed training time: {0:.2f} min'.format((toc-tic)/60.)
savefile(modelfile, model)
os.system('ls -lh ' + modelfile)
else:
model = loadfile(modelfile)
## validation
#=========================================
print 'validating trained model'
nValid = 5000
valididx = np.random.permutation(viper.valid_feat.shape[0])[:nValid]
# valid_ypred = model.predict(viper.valid_feat[valididx])
valid_ypred = predict(model, viper.valid_feat[valididx], viper.yy[viper.valid_ctrids][valididx], viper.imH)
#- quantize patches based on testing scores
kmeans = KMeans(init='k-means++', n_clusters=10, n_init=10, verbose=1)
kmeans.fit(valid_ypred.reshape(nValid, 1))
#- crop patches for testing image
valid_patset = np.asarray(viper.get_patchset('valid'))[valididx]
#- save to result folder
os.system('rm '+save_path+'*.jpg')
for i in range(10):
idx = kmeans.labels_== i
if any(idx):
pats = immontage(list(valid_patset[idx]))
imsave(save_path+'{}.jpg'.format(kmeans.cluster_centers_[i]), pats)
print 'patchset {} saved'.format(i)
### testing
#===============
print 'testing'
# test_ypred = model.predict(viper.test_feat)
test_ypred = predict(model, viper.test_feat, viper.yy_test[viper.test_ctrids], viper.imH)
## assign predicted scores to images
h, w = viper.imgs[0].shape[:2]
mh, mw = len(np.unique(viper.yy_test)), len(np.unique(viper.xx_test))
msk0 = np.zeros(mh*mw, dtype=np.float32)
msks = [msk0.copy() for im in viper.imgs]
showlist = []
n_test = len(test_ypred)
for i in range(n_test):
imgid = viper.test_imgids[i]
patid = viper.test_ctrids[i]
score = test_ypred[i]
msks[imgid][patid] = score
# resize predicted salience map to match image size
msks_rs = [mapresize(msk.reshape((mw, mh)).T, size=(h, w)) for msk in msks]
# msks_rs = msks
# save salience map for comparison
test_imids = np.asarray(np.unique(viper.test_imgids))
salmap_gt = np.asarray(viper.salmsks) #np.asarray([viper.salmsks[imid] for imid in test_imids])
salmap_pred = np.asarray(msks_rs) #np.asarray([msks_rs[imid]/255. for imid in test_imids])
savefile(save_path+'salmaps_comparison.pkl', [test_imids, salmap_gt, salmap_pred])
def test_knn():
datafile_viper = '../data_viper/viper.pkl'
viper = loadfile(datafile_viper)
viper = downsample_data(viper)
# from sklearn.neighbors import KNeighborsRegressor
# model = KNeighborsRegressor(n_neighbors=5, weights='uniform', metric='euclidean')
# model.fit(viper.train_feat, viper.train_y)
from sklearn.neighbors import KDTree
# divide into stripes
nStripe = 10
y_max = viper.yy.max()
y_min = viper.yy.min()
y_len = np.int((y_max - y_min)/10.)
y_centers = np.round(np.linspace(y_min+y_len, y_max-y_len, nStripe))
k = 5
y_ctr = y_centers[k]
stripe_idx = np.where((viper.yy[viper.train_ctrids] >= y_ctr-y_len) & (viper.yy[viper.train_ctrids] < y_ctr+y_len))[0]
model = KDTree(viper.train_feat[stripe_idx, :288], metric='euclidean')
train_patset = viper.get_patchset('train')
test_patset = viper.get_patchset('test')
test_ids = np.where((viper.yy[viper.test_ctrids] >= y_ctr-y_len) & (viper.yy[viper.test_ctrids] < y_ctr+y_len))[0]
np.random.shuffle(test_ids)
for i in test_ids:
get_testrect = lambda i: [viper.xx[viper.test_ctrids[i]] - viper.patL/2,
viper.yy[viper.test_ctrids[i]] - viper.patL/2,
viper.patL, viper.patL]
get_trainrect = lambda i: [viper.xx[viper.train_ctrids[i]] - viper.patL/2,
viper.yy[viper.train_ctrids[i]] - viper.patL/2,
viper.patL, viper.patL]
gray2color = lambda grayim: np.dstack((grayim, grayim, grayim))
imlist = []
patlist = []
maplist = []
patlist.append(imresize(test_patset[i], size=(100, 100)))
imlist.append(drawrect(viper.imgs[viper.test_imgids[i]], get_testrect(i)))
maplist.append(viper.salmsks[viper.test_imgids[i]])
dist, ind = model.query(viper.test_feat[i, :288], k=30, return_distance=True)
print viper.test_y[i]
hist = np.histogram(viper.train_y[stripe_idx[ind[0]]])
print hist[0]
print hist[1]
print dist
for id in stripe_idx[ind[0]]:
patlist.append(imresize(train_patset[id], size=(100, 100)))
imlist.append(drawrect(viper.imgs[viper.train_imgids[id]], get_trainrect(id)))
maplist.append(viper.salmsks[viper.train_imgids[id]])
pats = immontage(patlist)
imgs = immontage(imlist)
maps = immontage(maplist)
imsave('tmp1.jpg', pats)
imsave('tmp2.jpg', imgs)
imsave('tmp3.jpg', maps)
raw_input()
os.system('xdg-open tmp1.jpg')
def main():
''' pipeline for supervised salience training '''
if os.path.isdir('../data_viper/'):
datapath = '../data_viper/'
else:
datapath = '../data/'
# prepare training data for supervised salience training
datafile_viper = datapath + 'viper.pkl'
if not os.path.isfile(datafile_viper):
viper = DataMan_viper_small()
viper.make_data()
savefile(datafile_viper, viper)
viper = loadfile(datafile_viper)
viper = change_label(viper)
viper.train_feat = viper.get_pixeldata('train')
viper.valid_feat = viper.get_pixeldata('valid')
viper.test_feat = viper.get_pixeldata('test')
bs = 100
imL = 10
nfilter1 = 16
filterL = 3
x = T.tensor4(name='x', dtype=theano.config.floatX)
y = T.ivector(name='y')
# layer0 = x.reshape((bs, 3, imL, imL))
conv1 = ConvLayer(input=x, image_shape=(bs, 3, imL, imL),
filter_shape=(nfilter1, 3, filterL, filterL),
flatten=True,
actfun=tanh,
tag='_convpool1')
# outL = np.floor((imL-filterL+1.)/recfield).astype(np.int)
outL = imL-filterL+1
# nfilter3 = 16
# filterL3 = 3
# conv3 = ConvLayer(input=conv2.output(), image_shape=(bs, nfilter2, outL2, outL2),
# filter_shape=(nfilter3, nfilter2, filterL3, filterL3),
# flatten=True,
# actfun=tanh,
# tag='_conv3')
#
# outL3 = outL2-filterL3+1
fc2 = FCLayer(input=conv1.output(), n_in=nfilter1*outL*outL, n_out=256, actfun=tanh, tag='_fc2')
fc3 = FCLayer(input=fc2.output(), n_in=256, n_out=10, actfun=sigmoid, tag='_fc3')
params_cmb = conv1.params + fc2.params + fc3.params
# ypred = fc3.output().flatten()
ypred = fc3.output()
model = GeneralModel(input=x, data=viper, output=ypred,
target=y, params=params_cmb,
regularizers=0,
cost_func=negative_log_likelihood,
error_func=sqr_error,
batch_size=bs)
sgd = sgd_optimizer(data=viper,
model=model,
batch_size=bs,
learning_rate=0.001,
n_epochs=500)
sgd.fit_viper()
filepath = datapath + 'model/model.pkl'
savefile(filepath, model)
os.system('ls -lh ' + filepath)