def test_zf_saliency(dataSet='mnist', stride=2, patchSz=5):
if dataSet == 'mnist':
defFile = '/work4/pulkitag-code/pkgs/caffe-v2-2/modelFiles/mnist/hdf5_test/lenet.prototxt'
modelFile, _ = mp.get_model_mean_file('lenet')
net = mp.MyNet(defFile, modelFile, isGPU=False)
N = net.get_batchsz()
net.set_preprocess(chSwap=None,
imageDims=(28, 28, 1),
isBlobFormat=True)
h5File = '/data1/pulkitag/mnist/h5store/test/batch1.h5'
fid = h5py.File(h5File, 'r')
data = fid['data']
data = data[0:N]
#Do the saliency
imSal, score = mpu.zf_saliency(net,
data,
10,
'ip2',
patchSz=patchSz,
stride=stride)
gtLabels = fid['label']
else:
netName = 'bvlcAlexNet'
opLayer = 'fc8'
defFile = mp.get_layer_def_files(netName, layerName=opLayer)
modelFile, meanFile = mp.get_model_mean_file(netName)
net = mp.MyNet(defFile, modelFile)
net.set_preprocess(imageDims=(256, 256, 3),
meanDat=meanFile,
rawScale=255,
isBlobFormat=True)
ilDat = mpu.ILSVRC12Reader()
ilDat.set_count(2)
data, gtLabels, syn, words = ilDat.read()
data = data.reshape((1, data.shape[0], data.shape[1], data.shape[2]))
data = data.transpose((0, 3, 1, 2))
print data.shape
imSal, score = mpu.zf_saliency(net,
data,
1000,
'fc8',
patchSz=patchSz,
stride=stride)
pdLabels = np.argmax(score.squeeze(), axis=1)
return data, imSal, pdLabels, gtLabels
def get_scale_net(isGPU=True, deviceId=0):
netName = cfg.SCALE_MODEL.NET
defFile = cfg.SCALE_MODEL.PROTO
net = mp.MyNet(defFile, netName, isGPU=isGPU, deviceId=deviceId)
#Set preprocessing in the net
net.set_preprocess(meanDat=(115.2254, 123.9648, 124.2966))
return net
def reconstruct_optimal_input(exp, modelIter, im, recLayer='conv1',
imH=101, imW=101, cropH=101, cropW=101, channels=3,
meanFile=None, lamda=1e-8, batchSz=1, **kwargs):
exp = copy.deepcopy(exp)
kwargs['delAbove'] = recLayer
#Setup the original network
origNet = mpu.CaffeTest.from_caffe_exp(exp)
origNet.setup_network(opNames=recLayer, imH=imH, imW=imW, cropH=cropH, cropW=cropW,
modelIterations=modelIter, batchSz=batchSz,
isAccuracyTest=False, meanFile=meanFile, **kwargs)
#Get the size of the features in the layer that needs to be reconstructed
#Shape of the layer to be reconstructed
blob = origNet.net_.net.blobs[recLayer]
initBlobDat = np.zeros((blob.num, blob.channels, blob.height, blob.width)).astype('float32')
blobLbl = np.zeros((blob.num, 1, 1, 1)).astype('float32')
recShape = (blob.num, blob.channels, blob.height, blob.width)
#Get the initial layer features
print "Extracting Initial Features"
blobDat = np.zeros((blob.num, blob.channels, blob.height, blob.width)).astype('float32')
if im.ndim == 3:
imIp = im.reshape((batchSz,) + im.shape)
else:
imIp = im
feats = origNet.net_.forward_all(blobs=[recLayer], data=imIp)
blobDat = feats[recLayer]
#imDat = np.asarray(imDat)
#Get the net for reconstruvtions
#print (exp.expFile_.netDef_.get_all_layernames())
recProto = mpu.ProtoDef.recproto_from_proto(exp.expFile_.netDef_, featDims=recShape,
imSz=[[channels, cropH, cropW]], batchSz=batchSz, **kwargs)
recProto.write(exp.files_['netdefRec'])
recModel = exp.get_snapshot_name(modelIter)
recNet = edict()
recNet.net_ = mp.MyNet(exp.files_['netdefRec'], recModel, caffe.TRAIN)
#recNet = mpu.CaffeTest.from_model(exp.files_['netdefRec'], recModel)
#kwargs['dataLayerNames'] = ['data']
#kwargs['newDataLayerNames'] = None
#recNet.setup_network(opNames=[recLayer], imH=imH, imW=imW, cropH=cropH, cropW=cropW,
# modelIterations=modelIter, isAccuracyTest=False, meanFile=meanFile,
# testMode=False, **kwargs)
recNet.net_.net.set_force_backward(recLayer)
#Start the reconstruction
ch,h,w = imIp.shape[3], imIp.shape[1], imIp.shape[2]
imRec = 255*np.random.random((batchSz,ch,h,w)).astype('float32')
print imRec.shape, blobDat.shape, blobLbl.shape
sol = scipy.optimize.fmin_l_bfgs_b(rec_fun_grad, imRec.flatten(),args=[recNet, blobDat, blobLbl, imRec.shape, lamda], maxfun=1000, factr=1e+7,pgtol=1e-07, iprint=0, disp=1)
imRec = np.reshape(sol[0],((batchSz,ch,h,w)))
#imRec = im2visim(np.copy(imRec))
#imGt = im2visim(np.copy(imDat))
return imRec, imGt
def make_deploy_net(exp, numIter=60000):
imSz = exp.cPrms_.nwPrms['ipImSz']
imSz = [[3, imSz, imSz]]
exp.make_deploy(dataLayerNames=['window_data'],
newDataLayerNames=['data'], delAbove='elevation_fc',
imSz=imSz, batchSz=256, delLayers=['slice_label'])
modelName = exp.get_snapshot_name(numIter=numIter)
if not osp.exists(modelName):
print ('ModelFile %s doesnot exist' % modelName)
return None
net = mp.MyNet(exp.files_['netdefDeploy'], modelName)
#Set preprocessing
net.set_preprocess(ipName='data', chSwap=None, noTransform=True)
return net
def vis_weights(prms, cPrms, numIter, titleName='Streetview Weights'):
deployDir = osp.join(prms.paths.code.dr, 'base_files', 'deploy')
ax1 = plt.subplot(1,1,1)
exp = se.setup_experiment(prms, cPrms)
modelFile = exp.get_snapshot_name(numIter=numIter)
defFile = exp.files_['netdef']
imSz = cPrms.nwPrms.imSz
visDef = mpu.ProtoDef.visproto_from_proto(defFile, imSz=[[3, imSz, imSz]])
deployFile = osp.join(deployDir, 'vis_deploy.prototxt')
visDef.del_layer('slice_pair')
visDef.del_layer('slice_label')
visDef.write(deployFile)
net = mp.MyNet(deployFile, modelFile)
net.vis_weights('conv1', ax=ax1, titleName='Streetview Weights')
def get_pose_net(isGPU=True, deviceId=0):
netName = cfg.POSE_MODEL.NET
defFile = cfg.POSE_MODEL.PROTO
metaFile = cfg.POSE_MODEL.META
net = mp.MyNet(defFile, netName, isGPU=isGPU, deviceId=deviceId)
#Set preprocessing in the net
#As of now the ief net takes RGB images, but the scale net takes BGR
net.set_preprocess(chSwap=None,
meanDat=(117.3785, 117.6438, 110.1771),
ipName='image')
net.set_preprocess(ipName='kp_pos', noTransform=True)
net.set_preprocess(ipName='label', noTransform=True)
#Get the metadata
metaData = pickle.load(open(metaFile, 'r'))
return net, metaData
def test_convert_features():
defFile = '/data1/pulkitag/caffe_models/bvlc_reference/caffenet_deploy.prototxt'
netFile = '/data1/pulkitag/others/tmp.caffemodel'
#netFile = '/data1/pulkitag/others/alex-matconvnet.caffemodel'
net = mp.MyNet(defFile, netFile)
net.set_preprocess(isBlobFormat=True, chSwap=None)
imData = sio.loadmat('/data1/pulkitag/others/ref_imdata.mat',
squeeze_me=True)
imData = imData['im']
imData = imData.transpose((3, 2, 0, 1))
imData = imData[:, [2, 1, 0], :, :]
imData = imData[0:10]
op = net.forward_all(
blobs=['fc8', 'conv1', 'data', 'conv2', 'conv5', 'fc6', 'fc7'],
**{'data': imData})
pdb.set_trace()
def save_for_matconvnet(modelIter=60000,
isSlowness=False,
concatLayer='fc6',
trnSeq=[],
convConcat=False):
modelIter = modelIter
outDir = '/data1/pulkitag/others/joao/'
if trnSeq != []:
trnStr = 'trnSeq' + ''.join(['-%d' % t for t in trnSeq])
else:
trnStr = ''
print(trnStr)
if not isSlowness:
prms = ku.get_prms(poseType='sigMotion',
maxFrameDiff=7,
imSz=None,
isNewExpDir=True,
trnSeq=trnSeq)
cPrms = get_caffe_prms(concatLayer=concatLayer, convConcat=convConcat)
outName = 'kitti-' + trnStr + '-iter' + '%d.mat' % modelIter
else:
prms = ku.get_prms(poseType='slowness',
maxFrameDiff=7,
imSz=None,
lossType='contrastive',
nrmlzType='none',
trnSeq=trnSeq)
cPrms = get_caffe_prms(concatLayer=concatLayer,
contrastiveMargin=1,
convConcat=convConcat)
outName = 'kitti_slowness_concat%s_ctMrgn%d-%d.mat' % (concatLayer, 1,
modelIter)
exp = setup_experiment(prms, cPrms)
snapFile = exp.get_snapshot_name(modelIter)
defFile = '/work4/pulkitag-code/pkgs/caffe-v2-2/modelFiles/kitti/base_files/kitti_finetune_fc6_deploy_input.prototxt'
print snapFile
net = mp.MyNet(defFile, snapFile)
outName = os.path.join(outDir, outName)
matlabRefFile = '/home/carreira/imagenet-caffe-alex.mat'
mpio.save_weights_for_matconvnet(net, outName, matlabRefFile=matlabRefFile)
def setup_net(self, **kwargs):
if self.net_ is None:
self.make(**kwargs)
snapName = self.get_snapshot_name(kwargs['modelIter'])
self.net_ = mp.MyNet(self.files_['netdef'], snapName)
def vis_convert():
defFile = '/data1/pulkitag/caffe_models/bvlc_reference/caffenet_deploy.prototxt'
modelFile = '/data1/pulkitag/others/alex-matconvnet.caffemodel'
net = mp.MyNet(defFile, modelFile)
net.vis_weights('conv1')
def get_accuracy(numIter=30000,
imSz=256,
poseType='euler',
nrmlzType='zScoreScaleSeperate',
isScratch=True,
concatLayer='pool5',
numBatches=10):
'''
Determines the accuracy of the network in predicting stuff
'''
wFile, defFile = get_weight_proto_file(numIter=numIter,
imSz=imSz,
poseType=poseType,
nrmlzType=nrmlzType,
isScratch=isScratch,
concatLayer=concatLayer,
isDeploy=True)
print "Intializing Network"
net = mp.MyNet(defFile, wFile)
expName = 'consequent_pose-%s_nrmlz-%s_imSz%d' % (poseType, nrmlzType,
imSz)
lblNames = ['translation_label', 'euler_label']
predNames = ['translation_fc7', 'euler_fc7']
data = {}
for name in lblNames + predNames:
data[name] = []
print "Calculating Features"
for i in range(numBatches):
blobs = net.forward_all(blobs=lblNames + predNames, noInputs=True)
for name in lblNames + predNames:
data[name].append(blobs[name].squeeze())
#pdb.set_trace()
for name in lblNames + predNames:
data[name] = np.concatenate(data[name], axis=0)
print "Plotting Results"
plt.ion()
figT = plt.figure()
plt.title('Relative Translation')
plot_triplets(data['translation_label'],
fig=figT,
isDashed=True,
linewidth=1.0,
labels=['gtDeltaX', 'gtDeltaY', 'gtDeltaZ'])
plot_triplets(data['translation_fc7'],
fig=figT,
isDashed=False,
linewidth=1.0,
labels=['predDeltaX', 'predDeltaY', 'predDeltaZ'])
#Z, Y and X have been flipped - it should be X, Y, Z
figR = plt.figure()
plt.title('Relative Rotation')
plot_triplets(data['euler_label'],
fig=figR,
isDashed=True,
linewidth=1.0,
labels=['gtThetaX', 'gtThetaY', 'gtThetaZ'])
plot_triplets(data['euler_fc7'],
fig=figR,
isDashed=False,
linewidth=1.0,
labels=['predThetaX', 'predThetaY', 'predThetaZ'])
def save_alexnet_york_feats(dataset='york'):
#Read the images
dataDir = '/work5/pulkitag/data_sets/streetview'
if dataset == 'york':
dirName = '/work5/pulkitag/data_sets/streetview/york/York_VP_imOnly'
outDir = '/work5/pulkitag/data_sets/streetview/york/feats-alexnet/'
elif dataset == 'york_others':
dirName = '/work5/pulkitag/data_sets/streetview/york_and_others/tvp_pku_york_ims'
outDir = '/work5/pulkitag/data_sets/streetview/york_and_others/feats-alexnet/'
elif dataset == 'gen_test':
dirName = '/work5/pulkitag/data_sets/streetview/gen_test/gen_test_imgs'
outDir = '/work5/pulkitag/data_sets/streetview/gen_test/feats-alexnet/'
elif dataset == 'places':
dirName = osp.join(dataDir, 'places', 'images')
outDir = osp.join(dataDir, 'places', 'feats-alexnet')
if not osp.exists(outDir):
os.makedirs(outDir)
prefix = [f[0:-4] for f in os.listdir(dirName) if '.jpg' in f]
imFile = [osp.join(dirName, p + '.jpg') for p in prefix]
outName = [osp.join(outDir, p + '.pkl') for p in prefix]
#Setup the Net
mainDataDr = cfg.STREETVIEW_DATA_MAIN
meanFile = osp.join(mainDataDr,
'pulkitag/caffe_models/ilsvrc2012_mean.binaryproto')
netFile = osp.join(
mainDataDr,
'pulkitag/caffe_models/bvlc_reference/bvlc_reference_caffenet.caffemodel'
)
defFile = osp.join(
mainDataDr,
'pulkitag/caffe_models/bvlc_reference/caffenet_full_deploy.prototxt')
batchSz = 256
testNet = mp.MyNet(defFile, netFile, caffe.TEST)
testNet.set_preprocess(ipName='data',
isBlobFormat=False,
imageDims=(227, 227, 3),
cropDims=(227, 227),
chSwap=(2, 1, 0),
rawScale=None,
meanDat=meanFile)
#Send images and get features
for st in range(0, len(imFile), batchSz):
en = min(len(imFile), st + batchSz)
ims = []
skipList = []
for i in range(st, en):
im = scm.imread(imFile[i])
im = scm.imresize(im, (227, 227))
if im.ndim == 2:
skipList.append(i)
print('SKIPPING GRAY SCALE IMAGE')
continue
if im.ndim == 3 and im.shape[2] == 1:
skipList.append(i)
print('SKIPPING GRAY SCALE IMAGE')
continue
ims.append(im.reshape((1, ) + im.shape))
ims = np.concatenate(ims)
print ims.shape
feats = testNet.forward_all(blobs=['fc7'], **{'data': ims})
count = 0
for i in range(st, en):
#pickle.dump({'pool5': feats['pool5'][i-st], 'fc6': feats['fc6'][i-st],
# 'fc7': feats['fc7'][i-st]}, open(outName[i], 'w'))
if i in skipList:
continue
pickle.dump({'fc7': feats['fc7'][count]}, open(outName[i], 'w'))
count += 1
