def test_zf_saliency(dataSet='mnist', stride=2, patchSz=5): if dataSet == 'mnist': defFile = '/work4/pulkitag-code/pkgs/caffe-v2-2/modelFiles/mnist/hdf5_test/lenet.prototxt' modelFile, _ = mp.get_model_mean_file('lenet') net = mp.MyNet(defFile, modelFile, isGPU=False) N = net.get_batchsz() net.set_preprocess(chSwap=None, imageDims=(28, 28, 1), isBlobFormat=True) h5File = '/data1/pulkitag/mnist/h5store/test/batch1.h5' fid = h5py.File(h5File, 'r') data = fid['data'] data = data[0:N] #Do the saliency imSal, score = mpu.zf_saliency(net, data, 10, 'ip2', patchSz=patchSz, stride=stride) gtLabels = fid['label'] else: netName = 'bvlcAlexNet' opLayer = 'fc8' defFile = mp.get_layer_def_files(netName, layerName=opLayer) modelFile, meanFile = mp.get_model_mean_file(netName) net = mp.MyNet(defFile, modelFile) net.set_preprocess(imageDims=(256, 256, 3), meanDat=meanFile, rawScale=255, isBlobFormat=True) ilDat = mpu.ILSVRC12Reader() ilDat.set_count(2) data, gtLabels, syn, words = ilDat.read() data = data.reshape((1, data.shape[0], data.shape[1], data.shape[2])) data = data.transpose((0, 3, 1, 2)) print data.shape imSal, score = mpu.zf_saliency(net, data, 1000, 'fc8', patchSz=patchSz, stride=stride) pdLabels = np.argmax(score.squeeze(), axis=1) return data, imSal, pdLabels, gtLabels
def get_scale_net(isGPU=True, deviceId=0): netName = cfg.SCALE_MODEL.NET defFile = cfg.SCALE_MODEL.PROTO net = mp.MyNet(defFile, netName, isGPU=isGPU, deviceId=deviceId) #Set preprocessing in the net net.set_preprocess(meanDat=(115.2254, 123.9648, 124.2966)) return net
def reconstruct_optimal_input(exp, modelIter, im, recLayer='conv1', imH=101, imW=101, cropH=101, cropW=101, channels=3, meanFile=None, lamda=1e-8, batchSz=1, **kwargs): exp = copy.deepcopy(exp) kwargs['delAbove'] = recLayer #Setup the original network origNet = mpu.CaffeTest.from_caffe_exp(exp) origNet.setup_network(opNames=recLayer, imH=imH, imW=imW, cropH=cropH, cropW=cropW, modelIterations=modelIter, batchSz=batchSz, isAccuracyTest=False, meanFile=meanFile, **kwargs) #Get the size of the features in the layer that needs to be reconstructed #Shape of the layer to be reconstructed blob = origNet.net_.net.blobs[recLayer] initBlobDat = np.zeros((blob.num, blob.channels, blob.height, blob.width)).astype('float32') blobLbl = np.zeros((blob.num, 1, 1, 1)).astype('float32') recShape = (blob.num, blob.channels, blob.height, blob.width) #Get the initial layer features print "Extracting Initial Features" blobDat = np.zeros((blob.num, blob.channels, blob.height, blob.width)).astype('float32') if im.ndim == 3: imIp = im.reshape((batchSz,) + im.shape) else: imIp = im feats = origNet.net_.forward_all(blobs=[recLayer], data=imIp) blobDat = feats[recLayer] #imDat = np.asarray(imDat) #Get the net for reconstruvtions #print (exp.expFile_.netDef_.get_all_layernames()) recProto = mpu.ProtoDef.recproto_from_proto(exp.expFile_.netDef_, featDims=recShape, imSz=[[channels, cropH, cropW]], batchSz=batchSz, **kwargs) recProto.write(exp.files_['netdefRec']) recModel = exp.get_snapshot_name(modelIter) recNet = edict() recNet.net_ = mp.MyNet(exp.files_['netdefRec'], recModel, caffe.TRAIN) #recNet = mpu.CaffeTest.from_model(exp.files_['netdefRec'], recModel) #kwargs['dataLayerNames'] = ['data'] #kwargs['newDataLayerNames'] = None #recNet.setup_network(opNames=[recLayer], imH=imH, imW=imW, cropH=cropH, cropW=cropW, # modelIterations=modelIter, isAccuracyTest=False, meanFile=meanFile, # testMode=False, **kwargs) recNet.net_.net.set_force_backward(recLayer) #Start the reconstruction ch,h,w = imIp.shape[3], imIp.shape[1], imIp.shape[2] imRec = 255*np.random.random((batchSz,ch,h,w)).astype('float32') print imRec.shape, blobDat.shape, blobLbl.shape sol = scipy.optimize.fmin_l_bfgs_b(rec_fun_grad, imRec.flatten(),args=[recNet, blobDat, blobLbl, imRec.shape, lamda], maxfun=1000, factr=1e+7,pgtol=1e-07, iprint=0, disp=1) imRec = np.reshape(sol[0],((batchSz,ch,h,w))) #imRec = im2visim(np.copy(imRec)) #imGt = im2visim(np.copy(imDat)) return imRec, imGt
def make_deploy_net(exp, numIter=60000): imSz = exp.cPrms_.nwPrms['ipImSz'] imSz = [[3, imSz, imSz]] exp.make_deploy(dataLayerNames=['window_data'], newDataLayerNames=['data'], delAbove='elevation_fc', imSz=imSz, batchSz=256, delLayers=['slice_label']) modelName = exp.get_snapshot_name(numIter=numIter) if not osp.exists(modelName): print ('ModelFile %s doesnot exist' % modelName) return None net = mp.MyNet(exp.files_['netdefDeploy'], modelName) #Set preprocessing net.set_preprocess(ipName='data', chSwap=None, noTransform=True) return net
def vis_weights(prms, cPrms, numIter, titleName='Streetview Weights'): deployDir = osp.join(prms.paths.code.dr, 'base_files', 'deploy') ax1 = plt.subplot(1,1,1) exp = se.setup_experiment(prms, cPrms) modelFile = exp.get_snapshot_name(numIter=numIter) defFile = exp.files_['netdef'] imSz = cPrms.nwPrms.imSz visDef = mpu.ProtoDef.visproto_from_proto(defFile, imSz=[[3, imSz, imSz]]) deployFile = osp.join(deployDir, 'vis_deploy.prototxt') visDef.del_layer('slice_pair') visDef.del_layer('slice_label') visDef.write(deployFile) net = mp.MyNet(deployFile, modelFile) net.vis_weights('conv1', ax=ax1, titleName='Streetview Weights')
def get_pose_net(isGPU=True, deviceId=0): netName = cfg.POSE_MODEL.NET defFile = cfg.POSE_MODEL.PROTO metaFile = cfg.POSE_MODEL.META net = mp.MyNet(defFile, netName, isGPU=isGPU, deviceId=deviceId) #Set preprocessing in the net #As of now the ief net takes RGB images, but the scale net takes BGR net.set_preprocess(chSwap=None, meanDat=(117.3785, 117.6438, 110.1771), ipName='image') net.set_preprocess(ipName='kp_pos', noTransform=True) net.set_preprocess(ipName='label', noTransform=True) #Get the metadata metaData = pickle.load(open(metaFile, 'r')) return net, metaData
def test_convert_features(): defFile = '/data1/pulkitag/caffe_models/bvlc_reference/caffenet_deploy.prototxt' netFile = '/data1/pulkitag/others/tmp.caffemodel' #netFile = '/data1/pulkitag/others/alex-matconvnet.caffemodel' net = mp.MyNet(defFile, netFile) net.set_preprocess(isBlobFormat=True, chSwap=None) imData = sio.loadmat('/data1/pulkitag/others/ref_imdata.mat', squeeze_me=True) imData = imData['im'] imData = imData.transpose((3, 2, 0, 1)) imData = imData[:, [2, 1, 0], :, :] imData = imData[0:10] op = net.forward_all( blobs=['fc8', 'conv1', 'data', 'conv2', 'conv5', 'fc6', 'fc7'], **{'data': imData}) pdb.set_trace()
def save_for_matconvnet(modelIter=60000, isSlowness=False, concatLayer='fc6', trnSeq=[], convConcat=False): modelIter = modelIter outDir = '/data1/pulkitag/others/joao/' if trnSeq != []: trnStr = 'trnSeq' + ''.join(['-%d' % t for t in trnSeq]) else: trnStr = '' print(trnStr) if not isSlowness: prms = ku.get_prms(poseType='sigMotion', maxFrameDiff=7, imSz=None, isNewExpDir=True, trnSeq=trnSeq) cPrms = get_caffe_prms(concatLayer=concatLayer, convConcat=convConcat) outName = 'kitti-' + trnStr + '-iter' + '%d.mat' % modelIter else: prms = ku.get_prms(poseType='slowness', maxFrameDiff=7, imSz=None, lossType='contrastive', nrmlzType='none', trnSeq=trnSeq) cPrms = get_caffe_prms(concatLayer=concatLayer, contrastiveMargin=1, convConcat=convConcat) outName = 'kitti_slowness_concat%s_ctMrgn%d-%d.mat' % (concatLayer, 1, modelIter) exp = setup_experiment(prms, cPrms) snapFile = exp.get_snapshot_name(modelIter) defFile = '/work4/pulkitag-code/pkgs/caffe-v2-2/modelFiles/kitti/base_files/kitti_finetune_fc6_deploy_input.prototxt' print snapFile net = mp.MyNet(defFile, snapFile) outName = os.path.join(outDir, outName) matlabRefFile = '/home/carreira/imagenet-caffe-alex.mat' mpio.save_weights_for_matconvnet(net, outName, matlabRefFile=matlabRefFile)
def setup_net(self, **kwargs): if self.net_ is None: self.make(**kwargs) snapName = self.get_snapshot_name(kwargs['modelIter']) self.net_ = mp.MyNet(self.files_['netdef'], snapName)
def vis_convert(): defFile = '/data1/pulkitag/caffe_models/bvlc_reference/caffenet_deploy.prototxt' modelFile = '/data1/pulkitag/others/alex-matconvnet.caffemodel' net = mp.MyNet(defFile, modelFile) net.vis_weights('conv1')
def get_accuracy(numIter=30000, imSz=256, poseType='euler', nrmlzType='zScoreScaleSeperate', isScratch=True, concatLayer='pool5', numBatches=10): ''' Determines the accuracy of the network in predicting stuff ''' wFile, defFile = get_weight_proto_file(numIter=numIter, imSz=imSz, poseType=poseType, nrmlzType=nrmlzType, isScratch=isScratch, concatLayer=concatLayer, isDeploy=True) print "Intializing Network" net = mp.MyNet(defFile, wFile) expName = 'consequent_pose-%s_nrmlz-%s_imSz%d' % (poseType, nrmlzType, imSz) lblNames = ['translation_label', 'euler_label'] predNames = ['translation_fc7', 'euler_fc7'] data = {} for name in lblNames + predNames: data[name] = [] print "Calculating Features" for i in range(numBatches): blobs = net.forward_all(blobs=lblNames + predNames, noInputs=True) for name in lblNames + predNames: data[name].append(blobs[name].squeeze()) #pdb.set_trace() for name in lblNames + predNames: data[name] = np.concatenate(data[name], axis=0) print "Plotting Results" plt.ion() figT = plt.figure() plt.title('Relative Translation') plot_triplets(data['translation_label'], fig=figT, isDashed=True, linewidth=1.0, labels=['gtDeltaX', 'gtDeltaY', 'gtDeltaZ']) plot_triplets(data['translation_fc7'], fig=figT, isDashed=False, linewidth=1.0, labels=['predDeltaX', 'predDeltaY', 'predDeltaZ']) #Z, Y and X have been flipped - it should be X, Y, Z figR = plt.figure() plt.title('Relative Rotation') plot_triplets(data['euler_label'], fig=figR, isDashed=True, linewidth=1.0, labels=['gtThetaX', 'gtThetaY', 'gtThetaZ']) plot_triplets(data['euler_fc7'], fig=figR, isDashed=False, linewidth=1.0, labels=['predThetaX', 'predThetaY', 'predThetaZ'])
def save_alexnet_york_feats(dataset='york'): #Read the images dataDir = '/work5/pulkitag/data_sets/streetview' if dataset == 'york': dirName = '/work5/pulkitag/data_sets/streetview/york/York_VP_imOnly' outDir = '/work5/pulkitag/data_sets/streetview/york/feats-alexnet/' elif dataset == 'york_others': dirName = '/work5/pulkitag/data_sets/streetview/york_and_others/tvp_pku_york_ims' outDir = '/work5/pulkitag/data_sets/streetview/york_and_others/feats-alexnet/' elif dataset == 'gen_test': dirName = '/work5/pulkitag/data_sets/streetview/gen_test/gen_test_imgs' outDir = '/work5/pulkitag/data_sets/streetview/gen_test/feats-alexnet/' elif dataset == 'places': dirName = osp.join(dataDir, 'places', 'images') outDir = osp.join(dataDir, 'places', 'feats-alexnet') if not osp.exists(outDir): os.makedirs(outDir) prefix = [f[0:-4] for f in os.listdir(dirName) if '.jpg' in f] imFile = [osp.join(dirName, p + '.jpg') for p in prefix] outName = [osp.join(outDir, p + '.pkl') for p in prefix] #Setup the Net mainDataDr = cfg.STREETVIEW_DATA_MAIN meanFile = osp.join(mainDataDr, 'pulkitag/caffe_models/ilsvrc2012_mean.binaryproto') netFile = osp.join( mainDataDr, 'pulkitag/caffe_models/bvlc_reference/bvlc_reference_caffenet.caffemodel' ) defFile = osp.join( mainDataDr, 'pulkitag/caffe_models/bvlc_reference/caffenet_full_deploy.prototxt') batchSz = 256 testNet = mp.MyNet(defFile, netFile, caffe.TEST) testNet.set_preprocess(ipName='data', isBlobFormat=False, imageDims=(227, 227, 3), cropDims=(227, 227), chSwap=(2, 1, 0), rawScale=None, meanDat=meanFile) #Send images and get features for st in range(0, len(imFile), batchSz): en = min(len(imFile), st + batchSz) ims = [] skipList = [] for i in range(st, en): im = scm.imread(imFile[i]) im = scm.imresize(im, (227, 227)) if im.ndim == 2: skipList.append(i) print('SKIPPING GRAY SCALE IMAGE') continue if im.ndim == 3 and im.shape[2] == 1: skipList.append(i) print('SKIPPING GRAY SCALE IMAGE') continue ims.append(im.reshape((1, ) + im.shape)) ims = np.concatenate(ims) print ims.shape feats = testNet.forward_all(blobs=['fc7'], **{'data': ims}) count = 0 for i in range(st, en): #pickle.dump({'pool5': feats['pool5'][i-st], 'fc6': feats['fc6'][i-st], # 'fc7': feats['fc7'][i-st]}, open(outName[i], 'w')) if i in skipList: continue pickle.dump({'fc7': feats['fc7'][count]}, open(outName[i], 'w')) count += 1