def feat_extract(device_id=0, img_list_path='', batchsize=290, flag='', net='./deploy_33.prototxt', model='../snapshot/model_iter_100000.caffemodel', iter=100000 , blob='normed_feature'):
f_path = './'+'feat_'+os.path.basename(img_list_path).split('.')[0]+flag+'/iter'+str(iter)+'/'+blob
if not os.path.exists(f_path):
os.makedirs(f_path)
dp = DataProvider(batchsize=batchsize, path=img_list_path)
img_list_len=len(dp.img_dict)
net = caffe.Net(net, model, caffe.TEST)
caffe.set_device(device_id)
caffe.set_mode_gpu()
feature = np.zeros((batchsize*100,net.blobs[blob].data.shape[1]))
img_name = []
index = 0
epoch = 0
counter = 1
while epoch==0:
img_np, labels, epoch, img_name_batch= dp.get_batch_vec()
net.blobs['data'].data[...] = img_np
net.forward()
index += 1
feature[batchsize*(index-1):batchsize*index,:] = net.blobs[blob].data.reshape(batchsize,-1)
img_name+=img_name_batch
if epoch==1:
path = os.path.join(f_path, str(counter)+'.npz')
np.savez(path,feat=feature[:(img_list_len-(counter-1)*batchsize*100),:],img=img_name)
elif index == 100:
path = os.path.join(f_path, str(counter)+'.npz')
np.savez(path,feat=feature,img=img_name)
index = 0
img_name = []
counter +=1
def feat_extract(device_id=7, img_list_path='', batchsize=96, net='./deploy_no_part.prototxt', model='./models/models_iter_2000.caffemodel', iter=2000 , blob='fc7_n'):
f_path = '/data1/qtang/samsung/train_no_part/'+'feat_'+os.path.basename(img_list_path).split('.')[0]+'/iter'+str(iter)+'/'+blob
if not os.path.exists(f_path):
os.makedirs(f_path)
dp = DataProvider(batchsize=batchsize, path=img_list_path)
img_list_len=len(dp.img_dict)
net = caffe.Net(net, model, caffe.TEST)
caffe.set_device(device_id)
caffe.set_mode_gpu()
if blob == 'fc7_n':
feature = np.zeros((batchsize*100,298))
else:
feature = np.zeros((batchsize*100,4096))
img_name = []
index = 0
epoch = 0
counter = 1
while epoch==0:
img_np, labels, epoch, img_name_batch= dp.get_batch_vec()
net.blobs['top_0'].data[...] = img_np
net.blobs['top_1'].data[...] = labels
net.forward()
index += 1
feature[batchsize*(index-1):batchsize*index,:] = net.blobs[blob].data
img_name+=img_name_batch
if epoch==1:
path = os.path.join(f_path, str(counter)+'.npz')
np.savez(path,feat=feature[:(img_list_len-(counter-1)*batchsize*100),:],img=img_name)
elif index == 100:
path = os.path.join(f_path, str(counter)+'.npz')
np.savez(path,feat=feature,img=img_name)
index = 0
img_name = []
counter +=1
def feat_extract(device_id=2, img_list_path='', batchsize=256, net='./deploy_lda_siamese.prototxt', model='./197_iter_500.caffemodel', iter=500 , blob='pool6'):
f_path = './'+'feat_'+os.path.basename(img_list_path).split('.')[0]+'/iter'+str(iter)+'/'+blob
if not os.path.exists(f_path):
os.makedirs(f_path)
dp = DataProvider(batchsize=batchsize, path=img_list_path)
img_list_len=len(dp.img_dict)
net = caffe.Net(net, model, caffe.TEST)
caffe.set_device(device_id)
caffe.set_mode_gpu()
if blob == 'fc7_n':
feature = np.zeros((batchsize*100,298))
else:
feature = np.zeros((batchsize*100,8192))
img_name = []
block_label = []
index = 0
epoch = 0
counter = 1
while epoch==0:
img_np, labels, epoch, img_name_batch = dp.get_batch_vec()
net.blobs['top_0'].data[...] = img_np
net.forward()
index += 1
feature[batchsize*(index-1):batchsize*index,:] = net.blobs[blob].data.reshape(batchsize,-1)
img_name+=img_name_batch
block_label+=list(labels)
if epoch==1:
block_feat = feature[:(img_list_len-(counter-1)*batchsize*100),:]
block_feat /= np.sqrt((block_feat**2).sum(axis=1)).reshape(-1,1)
block_label = block_label[:(img_list_len-(counter-1)*batchsize*100)]
img_name = img_name[:(img_list_len-(counter-1)*batchsize*100)]
path = os.path.join(f_path, str(counter)+'.npz')
np.savez(path, feat=block_feat, img=img_name)
elif index == 100:
block_feat = feature
block_feat /= np.sqrt((block_feat**2).sum(axis=1)).reshape(-1,1)
path = os.path.join(f_path, str(counter)+'.npz')
np.savez(path,feat=block_feat,img=img_name)
index = 0
img_name = []
block_label = []
counter +=1
def test_acc(device_id, model, prefix=''):
net = caffe.Net('./part_val.prototxt', model, caffe.TEST)
caffe.set_device(device_id)
caffe.set_mode_gpu()
dp = DataProvider(batchsize = 64, img_size=227)
epoch = 0
loss_list = []
acc_layer = []
while epoch == 0:
img_nps_1, img_nps_2, img_nps_3, labels, epoch = dp.get_batch_vec()
net.blobs['top_0'].data[...] = img_nps_1
net.blobs['top_1'].data[...] = img_nps_2
net.blobs['top_2'].data[...] = img_nps_3
net.blobs['top_3'].data[...] = labels
net.forward()
loss_list.append(net.blobs['loss'].data)
acc_layer.append(net.blobs['accuracy'].data)
mean_loss = np.array(loss_list).mean()
acc = np.array(acc_layer).mean()
return acc, mean_loss
def feat_extract(device_id=2,
img_list_path='',
batchsize=64,
net='./part_val.prototxt',
model='./models/vggs_fc6weights/_iter_7600.caffemodel'):
f_path = '/data1/qtang/samsung/part_train/' + 'feat_' + os.path.basename(
img_list_path).split('.')[0] + '/vggs_fc6weights/iter7600/concate'
if not os.path.exists(f_path):
os.makedirs(f_path)
dp = DataProvider(batchsize=batchsize, path=img_list_path)
net = caffe.Net(net, model, caffe.TEST)
caffe.set_device(device_id)
caffe.set_mode_gpu()
feature = np.zeros((batchsize * 100, 12288))
img_name = []
index = 0
epoch = 0
counter = 1
while epoch == 0:
img_nps_1, img_nps_2, img_nps_3, labels, epoch, img_name_batch = dp.get_batch_vec(
)
net.blobs['top_0'].data[...] = img_nps_1
net.blobs['top_1'].data[...] = img_nps_2
net.blobs['top_2'].data[...] = img_nps_3
net.blobs['top_3'].data[...] = labels
net.forward()
index += 1
feature[batchsize * (index - 1):batchsize *
index, :] = net.blobs['concate'].data
img_name += img_name_batch
if epoch == 1:
path = os.path.join(f_path, str(counter) + '.npz')
np.savez(path, feat=feature[:index * batchsize, :], img=img_name)
elif index == 100:
path = os.path.join(f_path, str(counter) + '.npz')
np.savez(path, feat=feature, img=img_name)
index = 0
img_name = []
counter += 1
def feat_extract(device_id=2, img_list_path='', batchsize=48, net='./org_part_deploy.prototxt', iter=6000 , blob='fc8'):
f_path = '/data1/qtang/samsung/org_part/'+'feat_'+os.path.basename(img_list_path).split('.')[0]+'/iter'+str(iter)+'/'+blob
if not os.path.exists(f_path):
os.makedirs(f_path)
dp = DataProvider(batchsize=batchsize, path=img_list_path)
model='./models/_iter_%d.caffemodel'%iter
net = caffe.Net(net, model, caffe.TEST)
caffe.set_device(device_id)
caffe.set_mode_gpu()
if blob == 'concat2':
feature = np.zeros((batchsize*100,8192))
else:
feature = np.zeros((batchsize*100,298))
img_name = []
index = 0
epoch = 0
counter = 1
while epoch==0:
img_np_1,img_np_2,img_np_3,labels,img_np_org,epoch, img_name_batch= dp.get_batch_vec()
net.blobs['top_0'].data[...] = img_np_1
net.blobs['top_1'].data[...] = img_np_2
net.blobs['top_2'].data[...] = img_np_3
net.blobs['top_3'].data[...] = labels
net.blobs['top_4'].data[...] = img_np_org
net.forward()
index += 1
feature[batchsize*(index-1):batchsize*index,:] = net.blobs[blob].data
img_name+=img_name_batch
if epoch==1:
path = os.path.join(f_path, str(counter)+'.npz')
np.savez(path,feat=feature[:index*batchsize,:],img=img_name)
elif index == 100:
path = os.path.join(f_path, str(counter)+'.npz')
np.savez(path,feat=feature,img=img_name)
index = 0
img_name = []
counter +=1
import caffe
import numpy as np
from data_provider_layer import DataProvider
net=caffe.Net('./deploy_no_part.prototxt', './models/models_iter_8000.caffemodel', caffe.TEST)
caffe.set_mode_gpu()
caffe.set_device(2)
dp = DataProvider(batchsize=64)
img_len = len(dp.img_dict)
feat = np.zeros((img_len,298))
epoch = 0
batch_counter = 0
while epoch == 0:
img_np, labels, epoch, img_name = dp.get_batch_vec()
net.blobs['top_0'].data[...] = img_np
net.blobs['top_1'].data[...] = labels
net.forward()
batch_counter += 1
if epoch == 1:
feat[64*(batch_counter-1):,:] = net.blobs['fc7_n'].data[:img_len-64*batch_counter,:]
else:
feat[64*(batch_counter-1):64*batch_counter,:] = net.blobs['fc7_n'].data
feat /= np.sqrt((feat**2).sum(axis=1)).reshape(-1,1)
var = np.var(feat, axis=1)
mean = np.mean(feat, axis=1)
max_ = feat.max(axis=1)
min_ = feat.min(axis=1)
np.save('./var_no_part',var)
net = caffe.Net(
'./part_val.prototxt',
'/data1/qtang/samsung/part_train/models/new_fc6weights/_iter_8000.caffemodel',
caffe.TEST)
caffe.set_mode_gpu()
caffe.set_device(2)
dp = DataProvider(batchsize=64)
img_len = len(dp.img_dict)
feat = np.zeros((img_len, 298))
epoch = 0
batch_counter = 0
while epoch == 0:
img_np_1, img_np_2, img_np_3, labels, epoch, img_name = dp.get_batch_vec()
net.blobs['top_0'].data[...] = img_np_1
net.blobs['top_1'].data[...] = img_np_2
net.blobs['top_2'].data[...] = img_np_3
net.blobs['top_3'].data[...] = labels
net.forward()
batch_counter += 1
if epoch == 1:
feat[64 *
(batch_counter - 1):, :] = net.blobs['fc7'].data[:img_len - 64 *
batch_counter, :]
else:
feat[64 * (batch_counter - 1):64 *
batch_counter, :] = net.blobs['fc7'].data
feat /= np.sqrt((feat**2).sum(axis=1)).reshape(-1, 1)