def test(net_file,model_topdir,predict_file,gpunum,trialnum,outdir,keyword):
caffe.set_device(gpunum)
caffe.set_mode_gpu()
best_trial,best_iter = getBestRunAll(model_topdir,trialnum,'train.err',keyword)
model_dir = join(model_topdir,'trial'+str(best_trial))
model_file = os.path.join(model_dir,'train_iter_'+best_iter+'.caffemodel')
system(' '.join(['cp',model_file,join(outdir,'bestiter.caffemodel')]))
outfile = os.path.join(outdir,'bestiter.pred')
with open(join(outdir,'bestiter.info'),'w') as f:
f.write('best_trial\tbest_iter\n')
f.write('%d\t%s\n' % (best_trial,best_iter))
net = caffe.Net(net_file, model_file,caffe.TEST)
predict_dir = os.path.dirname(predict_file)
with open(predict_file,'r') as f:
files = [x.strip() for x in f]
with open(outfile,'w') as f:
for batchfile in files:
fi = h5py.File(batchfile, 'r')
dataset = np.asarray(fi['data'])
out = net.forward_all(data=dataset)
prob = np.vstack(np.asarray(out['prob']))
for out in prob:
f.write('%s\n' % '\t'.join([str(x) for x in out]))
def setCaffeMode(gpu, device = 0):
"""Initialise caffe"""
if gpu:
caffe.set_mode_gpu()
caffe.set_device(device)
else:
caffe.set_mode_cpu()
def justCheckGradients(solver_file,deploy_file,model_file):
model=model_file
print model
print os.path.exists(model);
caffe.set_device(1)
caffe.set_mode_gpu()
solver=caffe.SGDSolver(solver_file);
solver.net.forward();
net=caffe.Net(deploy_file,model);
print list(net._layer_names);
print net.blobs.keys();
# return
net.blobs['data'].data[...]=solver.net.blobs['data'].data;
net.blobs['thelabelscoarse'].data[...]=solver.net.blobs['thelabelscoarse'].data;
net.forward();
# print net.blobs['thelabelscoarse'].data[:10,0,0,0,0];
# print net.blobs['reshapefc8'].data[0,39,0,:]
net.backward();
# print net.blobs.keys();
layers_to_explore=['conv1','conv2','conv3','conv4','conv5','fc6_fix','fc7_fix','fc8_fix']
ratios=getRatios(net,layers_to_explore);
for layer_name in ratios.keys():
print layer_name,ratios[layer_name];
def model_creation_fromsuperclass(model_name):
caffe.set_device(1)
caffe.set_mode_gpu()
in_model_name = root + 'training/train_val_superevent.prototxt'
in_weight_name = root + 'snapshot/'+model_name
out_weight_name = root + 'snapshot/segment_fromsuperclass_frombegin_iter_100000_start.caffemodel'
print out_weight_name
in_net = caffe.Net(in_model_name,
in_weight_name,
caffe.TEST)
out_model_name = root + 'train_val_fromsuper.prototxt'
out_net = caffe.Net(out_model_name,
in_weight_name,
caffe.TEST)
# a = in_net.params['conv1'][0].data
# b = in_net.params['conv1'][1].data
# print a.shape, b.shape
a = in_net.params['fc8_multevent1'][0].data
b = np.zeros((23,a.shape[1]))
for i in xrange(23):
b[i,:] = a[dict_subcategory2[i],:]
out_net.params['fc8_multevent2'][0].data[...] = b
out_net.params['fc8_multevent2_p'][0].data[...] = b
a = in_net.params['fc8_multevent1'][1].data
b = np.zeros((23,))
for i in xrange(23):
b[i] = a[dict_subcategory2[i]]
out_net.params['fc8_multevent2'][1].data[...] = b
out_net.params['fc8_multevent2_p'][1].data[...] = b
out_net.save(out_weight_name)
def init(self):
image_net = self.image_net
caffe.set_device(0)
caffe.set_mode_gpu()
if image_net == 'caffenet':
convnet_proto = './caffe/models/bvlc_reference_caffenet/deploy.prototxt'
convnet_model = './caffe/models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel'
elif image_net == 'vggnet':
convnet_proto = './caffe/models/vggnet/VGG_ILSVRC_16_layers_deploy.prototxt'
convnet_model = './caffe/models/vggnet/VGG_ILSVRC_16_layers.caffemodel'
else:
raise StandardError('Unknown CNN %s' % image_net)
self.net = caffe.Net(convnet_proto, convnet_model, caffe.TEST)
if image_net == 'caffenet':
self.transformer = caffe.io.Transformer({'data': self.net.blobs['data'].data.shape})
self.transformer.set_transpose('data', (2, 0, 1))
self.transformer.set_mean('data', np.array([104, 117, 123]))
self.transformer.set_raw_scale('data', 255)
self.transformer.set_channel_swap('data', (2, 1, 0))
self.BATCH_SIZE = 100
self.net.blobs['data'].reshape(self.BATCH_SIZE, 3, 227, 227)
elif image_net == 'vggnet':
self.transformer = caffe.io.Transformer({'data': self.net.blobs['data'].data.shape})
self.transformer.set_transpose('data', (2, 0, 1))
self.transformer.set_mean('data', np.array([103.939, 116.779, 123.68]))
self.transformer.set_raw_scale('data', 255)
self.transformer.set_channel_swap('data', (2, 1, 0))
self.BATCH_SIZE = 100
self.net.blobs['data'].reshape(self.BATCH_SIZE, 3, 224, 224)
self.image_net = image_net
self.initialized = True
print "Done initializing image feature extractor"
def main(input, output, disp, gpu):
make_sure_path_exists(input)
make_sure_path_exists(output)
# should be picked up by caffe by default, but just in case
# add by macpod
if gpu:
caffe.set_mode_gpu();
caffe.set_device(0);
frame = np.float32(PIL.Image.open(input+'/0001.jpg'))
frame_i = 1
# let max nr of frames
nrframes =len([name for name in os.listdir('./input') if os.path.isfile(name)])
for i in xrange(frame_i,nrframes):
frame = deepdream(
net, frame, end = layersloop[frame_i % len(layersloop)], disp=disp, iter_n=5)
saveframe = output + "/%04d.jpg" % frame_i
PIL.Image.fromarray(np.uint8(frame)).save(saveframe)
newframe = input + "/%04d.jpg" % frame_i
frame = morphPicture(saveframe, newframe) # give it back 50% of original picture
frame = np.float32(frame)
frame_i += 1
def detectFace(img_path,threshold):
img = cv2.imread(img_path)
caffe_img = img.copy()-128
origin_h,origin_w,ch = caffe_img.shape
scales = tools.calculateScales(img)
out = []
for scale in scales:
hs = int(origin_h*scale)
ws = int(origin_w*scale)
scale_img = cv2.resize(caffe_img,(ws,hs))
scale_img = np.swapaxes(scale_img, 0, 2)
net_12.blobs['data'].reshape(1,3,ws,hs)
net_12.blobs['data'].data[...]=scale_img
caffe.set_device(0)
caffe.set_mode_gpu()
out_ = net_12.forward()
out.append(out_)
image_num = len(scales)
rectangles = []
for i in range(image_num):
cls_prob = out[i]['cls_score'][0][1]
roi = out[i]['conv4-2'][0]
out_h,out_w = cls_prob.shape
out_side = max(out_h,out_w)
rectangle = tools.detect_face_12net(cls_prob,roi,out_side,1/scales[i],origin_w,origin_h,threshold[0])
rectangles.extend(rectangle)
return rectangles
def evaluate_caffe_nn(train_data, valid_data, test_data):
import caffe
from caffe import layers as L, params as P
import caffe_utils as utils
def gen_net(net_path, data_shape, label_shape):
net = caffe.NetSpec()
net.data = L.Input(shape=dict(dim=list(data_shape)))
net.label = L.Input(shape=dict(dim=list(label_shape)))
net.fc0 = L.InnerProduct(net.data, num_output=30, weight_filler=dict(type='xavier'))
net.relu0 = L.ReLU(net.fc0, in_place=True)
net.output = L.InnerProduct(net.relu0, num_output=10, weight_filler=dict(type='xavier'))
net.loss = L.SoftmaxWithLoss(net.output, net.label)
net.accuracy = L.Accuracy(net.output, net.label)
with open(net_path, 'w') as f:
f.write(str(net.to_proto()))
def gen_solver(solver_path, net_path):
from caffe.proto import caffe_pb2
params = caffe_pb2.SolverParameter()
params.train_net = net_path
params.type = 'SGD'
params.momentum = 0.9
params.base_lr = 0.5
params.lr_policy = 'step'
params.gamma = 0.999
params.stepsize = 1
params.weight_decay = 0.0003
with open(solver_path, 'w') as f:
f.write(str(params))
batch_size = 1024
data_shape = (batch_size, train_data[0][0].size)
label_shape = (batch_size, 1)
train_blobs = utils.as_dl_blobs(train_data, batch_size, data_shape, label_shape)
valid_blobs = utils.as_dl_blobs(valid_data, batch_size, data_shape, label_shape)
test_blobs = utils.as_dl_blobs(test_data, batch_size, data_shape, label_shape)
net_path = 'temp/net.prototxt'
solver_path = 'temp/solver.txt'
gen_net(net_path, data_shape, label_shape)
gen_solver(solver_path, net_path)
caffe.set_device(0)
caffe.set_mode_gpu()
solver = caffe.SGDSolver(solver_path)
utils.train(solver, train_blobs, valid_blobs, 50, 5)
print('caffe nn: %.2f%%' % (utils.evaluate(solver, test_blobs)))
def __init__(self, weights_path, image_net_proto, lstm_net_proto,
vocab_path, device_id=-1):
if device_id >= 0:
caffe.set_mode_gpu()
caffe.set_device(device_id)
else:
caffe.set_mode_cpu()
# Setup image processing net.
phase = caffe.TEST
self.image_net = caffe.Net(image_net_proto, weights_path, phase)
image_data_shape = self.image_net.blobs['data'].data.shape
self.transformer = caffe.io.Transformer({'data': image_data_shape})
channel_mean = np.zeros(image_data_shape[1:])
channel_mean_values = [104, 117, 123]
assert channel_mean.shape[0] == len(channel_mean_values)
for channel_index, mean_val in enumerate(channel_mean_values):
channel_mean[channel_index, ...] = mean_val
self.transformer.set_mean('data', channel_mean)
self.transformer.set_channel_swap('data', (2, 1, 0))
self.transformer.set_transpose('data', (2, 0, 1))
# Setup sentence prediction net.
self.lstm_net = caffe.Net(lstm_net_proto, weights_path, phase)
self.vocab = ['<EOS>']
with open(vocab_path, 'r') as vocab_file:
self.vocab += [word.strip() for word in vocab_file.readlines()]
assert(self.vocab[1] == '<unk>')
self.vocab_inv = dict([(w,i) for i,w in enumerate(self.vocab)])
net_vocab_size = self.lstm_net.blobs['predict'].data.shape[2]
if len(self.vocab) != net_vocab_size:
raise Exception('Invalid vocab file: contains %d words; '
'net expects vocab with %d words' % (len(self.vocab), net_vocab_size))
def caffe_set_device(gpu=True, devid='0'): if gpu: caffe.set_mode_gpu() os.environ["CUDA_VISIBLE_DEVICES"] = devid caffe.set_device(int(devid)) else: caffe.set_mode_cpu()
def setup():
global resnet_mean
global resnet_net
global vqa_net
# data provider
vqa_data_provider_layer.CURRENT_DATA_SHAPE = EXTRACT_LAYER_SIZE
# mean substraction
blob = caffe.proto.caffe_pb2.BlobProto()
data = open( RESNET_MEAN_PATH , 'rb').read()
blob.ParseFromString(data)
resnet_mean = np.array( caffe.io.blobproto_to_array(blob)).astype(np.float32).reshape(3,224,224)
resnet_mean = np.transpose(cv2.resize(np.transpose(resnet_mean,(1,2,0)), (448,448)),(2,0,1))
# resnet
caffe.set_device(GPU_ID)
caffe.set_mode_gpu()
resnet_net = caffe.Net(RESNET_LARGE_PROTOTXT_PATH, RESNET_CAFFEMODEL_PATH, caffe.TEST)
# our net
vqa_net = caffe.Net(VQA_PROTOTXT_PATH, VQA_CAFFEMODEL_PATH, caffe.TEST)
# uploads
if not os.path.exists(UPLOAD_FOLDER):
os.makedirs(UPLOAD_FOLDER)
if not os.path.exists(VIZ_FOLDER):
os.makedirs(VIZ_FOLDER)
print 'Finished setup'
def train():
with open('./seg_low_res_model/proto_train.prototxt', 'w') as f:
f.write(str(segmodel.generate_model('train', config.N)))
caffe.set_device(config.gpu_id)
caffe.set_mode_gpu()
solver = caffe.get_solver('./seg_low_res_model/solver.prototxt')
if config.weights is not None:
solver.net.copy_from(config.weights)
cls_loss_avg = 0.0
avg_accuracy_all, avg_accuracy_pos, avg_accuracy_neg = 0.0, 0.0, 0.0
decay = 0.99
for it in range(config.max_iter):
solver.step(1)
cls_loss_val = solver.net.blobs['loss'].data
scores_val = solver.net.blobs['fcn_scores'].data.copy()
label_val = solver.net.blobs['label'].data.copy()
cls_loss_avg = decay*cls_loss_avg + (1-decay)*cls_loss_val
print('\titer = %d, cls_loss (cur) = %f, cls_loss (avg) = %f'
% (it, cls_loss_val, cls_loss_avg))
# Accuracy
accuracy_all, accuracy_pos, accuracy_neg = compute_accuracy(scores_val, label_val)
avg_accuracy_all = decay*avg_accuracy_all + (1-decay)*accuracy_all
avg_accuracy_pos = decay*avg_accuracy_pos + (1-decay)*accuracy_pos
avg_accuracy_neg = decay*avg_accuracy_neg + (1-decay)*accuracy_neg
print('\titer = %d, accuracy (cur) = %f (all), %f (pos), %f (neg)'
% (it, accuracy_all, accuracy_pos, accuracy_neg))
print('\titer = %d, accuracy (avg) = %f (all), %f (pos), %f (neg)'
% (it, avg_accuracy_all, avg_accuracy_pos, avg_accuracy_neg))
def __init__(self,params):
self.dimension = params['dimension']
self.dataset = params['dataset']
self.pooling = params['pooling']
# Read image lists
with open(params['query_list'],'r') as f:
self.query_names = f.read().splitlines()
with open(params['frame_list'],'r') as f:
self.database_list = f.read().splitlines()
# Parameters needed
self.layer = params['layer']
self.save_db_feats = params['database_feats']
# Init network
if params['gpu']:
caffe.set_mode_gpu()
caffe.set_device(0)
else:
caffe.set_mode_cpu()
print "Extracting from:", params['net_proto']
cfg.TEST.HAS_RPN = True
self.net = caffe.Net(params['net_proto'], params['net'], caffe.TEST)
def set_caffe_mode(gpu):
if gpu == 0: # cpu mode
caffe.set_mode_cpu()
else: # gpu mode
caffe.set_device(0)
caffe.set_mode_gpu()
return 0
def main(args_list):
args = parse_args(args_list)
print('Called with args:')
print(args)
if args.cfg_file is not None:
cfg_from_file(args.cfg_file)
if args.set_cfgs is not None:
cfg_from_list(args.set_cfgs)
cfg.GPU_ID = args.gpu_id
print('Using config:')
pprint.pprint(cfg)
while not os.path.exists(args.caffemodel) and args.wait:
print('Waiting for {} to exist...'.format(args.caffemodel))
time.sleep(10)
caffe.set_mode_gpu()
caffe.set_device(args.gpu_id)
net = caffe.Net(args.prototxt, args.caffemodel, caffe.TEST)
net.name = os.path.splitext(os.path.basename(args.caffemodel))[0]
imdb = get_imdb(args.imdb_name)
imdb.competition_mode(args.comp_mode)
if not cfg.TEST.HAS_RPN:
imdb.set_proposal_method(cfg.TEST.PROPOSAL_METHOD)
test_net(net, imdb, max_per_image=args.max_per_image, vis=args.vis)
def init_caffe_net(gpu_id, raw_image_size, crop_size, batch_size):
'''
Initialize caffe configuration.
The function is used to extract the RGB images.
If your dataset contains gray images, set the channels 3 to 1 and comment the image preprocessing in transposing and channels changing.
'''
caffe.set_mode_gpu()
caffe.set_device(int(gpu_id)) # {0, 1, 2, 3} to four GPUs you want to choose.
# The train_val.prototxt file defination.
model_def = '/home/u514/caffe-i/caffe-master/caffe/models/vgg/vgg_2048/deploy-bak.prototxt'
# The pre-trained model.
caffemodel = '/home/u514/caffe-i/caffe-master/caffe/models/vgg/vgg_2048/pretrain_ilsvrc2012_vgg_2048.caffemodel'
# The mean file of the image set used to train the model.
mean_file = '/home/u514/caffe-i/caffe-master/caffe/models/vgg/vgg_2048/vgg_mean.npy'
net = caffe.Net(model_def, caffemodel, caffe.TEST)
transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape})
transformer.set_transpose('data', (2, 0, 1)) # (H,W,C) to (C,H,W)
transformer.set_mean('data', np.load(mean_file).mean(1).mean(1))
transformer.set_raw_scale('data', int(raw_image_size))
transformer.set_channel_swap('data', (2, 1, 0)) # RGB to BGR
# Set batch size (default: 50).
net.blobs['data'].reshape(int(batch_size), 3, int(crop_size), int(crop_size))
return net, transformer
def __init__(self, model_def_file, pretrained_model_file, mean_file,
raw_scale, class_labels_file, bet_file, image_dim, gpu_mode, gpu_device):
logging.info('Loading net and associated files...')
if gpu_mode:
caffe.set_device(gpu_device)
caffe.set_mode_gpu()
else:
caffe.set_mode_cpu()
self.net = caffe.Classifier(
model_def_file, pretrained_model_file,
image_dims=(image_dim, image_dim), raw_scale=raw_scale,
mean=np.load(mean_file).mean(1).mean(1), channel_swap=(2, 1, 0)
)
with open(class_labels_file) as f:
labels_df = pd.DataFrame([
{
'synset_id': l.strip().split(' ')[0],
'name': ' '.join(l.strip().split(' ')[1:]).split(',')[0]
}
for l in f.readlines()
])
self.labels = labels_df.sort('synset_id')['name'].values
self.bet = cPickle.load(open(bet_file))
# A bias to prefer children nodes in single-chain paths
# I am setting the value to 0.1 as a quick, simple model.
# We could use better psychological models here...
self.bet['infogain'] -= np.array(self.bet['preferences']) * 0.1
def setup_net(self): caffe.set_mode_gpu() caffe.set_device(0) netFiles = cfg.get_caffe_net_files(self.prms_.netName) self.net_ = caffe.Net(netFiles.deployFile, netFiles.netFile, caffe.TEST) self.cls_ = cfg.dataset2classnames(self.prms_.trainDataSet)
def test(net_file,model_file,predict_file,gpunum,outdir,outputlayer):
caffe.set_device(gpunum)
caffe.set_mode_gpu()
if not exists(outdir):
makedirs(outdir)
outfile = os.path.join(outdir,'bestiter.pred')
outputlayer_split = outputlayer.split('_')
outputlayer_cnt = len(outputlayer_split)
flag = False
outdata = []
net = caffe.Net(realpath(net_file), realpath(model_file),caffe.TEST)
with open(predict_file,'r') as f:
files = [x.strip() for x in f]
with open(outfile,'w') as f:
for batchfile in files:
fi = h5py.File(batchfile, 'r')
dataset = np.asarray(fi['data'])
out = net.forward_all(data=dataset,blobs=outputlayer_split)
for i in range(outputlayer_cnt):
if not flag:
outdata.append( np.vstack(np.asarray(out[outputlayer_split[i]])) )
else:
outdata[i] = np.vstack((outdata[i],np.vstack(np.asarray(out[outputlayer_split[i]]))))
flag = True
for out in outdata[0]:
f.write('%s\n' % '\t'.join([str(x) for x in out]))
with open(join(outdir,'bestiter.pred.params.pkl'),'wb') as f:
cPickle.dump((outdata,outputlayer_split),f,protocol=cPickle.HIGHEST_PROTOCOL)
def main(args):
caffe.set_device(0)
caffe.set_mode_gpu()
blob, next_layer = parse_prototxt(args.model, args.layer)
net = caffe.Net(args.model, args.weights, caffe.TEST)
# Channelwise for conv
impact = np.zeros(net.blobs[blob].shape[1])
for i in xrange(args.num_iters):
net.forward()
f = net.blobs[blob].data.copy()
loss = net.blobs['loss'].data.copy()
for n in xrange(f.shape[1]):
net.blobs[blob].data[...] = f.copy()
net.blobs[blob].data[:, n] = 0
net.forward(start=next_layer)
delta = net.blobs['loss'].data - loss
impact[n] += delta.sum()
# Normalize
if args.normalize:
assert impact.max() > 0, "No neuron has positive impact"
scale = np.log(9) / impact.max()
impact *= scale
else:
batch_size = net.blobs[blob].shape[0]
impact /= (batch_size * args.num_iters)
# Save
np.save(args.output, impact)
def run_crfasrnn(inputfile, outputfile, gpudevice):
MODEL_FILE = 'TVG_CRFRNN_new_deploy.prototxt'
PRETRAINED = 'TVG_CRFRNN_COCO_VOC.caffemodel'
IMAGE_FILE = inputfile
if gpudevice > 0:
#Do you have GPU device?
has_gpu = 1
#which gpu device is available?
gpu_device=gpudevice#assume the first gpu device is available, e.g. Titan X
else:
has_gpu = 0
if has_gpu==1:
caffe.set_device(gpu_device)
caffe.set_mode_gpu()
tic()
net = caffe.Segmenter(MODEL_FILE, PRETRAINED,True)
toc()
else:
caffe.set_mode_cpu()
tic()
net = caffe.Segmenter(MODEL_FILE, PRETRAINED,False)
toc()
input_image = 255 * caffe.io.load_image(IMAGE_FILE)
width = input_image.shape[0]
height = input_image.shape[1]
maxDim = max(width,height)
image = PILImage.fromarray(np.uint8(input_image))
image = np.array(image)
pallete = getpallete(256)
mean_vec = np.array([103.939, 116.779, 123.68], dtype=np.float32)
reshaped_mean_vec = mean_vec.reshape(1, 1, 3);
# Rearrange channels to form BGR
im = image[:,:,::-1]
# Subtract mean
im = im - reshaped_mean_vec
# Pad as necessary
cur_h, cur_w, cur_c = im.shape
pad_h = 500 - cur_h
pad_w = 500 - cur_w
im = np.pad(im, pad_width=((0, pad_h), (0, pad_w), (0, 0)), mode = 'constant', constant_values = 0)
# Get predictions
segmentation = net.predict([im])
segmentation2 = segmentation[0:cur_h, 0:cur_w]
output_im = PILImage.fromarray(segmentation2)
output_im.putpalette(pallete)
plt.imshow(output_im)
plt.savefig(outputfile)
def main(args_list):
args = parse_args(args_list)
print('Called with args:')
print(args)
if args.cfg_file is not None:
cfg_from_file(args.cfg_file)
if args.set_cfgs is not None:
cfg_from_list(args.set_cfgs)
cfg.GPU_ID = args.GPU_ID
print('Using config:')
pprint.pprint(cfg)
if not args.randomize:
# fix the random seeds (numpy and caffe) for reproducibility
np.random.seed(cfg.RNG_SEED)
caffe.set_random_seed(cfg.RNG_SEED)
print 'Setting GPU device %d for training' % cfg.GPU_ID
caffe.set_mode_gpu()
caffe.set_device(cfg.GPU_ID)
imdb, roidb = combined_roidb(args.imdb_name)
print '{:d} roidb entries'.format(len(roidb))
output_dir = get_output_dir(imdb)
print 'Output will be saved to `{:s}`'.format(output_dir)
train_net(args.solver, roidb, output_dir,
pretrained_model=args.pretrained_model,
max_iters=args.max_iters)
def solve(self,max_iter):
caffe.set_mode_gpu()
if self.gpu_id is not None:
caffe.set_device(self.gpu_id)
solver = caffe.get_solver(self.solverfile)
for i in xrange( max_iter+1):
solver.step(1)
def predictImage(filename):
vgg_point_MODEL_FILE = 'model/deploy.prototxt'
vgg_point_PRETRAINED = 'model/68point_dlib_with_pose.caffemodel'
mean_filename='model/VGG_mean.binaryproto'
vgg_point_net=caffe.Net(vgg_point_MODEL_FILE,vgg_point_PRETRAINED,caffe.TEST)
# caffe.set_mode_cpu()
caffe.set_mode_gpu()
caffe.set_device(0)
f = open(filename)
line = f.readline()
index = 0
proto_data = open(mean_filename, "rb").read()
a = caffe.io.caffe_pb2.BlobProto.FromString(proto_data)
mean = caffe.io.blobproto_to_array(a)[0]
while line:
print index
line = line.strip()
info = line.split(' ')
imgPath = info[0]
print imgPath
num = 1
colorImage = cv2.imread(imgPath)
bboxs = detectFace(colorImage)
faceNum = bboxs.shape[0]
faces = np.zeros((1,3,vgg_height,vgg_width))
predictpoints = np.zeros((faceNum,pointNum*2))
predictpose = np.zeros((faceNum,3))
imgsize = np.zeros((2))
imgsize[0] = colorImage.shape[0]-1
imgsize[1] = colorImage.shape[1]-1
TotalSize = np.zeros((faceNum,2))
for i in range(0,faceNum):
TotalSize[i] = imgsize
for i in range(0,faceNum):
bbox = bboxs[i]
colorface = getRGBTestPart(bbox,M_left,M_right,M_top,M_bottom,colorImage,vgg_height,vgg_width)
normalface = np.zeros(mean.shape)
normalface[0] = colorface[:,:,0]
normalface[1] = colorface[:,:,1]
normalface[2] = colorface[:,:,2]
normalface = normalface - mean
faces[0] = normalface
blobName = '68point'
data4DL = np.zeros([faces.shape[0],1,1,1])
vgg_point_net.set_input_arrays(faces.astype(np.float32),data4DL.astype(np.float32))
vgg_point_net.forward()
predictpoints[i] = vgg_point_net.blobs[blobName].data[0]
blobName = 'poselayer'
pose_prediction = vgg_point_net.blobs[blobName].data
predictpose[i] = pose_prediction * 50
predictpoints = predictpoints * vgg_height/2 + vgg_width/2
level1Point = batchRecoverPart(predictpoints,bboxs,TotalSize,M_left,M_right,M_top,M_bottom,vgg_height,vgg_width)
show_image(colorImage, level1Point, bboxs, predictpose)
line = f.readline()
index = index + 1
def initialize_model(model, mode):
if mode is None:
mode = "cpu"
if "cpu" in mode.lower():
caffe.set_mode_cpu()
if "gpu" in mode.lower():
caffe.set_device(0)
caffe.set_mode_gpu()
if model is None:
model = "cafferef"
if "cafferef" in model.lower():
cnn_proto = caffe_root + "models/bvlc_reference_caffenet/deploy.prototxt"
cnn_model = caffe_root + "models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel"
imgnet_mean = np.load(caffe_root + "python/caffe/imagenet/ilsvrc_2012_mean.npy").mean(1).mean(1)
cnn_imgmean = imgnet_mean
cnn_imgsize = 227
if "vgg" in model.lower():
if "vgg16" in model.lower():
cnn_proto = caffe_root + "models/VGG_ILSVRC_16_layers/VGG_ILSVRC_16_layers_deploy.prototxt"
cnn_model = caffe_root + "models/VGG_ILSVRC_16_layers/VGG_ILSVRC_16_layers.caffemodel"
if "vgg19" in model.lower():
cnn_proto = caffe_root + "models/VGG_ILSVRC_19_layers/VGG_ILSVRC_19_layers_deploy.prototxt"
cnn_model = caffe_root + "models/VGG_ILSVRC_19_layers/VGG_ILSVRC_19_layers.caffemodel"
vgg_mean = np.array([103.939, 116.779, 123.68])
cnn_imgmean = vgg_mean
cnn_imgsize = 224
if "action" in model.lower():
cnn_proto = caffe_root + "models/action_cube/deploy_extractpred.prototxt"
cnn_model = caffe_root + "models/action_cube/action_cube.caffemodel"
cnn_imgmean = np.array([128, 128, 128])
cnn_imgsize = 227
cf = caffe_feat(cnn_proto, cnn_model, cnn_imgmean, cnn_imgsize)
return cf
def main(model_name, label_key, slice_type, new_db):
lmdb_train = os.path.join(TMP_DATA_DIR, 'lmdb_train_%s_%s' % (model_name, slice_type))
lmdb_train_lab = os.path.join(TMP_DATA_DIR, 'lmdb_train_lab_%s_%s' % (model_name, slice_type))
lmdb_test = os.path.join(TMP_DATA_DIR, 'lmdb_test_%s_%s' % (model_name, slice_type))
lmdb_test_lab = os.path.join(TMP_DATA_DIR, 'lmdb_test_lab_%s_%s' % (model_name, slice_type))
if new_db:
train_data = os.path.join(PROCESSED_DATA_DIR, 'local_train')
test_data = os.path.join(PROCESSED_DATA_DIR, 'local_test')
print 'Creating train LMDB'
create_lmdb.create_db(lmdb_train, lmdb_train_lab, train_data,
frames_preproc=PREPROC,
trans_generator=TRANS,
label_key=label_key,
slice_type=slice_type)
print 'Creating test LMDB'
create_lmdb.create_db(lmdb_test, lmdb_test_lab, test_data,
frames_preproc=PREPROC,
trans_generator=TRANS,
label_key=label_key,
slice_type=slice_type)
print 'Creating neural net'
full_model_name = '%s_%s_%s' % (model_name, label_key, slice_type)
nn_models.write_model(full_model_name, MODEL_DIR, MODEL,
lmdb_train, lmdb_train_lab, lmdb_test, lmdb_test_lab,
BATCH_SIZE)
print 'Training neural net'
caffe.set_device(DEVICE_ID)
caffe.set_mode_gpu()
solver_path = os.path.join(MODEL_DIR, '%s_solver.prototxt' % full_model_name)
solver.train(solver_path)
def init_net(gpu_id, solver_path, caffe_model): caffe.set_mode_gpu() caffe.set_device(int(gpu_id)) #net = caffe.Net(model_def, caffemodel, caffe.TRAIN) solver = caffe.SGDSolver(solver_path) solver.net.copy_from(caffe_model) return solver
def init_extraction_net(gpu_id=0):
#pycaffe_dir = os.path.dirname(__file__)
#caffe_root = '/opt/fast-rcnn/caffe-fast-rcnn/'
caffe_root = osp.abspath(osp.join(osp.dirname(__file__),'..'))
model = osp.join(caffe_root,'models/clothes.caffemodel')
prototxt = osp.join(caffe_root, 'models/deploy.prototxt')
#sys.path.insert(0, caffe_root + 'python')
if not osp.isfile(model):
raise IOError(('{:s} not found\n').format(model))
caffe.set_mode_gpu()
caffe.set_device(gpu_id)
net = caffe.Net(prototxt, model, caffe.TEST)
transformer = caffe.io.Transformer({'data':net.blobs['data'].data.shape})
# dim(channels,x,y)
transformer.set_transpose('data', (2,0,1))
#mean_file = osp.join(caffe_root,'python/caffe/face_mean.npy')
#transformer.set_mean('data', np.load(mean_file).mean(1).mean(1))
#mean = np.float32([104.0, 116.0, 122.0])
mean = np.float32([128.0, 128.0, 128.0])
transformer.set_mean('data', mean)
#transformer.set_raw_scale('data', 255)#set image data in [0,255]
#transformer.set_channel_swap('data', (2,1,0))
#googlenet 224, bvlc_caffenet 227
net.blobs['data'].reshape(1,3,224,224)
return net,transformer
def main():
from argparse import ArgumentParser
from os import path
parser = ArgumentParser()
parser.add_argument('prototxt')
parser.add_argument('-l', '--load', help='Load a caffemodel')
parser.add_argument('-d', '--data', default=None, help='Image list to use [default prototxt data]')
#parser.add_argument('-q', action='store_true', help='Quiet execution')
parser.add_argument('-sm', action='store_true', help='Summary only')
parser.add_argument('-q', action='store_true', help='Quiet execution')
parser.add_argument('-a', '--all', action='store_true', help='Show the statistic for all layers')
parser.add_argument('-nc', action='store_true', help='Do not use color')
parser.add_argument('-s', type=float, default=1.0, help='Scale the input [only custom data "-d"]')
parser.add_argument('-bs', type=int, default=16, help='Batch size [only custom data "-d"]')
parser.add_argument('-nit', type=int, default=10, help='Number of iterations')
parser.add_argument('--gpu', type=int, default=0, help='What gpu to run it on?')
args = parser.parse_args()
if args.q:
from os import environ
environ['GLOG_minloglevel'] = '2'
import caffe, load
from caffe import NetSpec, layers as L
caffe.set_mode_gpu()
if args.gpu is not None:
caffe.set_device(args.gpu)
model = load.ProtoDesc(args.prototxt)
net = NetSpec()
if args.data is not None:
fl = getFileList(args.data)
if len(fl) == 0:
print("Unknown data type for '%s'"%args.data)
exit(1)
from tempfile import NamedTemporaryFile
f = NamedTemporaryFile('w')
f.write('\n'.join([path.abspath(i)+' 0' for i in fl]))
f.flush()
net.data, net.label = L.ImageData(source=f.name, batch_size=args.bs, new_width=model.input_dim[-1], new_height=model.input_dim[-1], transform_param=dict(mean_value=[104,117,123], scale=args.s),ntop=2)
net.out = model(data=net.data, label=net.label)
else:
net.out = model()
n = netFromString('force_backward:true\n'+str(net.to_proto()), caffe.TRAIN )
if args.load is not None:
n.copy_from(args.load)
cvar = printMeanStddev(n, NIT=args.nit, show_all=args.all, show_color=not args.nc, quiet=args.sm)
cv, gr = computeGraidentRatio(n, NIT=args.nit)
print()
print(' Summary ')
print('-----------')
print()
print('layer name out cvar rate cvar rate mean')
for l in n._layer_names:
if l in cvar and l in cv and l in gr:
print('%-30s %10.2f %10.2f %10.2f'%(l, cvar[l], cv[l], gr[l]) )
def init_net():
cfg.TEST.HAS_RPN = True # Use RPN for proposals
args = parse_args()
# prototxt = os.path.join(cfg.MODELS_DIR, NETS[args.demo_net][0],
# 'faster_rcnn_alt_opt', 'faster_rcnn_test.pt')
prototxt = os.path.join('/home/dean/Documents/py-faster-rcnn/models/WIDER_FACE/VGG16/faster_rcnn_end2end',
'test.prototxt')
# caffemodel = os.path.join(cfg.DATA_DIR, 'faster_rcnn_models',
# NETS[args.demo_net][1])
caffemodel = os.path.join('/home/dean/Documents/py-faster-rcnn/output/faster_rcnn_end2end/voc_2007_train',
'vgg16_faster_rcnn_iter_50000.caffemodel')
if not os.path.isfile(caffemodel):
raise IOError(('{:s} not found.\nDid you run ./data/script/'
'fetch_faster_rcnn_models.sh?').format(caffemodel))
if args.cpu_mode:
caffe.set_mode_cpu()
else:
caffe.set_mode_gpu()
caffe.set_device(args.gpu_id)
cfg.GPU_ID = args.gpu_id
net = caffe.Net(prototxt, caffemodel, caffe.TEST)
print '\n\nLoaded network {:s}'.format(caffemodel)
im = 128 * np.ones((300, 500, 3), dtype=np.uint8)
for i in xrange(2):
_, _= im_detect(net, im)
return net
cfg.TEST.HAS_RPN = True # Use RPN for proposals
prototxt = os.path.join(cfg.MODELS_DIR, NETS[demo_net][0],
'faster_rcnn_alt_opt', 'faster_rcnn_test.pt')
caffemodel = os.path.join(cfg.DATA_DIR, 'faster_rcnn_models',
NETS[demo_net][1])
if not os.path.isfile(caffemodel):
raise IOError(('{:s} not found.\nDid you run ./data/script/'
'fetch_faster_rcnn_models.sh?').format(caffemodel))
if cpu_mode:
caffe.set_mode_cpu()
else:
caffe.set_mode_gpu()
caffe.set_device(gpu_id)
cfg.GPU_ID = gpu_id
net = caffe.Net(prototxt, caffemodel, caffe.TEST)
print '\n\nLoaded network {:s}'.format(caffemodel)
def detect_frcnn(url_or_np_array,save_data=False,filename=None):
print "detect_frcnn started"
# check if i get a url (= string) or np.ndarray
if filename:
full_image = cv2.imread(filename)
url = 'from_file'
elif isinstance(url_or_np_array, basestring):
response = requests.get(url_or_np_array) # download
full_image = cv2.imdecode(np.asarray(bytearray(response.content)), 1)
filename=url_or_np_array.replace('https://','').replace('http://','').replace('/','_')
def set_gpu(gpuID):
if gpuID >= 0:
caffe.set_mode_gpu()
caffe.set_device(gpuID)
else:
caffe.set_mode_cpu()
import cv2
import caffe
import numpy as np
from copy import deepcopy
import os
import matplotlib
import glob
caffe.set_mode_gpu()
caffe.set_device(0)
def compute_iou(box1, box2):
x1_1 = box1[0]
y1_1 = box1[1]
x2_1 = box1[2]
y2_1 = box1[3]
x1_2 = box2[0]
y1_2 = box2[1]
x2_2 = box2[2]
y2_2 = box2[3]
area_inter = float(
max(0,
min(x2_1, x2_2) - max(x1_1, x1_2)) * max(
(0, min(y2_1, y2_2) - max(y1_1, y1_2))))
area1 = float((x2_1 - x1_1) * (y2_1 - y1_1))
area2 = float((x2_2 - x1_2) * (y2_2 - y1_2))
return float(area_inter) / (area1 + area2 - area_inter)
def filter_boxes(boxes, iou_thre):
#[xmin,yin,xmax,ymax,score]
model = 'model'
if not os.path.exists(model):
print 'Creating model directory...\n'
os.symlink('{}/pascalpart-fcn16s/person/{}'.format(models, part), model)
loss = 'loss'
if not os.path.exists(loss):
print 'Creating loss directory...\n'
os.mkdir(loss)
if not os.path.exists(log):
print 'Creating training_log directory...\n'
os.mkdir(log)
# init
caffe.set_device(int(device))
caffe.set_mode_gpu()
solver = caffe.SGDSolver('{}/pascalpart-fcn16s/person/{}/solver.prototxt'.format(models, part))
if is_resume:
solver.net.copy_from('{}/train_iter_{}.caffemodel'.format(snapshot, iteration))
solver.restore('{}/train_iter_{}.solverstate'.format(snapshot, iteration))
else:
solver.net.copy_from('../../../pascalpart-fcn32s/person/{}/snapshot/train_iter_80000.caffemodel'.format(part))
# surgeries
interp_layers = [k for k in solver.net.params.keys() if 'up' in k]
surgery.interp(solver.net, interp_layers)
# scoring
val = np.loadtxt('{}/data/pascal/VOC/VOC2010/ImageSets/person/{}_val.txt'.format(caffe_root, part), dtype=str)
cfg.TEST.HAS_RPN = True # Use RPN for proposals
cfg.TEST.SCALE_MULTIPLE_OF = 32
cfg.TEST.SCALES = (352, )
args = parse_args()
rootdir = '/export/home/wjh/py-faster-rcnn/'
prototxt = rootdir + 'workdir_fg/testreg.prototxt'
caffemodel = rootdir + 'workdir_fg/output/zf_faster_rcnn_iter_200000.caffemodel'
if not os.path.isfile(caffemodel):
raise IOError(('{:s} not found.').format(caffemodel))
if args.cpu_mode:
caffe.set_mode_cpu()
else:
caffe.set_mode_gpu()
caffe.set_device(args.gpu_id)
cfg.GPU_ID = args.gpu_id
net = caffe.Net(prototxt, caffemodel, caffe.TEST)
print '\n\nLoaded network {:s}'.format(caffemodel)
im = 128 * np.ones((300, 500, 3), dtype=np.uint8)
for i in xrange(1):
_ = im_detectreg(net, im)
imdir = '/export/home/wjh/multi-task/fgtest/JPEGImages/'
annofile = '/export/home/wjh/multi-task/fgtest/test.bbox'
anno = np.loadtxt(annofile, dtype='str')
vis = False
ious = []
for i in range(anno.shape[0]):
#for i in range(100):
top = min(top, len(fwd)+len(bck))
T = sorted([np.mean(v) for v in fwd.values()] + [np.mean(v) for v in bck.values()])[-top]
T0 = time()-T0
print("%s%0.1f%s it / sec [%s%0.1f%s ms / it]"%(bcolors.BOLD+bcolors().FAIL, NIT / T0, bcolors.ENDC, bcolors.BOLD+bcolors().FAIL, 1000*T0 / NIT, bcolors.ENDC))
for n in L:
cf, cb = bcolors.OKGREEN, bcolors.OKGREEN
if np.mean(fwd[n]) >= T: cf = bcolors.BOLD+bcolors().FAIL
if np.mean(bck[n]) >= T: cb = bcolors.BOLD+bcolors().FAIL
print(' %30s \t %s%0.2f \261 %0.1f%s ms \t %s%0.2f \261 %0.1f%s ms'%(n, cf, 1000*np.mean(fwd[n]), 1000*np.std(fwd[n]), bcolors.ENDC, cb, 1000*np.mean(bck[n]), 1000*np.std(bck[n]), bcolors.ENDC))
if __name__ == "__main__":
import argparse
from os import path
import caffe
parser = argparse.ArgumentParser(description="Visualize decompositions on sintel")
parser.add_argument('input_dir', help='input directory')
parser.add_argument('-n', type=int, default=100, help='Number of iterations')
parser.add_argument('-t', type=int, default=5, help='Highlight the top t times')
parser.add_argument('-gpu', type=int, help='What GPU do we test on')
args = parser.parse_args()
caffe.set_mode_gpu()
if args.gpu is not None:
caffe.set_device(args.gpu)
if path.isfile(args.input_dir):
net = caffe.Net(args.input_dir, caffe.TRAIN)
else:
net = caffe.Net(args.input_dir+'trainval.prototxt', caffe.TRAIN)
time_net(net)
return self.diff
caffe_root = "/home/dlg/ssd_caffe/"
if os.path.isfile(
caffe_root +
'models/VGGNet/KITTI/SSD_600x150/VGG_KITTI_SSD_600x150_iter_60000.caffemodel'
):
print 'CaffeNet found.'
else:
print 'CaffeNet not found'
model_def = caffe_root + 'models/VGGNet/KITTI/SSD_600x150/deploy_large.prototxt'
model_weights = caffe_root + 'models/VGGNet/KITTI/SSD_600x150/VGG_KITTI_SSD_600x150_iter_60000.caffemodel'
net = caffe.Net(model_def, model_weights, caffe.TEST)
caffe.set_device(0) # if we have multiple GPUs, pick the first one
caffe.set_mode_gpu()
mu = np.array([104, 117, 123])
transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape})
transformer.set_transpose(
'data', (2, 0, 1)) # move image channels to outermost dimension
transformer.set_mean('data',
mu) # subtract the dataset-mean value in each channel
transformer.set_raw_scale('data', 255) # rescale from [0, 1] to [0, 255]
transformer.set_channel_swap('data',
(2, 1, 0)) # swap channels from RGB to BGR
net.blobs['data'].reshape(1, 3, 540, 960)
color = [(255, 0, 0), (0, 255, 0), (0, 0, 255)]
visualize_threshold = 0.6
import scipy.ndimage as nd
import PIL.Image
from IPython.display import clear_output, Image, display
from google.protobuf import text_format
import caffe
model_folders = [
'bvlc_googlenet', 'googlenet_places205', 'bvlc_reference_caffenet',
'finetune_BarryLyndon_8Sept2015', 'VGG_ILSVRC_16_layers',
'person_clothing_bigger_18Sept2015', 'bvlc_googlenet_person'
]
cluster_home_path = '/global/home/users/karlz'
if home_path == cluster_home_path:
caffe.set_mode_gpu()
caffe.set_device(0) # select GPU device if multiple devices exist
def showarray(a, fmt='jpeg'):
a = np.uint8(np.clip(a, 0, 255))
f = StringIO()
PIL.Image.fromarray(a).save(f, fmt)
display(Image(data=f.getvalue()))
def get_net(model_name):
model_folder = model_name
model_path = opj(home_path, 'caffe/models', model_folder)
net_fn = opj(model_path, 'deploy.prototxt')
param_fn = opj(model_path, 'model.caffemodel')
# Patching model to be able to compute gradients.
if h != w:
print 'filters need to be square'
raise
filt = upsample_filt(h)
net.params[l][0].data[range(m), range(k), :, :] = filt
# base net -- follow the editing model parameters example to make
# a fully convolutional VGG16 net.
# http://nbviewer.ipython.org/github/BVLC/caffe/blob/master/examples/net_surgery.ipynb
# base_weights = '5stage-vgg.caffemodel'
base_weights = '5stage-vgg.caffemodel'
# init
caffe.set_mode_gpu()
caffe.set_device(3)
solver = caffe.SGDSolver('solver.prototxt')
# do net surgery to set the deconvolution weights for bilinear interpolation
interp_layers = [k for k in solver.net.params.keys() if 'up' in k]
interp_surgery(solver.net, interp_layers)
# copy base weights for fine-tuning
#solver.restore('dsn-full-res-3-scales_iter_29000.solverstate')
solver.net.copy_from(base_weights)
# solve straight through -- a better approach is to define a solving loop to
# 1. take SGD steps
# 2. score the model by the test net `solver.test_nets[0]`
# 3. repeat until satisfied
import numpy as np
from PIL import Image
import pdb
import matplotlib.pyplot as plt
import sys
import time
import scipy.ndimage
import scipy.misc
sys.path.insert(0, 'caffe/install/python') # PATH TO CAFFE PYTHON INSTALL
import caffe
caffe.set_mode_gpu()
caffe.set_device(1)
MEAN_VALUE = np.array([103.939, 116.779, 123.68]) # BGR
MEAN_VALUE = MEAN_VALUE[:,None, None]
FINE_SCALE = np.array([1,3,1200,1600], dtype=np.float32)
COARSE_SCALE = np.array([1,3,600,800], dtype=np.float32)
class Salicon:
def __init__(self, prototxtpath='salicon.prototxt', model='salicon_osie.caffemodel'):
self.net = caffe.Net(prototxtpath, model, caffe.TEST)
def process_the_image(self, im):
# put channel dimension first
im = np.transpose(im, (2,0,1))
# switch to BGR
im = im[::-1, :, :]
# subtract mean
im = im - MEAN_VALUE
im = im[None,:]
im = im / 255 # convert to float precision
return im
def GetNet(net, weights, gpu):
caffe.set_device(gpu)
caffe.set_mode_gpu()
net = caffe.Net(net, weights, caffe.TEST)
#net = caffe.Net(net, caffe.TEST)
return net
dest='gid',
help='GPU device id to use [-1]',
default=-1,
type=int)
parser.add_argument('--net',
dest='net',
help='Network to use [vgg16]',
choices=NETS.keys(),
default='vgg16')
args = parser.parse_args()
start = datetime.now()
if args.gid >= 0:
caffe.set_mode_gpu()
caffe.set_device(args.gid)
cfg.GPU_ID = args.gid
else:
caffe.set_mode_cpu()
cfg.TEST.HAS_RPN = True # Use RPN for proposals
prototxt = os.path.join(cfg.MODELS_DIR, NETS[args.net][0],
'faster_rcnn_alt_opt', 'faster_rcnn_test.pt')
caffemodel = os.path.join(cfg.DATA_DIR, 'faster_rcnn_models',
NETS[args.net][1])
net = caffe.Net(prototxt, caffemodel, caffe.TEST)
video_reader = cv2.VideoCapture(args.input_video)
if video_reader.isOpened():
fourcc = int(video_reader.get(cv2.cv.CV_CAP_PROP_FOURCC))
import copy
import matplotlib
import pylab as plt
test_image = '../sample_image/ski.jpg'
#test_image = '../sample_image/upper.jpg'
#test_image = '../sample_image/upper2.jpg'
oriImg = cv.imread(test_image) # B,G,R order
f = plt.imshow(oriImg[:,:,[2,1,0]]) # reorder it before displaying
param, model = config_reader()
multiplier = [x * model['boxsize'] / oriImg.shape[0] for x in param['scale_search']]
if param['use_gpu']:
caffe.set_mode_gpu()
caffe.set_device(param['GPUdeviceNumber']) # set to your device!
else:
caffe.set_mode_cpu()
net = caffe.Net(model['deployFile'], model['caffemodel'], caffe.TEST)
heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 19))
paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 38))
# first figure shows padded images
f, axarr = plt.subplots(1, len(multiplier))
f.set_size_inches((20, 5))
# second figure shows heatmaps
f2, axarr2 = plt.subplots(1, len(multiplier))
f2.set_size_inches((20, 5))
# third figure shows PAFs
def extract_tubelets(dname, gpu=-1, redo=False):
"""Extract the tubelets for a given dataset
args:
- dname: dataset name (example: 'JHMDB')
- gpu (default -1): use gpu given in argument, or use cpu if -1
- redo: wheter or not to recompute already computed files
save a pickle file for each frame
the file contains a tuple (dets, dets_all)
- dets is a numpy array with 2+4*K columns containing the tubelets starting at this frame after per-class nms at 0.45 and thresholding the scores at 0.01
the columns are <label> <score> and then <x1> <y1> <x2> <y2> for each of the frame in the tubelet
- dets_all contains the tubelets obtained after a global nms at 0.7 and thresholding the scores at 0.01
it is a numpy arrray with 4*K + L + 1 containing the coordinates of the tubelets and the scores for all labels
note: this version is inefficient: it is better to estimate the per-frame features once
"""
d = GetDataset(dname)
if gpu >= 0:
caffe.set_mode_gpu()
caffe.set_device(gpu)
model_dir = os.path.join(os.path.dirname(__file__),
'../models/ACT-detector/', dname)
output_dir = os.path.join(os.path.dirname(__file__),
'../results/ACT-detector/', dname)
# load the RGB network
rgb_proto = os.path.join(model_dir, "deploy_RGB.prototxt")
rgb_model = os.path.join(model_dir,
"../generated_AVA_iter_118662.caffemodel")
net_rgb = caffe.Net(rgb_proto, caffe.TEST, weights=rgb_model)
# load the FLOW5 network
flo_proto = os.path.join(model_dir, "deploy_FLOW5.prototxt")
flo_model = os.path.join(model_dir,
"../generated_AVA_iter_59463.caffemodel")
net_flo = caffe.Net(flo_proto, caffe.TEST, weights=flo_model)
vlist = d.test_vlist()
for iv, v in enumerate(vlist):
print("Processing video {:d}/{:d}: {:s}".format(iv + 1, len(vlist), v))
h, w = d.resolution(v)
# network output is normalized between 0,1 ; so we will multiply it by the following array
resolution_array = np.array([w, h, w, h] * K, dtype=np.float32)
# now process each frame
for i in xrange(1, 1 + d.nframes(v) - K + 1):
outfile = os.path.join(output_dir, d.frame_format(v, i) + ".pkl")
# skip if already computed
if os.path.isfile(outfile) and not redo:
continue
# read the frames for the forward
kwargs_rgb = {}
kwargs_flo = {}
for j in xrange(K):
cap = cv2.VideoCapture(d.vidfile(v, 0))
#print(frame)
#print(int(cap.get(7)))
cap.set(1, i + j - 1)
im = cap.read()[1]
cap.release()
#im = cv2.imread(d.imfile(v, i + j))
if im is None:
print "Image {:s} does not exist".format(d.imfile(
v, i + j))
return
imscale = cv2.resize(im, (IMGSIZE, IMGSIZE),
interpolation=cv2.INTER_LINEAR)
kwargs_rgb['data_stream' + str(j)] = np.transpose(
imscale - MEAN, (2, 0, 1))[None, :, :, :]
imf = [
cv2.imread(
d.flowfile(
v.split(".")[0], min(d.nframes(v), i + j + iflow)))
for iflow in xrange(NFLOWS)
]
if np.any(imf) is None:
print "Flow image {:s} does not exist".format(
d.flowfile(v, i + j))
return
imscalef = [
cv2.resize(im, (IMGSIZE, IMGSIZE),
interpolation=cv2.INTER_LINEAR) for im in imf
]
timscale = [
np.transpose(im - MEAN, (2, 0, 1))[None, :, :, :]
for im in imscalef
]
kwargs_flo['data_stream' + str(j) + 'flow'] = np.concatenate(
timscale, axis=1)
# compute rgb and flow scores
# two forward passes: one for the rgb and one for the flow
net_rgb.forward(
end="mbox_conf_flatten",
**kwargs_rgb) # forward of rgb with confidence and regression
net_flo.forward(
end="mbox_conf_flatten", **
kwargs_flo) # forward of flow5 with confidence and regression
# compute late fusion of rgb and flow scores (keep regression from rgb)
# use net_rgb for standard detections, net_flo for having all boxes
scores = 0.5 * (net_rgb.blobs['mbox_conf_flatten'].data +
net_flo.blobs['mbox_conf_flatten'].data)
net_rgb.blobs['mbox_conf_flatten'].data[...] = scores
net_flo.blobs['mbox_conf_flatten'].data[...] = scores
net_flo.blobs['mbox_loc'].data[
...] = net_rgb.blobs['mbox_loc'].data
# two forward passes, only for the last layer
# dets is the detections after per-class NMS and thresholding (stardard)
# dets_all contains all the scores and regressions for all tubelets
dets = net_rgb.forward(
start='detection_out')['detection_out'][0, 0, :, 1:]
dets_all = net_flo.forward(
start='detection_out_full')['detection_out_full'][0, 0, :, 1:]
# parse detections with per-class NMS
if dets.shape[0] == 1 and np.all(dets == -1):
dets = np.empty((0, dets.shape[1]), dtype=np.float32)
dets[:,
2:] *= resolution_array # network output was normalized in [0..1]
dets[:,
0] -= 1 # label 0 was background, come back to label in [0..nlabels-1]
dets[:, 2::2] = np.maximum(0, np.minimum(w, dets[:, 2::2]))
dets[:, 3::2] = np.maximum(0, np.minimum(h, dets[:, 3::2]))
# parse detections with global NMS at 0.7 (top 300)
# coordinates were normalized in [0..1]
dets_all[:, 0:4 * K] *= resolution_array
dets_all[:, 0:4 * K:2] = np.maximum(
0, np.minimum(w, dets_all[:, 0:4 * K:2]))
dets_all[:, 1:4 * K:2] = np.maximum(
0, np.minimum(h, dets_all[:, 1:4 * K:2]))
idx = nms_tubelets(
np.concatenate(
(dets_all[:, :4 * K],
np.max(dets_all[:, 4 * K + 1:], axis=1)[:, None]),
axis=1), 0.7, 300)
dets_all = dets_all[idx, :]
# save file
if not os.path.isdir(os.path.dirname(outfile)):
os.system('mkdir -p ' + os.path.dirname(outfile))
with open(outfile, 'wb') as fid:
pickle.dump((dets, dets_all), fid)
if __name__ == '__main__':
parameter = np.load(cfg.STD_MEAN)
cfg.CENTERX_STD = parameter[0]
cfg.CENTERX_MEAN = parameter[1]
cfg.CENTERY_STD = parameter[2]
cfg.CENTERY_MEAN = parameter[3]
cfg.WIDTH_STD = parameter[4]
cfg.WIDTH_MEAN = parameter[5]
cfg.HEIGHT_STD = parameter[6]
cfg.HEIGHT_MEAN = parameter[7]
caffe.set_mode_gpu()
caffe.set_device(cfg.GPU_ID)
# image_source = osp.join(cfg.ROOT_DIR, 'testimage/1.jpg')
cfg.TEST_PROTOTXT = osp.join(cfg.ROOT_DIR, 'test.prototxt')
cfg.TEST_MODEL = osp.join(cfg.ROOT_DIR, 'demo.caffemodel')
# net = caffe.Net(cfg.TEST_PROTOTXT, cfg.TEST_MODEL, caffe.TEST)
# test_image(net, image_source, vis=True, save=False)
for index in range(1, 81):
cfg.IMAGE_NUMBER = index
image_source = osp.join(cfg.ROOT_DIR + '/testimage',
str(index) + cfg.IMAGE_TYPE)
net = caffe.Net(cfg.TEST_PROTOTXT, cfg.TEST_MODEL, caffe.TEST)
if osp.exists(image_source):
test_image(net, image_source, vis=False, save=True)
import caffe
import surgery, score
import numpy as np
import os
import setproctitle
setproctitle.setproctitle(os.path.basename(os.getcwd()))
weights = '../voc-fcn32s/voc-fcn32s.caffemodel'
# init
caffe.set_device(int(sys.argv[1]))
caffe.set_mode_gpu()
solver = caffe.SGDSolver('solver.prototxt')
solver.net.copy_from(weights)
# surgeries
interp_layers = [k for k in solver.net.params.keys() if 'up' in k]
surgery.interp(solver.net, interp_layers)
# scoring
val = np.loadtxt('../data/segvalid11.txt', dtype=str)
for _ in range(25):
solver.step(4000)
score.seg_tests(solver, False, val, layer='score')
from multiprocessing import Pool
import os
import h5py
import numpy as np
import scipy.io as scio
from scipy import ndimage as nd
# Make sure that caffe is on the python path:
#caffe_root = '/usr/local/caffe3/' # this is the path in GPU server
caffe_root = '/home/dongnie/caffe3D/' # this is the path in GPU server
import sys
sys.path.insert(0, caffe_root + 'python')
print caffe_root + 'python'
import caffe
caffe.set_device(3) #very important
caffe.set_mode_gpu()
### load the solver and create train and test nets
solver = None # ignore this workaround for lmdb data (can't instantiate two solvers on the same data)
#solver = caffe.SGDSolver('infant_fcn_solver.prototxt') #for training
protopath = '/home/dongnie/caffe3D/examples/prostate/'
mynet = caffe.Net(protopath + 'prostate_deploy_v12_1.prototxt',
protopath + 'prostate_fcn_v12_1_iter_100000.caffemodel',
caffe.TEST)
print("blobs {}\nparams {}".format(mynet.blobs.keys(), mynet.params.keys()))
d1 = 32
d2 = 32
d3 = 32
dFA = [d1, d2, d3]
dSeg = [24, 24, 24]
if __name__ == '__main__':
cfg_from_file("models/pvanet/cfgs/submit_1019.yml")
output_name = "v13-100k"
data_name = "Thailand"
CLASSES_main = CLASS_SETS["coco"]
CLASSES_sub = CLASS_SETS['vehicle-types']
prototxt = "/root/pva-faster-rcnn/models/pvanet/lite/hierachy/v1_test.prototxt"
caffemodel = "/root/pva-faster-rcnn/models/hierarchy/v13/v13_iter_100000.caffemodel"
FPS_rate = 1
GPU_ID = 2
imdb = get_imdb(data_name, CLASSES_main)
OUTPUT_DIR = os.path.join(imdb._data_path, "res", output_name)
if not os.path.exists(OUTPUT_DIR):
os.makedirs(OUTPUT_DIR)
if not os.path.isfile(caffemodel):
raise IOError(('Caffemodel: {:s} not found').format(caffemodel))
caffe.set_mode_gpu()
caffe.set_device(GPU_ID)
cfg.GPU_ID = GPU_ID
net = caffe.Net(prototxt, caffemodel, caffe.TEST)
print '\n\nLoaded network {:s}'.format(caffemodel)
print("PVANET Loaded")
print("Start Detecting")
write_testing_results_file(net, imdb, FPS_rate, OUTPUT_DIR, CLASSES_main)
def main(model, weights, K, num_act, num_step, num_iter,
gpu, data, mean, video):
font = ImageFont.truetype('/usr/share/fonts/dejavu/DejaVuSans.ttf', 20)
caffe.set_mode_gpu()
caffe.set_device(gpu)
if model == 1:
data_net_file, net_proto = N.create_netfile(model,
data, mean, K + num_step, K, 1, num_act, num_step=num_step, mode='data')
test_net_file, net_proto = N.create_netfile(model, data, mean, K, K,
1, num_act, num_step=1, mode='test')
data_net = caffe.Net(data_net_file, caffe.TEST)
test_net = caffe.Net(test_net_file, caffe.TEST)
test_net.copy_from(weights)
else:
data_net_file, net_proto = N.create_netfile(model,
data, mean, K + num_step, K, 1, num_act, num_step=num_step, mode='data')
encoder_net_file, net_proto = N.create_netfile(model, data, mean, K, K,
1, num_act, num_step=0, mode='test_encode')
decoder_net_file, net_proto = N.create_netfile(model, data, mean, 1, 0,
1, num_act, num_step=1, mode='test_decode')
data_net = caffe.Net(data_net_file, caffe.TEST)
encoder_net = caffe.Net(encoder_net_file, caffe.TEST)
decoder_net = caffe.Net(decoder_net_file, caffe.TEST)
decoder_net.copy_from(weights)
encoder_net.share_with(decoder_net)
mean_blob = caffe.proto.caffe_pb2.BlobProto()
mean_bin = open(mean, 'rb').read()
mean_blob.ParseFromString(mean_bin)
mean_arr = caffe.io.blobproto_to_array(mean_blob).squeeze()
if video:
sp.call(['rm', '-rf', video])
sp.call(['mkdir', '-p', video])
for i in range(0, num_iter):
print("iteration " + str(i) + "/" + str(num_iter))
data_net.forward()
data_blob = data_net.blobs['data'].data
act_blob = data_net.blobs['act'].data
if model == 1:
test_net.blobs['data'].data[:] = data_blob[:, 0:K, :, :, :]
test_net.blobs['act'].data[:] = act_blob[:, K-1, :]
net = test_net
elif model == 2:
clip_blob = data_net.blobs['clip']
encoder_net.blobs['data'].data[:] = data_blob[0:K, :, :, :, :]
encoder_net.blobs['clip'].data[:] = 1
encoder_net.blobs['clip'].data[0, :] = 0
encoder_net.blobs['h-00'].data[:] = 0
encoder_net.blobs['c-00'].data[:] = 0
encoder_net.forward()
decoder_net.blobs['h-00'].data[:] = encoder_net.blobs['h'+tag(K)].data[:]
decoder_net.blobs['c-00'].data[:] = encoder_net.blobs['c'+tag(K)].data[:]
decoder_net.blobs['clip'].data[:] = 1
decoder_net.blobs['act'].data[:] = act_blob[K-1, :, :]
net = decoder_net
pred_data = np.zeros((3, 210, 160), np.float)
true_data = np.zeros((3, 210, 160), np.float)
imgs = []
for step in range(0, num_step):
net.forward()
if model == 1:
pred_data[:] = net.blobs['x_hat'+tag(K+1)].data[:]
true_data[:] = data_net.blobs['data'].data[:, K+step, :, :, :]
elif model == 2:
pred_data[:] = net.blobs['x_hat'+tag(1)].data[:]
true_data[:] = data_net.blobs['data'].data[K+step, :, :, :, :]
pred_img = post_process(pred_data, mean_arr, 1./255)
true_img = post_process(true_data, mean_arr, 1./255)
# display
show_img = np.hstack((pred_img, true_img))
top_pad = np.zeros((35, show_img.shape[1], show_img.shape[2]), np.uint8)
show_img = np.vstack((top_pad, show_img))
imgs.append(show_img)
img = Image.fromarray(show_img[:, :, ::-1])
# draw = ImageDraw.Draw(img)
# draw.text((10, 10), 'Step:' + str(step), fill=(255, 255, 255), font=font)
img.save('iter_' + str(i) + '.png')
# cv2.imshow('Display', np.array(img))
# key = cv2.waitKey(40)
if video:
file_name = video+'/{:0>3d}-{:0>5d}.png'.format(i, step)
b, g, r = img.split()
Image.merge("RGB", (r, g, b)).save(file_name)
if step < num_step - 1:
if model == 1:
net.blobs['data'].data[:, 0:K-1, :, :, :] = net.blobs['data'].data[:, 1:K, :, :, :]
net.blobs['data'].data[:, K-1, :, :, :] = pred_data[:]
net.blobs['act'].data[:] = act_blob[:, K+step, :]
elif model == 2:
net.blobs['h-00'].data[:] = net.blobs['h-01'].data[:]
net.blobs['c-00'].data[:] = net.blobs['c-01'].data[:]
net.blobs['data'].data[:] = pred_data[:]
net.blobs['act'].data[:] = act_blob[K+step, :, :]
imgs = np.concatenate(imgs, axis=2)
img = Image.fromarray(show_img[:,:,::-1])
img.save('final.png')
if video:
sp.call(['ffmpeg', '-pattern_type', 'glob', '-r', '15', '-i', video+'/*.png', '-qscale', '0', video+'.mp4'])
def main():
gpu = 0
net_type = 'googlenet'
data_desc = 'train_heldout'
# TODO: Change this to the directory in which learned masks are stored
#mask_dir = '/data/ruthfong/neural_coding/pycaffe_results/googlenet_train_heldout_given_grad_1_norm_0/min_top0_prob_blur/lr_-1.00_l1_lambda_-4.00_tv_lambda_-inf_l1_lambda_2_-2.00_beta_3.00_mask_scale_8_blur_mask_5_jitter_4_noise_-inf_num_iters_300_tv2_mask_init'
mask_dir = '/data/ruthfong/neural_coding/results_reb/occ_masks_imagenet_googlenet_train_heldout_defaults'
# TODO: Change this to desired output directorya (subdirs will be created)
out_dir = '/users/ruthfong/neural_coding/deletion_game'
mask_flip = True
bb_method = 'min_max_diff'
threshold = 0.01
caffe.set_device(gpu)
caffe.set_mode_gpu()
if data_desc == 'train_heldout':
(paths, labels) = read_imdb(
'../../../data/ilsvrc12/annotated_train_heldout_imdb.txt')
elif data_desc == 'val':
(paths, labels) = read_imdb('../../../data/ilsvrc12/val_imdb.txt')
elif data_desc == 'animal_parts':
(paths, labels) = read_imdb(
'../../../data/ilsvrc12/animal_parts_require_both_min_10_imdb.txt')
mask_paths = [
os.path.join(mask_dir, '%d.npy' % x) for x in range(len(labels))
]
net = get_net(net_type)
net_transformer = get_ILSVRC_net_transformer(net)
alphas = np.arange(0, 1, 0.01)
#heatmap_types = ['mask', 'saliency', 'guided_backprop', 'excitation_backprop', 'contrast_excitation_backprop']
heatmap_types = ['occlusion', 'grad_cam']
num_imgs = len(labels)
best_bb_sizes = np.zeros((len(heatmap_types), num_imgs))
for i in range(num_imgs):
start = time.time()
img = net_transformer.preprocess('data', caffe.io.load_image(paths[i]))
null_img = net_transformer.preprocess('data',
get_blurred_img(paths[i]))
target = np.zeros(1000)
target[labels[i]] = 1
orig_score = forward_pass(net, img, target)
null_score = forward_pass(net, null_img, target)
for hh in range(len(heatmap_types)):
h = heatmap_types[hh]
resize = None
if h == 'mask' or h == 'occlusion':
heatmap = np.load(mask_paths[i])
if mask_flip:
heatmap = 1 - heatmap
else:
top_name = 'loss3/classifier'
if h == 'excitation_backprop':
bottom_name = 'pool2/3x3_s2'
norm_deg = -1
resize = (224, 224)
elif h == 'contrast_excitation_backprop':
bottom_name = 'pool2/3x3_s2'
norm_deg = -2
resize = (224, 224)
elif h == 'grad_cam':
bottom_name = 'inception_4e/output'
norm_deg = None
resize = (224, 224)
else:
bottom_name = 'data'
norm_deg = np.inf
heatmap = compute_heatmap(net,
net_transformer,
paths[i],
labels[i],
h,
top_name,
top_name,
outputBlobName=bottom_name,
outputLayerName=bottom_name,
norm_deg=norm_deg,
gpu=gpu)
bb_sizes = np.zeros(len(alphas))
scores = np.zeros(len(alphas))
for j in range(len(alphas)):
alpha = alphas[j]
(x0, y0, x1, y1) = getbb_from_heatmap(heatmap,
alpha,
method=bb_method,
resize=resize)
mask = np.ones((224, 224))
mask[y0:y1, x0:x1] = 0
bb_size = (y1 - y0) * (x1 - x0)
bb_sizes[j] = bb_size
comp_img = img * mask + null_img * (1 - mask)
masked_score = forward_pass(net, comp_img, target)
scores[j] = (masked_score - null_score) / float(orig_score -
null_score)
# alphas_scores_path = os.path.join(out_dir, '%s_%s/%s/alpha_scores_num_imgs_%d_%s.txt' % (net_type, data_desc,
# mask_rel_dir, num_imgs, h))
# bb_sizes_path = os.path.join(out_dir, '%s_%s/%s/bb_sizes_num_imgs_%d_%s.txt' % (net_type, data_desc,
# mask_rel_dir, num_imgs, h))
alphas_scores_path = os.path.join(
out_dir, '%s_%s/alpha_scores_num_imgs_%d_%s.txt' %
(net_type, data_desc, num_imgs, h))
bb_sizes_path = os.path.join(
out_dir, '%s_%s/bb_sizes_num_imgs_%d_%s.txt' %
(net_type, data_desc, num_imgs, h))
directory = os.path.dirname(alphas_scores_path)
if not os.path.exists(directory):
os.makedirs(directory)
f = open(alphas_scores_path, 'a')
np.savetxt(f, scores[None, :])
f.close()
f = open(bb_sizes_path, 'a')
np.savetxt(f, bb_sizes[None, :])
f.close()
try:
best_i = np.where(scores <= threshold)[0][-1]
except:
best_i = np.argmin(scores)
best_bb_sizes[hh][i] = bb_sizes[best_i]
print 'Image %d took %f' % (i, time.time() - start)
if i % 50 == 0:
print 'heatmap_type', 'best_bb_size_mean', 'best_bb_size_std'
for hh in range(len(heatmap_types)):
print heatmap_types[hh], best_bb_sizes[hh][:(
i + 1)].mean(), best_bb_sizes[hh][:(i + 1)].std()
if print_:
# print "confidence mean",yp_np.mean(),yp_np.max(),yp_np.min()
print "using ", ind.sum(
), " unlabeled data ", " of accuracy ", accuracy
return yp[ind], ind, accuracy
if __name__ == '__main__':
# os.chdir('/home/kashefy/src/caffe/')
os.chdir('/home/jianqiao/Caffe/caffe-master/')
test_batchsz = 1000
train_batchsz = 128
caffe.set_mode_gpu()
caffe.set_device(0) # for gpu mode
# caffe.set_mode_cpu()
TOTAL_NUM = 50000
# (labelset,cifar_table )=create_labelset(TOTAL_NUM, prototxt)
# (labelset,cifar_table )=pkl.load(open('examples/cifar100/labelset.pkl','rb'))
filt_hist = []
start_index = 0
end_index = start_index + train_batchsz
unlabeled = rand(train_batchsz) > 0
yd = 100
label_ratio = 0.1
rnd.seed(100)
labeled_label, num_features_label = ndimage.label(label_temp, structure=s)
for i in range(1,num_features_prediction+1):
intersection=np.sum(labeled_prediction[label==1]==i)
if intersection>0:
TP+=1
else:
FP+=1
for i in range(1,num_features_label+1):
intersection=np.sum(labeled_label[prediction==1]==i)
if intersection==0:
FN=+1
return TP,FN,FP
caffe.set_mode_gpu()
caffe.set_device(0);
# load net
snapshot_path = 'snapshot/train_iter_120000.caffemodel'
net = caffe.Net('deploy.prototxt', snapshot_path, caffe.TEST)
# Initialize data index
maxIdx = len(net.layers[0].indices)
# Lists to hold segmentation precision and recall, detection precisio and recall
# and intersection over union values
prec = []
rec = []
precd = []
recd = []
iu=[]
def main(argv):
parser = ArgumentParser(argv[0], description=__doc__)
parser.add_argument('--model',
'-m',
type=str,
default='models/1_layer_ride/rim.10082016.221222.xpck')
parser.add_argument('--data',
'-d',
type=str,
default='data/BSDS_Cropped/img_data.mat')
parser.add_argument('--noise_std', '-n', type=int, default=-1)
parser.add_argument('--momentum', '-M', type=float, default=0.9)
parser.add_argument('--lr', '-l', type=float, default=5.0)
parser.add_argument('--niter', '-N', type=int, default=400)
parser.add_argument('--path', '-p', type=str, default='map_single_pixel/')
parser.add_argument('--mode',
'-q',
type=str,
default='CPU',
choices=['CPU', 'GPU'])
parser.add_argument('--device', '-D', type=int, default=0)
parser.add_argument('--size', '-s', type=int, default=160)
parser.add_argument('--samples', '-y', type=float, default=0.4)
parser.add_argument('--image_num', '-K', type=int, default=2)
parser.add_argument('--resume', '-r', type=int, default=-1)
parser.add_argument('--flip', '-f', type=int, default=0)
parser.add_argument('--ent_max', '-e', type=float, default=100.0)
args = parser.parse_args(argv[1:])
niter = args.niter
lr = args.lr
N = args.size
K = args.image_num
noise_std = args.noise_std
path = args.path
if not os.path.exists(path):
os.makedirs(path)
print 'Measurement Rate', args.samples
print 'Noise Level', noise_std
# select CPU or GPU for caffe
if args.mode.upper() == 'GPU':
print "setting the GPU mode"
caffe.set_mode_gpu()
caffe.set_device(args.device)
else:
caffe.set_mode_cpu()
if noise_std > -1:
noise_std = float(noise_std) * 10 / 255.0
path += str(args.noise_std) + '/'
if args.samples > -1:
path += str(args.samples) + '/'
if not os.path.exists(path):
os.makedirs(path)
sys.stdout = open(path + 'log.txt', 'w')
# load data
if args.data.lower()[-4:] in ['.gif', '.png', '.jpg', 'jpeg']:
images = plt.imread(args.data)
img = rgb2gray(images[:args.size, :args.size])
img = img.astype('float64')
else:
images = loadmat(args.data)['data']
images = images.astype('float64')
img = images[:K, :args.size, :args.size]
# load model
print 'Loading model'
experiment = Experiment(args.model)
model = experiment['model']
input_mask = model.input_mask
output_mask = model.output_mask
Phi = np.load('map_single_pixel/Phi_g_' + str(N) +
'.npy')[1:int(args.samples * args.size**2), :]
del images
for k in range(K):
if not os.path.exists(path + str(k) + '/'):
os.makedirs(path + str(k) + '/')
mplimg.imsave(path + str(k) + '/original_img',
img[k].squeeze(),
cmap=cm.gray)
y = np.dot(Phi, img.reshape(K, -1).transpose())
np.random.seed(123)
if noise_std > -1:
y += noise_std * np.random.randn(*y.shape)
M = y.shape[0]
print 'Number of measurements', M
#Initializing
np.random.seed(123)
init_img = np.random.rand(N**2, K)
prev_grad = 0
if args.resume > 0:
init_img = np.load(path + 'cleaned_img/' + str(args.resume) + '.npy')
for k in range(K):
mplimg.imsave(path + str(k) + '/init_img',
init_img[:, k].reshape(N, N),
cmap=cm.gray)
psnr_list = [[] for i in range(K)]
ssim_list = [[] for i in range(K)]
for i in range(args.niter):
j = args.flip * i
f, grad_img, whitened_img = model.gradient(
init_img.transpose().reshape(K, N, N,
1)[:, ::(-1)**j, ::(-1)**(j / 2), :],
precond=None,
niter=i,
path=path,
ent_max=args.ent_max)
df_dh = grad_img[:, ::(-1)**j, ::(-1)**(j / 2), :].reshape(
K, -1).transpose()
print i, 'f', f.sum(), 'df_dh', np.abs(df_dh).sum(),
prev_grad = args.momentum * prev_grad + df_dh
x_up = init_img + lr * (prev_grad)
init_img = x_up - np.dot(Phi.transpose(), np.dot(Phi, x_up) - y)
init_img = np.clip(init_img, 0.0, 1.0)
if i % 10 == 0:
for k in range(K):
mplimg.imsave(path + str(k) + '/img' + str(i),
init_img[:, k].reshape(N, N),
cmap=cm.gray)
l = linalg.norm(y - np.dot(Phi, init_img))
print 'l', l
#For Saving Gradient Image
# if (i%10==0):
# for k in range(K):
# fig2 = plt.figure(2)
# plt.imshow(df_dh[:,k].reshape(N,N),vmin = -0.02,vmax=0.02)
# plt.colorbar()
# plt.savefig(path+str(k)+'/grad_img'+str(i))
# plt.close(fig2)
# if (i%1 ==0):
# for k in range(K):
# fig1 = plt.figure(1)
# plt.imshow(f[:,k].reshape(N,N))
# plt.colorbar()
# plt.savefig(path+str(k)+'/loglik_img'+str(i))
# plt.close(fig1)
m = 2 #Margin to remove for comparision
for k in range(K):
ssim1 = ssim(init_img[:, k].reshape(N, N)[m:-m, m:-m],
img[k].squeeze()[m:-m, m:-m],
dynamic_range=img.min() - img.max())
psnr1 = psnr(init_img[:, k].reshape(N, N)[m:-m, m:-m],
img[k].squeeze()[m:-m, m:-m],
dynamic_range=img.min() - img.max())
ssim_list[k].append(ssim1)
psnr_list[k].append(psnr1)
if not os.path.exists(path + 'cleaned_img/'):
os.makedirs(path + 'cleaned_img/')
np.save(path + 'cleaned_img/ssim_list', ssim_list)
np.save(path + 'cleaned_img/psnr_list', psnr_list)
print k, 'ssim', ssim1, 'psnr', psnr1
if (i % 50 == 0): #Storing npy files
np.save(path + 'cleaned_img/' + str(i), init_img)
return 0
def main(config):
caffe.set_device(config['device_id'])
caffe.set_mode_gpu()
# step 1. configure cnn
if config['arch']=='A4,G1':
# an ensemble of alexnet and googlenet
models=[
caffe.Net('deploy-alexnet_full_conv.prototxt','minc-alexnet_full_conv.caffemodel',caffe.TEST),
caffe.Net('deploy-googlenet_full_conv_no_pooling.prototxt','minc-googlenet_full_conv.caffemodel',caffe.TEST)
]
# alexnet needs a padded input to get a full-frame prediction
input_padding=[97,0]
# nominal footprint is 46.4% for A4, 23.2% for G1
scales=[256/550.0,256/1100.0]
bgr_mean=numpy.array([104,117,124],dtype=numpy.float32)
# inputs must be a multiple of the stride
# otherwise, the full-frame prediction will be shifted
effective_stride=[32,32]
# TODO: A4 needs spatial oversampling (# shifts = 2)
else:
raise NotImplementedError
# step 2. configure crf
if config['crf']=='1':
# these are the CRF parameters for MINC (see supplemental)
# the parameters can have a big impact on the output
# so they should be tuned for the target domain
# (MINC note: this code is not exactly the same as the
# original MINC code so these parameters will not
# generate the exact same results)
crf_params={
"bilateral_pairwise_weight": 5.0, # w_p
"bilateral_theta_xy": 0.1, # \theta_p
"bilateral_theta_lab_l": 20.0, # \theta_L
"bilateral_theta_lab_ab": 5.0, # \theta_ab
"n_crf_iters": 10,
"unary_prob_padding": 1e-05,
}
elif config['crf']=='matclass':
# new CRF parameters
crf_params={
"bilateral_pairwise_weight": 8.0, # w_p
"bilateral_theta_xy": 0.5, # \theta_p
"bilateral_theta_lab_l": 0.5, # \theta_L
"bilateral_theta_lab_ab": 3.0, # \theta_ab
"n_crf_iters": 10,
"unary_prob_padding": 1e-05,
}
else:
raise NotImplementedError
pad_value=bgr_mean[::-1]/255.0
# step 3. extract class prediction maps
for ipath in config['input']:
# read image
original=imageutils.read(ipath)
z=config['min_dim']/float(min(original.shape[:2]))
crf_shape=(23,int(round(original.shape[0]*z)),int(round(original.shape[1]*z)))
# predict 6 maps: 3 scales for each model
maps=[]
for index,model in enumerate(models):
p=input_padding[index]
s=scales[index]
for index2,multiscale in enumerate([0.7071067811865476,1.0,1.4142135623730951]):
# resample the input so it is a multiple of the stride
# and the receptive field matches the nominal footprint
scale_factor=(256/s)/float(min(original.shape[:2]))
scaled_size=[nearest_multiple(original.shape[i]*scale_factor*multiscale,effective_stride[index]) for i in range(2)]
scaled=imageutils.resize(original,scaled_size)
if p>0:
# add input padding for alexnet
pad=numpy.ones((scaled.shape[0]+2*p,scaled.shape[1]+2*p,scaled.shape[2]),dtype=scaled.dtype)*pad_value
pad[p:-p,p:-p]=scaled
scaled=pad
# predict and resample the map to be the correct size
data=preprocess_and_reshape(scaled,model,bgr_mean=bgr_mean)
output=model.forward_all(data=data)['prob'][0]
output=scipy.ndimage.interpolation.zoom(output,[1.0,crf_shape[1]/float(output.shape[1]),crf_shape[2]/float(output.shape[2])],order=1)
maps.append(output)
# step 4. average all maps
crf_map=numpy.asarray(maps).mean(axis=0)
if False:
# output extra maps for debugging
for i,x,j in [(i,x,j) for i,x in enumerate(maps) for j in range(23)]: imageutils.write('zzz_map_{}{}.jpg'.format(dataset.NETCAT_TO_NAME[j],i),x[j])
for j in range(23): imageutils.write('zzz_mean_map_{}.jpg'.format(dataset.NETCAT_TO_NAME[j],i),crf_map[j])
imageutils.write('zzz_naive_labels.png',labels_to_color(numpy.argsort(-crf_map.reshape(23,-1).T).T.reshape(*crf_map.shape)[0]))
crf_color=imageutils.resize(original,(crf_shape[1],crf_shape[2]))
assert crf_color.shape[0]==crf_map.shape[1] and crf_color.shape[1]==crf_map.shape[2]
# step 5. dense crf
#lcrf=densecrf_matclass.general_densecrf.LearnableDenseCRF(crf_color,crf_map,crf_params)
lcrf=densecrf_matclass.densecrf.LearnableDenseCRF(crf_color,crf_map,crf_params)
labels_crf=lcrf.map(crf_params)
# step 6. visualize with color labels
result=labels_to_color(labels_crf)
if os.path.exists(config['output']) and os.path.isdir(config['output']):
opath=os.path.join(config['output'],os.path.splitext(os.path.split(ipath)[1])[0])+'.png'
else:
opath=config['output']
assert not os.path.exists(opath)
imageutils.write(opath,result)
print(opath)
def each_video(results_folder, root_folder, video):
caffe.set_device(1) #gpu device number
caffe.set_mode_gpu()
ucf_mean_RGB = np.zeros((3, 1, 1))
ucf_mean_RGB[0, :, :] = 75
ucf_mean_RGB[1, :, :] = 75
ucf_mean_RGB[2, :, :] = 101
# 75,75,101 are the mean values for three channels
transformer_RGB = initialize_transformer(ucf_mean_RGB, False)
#Models and weights
svrcnet_model = '/path/to/deploy/folder/SVRCNet-workflow-deploy.prototxt'
svrcnet_caffemodel = '/path/to/caffemodel/folder/name.caffemodel'
svrcnet_net = caffe.Net(svrcnet_model, svrcnet_caffemodel, caffe.TEST)
# create files to save results
filename = results_folder + video + "_phase.txt"
fw = codecs.open(filename, "w", "utf-8-sig")
filename_pre = results_folder + video + "_phase_prediction.txt"
fw_pre = codecs.open(filename_pre, "w", "utf-8-sig")
# load test data path
frames = sorted(glob.glob('%s%s/*.jpg' % (root_folder, video)),
key=os.path.getmtime)
clip_length = 3
total_num = len(frames) #frame number of each video
frames = frames[0:total_num - 1]
frames_num = total_num - 1
print frames_num
action_hash = pickle.load(open('/path/to/action_hash_rev_cholec80.p',
'rb'))
# initialize the model
predictions_SVRCNet = SVRCNet_test_video(frames, svrcnet_net,
transformer_RGB, False)
del svrcnet_net
#test
headlines = ''.join(['Frame', '\t', 'Phase', '\n'])
fw.writelines(headlines)
for i in range(frames_num):
frame_prediction = np.zeros((1, 8))
frame_prediction = predictions_SVRCNet[i, :]
max_label = frame_prediction.argmax()
top_inds = frame_prediction.argsort()[::-1][:3]
pre = zip(frame_prediction[top_inds], top_inds)
index = i * 25
fileline_v2 = ''.join([str(index), '\t', action_hash[max_label], '\n'])
fw.writelines(fileline_v2)
fileline_v2_pre = ''.join([
str(i), '\t',
str(frame_prediction[0]), '\t',
str(frame_prediction[1]), '\t',
str(frame_prediction[2]), '\t',
str(frame_prediction[3]), '\t',
str(frame_prediction[4]), '\t',
str(frame_prediction[5]), '\t',
str(frame_prediction[6]), '\n'
])
fw_pre.writelines(fileline_v2_pre)
fw.close()
fw_pre.close()
def classify(net_config_location, net_weights, image_location, alpha_channel, use_gpu, win_size,crop_size):
if use_gpu:
caffe.set_device(0)
caffe.set_mode_gpu()
else:
caffe.set_mode_cpu()
f = open(net_config_location, 'r')
s = f.read()
if (alpha_channel):
append = 'input: "data" input_dim: 1 input_dim: 4 input_dim: 512 input_dim: 512 \n'
else:
append = 'input: "data" input_dim: 1 input_dim: 3 input_dim: 512 input_dim: 512 \n'
s = append + s
f = open('temp.prototxt', 'w')
f.write(s)
f.close()
net = caffe.Net('temp.prototxt', net_weights, caffe.TEST)
print 'Opening image...'
#open image
in_ = getimage(image_location,alpha_channel)
print 'Image opened...'
#height and width of original image
orig_img_h = in_.shape[1]
orig_img_w = in_.shape[2]
#size of valid output patch
out_size = win_size-2*crop_size
#number of patches horizontally and vertically
n_patch_horiz = int(math.ceil(orig_img_w/float(out_size)))
n_patch_vert = int(math.ceil(orig_img_h/float(out_size)))
#pad image...
#how much to pad?
pad_w_before = crop_size
pad_h_before = crop_size
pad_w_after = n_patch_horiz*out_size + crop_size - orig_img_w
pad_h_after = n_patch_vert*out_size + crop_size - orig_img_h
#do padding
in_ = np.pad(in_, ((0,0), (pad_h_before,pad_h_after), (pad_w_before,pad_w_after)), mode='symmetric')
# shape for input (data blob is N x C x H x W), set data
if alpha_channel:
channels=4
else:
channels=3
net.blobs['data'].reshape(1, channels, win_size, win_size)
print 'Predicting...'
rows = []
for i in range(0,n_patch_vert):
patches_in_row = []
for j in range(0,n_patch_horiz):
input_ = in_[:,out_size*i:out_size*i+win_size,out_size*j:out_size*j+win_size]
net.blobs['data'].data[...] = input_
# run net prediction
net.forward()
patch_out = net.blobs['prob'].data[0]
#compute offset in case output patch provided by the network
#is larger than it should be
h_offset = (net.blobs['prob'].data[0].shape[1] - out_size)/2
w_offset = (net.blobs['prob'].data[0].shape[2] - out_size)/2
#crop
patch_out = patch_out[:,h_offset:h_offset+out_size,w_offset:w_offset+out_size]
patches_in_row.append(np.copy(patch_out))
row = np.concatenate(patches_in_row,2)
rows.append(np.copy(row))
entire_output = np.concatenate(rows,1)
#remove excess border
output = entire_output[:,0:orig_img_h,0:orig_img_w]
#out.astype('double').tofile("prob.dat");
if output.shape[0]==1:
pred = np.rint(np.squeeze(output)).astype(np.uint8)
else:
pred = output.argmax(axis=0).astype(np.uint8)
print 'Done predicting.'
return pred
gpu_device = 1
start_epoch = 1
epoch_step = 2
model_fold = '../code/model/new_experiment'
net_style = 'test'
epoch_num = 10
model_fold = os.path.join(model_fold, net_style)
model = prepare_model(model_fold)
model['fea_name'] = 'avg_norm'
model['model_file'] = os.path.join(model_fold, 'test')
model['weight_file'] = os.path.join(model_fold,
"epoch_%d.caffemodel" % epoch_num)
caffe.set_mode_gpu()
caffe.set_device(start_epoch)
net = caffe.Net(model[''])
data_folder = '/home1/qcz/DataSet'
dataset_train, dataset_test = 'paris6k', 'oxford5k'
gnd_test = os.path.join(data_folder, dataset_test,
'gnd_' + dataset_test + '.mat')
gnd_test = scio.loadmat(gnd_test)
qidx = gnd_test['qidx']
imlist = gnd_test['imlist']
gnd = gnd_test['gnd']
rank_path = '/home1/qcz/qcz_pro/building_train/code/ranks.mat'
ranks = scio.loadmat(rank_path)
ranks = ranks['ranksOxLw']
[map, aps] = compute_map(ranks, gnd)
gnd_train = os.path.join(data_folder, dataset_train,
def train(args):
# generate and store training data online
Data_Q = Queue(50)
Label_Q = Queue(50)
pData = Process(target=genData,
args=(Data_Q, Label_Q, args.batchsize, args.crop_H,
args.crop_W))
pData.start()
caffe.set_mode_gpu()
caffe.set_device(args.GPU_ID)
logging.basicConfig(filename=args.log, level=logging.INFO)
solver = caffe.AdamSolver(args.solver)
# initialization
if args.init_model:
logging.info('loading' + args.init_model)
solver.net.copy_from(args.init_model)
logging.info('loaded')
solver.net.params['init_norm'][0].data[...] = args.init_p
solver.net.params['init_pool'][0].data[...] = args.init_p
norm_param_ul = ['norm_%d' % i for i in xrange(1, args.T)]
pool_param_ul = ['pool_%d' % i for i in xrange(1, args.T)]
for i in xrange(args.T - 1):
solver.net.params[norm_param_ul[i]][0].data[...] = args.init_p
solver.net.params[pool_param_ul[i]][0].data[...] = args.init_p
solver.net.blobs['data'].reshape(args.batchsize, 1, args.crop_H,
args.crop_W)
solver.net.blobs['label'].reshape(args.batchsize, 1, args.crop_H,
args.crop_W)
# training process
epoch = args.num_epochs
epoch_iters = 750
step_sum = epoch_iters * epoch + 1
step = 0
valid_acc = 0
aver_psnr = 0.0
best_psnr = 0.0
best_iter = 0
best_each_psnr = [0 for i in xrange(19)]
while step < step_sum:
if Data_Q.empty() != True:
# training
solver.net.blobs['data'].data[...] = Data_Q.get()
solver.net.blobs['label'].data[...] = Label_Q.get()
solver.step(1)
p = solver.net.params['init_norm'][0].data[...]
log_out = 'iter: %d p: %.4f loss: %f' % (
step, p, solver.net.blobs['loss'].data)
logging.info(log_out)
step += 1
# evaluated on validation or test set.
if step % epoch_iters == 0:
psnr, aver_psnr, psnr_set5, psnr_set14 = test_valid(solver.net)
if (aver_psnr > best_psnr):
best_psnr = aver_psnr
best_each_psnr = psnr
best_iter = step
log_out = 'test: iter: %d aver_psnr/best_psnr %f/%f psnr_set5: %f psnr_set14: %f\n' % (
step, aver_psnr, best_psnr, psnr_set5, psnr_set14)
logging.info(log_out)
log_psnr = 'test: iter: %d ' % step
log_best_psnr = 'best: iter: %d ' % best_iter
for j in xrange(19):
log_psnr = log_psnr + '%d: %f ' % (j, psnr[j - 1])
log_best_psnr = log_best_psnr + '%d: %f ' % (
j, best_each_psnr[j - 1])
logging.info(log_psnr)
logging.info(log_best_psnr)
solver.net.blobs['data'].reshape(args.batchsize, 1,
args.crop_H, args.crop_W)
solver.net.blobs['label'].reshape(args.batchsize, 1,
args.crop_H, args.crop_W)
def initCaffe(opt):
caffe.set_mode_gpu()
gpu_id = 0
caffe.set_device(gpu_id)
net = caffe.Net(opt.modelDefFile, opt.modelFile, caffe.TEST)
return net
in_ = np.array(im, dtype=np.float32)
in_ = in_[:,:,::-1]
in_ -= np.array((104.00698793,116.66876762,122.67891434))
in_ = in_.transpose((2,0,1))
return in_
video_name = sys.argv[1]
caffe_model = '../model/SegFlow.caffemodel'
deploy_proto = '../model/deploy.prototxt'
file_out = '../results/' + video_name[8:-4]
device_id = 1
# init network
caffe.set_device(device_id)
caffe.set_mode_gpu()
net = caffe.Net(deploy_proto , caffe_model, caffe.TEST)
if os.path.exists(file_out) == False:
os.mkdir(file_out)
vid = cv2.VideoCapture(video_name)
flag, frame = vid.read()
print(flag)
num = 0
while flag:
num = num + 1
img1 = img_transform(frame)
