def flowWarp(img, flow):
import caffe
width = img.shape[1]
height = img.shape[0]
print 'processing (%dx%d)' % (width, height)
defFile = tempFilename('.prototxt')
preprocessFile('/home/ilge/hackathon2/common/prototmp/apply_flow.prototmp',
defFile, {
'WIDTH': width,
'HEIGHT': height
})
caffe.set_logging_disabled()
caffe.set_mode_gpu()
net = caffe.Net(defFile, caffe.TEST)
os.remove(defFile)
print 'network forward pass'
img_input = img[np.newaxis, :, :, :].transpose(0, 3, 1, 2)
flow_input = flow[np.newaxis, :, :, :].transpose(0, 3, 1, 2)
net.blobs['image'].reshape(*img_input.shape)
net.blobs['image'].data[...] = img_input
net.blobs['flow'].reshape(*flow_input.shape)
net.blobs['flow'].data[...] = flow_input
net.forward()
output = net.blobs['output'].data[...].transpose(0, 2, 3, 1).squeeze()
return output
def __init__(self, net_proto, net_weights, device_id, input_size=None):
caffe.set_logging_disabled()
if device_id >= 0:
caffe.set_mode_gpu()
caffe.set_device(device_id)
else:
caffe.set_mode_cpu()
self._net = caffe.Net(net_proto, net_weights, caffe.TEST)
input_shape = self._net.blobs['data'].data.shape
if input_size is not None:
input_shape = input_shape[:2] + input_size
transformer = caffe.io.Transformer({'data': input_shape})
if self._net.blobs['data'].data.shape[1] == 3:
transformer.set_transpose(
'data',
(2, 0, 1)) # move image channels to outermost dimension
transformer.set_mean('data', np.array(
[104, 117,
123])) # subtract the dataset-mean value in each channel
else:
pass # non RGB data need not use transformer
self._transformer = transformer
self._sample_shape = self._net.blobs['data'].data.shape
def load_model(width, height):
vars = {}
vars['TARGET_WIDTH'] = width
vars['TARGET_HEIGHT'] = height
divisor = 64.
vars['ADAPTED_WIDTH'] = int(ceil(width / divisor) * divisor)
vars['ADAPTED_HEIGHT'] = int(ceil(height / divisor) * divisor)
vars['SCALE_WIDTH'] = width / float(vars['ADAPTED_WIDTH'])
vars['SCALE_HEIGHT'] = height / float(vars['ADAPTED_HEIGHT'])
tmp = tempfile.NamedTemporaryFile(mode='w', delete=False)
proto = open(deployproto).readlines()
for line in proto:
for key, value in vars.items():
tag = "$%s$" % key
line = line.replace(tag, str(value))
tmp.write(line)
tmp.flush()
caffe.set_logging_disabled()
caffe.set_device(0)
caffe.set_mode_gpu()
net = caffe.Net(tmp.name, caffemodel, caffe.TEST)
print('Network forward pass using %s.' % caffemodel)
return net
def flowWarp(img, flow):
import caffe
width = img.shape[1]
height = img.shape[0]
print 'processing (%dx%d)' % (width, height)
defFile = tempFilename('.prototxt')
preprocessFile('/home/ilge/hackathon2/common/prototmp/apply_flow.prototmp', defFile, {'WIDTH': width, 'HEIGHT': height})
caffe.set_logging_disabled()
caffe.set_mode_gpu()
net = caffe.Net(
defFile,
caffe.TEST
)
os.remove(defFile)
print 'network forward pass'
img_input = img[np.newaxis, :, :, :].transpose(0, 3, 1, 2)
flow_input = flow[np.newaxis, :, :, :].transpose(0, 3, 1, 2)
net.blobs['image'].reshape(*img_input.shape)
net.blobs['image'].data[...] = img_input
net.blobs['flow'].reshape(*flow_input.shape)
net.blobs['flow'].data[...] = flow_input
net.forward()
output = net.blobs['output'].data[...].transpose(0, 2, 3, 1).squeeze()
return output
def __init__(self, height, width, caffemodel, deployproto, gpu=0):
import caffe
divisor = 64.
vars = {
'TARGET_WIDTH': width,
'TARGET_HEIGHT': height,
'ADAPTED_WIDTH': int(ceil(width / divisor) * divisor),
'ADAPTED_HEIGHT': int(ceil(height / divisor) * divisor)
}
vars['SCALE_WIDTH'] = width / float(vars['ADAPTED_WIDTH'])
vars['SCALE_HEIGHT'] = height / float(vars['ADAPTED_HEIGHT'])
tmp = tempfile.NamedTemporaryFile(mode='w', delete=False)
proto = open(deployproto).readlines()
for line in proto:
for key, value in vars.items():
tag = "$%s$" % key
line = line.replace(tag, str(value))
tmp.write(line)
tmp.flush()
caffe.set_logging_disabled()
caffe.set_device(gpu)
caffe.set_mode_gpu()
self.net = caffe.Net(tmp.name, caffemodel, caffe.TEST)
def testFiles(self, filelist, iter, output=False, definition=None, vars={}):
modelFile, iter = self.getModelFile(iter)
assert(output)
vars['output'] = True
vars['prefix'] = iter
vars['dataset'] = os.path.splitext(os.path.basename(filelist))[0]
self.makeScratchDir()
if definition is None: definition = 'testsingle'
proto = self.findProto(definition)
if output and 'dataset' in vars:
outPath = '%s/output_%d_%s' % (self._path, iter, vars['dataset'])
if os.path.isdir(outPath):
if self._unattended or tb.queryYesNo('Output folder %s exists, do you want to delete it first?' % os.path.basename(outPath)):
os.system('rm -rf %s' % outPath)
data_list = IO.readTupleList(filelist)
print data_list
# Run a new net for every sample:
for idx, line in enumerate(data_list):
num_blobs = len(line)
input_data = []
for blob_idx in range(num_blobs):
img = IO.readImage(line[blob_idx])
#print(img.shape)
input_data.append(img[np.newaxis, :, :, :].transpose(0, 3, 1, 2)[:,[2,1,0],:,:])
#print(input_data[-1].shape)
vars['width'] = input_data[0].shape[3]
vars['height'] = input_data[0].shape[2]
vars['basename'] = 'b%03d' % idx
finalProto = self.makeScratchPrototxt(proto, vars)
caffe.set_logging_disabled()
caffe.set_mode_gpu()
net = caffe.Net(finalProto, modelFile,caffe.TEST)
print 'Network forward pass (%d of %d). %d inputs of shapes:' % (idx+1, len(data_list), num_blobs)
for blob_idx in range(num_blobs):
print(" " + str(input_data[blob_idx].shape))
if not len(net.inputs) == len(line):
raise Exception('Net has %d inputs and in file list there are %d' % (len(net.inputs), len(line)))
input_dict = {}
for blob_idx in range(num_blobs):
input_dict[net.inputs[blob_idx]] = input_data[blob_idx]
net.forward(**input_dict)
print 'Iteration was %d' %iter
def process(set, ent, inputEnt):
highres = misc.imread(ent, flatten=False).astype(np.float32)
lowres = misc.imread(inputEnt, flatten=False).astype(np.float32)
width = highres.shape[1]
height = highres.shape[0]
print "processing %s (%dx%d)" % (ent, width, height)
defFile = "scratch/test_SR_deploy.prototxt"
pycnn.preprocessFile("deploy.prototmp", defFile, {"WIDTH": width, "HEIGHT": height})
if "youtube" in set:
print "using youtube mean"
mean_bgr = tb.readFloat("/misc/lmbraid17/ilge/caffe/superresolution/datasets/youtube/test/mean3.float3").astype(
np.float32
)
else:
mean_bgr = tb.readFloat("/home/ilge/data/caffe/superresolution/datasets/coco/mean.float3").astype(np.float32)
mean_bgr = cv2.resize(mean_bgr, (width, height), interpolation=cv2.INTER_CUBIC)
mean_bgr_lowres = cv2.resize(mean_bgr, (width / 4, height / 4), interpolation=cv2.INTER_CUBIC)
highres_nomean_bgr = highres[:, :, (2, 1, 0)] - mean_bgr
lowres_nomean_bgr = lowres[:, :, (2, 1, 0)] - mean_bgr_lowres
caffe.set_phase_test()
caffe.set_mode_gpu()
caffe.set_logging_disabled()
net = caffe.Net(defFile, modelFile)
print "network forward pass"
blobs = net.forward(
highres=np.asarray([net.preprocess("highres", highres_nomean_bgr / 255.0)]),
lowres=np.asarray([net.preprocess("lowres", lowres_nomean_bgr / 255.0)]),
)
output_bgr = 255.0 * blobs["output"].transpose(0, 2, 3, 1).squeeze()
output_bgr += mean_bgr
output_bgr[output_bgr < 0] = 0
output_bgr[output_bgr > 255] = 255
os.system("mkdir -p %s/%s" % (out_dir, set))
basename = os.path.basename(ent)[:-4].replace("_GT", "")
misc.imsave("%s/%s/%s-gt.png" % (out_dir, set, basename), highres)
misc.imsave("%s/%s/%s-recon.png" % (out_dir, set, basename), output_bgr[:, :, (2, 1, 0)])
# nn, li, cu = tb.computeBasePSNRs(ent, downsampledFilename=inputEnt)
nn = tb.PSNR()
li = tb.PSNR()
cu = tb.PSNR()
psnr = tb.PSNR()
psnr.set(blobs["psnr"][0, 0, 0, 0], blobs["psnr_y"][0, 0, 0, 0])
print "nn=%5s, li=%5s, cu=%5s, net=%5s" % (nn, li, cu, psnr)
return (nn, li, cu, psnr)
def __init__(self, values_to_load='compute_image.csv'):
caffe.set_logging_disabled()
caffe.set_device(0)
caffe.set_mode_gpu()
self.check = False
self.tmp = None
self.net = None
self.colorwheel = self.make_color_wheel()
if os.path.isfile(values_to_load):
self.init_class_from_csv(values_to_load)
def process(set,ent,inputEnt):
highres = misc.imread(ent, flatten=False).astype(np.float32)
lowres = misc.imread(inputEnt, flatten=False).astype(np.float32)
width = highres.shape[1]
height = highres.shape[0]
print 'processing %s (%dx%d)' % (ent, width, height)
defFile = 'scratch/test_SR_deploy.prototxt'
pycnn.preprocessFile('deploy.prototmp', defFile, {'WIDTH': width, 'HEIGHT': height})
if 'youtube' in set:
print 'using youtube mean'
mean_bgr = tb.readFloat("/misc/lmbraid17/ilge/caffe/superresolution/datasets/youtube/test/mean3.float3").astype(np.float32)
else:
mean_bgr = tb.readFloat("/home/ilge/data/caffe/superresolution/datasets/coco/mean.float3").astype(np.float32)
mean_bgr = cv2.resize(mean_bgr, (width, height), interpolation=cv2.INTER_CUBIC)
mean_bgr_lowres = cv2.resize(mean_bgr, (width/4, height/4), interpolation=cv2.INTER_CUBIC)
highres_nomean_bgr = highres[:, :, (2, 1, 0)] - mean_bgr
lowres_nomean_bgr = lowres[:, :, (2, 1, 0)] - mean_bgr_lowres
caffe.set_phase_test()
caffe.set_mode_gpu()
caffe.set_logging_disabled()
net = caffe.Net(
defFile,
modelFile
)
print 'network forward pass'
blobs = net.forward(highres=np.asarray([net.preprocess('highres', highres_nomean_bgr / 255.0)]),lowres=np.asarray([net.preprocess('lowres', lowres_nomean_bgr / 255.0)]))
output_bgr = 255.0 * blobs['output'].transpose(0, 2, 3, 1).squeeze()
output_bgr += mean_bgr
output_bgr[output_bgr < 0] = 0
output_bgr[output_bgr > 255] = 255
os.system('mkdir -p %s/%s' % (out_dir, set))
basename = os.path.basename(ent)[:-4].replace('_GT', '')
misc.imsave('%s/%s/%s-gt.png' % (out_dir, set, basename), highres)
misc.imsave('%s/%s/%s-recon.png' % (out_dir, set, basename), output_bgr[:, :, (2, 1, 0)])
#nn, li, cu = tb.computeBasePSNRs(ent, downsampledFilename=inputEnt)
nn = tb.PSNR(); li=tb.PSNR(); cu=tb.PSNR()
psnr = tb.PSNR()
psnr.set(blobs['psnr'][0, 0, 0, 0], blobs['psnr_y'][0, 0, 0, 0])
print 'nn=%5s, li=%5s, cu=%5s, net=%5s' % (nn, li, cu, psnr)
return (nn, li, cu, psnr)
def caffeNet(modelFile=None, prototmp=None, inputs={}, phase=None, logging=False):
if phase is None: phase=caffe.TEST
if prototmp is not None:
modelFile = tempFilename('.prototmp')
open(modelFile,'w').write(prototmp)
if not logging: caffe.set_logging_disabled()
caffe.set_mode_gpu()
net = caffe.Net(
modelFile,
phase
)
if prototmp is not None:
os.remove(modelFile)
for name, value in inputs.iteritems():
value = value[np.newaxis, :, :, :].transpose(0, 3, 1, 2)
net.blobs[name].reshape(*value.shape)
net.blobs[name].data[...] = value
return net
vars['ADAPTED_HEIGHT'] = int(ceil(height / divisor) * divisor)
vars['SCALE_WIDTH'] = width / float(vars['ADAPTED_WIDTH'])
vars['SCALE_HEIGHT'] = height / float(vars['ADAPTED_HEIGHT'])
tmp = tempfile.NamedTemporaryFile(mode='w', delete=True)
proto = open(args.deployproto).readlines()
for line in proto:
for key, value in vars.items():
tag = "$%s$" % key
line = line.replace(tag, str(value))
tmp.write(line)
tmp.flush()
if not args.verbose:
caffe.set_logging_disabled()
caffe.set_device(args.gpu)
caffe.set_mode_gpu()
net = caffe.Net(tmp.name, args.caffemodel, caffe.TEST)
input_dict = {}
for blob_idx in range(num_blobs):
input_dict[net.inputs[blob_idx]] = input_data[blob_idx]
#
# There is some non-deterministic nan-bug in caffe
# it seems to be a race-condition
#
print('Network forward pass using %s.' % args.caffemodel)
i = 1
while i <= 5:
def calculate_flow(im1=None, im2=None, caffemodel='/home/yi/code/video_motion3/flownet2/models/FlowNet2/FlowNet2_weights.caffemodel.h5',
deployproto='/home/yi/code/video_motion3/flownet2/models/FlowNet2/FlowNet2_deploy.prototxt.template'):
if not os.path.exists(caffemodel):
raise BaseException('caffemodel does not exist: ' + caffemodel)
if not os.path.exists(deployproto):
raise BaseException('deploy-proto does not exist: ' + deployproto)
# im1 = misc.imread('flownet2/viper_example/042_00742.jpg')
# im2 = misc.imread('flownet2/viper_example/042_00743.jpg')
num_blobs = 2
input_data = []
if len(im1.shape) < 3:
input_data.append(im1[np.newaxis, np.newaxis, :, :])
else:
input_data.append(im1[np.newaxis, :, :, :].transpose(0, 3, 1, 2)[:, [2, 1, 0], :, :])
if len(im2.shape) < 3:
input_data.append(im2[np.newaxis, np.newaxis, :, :])
else:
input_data.append(im2[np.newaxis, :, :, :].transpose(0, 3, 1, 2)[:, [2, 1, 0], :, :])
width = input_data[0].shape[3]
height = input_data[0].shape[2]
vars = {}
vars['TARGET_WIDTH'] = width
vars['TARGET_HEIGHT'] = height
divisor = 64.
vars['ADAPTED_WIDTH'] = int(ceil(width/divisor) * divisor)
vars['ADAPTED_HEIGHT'] = int(ceil(height/divisor) * divisor)
vars['SCALE_WIDTH'] = width / float(vars['ADAPTED_WIDTH']);
vars['SCALE_HEIGHT'] = height / float(vars['ADAPTED_HEIGHT']);
tmp = tempfile.NamedTemporaryFile(mode='w', delete=True)
proto = open(deployproto).readlines()
for line in proto:
for key, value in vars.items():
tag = "$%s$" % key
line = line.replace(tag, str(value))
tmp.write(line)
tmp.flush()
caffe.set_logging_disabled()
caffe.set_device(0)
caffe.set_mode_gpu()
net = caffe.Net(tmp.name, caffemodel, caffe.TEST)
input_dict = {}
for blob_idx in range(num_blobs):
input_dict[net.inputs[blob_idx]] = input_data[blob_idx]
print('Network forward pass using %s.' % caffemodel)
i = 1
while i<=5:
i+=1
net.forward(**input_dict)
containsNaN = False
for name in net.blobs:
blob = net.blobs[name]
has_nan = np.isnan(blob.data[...]).any()
if has_nan:
print('blob %s contains nan' % name)
containsNaN = True
if not containsNaN:
print('Succeeded.')
break
else:
print('**************** FOUND NANs, RETRYING ****************')
flow = np.squeeze(net.blobs['predict_flow_final'].data).transpose(1, 2, 0)
return flow
vars['SCALE_HEIGHT'] = height / float(vars['ADAPTED_HEIGHT']);
tmp = tempfile.NamedTemporaryFile(mode='w', delete=True)
proto = open(args.deployproto).readlines()
for line in proto:
for key, value in vars.items():
tag = "$%s$" % key
line = line.replace(tag, str(value))
tmp.write(line)
tmp.flush()
if not args.verbose:
caffe.set_logging_disabled()
caffe.set_device(args.gpu)
caffe.set_mode_gpu()
net = caffe.Net(tmp.name, args.caffemodel, caffe.TEST)
input_dict = {}
for blob_idx in range(num_blobs):
input_dict[net.inputs[blob_idx]] = input_data[blob_idx]
#
# There is some non-deterministic nan-bug in caffe
# it seems to be a race-condition
#
print('Network forward pass using %s.' % args.caffemodel)
i = 1
while i<=5:
def testFiles(self,
filelist,
iter,
output=False,
definition=None,
vars={}):
modelFile, iter = self.getModelFile(iter)
assert (output)
vars['output'] = True
vars['prefix'] = iter
vars['dataset'] = os.path.splitext(os.path.basename(filelist))[0]
self.makeScratchDir()
if definition is None: definition = 'testsingle'
proto = self.findProto(definition)
if output and 'dataset' in vars:
outPath = '%s/output_%d_%s' % (self._path, iter, vars['dataset'])
if os.path.isdir(outPath):
if self._unattended or tb.queryYesNo(
'Output folder %s exists, do you want to delete it first?'
% os.path.basename(outPath)):
os.system('rm -rf %s' % outPath)
data_list = IO.readTupleList(filelist)
print data_list
# Run a new net for every sample:
for idx, line in enumerate(data_list):
num_blobs = len(line)
input_data = []
for blob_idx in range(num_blobs):
img = IO.readImage(line[blob_idx])
#print(img.shape)
input_data.append(img[np.newaxis, :, :, :].transpose(
0, 3, 1, 2)[:, [2, 1, 0], :, :])
#print(input_data[-1].shape)
vars['width'] = input_data[0].shape[3]
vars['height'] = input_data[0].shape[2]
vars['basename'] = 'b%03d' % idx
finalProto = self.makeScratchPrototxt(proto, vars)
caffe.set_logging_disabled()
caffe.set_mode_gpu()
net = caffe.Net(finalProto, modelFile, caffe.TEST)
print 'Network forward pass (%d of %d). %d inputs of shapes:' % (
idx + 1, len(data_list), num_blobs)
for blob_idx in range(num_blobs):
print(" " + str(input_data[blob_idx].shape))
if not len(net.inputs) == len(line):
raise Exception(
'Net has %d inputs and in file list there are %d' %
(len(net.inputs), len(line)))
input_dict = {}
for blob_idx in range(num_blobs):
input_dict[net.inputs[blob_idx]] = input_data[blob_idx]
net.forward(**input_dict)
print 'Iteration was %d' % iter
def run_model_multiples(prototxt,
weights,
listfile,
output_dir,
blobs=['warp_loss2'],
save_images=False,
start=0,
flow_loss=False):
"""
Input : prototxt, weights
listfile : list of (image1, image2)
output_dir : place where the computed flows will be saved
outfile : csv file with losses for each image pair will be saved
Call this function with a prototxt that work without LMDB, and that has only two images in input
"""
import os, sys, numpy as np
import argparse
from scipy import misc
import caffe
import tempfile
from math import ceil
if (not os.path.exists(weights)):
raise BaseException('caffemodel does not exist: ' + weights)
if (not os.path.exists(prototxt)):
raise BaseException('deploy-proto does not exist: ' + prototxt)
if (not os.path.exists(listfile)):
raise BaseException('listfile does not exist: ' + listfile)
def readTupleList(filename):
list = []
for line in open(filename).readlines():
if line.strip() != '':
list.append(line.split())
return list
ops = readTupleList(listfile)
width = -1
height = -1
output = []
first_line = [
'image0', 'image1', 'mean_flow', 'median_flow', 'L2_distance'
]
output.append(['image0', 'image1', 'real_flow', 'estimated_flow'] + blobs)
output_file = open(output_dir + '/list_out.txt', 'w')
output_file.write(",".join(first_line))
output_file.write("\n")
n = len(ops)
for i, ent in list(enumerate(ops))[start:]:
print("processing", i)
flush()
print('Processing tuple:', ent)
num_blobs = 2
if flow_loss:
num_blobs = 3
input_data = []
img0 = misc.imread(ent[0])
if len(img0.shape) < 3:
input_data.append(img0[np.newaxis, np.newaxis, :, :])
else:
input_data.append(img0[np.newaxis, :, :, :].transpose(
0, 3, 1, 2)[:, [2, 1, 0], :, :])
img1 = misc.imread(ent[1])
if len(img1.shape) < 3:
input_data.append(img1[np.newaxis, np.newaxis, :, :])
else:
input_data.append(img1[np.newaxis, :, :, :].transpose(
0, 3, 1, 2)[:, [2, 1, 0], :, :])
if flow_loss:
flow = readFlow(ent[2])
input_data.append(flow[np.newaxis, :, :, :].transpose(0, 3, 1, 2))
if width != input_data[0].shape[3] or height != input_data[0].shape[2]:
width = input_data[0].shape[3]
height = input_data[0].shape[2]
vars = {}
vars['TARGET_WIDTH'] = width
vars['TARGET_HEIGHT'] = height
divisor = 64.
vars['ADAPTED_WIDTH'] = int(ceil(width / divisor) * divisor)
vars['ADAPTED_HEIGHT'] = int(ceil(height / divisor) * divisor)
vars['SCALE_WIDTH'] = width / float(vars['ADAPTED_WIDTH'])
vars['SCALE_HEIGHT'] = height / float(vars['ADAPTED_HEIGHT'])
tmp = tempfile.NamedTemporaryFile(mode='w', delete=False)
proto = open(prototxt).readlines()
for line in proto:
for key, value in vars.items():
tag = "$%s$" % key
line = line.replace(tag, str(value))
tmp.write(line)
tmp.flush()
caffe.set_logging_disabled()
caffe.set_mode_gpu()
net = caffe.Net(tmp.name, weights, caffe.TEST)
blobs = net.outputs
print("blobs", blobs)
flush()
if i == 0:
output.append(['image0', 'image1', 'real_flow', 'estimated_flow'] +
blobs)
input_dict = {}
for blob_idx in range(num_blobs):
input_dict[net.inputs[blob_idx]] = input_data[blob_idx]
#input_data
# There is some non-deterministic nan-bug in caffe
#
print('Network forward pass using %s.' % weights)
j = 1
while j <= 5:
j += 1
net.forward(**input_dict)
containsNaN = False
for name in net.blobs:
blob = net.blobs[name]
has_nan = np.isnan(blob.data[...]).any()
if has_nan:
print('blob %s contains nan' % name)
containsNaN = True
if not containsNaN:
print('Succeeded.')
break
else:
print('**************** FOUND NANs, RETRYING ****************')
flow_blob = np.squeeze(net.blobs['predict_flow_final'].data).transpose(
1, 2, 0)
output_flow_file = output_dir + "%04d_flow" % i
print("saving flow at %s" % output_flow_file)
writeFlow(output_flow_file + ".flo", flow_blob)
out_line = [ent[0], ent[1], output_flow_file]
out_line.append("%.3f" % np.median(flow_blob))
out_line.append("%.3f" % np.mean(flow_blob))
#for blob in blobs:
# out_line.append(str(net.blobs[blob].data))
if save_images:
save_flow_image(flow_blob, output_flow_file + ".png")
misc.imsave(output_dir + "%04d_img0.jpg" % i, img0)
misc.imsave(output_dir + "%04d_img1.jpg" % i, img1)
expected_img0 = apply_flow_reverse(img1, flow_blob)
misc.imsave(output_dir + "%04d_img0_expected.jpg" % i,
expected_img0)
l2_loss = np.linalg.norm(img0 - expected_img0)
out_line.append("%.3f" % l2_loss)
output.append(out_line)
print(out_line)
output_file.write(",".join(out_line))
output_file.write("\n")
output_file.flush()
def run_model(prototxt,
weights,
img0_p,
img1_p,
out_p,
verbose=False,
write_flow_image=True):
if (not os.path.exists(weights)):
raise BaseException('caffemodel does not exist: ' + weights)
if (not os.path.exists(prototxt)):
raise BaseException('deploy-proto does not exist: ' + prototxt)
if (not os.path.exists(img0_p)):
raise BaseException('img0 does not exist: ' + img0_p)
if (not os.path.exists(img1_p)):
raise BaseException('img1 does not exist: ' + img1_p)
print("starting run_model")
num_blobs = 2
input_data = []
img0 = misc.imread(img0_p)
if len(img0.shape) < 3:
input_data.append(img0[np.newaxis, np.newaxis, :, :])
else:
input_data.append(img0[np.newaxis, :, :, :].transpose(
0, 3, 1, 2)[:, [2, 1, 0], :, :])
img1 = misc.imread(img1_p)
if len(img1.shape) < 3:
input_data.append(img1[np.newaxis, np.newaxis, :, :])
else:
input_data.append(img1[np.newaxis, :, :, :].transpose(
0, 3, 1, 2)[:, [2, 1, 0], :, :])
width = input_data[0].shape[3]
height = input_data[0].shape[2]
vars = {}
vars['TARGET_WIDTH'] = width
vars['TARGET_HEIGHT'] = height
print(width, height)
divisor = 64.
vars['ADAPTED_WIDTH'] = int(ceil(width / divisor) * divisor)
vars['ADAPTED_HEIGHT'] = int(ceil(height / divisor) * divisor)
vars['SCALE_WIDTH'] = width / float(vars['ADAPTED_WIDTH'])
vars['SCALE_HEIGHT'] = height / float(vars['ADAPTED_HEIGHT'])
tmp = tempfile.NamedTemporaryFile(mode='w', delete=True)
proto = open(prototxt).readlines()
for line in proto:
for key, value in vars.items():
tag = "$%s$" % key
line = line.replace(tag, str(value))
tmp.write(line)
tmp.flush()
if not verbose:
caffe.set_logging_disabled()
#caffe.set_device(args.gpu)
caffe.set_mode_gpu()
net = caffe.Net(tmp.name, weights, caffe.TEST)
input_dict = {}
for blob_idx in range(num_blobs):
input_dict[net.inputs[blob_idx]] = input_data[blob_idx]
#
# There is some non-deterministic nan-bug in caffe
# it seems to be a race-condition
#
print('Network forward pass using %s.' % weights)
i = 1
while i <= 5:
i += 1
net.forward(**input_dict)
containsNaN = False
for name in net.blobs:
blob = net.blobs[name]
has_nan = np.isnan(blob.data[...]).any()
if has_nan:
print('blob %s contains nan' % name)
containsNaN = True
if not containsNaN:
print('Succeeded.')
break
else:
print('**************** FOUND NANs, RETRYING ****************')
blob = np.squeeze(net.blobs['predict_flow_final'].data).transpose(1, 2, 0)
writeFlow(out_p, blob)
if write_flow_image:
print("saving flow image, and warp image")
save_flow_image(blob, out_p + ".png")
# save warp
expected_img0 = apply_flow_reverse(img1, blob)
misc.imsave(out_p + "_warp.png", expected_img0)
def motion_analysis_flownet2(args):
data_base_dir = args.dir
folder = data_base_dir + 'classifiedgood/'
outputfolder = data_base_dir + 'opticalflow2/'
if os.path.isdir(outputfolder):
rmtree(outputfolder)
os.mkdir(outputfolder)
# sample two images to initialize the blob dimensions
imglist = os.listdir(folder)
imglist.sort()
img0 = misc.imread(folder + imglist[0])
img1 = misc.imread(folder + imglist[1])
# prepare x axis and y axis for faster computation
x_axis = np.arange(0, img0.shape[0], dtype=np.int)
x_axis = repmat(x_axis[:, np.newaxis], 1, img0.shape[1])
y_axis = np.arange(0, img0.shape[1], dtype=np.int)
y_axis = repmat(y_axis, img0.shape[0], 1)
def warp_NCC_opt(img0, img1, flow):
nx = (x_axis + flow[:, :, 1]).astype(int)
ny = (y_axis + flow[:, :, 0]).astype(int)
pos0 = (nx <= img1.shape[0] - 1) & (nx >= 0) & (
ny <= img1.shape[1] - 1) & (ny >= 0)
patch0 = img0[pos0]
pos1x = nx[pos0]
pos1y = ny[pos0]
nn = [pos1x, pos1y]
patch1 = img1[nn]
return correlation_coefficient(patch0, patch1)
input_data = []
if len(img0.shape) < 3:
input_data.append(img0[np.newaxis, np.newaxis, :, :])
else:
input_data.append(img0[np.newaxis, :, :, :].transpose(
0, 3, 1, 2)[:, [2, 1, 0], :, :])
print('input blob 0 :'),
print(input_data[0].shape)
if len(img1.shape) < 3:
input_data.append(img1[np.newaxis, np.newaxis, :, :])
else:
input_data.append(img1[np.newaxis, :, :, :].transpose(
0, 3, 1, 2)[:, [2, 1, 0], :, :])
print('input blob 1 :'),
print(input_data[1].shape)
width = input_data[0].shape[3]
height = input_data[0].shape[2]
vars = {}
vars['TARGET_WIDTH'] = width
vars['TARGET_HEIGHT'] = height
divisor = 64.
vars['ADAPTED_WIDTH'] = int(ceil(width / divisor) * divisor)
vars['ADAPTED_HEIGHT'] = int(ceil(height / divisor) * divisor)
vars['SCALE_WIDTH'] = width / float(vars['ADAPTED_WIDTH'])
vars['SCALE_HEIGHT'] = height / float(vars['ADAPTED_HEIGHT'])
tmp = tempfile.NamedTemporaryFile(mode='w', delete=True)
proto = open(args.deployproto).readlines()
for line in proto:
for key, value in vars.items():
tag = "$%s$" % key
line = line.replace(tag, str(value))
tmp.write(line)
tmp.flush()
if not args.verbose:
caffe.set_logging_disabled()
caffe.set_device(args.gpu)
caffe.set_mode_gpu()
net = caffe.Net(tmp.name, args.caffemodel, caffe.TEST)
# calculate sharpness and sort by sharpness
sharpness = []
print('Calculating sharpness ...')
dl = DoubleList()
D = {}
for f in imglist:
dl.append(f)
D[f] = dl.tail
for i in range(len(imglist)):
print('%d / %d' % (i, len(imglist)))
img = misc.imread(folder + imglist[i], flatten=True)
sobelx = cv2.Sobel(img, cv2.CV_64F, 1, 0, ksize=5)
sobely = cv2.Sobel(img, cv2.CV_64F, 0, 1, ksize=5)
sharpness.append((np.sum(sobelx**2) + np.sum(sobely**2)) / img.size)
order = np.argsort(sharpness)
## load img0 and img1 iteratively from folder
score = []
keyframes = []
for j in range(len(imglist)):
print('Homography estimation: %d / %d sharpness = %.2f' %
(j, len(imglist), sharpness[order[j]]))
if order[j] == 0 or order[j] == len(imglist) - 1:
keyframes.append(imglist[order[j]])
score.append(0.9)
continue
ID = order[j]
PrevName = D[imglist[ID]].prev.data
NextName = D[imglist[ID]].next.data
Prev = misc.imread(folder + PrevName, mode='RGB')
Next = misc.imread(folder + NextName, mode='RGB')
input_dict = {}
input_data = []
if len(Prev.shape) < 3:
input_data.append(Prev[np.newaxis, np.newaxis, :, :])
else:
input_data.append(Prev[np.newaxis, :, :, :].transpose(
0, 3, 1, 2)[:, [2, 1, 0], :, :])
if len(Next.shape) < 3:
input_data.append(Next[np.newaxis, np.newaxis, :, :])
else:
input_data.append(Next[np.newaxis, :, :, :].transpose(
0, 3, 1, 2)[:, [2, 1, 0], :, :])
input_dict[net.inputs[0]] = input_data[0]
input_dict[net.inputs[1]] = input_data[1]
print('Network forward pass using %s and %s' % (PrevName, NextName))
i = 1
while i <= 5:
i += 1
net.forward(**input_dict)
containsNaN = False
for name in net.blobs:
blob = net.blobs[name]
has_nan = np.isnan(blob.data[...]).any()
if has_nan:
print('blob %s contains nan' % name)
containsNaN = True
if not containsNaN:
print('Succeeded.')
break
else:
print('**************** FOUND NANs, RETRYING ****************')
flow = np.squeeze(net.blobs['predict_flow_final'].data).transpose(
1, 2, 0)
#s = warp_NCC(img0=rgb2gray(Prev), img1=rgb2gray(Next), flow=flow)
s = warp_NCC_opt(img0=rgb2gray(Prev), img1=rgb2gray(Next), flow=flow)
#warpped = warpFlow(rgb2gray(Next), flow)
#s = correlation_coefficient(warpped, rgb2gray(Prev))
print('correlation coefficient = %.4f' % s)
if s > 0.98:
dl.remove_byaddress(D[imglist[ID]])
print('Redundant: ' + imglist[ID])
else:
keyframes.append(imglist[order[j]])
score.append(s)
print('Keyframe: ' + imglist[ID])
ordkeyframes = np.argsort(keyframes)
keyframes.sort()
score = np.asarray(score)
score = score[np.asarray(ordkeyframes)]
offilename = data_base_dir + 'opticalflowscore2.txt'
if os.path.exists(offilename):
os.remove(offilename)
offile = open(offilename, 'w')
for i in range(len(keyframes)):
offile.write('%s %.4f\n' % (keyframes[i], score[i]))
os.system('cp ' + folder + keyframes[i] + ' ' + outputfolder +
keyframes[i])
analyze_score(data_base_dir=data_base_dir,
fname='opticalflowscore2',
imgnames=keyframes,
score=score)
del net
