Example #1
0
def flowWarp(img, flow):
    import caffe
    width = img.shape[1]
    height = img.shape[0]

    print 'processing (%dx%d)' % (width, height)

    defFile = tempFilename('.prototxt')
    preprocessFile('/home/ilge/hackathon2/common/prototmp/apply_flow.prototmp',
                   defFile, {
                       'WIDTH': width,
                       'HEIGHT': height
                   })

    caffe.set_logging_disabled()
    caffe.set_mode_gpu()
    net = caffe.Net(defFile, caffe.TEST)

    os.remove(defFile)

    print 'network forward pass'

    img_input = img[np.newaxis, :, :, :].transpose(0, 3, 1, 2)
    flow_input = flow[np.newaxis, :, :, :].transpose(0, 3, 1, 2)

    net.blobs['image'].reshape(*img_input.shape)
    net.blobs['image'].data[...] = img_input
    net.blobs['flow'].reshape(*flow_input.shape)
    net.blobs['flow'].data[...] = flow_input

    net.forward()
    output = net.blobs['output'].data[...].transpose(0, 2, 3, 1).squeeze()

    return output
Example #2
0
    def __init__(self, net_proto, net_weights, device_id, input_size=None):
        caffe.set_logging_disabled()

        if device_id >= 0:
            caffe.set_mode_gpu()
            caffe.set_device(device_id)
        else:
            caffe.set_mode_cpu()
        self._net = caffe.Net(net_proto, net_weights, caffe.TEST)

        input_shape = self._net.blobs['data'].data.shape

        if input_size is not None:
            input_shape = input_shape[:2] + input_size

        transformer = caffe.io.Transformer({'data': input_shape})

        if self._net.blobs['data'].data.shape[1] == 3:
            transformer.set_transpose(
                'data',
                (2, 0, 1))  # move image channels to outermost dimension
            transformer.set_mean('data', np.array(
                [104, 117,
                 123]))  # subtract the dataset-mean value in each channel
        else:
            pass  # non RGB data need not use transformer

        self._transformer = transformer
        self._sample_shape = self._net.blobs['data'].data.shape
Example #3
0
def load_model(width, height):
    vars = {}
    vars['TARGET_WIDTH'] = width
    vars['TARGET_HEIGHT'] = height

    divisor = 64.
    vars['ADAPTED_WIDTH'] = int(ceil(width / divisor) * divisor)
    vars['ADAPTED_HEIGHT'] = int(ceil(height / divisor) * divisor)

    vars['SCALE_WIDTH'] = width / float(vars['ADAPTED_WIDTH'])
    vars['SCALE_HEIGHT'] = height / float(vars['ADAPTED_HEIGHT'])

    tmp = tempfile.NamedTemporaryFile(mode='w', delete=False)
    proto = open(deployproto).readlines()
    for line in proto:
        for key, value in vars.items():
            tag = "$%s$" % key
            line = line.replace(tag, str(value))
        tmp.write(line)
    tmp.flush()

    caffe.set_logging_disabled()
    caffe.set_device(0)
    caffe.set_mode_gpu()
    net = caffe.Net(tmp.name, caffemodel, caffe.TEST)
    print('Network forward pass using %s.' % caffemodel)

    return net
Example #4
0
def flowWarp(img, flow):
    import caffe
    width = img.shape[1]
    height = img.shape[0]

    print 'processing (%dx%d)' % (width, height)

    defFile = tempFilename('.prototxt')
    preprocessFile('/home/ilge/hackathon2/common/prototmp/apply_flow.prototmp', defFile, {'WIDTH': width, 'HEIGHT': height})

    caffe.set_logging_disabled()
    caffe.set_mode_gpu()
    net = caffe.Net(
        defFile,
        caffe.TEST
    )

    os.remove(defFile)

    print 'network forward pass'

    img_input = img[np.newaxis, :, :, :].transpose(0, 3, 1, 2)
    flow_input = flow[np.newaxis, :, :, :].transpose(0, 3, 1, 2)

    net.blobs['image'].reshape(*img_input.shape)
    net.blobs['image'].data[...] = img_input
    net.blobs['flow'].reshape(*flow_input.shape)
    net.blobs['flow'].data[...] = flow_input

    net.forward()
    output = net.blobs['output'].data[...].transpose(0, 2, 3, 1).squeeze()

    return output
Example #5
0
    def __init__(self, height, width, caffemodel, deployproto, gpu=0):
        import caffe

        divisor = 64.
        vars = {
            'TARGET_WIDTH': width,
            'TARGET_HEIGHT': height,
            'ADAPTED_WIDTH': int(ceil(width / divisor) * divisor),
            'ADAPTED_HEIGHT': int(ceil(height / divisor) * divisor)
        }
        vars['SCALE_WIDTH'] = width / float(vars['ADAPTED_WIDTH'])
        vars['SCALE_HEIGHT'] = height / float(vars['ADAPTED_HEIGHT'])

        tmp = tempfile.NamedTemporaryFile(mode='w', delete=False)
        proto = open(deployproto).readlines()
        for line in proto:
            for key, value in vars.items():
                tag = "$%s$" % key
                line = line.replace(tag, str(value))
            tmp.write(line)
        tmp.flush()

        caffe.set_logging_disabled()
        caffe.set_device(gpu)
        caffe.set_mode_gpu()
        self.net = caffe.Net(tmp.name, caffemodel, caffe.TEST)
Example #6
0
    def testFiles(self, filelist, iter, output=False, definition=None, vars={}):
        modelFile, iter = self.getModelFile(iter)

        assert(output)
        vars['output'] = True
        vars['prefix'] = iter

        vars['dataset'] = os.path.splitext(os.path.basename(filelist))[0]

        self.makeScratchDir()

        if definition is None: definition = 'testsingle'
        proto = self.findProto(definition)

        if output and 'dataset' in vars:
            outPath = '%s/output_%d_%s' % (self._path, iter, vars['dataset'])
            if os.path.isdir(outPath):
                if self._unattended or tb.queryYesNo('Output folder %s exists, do you want to delete it first?' % os.path.basename(outPath)):
                    os.system('rm -rf %s' % outPath)

        data_list = IO.readTupleList(filelist)

        print data_list

        # Run a new net for every sample:
        for idx, line in enumerate(data_list):
        
            num_blobs = len(line)
            input_data = []
            for blob_idx in range(num_blobs):
                img = IO.readImage(line[blob_idx])
                #print(img.shape)
                input_data.append(img[np.newaxis, :, :, :].transpose(0, 3, 1, 2)[:,[2,1,0],:,:])
                #print(input_data[-1].shape)

            vars['width'] = input_data[0].shape[3]
            vars['height'] = input_data[0].shape[2]
            vars['basename'] = 'b%03d' % idx
            
            finalProto = self.makeScratchPrototxt(proto, vars)

            caffe.set_logging_disabled()
            caffe.set_mode_gpu()
            net = caffe.Net(finalProto, modelFile,caffe.TEST)

            print 'Network forward pass (%d of %d). %d inputs of shapes:' % (idx+1, len(data_list), num_blobs)
            for blob_idx in range(num_blobs):
                print("  " + str(input_data[blob_idx].shape))

            if not len(net.inputs) == len(line):
                raise Exception('Net has %d inputs and in file list there are %d' % (len(net.inputs), len(line)))

            input_dict = {}
            for blob_idx in range(num_blobs):
                input_dict[net.inputs[blob_idx]] = input_data[blob_idx]

            net.forward(**input_dict)

        print 'Iteration was %d' %iter
Example #7
0
def process(set, ent, inputEnt):
    highres = misc.imread(ent, flatten=False).astype(np.float32)
    lowres = misc.imread(inputEnt, flatten=False).astype(np.float32)
    width = highres.shape[1]
    height = highres.shape[0]

    print "processing %s (%dx%d)" % (ent, width, height)

    defFile = "scratch/test_SR_deploy.prototxt"
    pycnn.preprocessFile("deploy.prototmp", defFile, {"WIDTH": width, "HEIGHT": height})

    if "youtube" in set:
        print "using youtube mean"
        mean_bgr = tb.readFloat("/misc/lmbraid17/ilge/caffe/superresolution/datasets/youtube/test/mean3.float3").astype(
            np.float32
        )
    else:
        mean_bgr = tb.readFloat("/home/ilge/data/caffe/superresolution/datasets/coco/mean.float3").astype(np.float32)

    mean_bgr = cv2.resize(mean_bgr, (width, height), interpolation=cv2.INTER_CUBIC)
    mean_bgr_lowres = cv2.resize(mean_bgr, (width / 4, height / 4), interpolation=cv2.INTER_CUBIC)

    highres_nomean_bgr = highres[:, :, (2, 1, 0)] - mean_bgr
    lowres_nomean_bgr = lowres[:, :, (2, 1, 0)] - mean_bgr_lowres

    caffe.set_phase_test()
    caffe.set_mode_gpu()
    caffe.set_logging_disabled()
    net = caffe.Net(defFile, modelFile)

    print "network forward pass"
    blobs = net.forward(
        highres=np.asarray([net.preprocess("highres", highres_nomean_bgr / 255.0)]),
        lowres=np.asarray([net.preprocess("lowres", lowres_nomean_bgr / 255.0)]),
    )

    output_bgr = 255.0 * blobs["output"].transpose(0, 2, 3, 1).squeeze()
    output_bgr += mean_bgr
    output_bgr[output_bgr < 0] = 0
    output_bgr[output_bgr > 255] = 255

    os.system("mkdir -p %s/%s" % (out_dir, set))
    basename = os.path.basename(ent)[:-4].replace("_GT", "")
    misc.imsave("%s/%s/%s-gt.png" % (out_dir, set, basename), highres)
    misc.imsave("%s/%s/%s-recon.png" % (out_dir, set, basename), output_bgr[:, :, (2, 1, 0)])

    # nn, li, cu = tb.computeBasePSNRs(ent, downsampledFilename=inputEnt)
    nn = tb.PSNR()
    li = tb.PSNR()
    cu = tb.PSNR()

    psnr = tb.PSNR()
    psnr.set(blobs["psnr"][0, 0, 0, 0], blobs["psnr_y"][0, 0, 0, 0])

    print "nn=%5s, li=%5s, cu=%5s, net=%5s" % (nn, li, cu, psnr)

    return (nn, li, cu, psnr)
 def __init__(self, values_to_load='compute_image.csv'):
     caffe.set_logging_disabled()
     caffe.set_device(0)
     caffe.set_mode_gpu()
     self.check = False
     self.tmp = None
     self.net = None
     self.colorwheel = self.make_color_wheel()
     if os.path.isfile(values_to_load):
         self.init_class_from_csv(values_to_load)
Example #9
0
def process(set,ent,inputEnt):
    highres = misc.imread(ent, flatten=False).astype(np.float32)
    lowres = misc.imread(inputEnt, flatten=False).astype(np.float32)
    width = highres.shape[1]
    height = highres.shape[0]

    print 'processing %s (%dx%d)' % (ent, width, height)

    defFile = 'scratch/test_SR_deploy.prototxt'
    pycnn.preprocessFile('deploy.prototmp', defFile, {'WIDTH': width, 'HEIGHT': height})

    if 'youtube' in set:
        print 'using youtube mean'
        mean_bgr = tb.readFloat("/misc/lmbraid17/ilge/caffe/superresolution/datasets/youtube/test/mean3.float3").astype(np.float32)
    else:
        mean_bgr = tb.readFloat("/home/ilge/data/caffe/superresolution/datasets/coco/mean.float3").astype(np.float32)

    mean_bgr = cv2.resize(mean_bgr, (width, height), interpolation=cv2.INTER_CUBIC)
    mean_bgr_lowres = cv2.resize(mean_bgr, (width/4, height/4), interpolation=cv2.INTER_CUBIC)

    highres_nomean_bgr = highres[:, :, (2, 1, 0)] - mean_bgr
    lowres_nomean_bgr = lowres[:, :, (2, 1, 0)] - mean_bgr_lowres

    caffe.set_phase_test()
    caffe.set_mode_gpu()
    caffe.set_logging_disabled()
    net = caffe.Net(
       defFile,
       modelFile
    )

    print 'network forward pass'
    blobs = net.forward(highres=np.asarray([net.preprocess('highres', highres_nomean_bgr / 255.0)]),lowres=np.asarray([net.preprocess('lowres', lowres_nomean_bgr / 255.0)]))

    output_bgr = 255.0 * blobs['output'].transpose(0, 2, 3, 1).squeeze()
    output_bgr += mean_bgr
    output_bgr[output_bgr < 0] = 0
    output_bgr[output_bgr > 255] = 255

    os.system('mkdir -p %s/%s' % (out_dir, set))
    basename = os.path.basename(ent)[:-4].replace('_GT', '')
    misc.imsave('%s/%s/%s-gt.png' % (out_dir, set, basename), highres)
    misc.imsave('%s/%s/%s-recon.png' % (out_dir, set, basename), output_bgr[:, :, (2, 1, 0)])

    #nn, li, cu = tb.computeBasePSNRs(ent, downsampledFilename=inputEnt)
    nn = tb.PSNR(); li=tb.PSNR(); cu=tb.PSNR()

    psnr = tb.PSNR()
    psnr.set(blobs['psnr'][0, 0, 0, 0],  blobs['psnr_y'][0, 0, 0, 0])

    print 'nn=%5s, li=%5s, cu=%5s, net=%5s' % (nn, li, cu, psnr)

    return (nn, li, cu, psnr)
Example #10
0
def caffeNet(modelFile=None, prototmp=None, inputs={}, phase=None, logging=False):
    if phase is None: phase=caffe.TEST
    if prototmp is not None:
        modelFile = tempFilename('.prototmp')
        open(modelFile,'w').write(prototmp)

    if not logging: caffe.set_logging_disabled()
    caffe.set_mode_gpu()
    net = caffe.Net(
        modelFile,
        phase
    )

    if prototmp is not None:
        os.remove(modelFile)

    for name, value in inputs.iteritems():
        value = value[np.newaxis, :, :, :].transpose(0, 3, 1, 2)
        net.blobs[name].reshape(*value.shape)
        net.blobs[name].data[...] = value

    return net
Example #11
0
        vars['ADAPTED_HEIGHT'] = int(ceil(height / divisor) * divisor)

        vars['SCALE_WIDTH'] = width / float(vars['ADAPTED_WIDTH'])
        vars['SCALE_HEIGHT'] = height / float(vars['ADAPTED_HEIGHT'])

        tmp = tempfile.NamedTemporaryFile(mode='w', delete=True)
        proto = open(args.deployproto).readlines()
        for line in proto:
            for key, value in vars.items():
                tag = "$%s$" % key
                line = line.replace(tag, str(value))
            tmp.write(line)
        tmp.flush()

        if not args.verbose:
            caffe.set_logging_disabled()
        caffe.set_device(args.gpu)
        caffe.set_mode_gpu()
        net = caffe.Net(tmp.name, args.caffemodel, caffe.TEST)

    input_dict = {}
    for blob_idx in range(num_blobs):
        input_dict[net.inputs[blob_idx]] = input_data[blob_idx]

    #
    # There is some non-deterministic nan-bug in caffe
    # it seems to be a race-condition
    #
    print('Network forward pass using %s.' % args.caffemodel)
    i = 1
    while i <= 5:
Example #12
0
def calculate_flow(im1=None, im2=None, caffemodel='/home/yi/code/video_motion3/flownet2/models/FlowNet2/FlowNet2_weights.caffemodel.h5',
                   deployproto='/home/yi/code/video_motion3/flownet2/models/FlowNet2/FlowNet2_deploy.prototxt.template'):

    if not os.path.exists(caffemodel):
        raise BaseException('caffemodel does not exist: ' + caffemodel)
    if not os.path.exists(deployproto):
        raise BaseException('deploy-proto does not exist: ' + deployproto)

    # im1 = misc.imread('flownet2/viper_example/042_00742.jpg')
    # im2 = misc.imread('flownet2/viper_example/042_00743.jpg')

    num_blobs = 2
    input_data = []
    if len(im1.shape) < 3:
        input_data.append(im1[np.newaxis, np.newaxis, :, :])
    else:
        input_data.append(im1[np.newaxis, :, :, :].transpose(0, 3, 1, 2)[:, [2, 1, 0], :, :])
    if len(im2.shape) < 3:
        input_data.append(im2[np.newaxis, np.newaxis, :, :])
    else:
        input_data.append(im2[np.newaxis, :, :, :].transpose(0, 3, 1, 2)[:, [2, 1, 0], :, :])

    width = input_data[0].shape[3]
    height = input_data[0].shape[2]
    vars = {}
    vars['TARGET_WIDTH'] = width
    vars['TARGET_HEIGHT'] = height

    divisor = 64.
    vars['ADAPTED_WIDTH'] = int(ceil(width/divisor) * divisor)
    vars['ADAPTED_HEIGHT'] = int(ceil(height/divisor) * divisor)

    vars['SCALE_WIDTH'] = width / float(vars['ADAPTED_WIDTH']);
    vars['SCALE_HEIGHT'] = height / float(vars['ADAPTED_HEIGHT']);

    tmp = tempfile.NamedTemporaryFile(mode='w', delete=True)

    proto = open(deployproto).readlines()
    for line in proto:
        for key, value in vars.items():
            tag = "$%s$" % key
            line = line.replace(tag, str(value))

        tmp.write(line)

    tmp.flush()

    caffe.set_logging_disabled()
    caffe.set_device(0)
    caffe.set_mode_gpu()
    net = caffe.Net(tmp.name, caffemodel, caffe.TEST)

    input_dict = {}
    for blob_idx in range(num_blobs):
        input_dict[net.inputs[blob_idx]] = input_data[blob_idx]

    print('Network forward pass using %s.' % caffemodel)
    i = 1
    while i<=5:
        i+=1

        net.forward(**input_dict)

        containsNaN = False
        for name in net.blobs:
            blob = net.blobs[name]
            has_nan = np.isnan(blob.data[...]).any()

            if has_nan:
                print('blob %s contains nan' % name)
                containsNaN = True

        if not containsNaN:
            print('Succeeded.')
            break
        else:
            print('**************** FOUND NANs, RETRYING ****************')

    flow = np.squeeze(net.blobs['predict_flow_final'].data).transpose(1, 2, 0)
    return flow
Example #13
0
vars['SCALE_HEIGHT'] = height / float(vars['ADAPTED_HEIGHT']);

tmp = tempfile.NamedTemporaryFile(mode='w', delete=True)

proto = open(args.deployproto).readlines()
for line in proto:
    for key, value in vars.items():
        tag = "$%s$" % key
        line = line.replace(tag, str(value))

    tmp.write(line)

tmp.flush()

if not args.verbose:
    caffe.set_logging_disabled()
caffe.set_device(args.gpu)
caffe.set_mode_gpu()
net = caffe.Net(tmp.name, args.caffemodel, caffe.TEST)

input_dict = {}
for blob_idx in range(num_blobs):
    input_dict[net.inputs[blob_idx]] = input_data[blob_idx]

#
# There is some non-deterministic nan-bug in caffe
# it seems to be a race-condition 
#
print('Network forward pass using %s.' % args.caffemodel)
i = 1
while i<=5:
Example #14
0
    def testFiles(self,
                  filelist,
                  iter,
                  output=False,
                  definition=None,
                  vars={}):
        modelFile, iter = self.getModelFile(iter)

        assert (output)
        vars['output'] = True
        vars['prefix'] = iter

        vars['dataset'] = os.path.splitext(os.path.basename(filelist))[0]

        self.makeScratchDir()

        if definition is None: definition = 'testsingle'
        proto = self.findProto(definition)

        if output and 'dataset' in vars:
            outPath = '%s/output_%d_%s' % (self._path, iter, vars['dataset'])
            if os.path.isdir(outPath):
                if self._unattended or tb.queryYesNo(
                        'Output folder %s exists, do you want to delete it first?'
                        % os.path.basename(outPath)):
                    os.system('rm -rf %s' % outPath)

        data_list = IO.readTupleList(filelist)

        print data_list

        # Run a new net for every sample:
        for idx, line in enumerate(data_list):

            num_blobs = len(line)
            input_data = []
            for blob_idx in range(num_blobs):
                img = IO.readImage(line[blob_idx])
                #print(img.shape)
                input_data.append(img[np.newaxis, :, :, :].transpose(
                    0, 3, 1, 2)[:, [2, 1, 0], :, :])
                #print(input_data[-1].shape)

            vars['width'] = input_data[0].shape[3]
            vars['height'] = input_data[0].shape[2]
            vars['basename'] = 'b%03d' % idx

            finalProto = self.makeScratchPrototxt(proto, vars)

            caffe.set_logging_disabled()
            caffe.set_mode_gpu()
            net = caffe.Net(finalProto, modelFile, caffe.TEST)

            print 'Network forward pass (%d of %d). %d inputs of shapes:' % (
                idx + 1, len(data_list), num_blobs)
            for blob_idx in range(num_blobs):
                print("  " + str(input_data[blob_idx].shape))

            if not len(net.inputs) == len(line):
                raise Exception(
                    'Net has %d inputs and in file list there are %d' %
                    (len(net.inputs), len(line)))

            input_dict = {}
            for blob_idx in range(num_blobs):
                input_dict[net.inputs[blob_idx]] = input_data[blob_idx]

            net.forward(**input_dict)

        print 'Iteration was %d' % iter
Example #15
0
def run_model_multiples(prototxt,
                        weights,
                        listfile,
                        output_dir,
                        blobs=['warp_loss2'],
                        save_images=False,
                        start=0,
                        flow_loss=False):
    """
    Input : prototxt, weights
    listfile : list of (image1, image2)
    
    output_dir : place where the computed flows will be saved
    outfile : csv file with losses for each image pair will be saved
    
        Call this function with a prototxt that work without LMDB, and that has only two images in input
    
    """

    import os, sys, numpy as np
    import argparse
    from scipy import misc
    import caffe
    import tempfile
    from math import ceil

    if (not os.path.exists(weights)):
        raise BaseException('caffemodel does not exist: ' + weights)
    if (not os.path.exists(prototxt)):
        raise BaseException('deploy-proto does not exist: ' + prototxt)
    if (not os.path.exists(listfile)):
        raise BaseException('listfile does not exist: ' + listfile)

    def readTupleList(filename):
        list = []
        for line in open(filename).readlines():
            if line.strip() != '':
                list.append(line.split())

        return list

    ops = readTupleList(listfile)

    width = -1
    height = -1

    output = []

    first_line = [
        'image0', 'image1', 'mean_flow', 'median_flow', 'L2_distance'
    ]
    output.append(['image0', 'image1', 'real_flow', 'estimated_flow'] + blobs)

    output_file = open(output_dir + '/list_out.txt', 'w')

    output_file.write(",".join(first_line))
    output_file.write("\n")

    n = len(ops)

    for i, ent in list(enumerate(ops))[start:]:
        print("processing", i)
        flush()
        print('Processing tuple:', ent)

        num_blobs = 2
        if flow_loss:
            num_blobs = 3

        input_data = []
        img0 = misc.imread(ent[0])
        if len(img0.shape) < 3:
            input_data.append(img0[np.newaxis, np.newaxis, :, :])
        else:
            input_data.append(img0[np.newaxis, :, :, :].transpose(
                0, 3, 1, 2)[:, [2, 1, 0], :, :])
        img1 = misc.imread(ent[1])
        if len(img1.shape) < 3:
            input_data.append(img1[np.newaxis, np.newaxis, :, :])
        else:
            input_data.append(img1[np.newaxis, :, :, :].transpose(
                0, 3, 1, 2)[:, [2, 1, 0], :, :])

        if flow_loss:
            flow = readFlow(ent[2])
            input_data.append(flow[np.newaxis, :, :, :].transpose(0, 3, 1, 2))

        if width != input_data[0].shape[3] or height != input_data[0].shape[2]:
            width = input_data[0].shape[3]
            height = input_data[0].shape[2]

            vars = {}
            vars['TARGET_WIDTH'] = width
            vars['TARGET_HEIGHT'] = height

            divisor = 64.
            vars['ADAPTED_WIDTH'] = int(ceil(width / divisor) * divisor)
            vars['ADAPTED_HEIGHT'] = int(ceil(height / divisor) * divisor)

            vars['SCALE_WIDTH'] = width / float(vars['ADAPTED_WIDTH'])
            vars['SCALE_HEIGHT'] = height / float(vars['ADAPTED_HEIGHT'])

            tmp = tempfile.NamedTemporaryFile(mode='w', delete=False)

            proto = open(prototxt).readlines()
            for line in proto:
                for key, value in vars.items():
                    tag = "$%s$" % key
                    line = line.replace(tag, str(value))

                tmp.write(line)

            tmp.flush()

        caffe.set_logging_disabled()
        caffe.set_mode_gpu()
        net = caffe.Net(tmp.name, weights, caffe.TEST)
        blobs = net.outputs
        print("blobs", blobs)
        flush()
        if i == 0:
            output.append(['image0', 'image1', 'real_flow', 'estimated_flow'] +
                          blobs)

        input_dict = {}
        for blob_idx in range(num_blobs):
            input_dict[net.inputs[blob_idx]] = input_data[blob_idx]

        #input_data
        # There is some non-deterministic nan-bug in caffe
        #
        print('Network forward pass using %s.' % weights)
        j = 1
        while j <= 5:
            j += 1

            net.forward(**input_dict)

            containsNaN = False
            for name in net.blobs:
                blob = net.blobs[name]
                has_nan = np.isnan(blob.data[...]).any()

                if has_nan:
                    print('blob %s contains nan' % name)
                    containsNaN = True

            if not containsNaN:
                print('Succeeded.')
                break
            else:
                print('**************** FOUND NANs, RETRYING ****************')

        flow_blob = np.squeeze(net.blobs['predict_flow_final'].data).transpose(
            1, 2, 0)
        output_flow_file = output_dir + "%04d_flow" % i
        print("saving flow at %s" % output_flow_file)

        writeFlow(output_flow_file + ".flo", flow_blob)
        out_line = [ent[0], ent[1], output_flow_file]

        out_line.append("%.3f" % np.median(flow_blob))
        out_line.append("%.3f" % np.mean(flow_blob))

        #for blob in blobs:
        #    out_line.append(str(net.blobs[blob].data))

        if save_images:
            save_flow_image(flow_blob, output_flow_file + ".png")

            misc.imsave(output_dir + "%04d_img0.jpg" % i, img0)
            misc.imsave(output_dir + "%04d_img1.jpg" % i, img1)

            expected_img0 = apply_flow_reverse(img1, flow_blob)
            misc.imsave(output_dir + "%04d_img0_expected.jpg" % i,
                        expected_img0)

            l2_loss = np.linalg.norm(img0 - expected_img0)

            out_line.append("%.3f" % l2_loss)

        output.append(out_line)
        print(out_line)

        output_file.write(",".join(out_line))
        output_file.write("\n")
        output_file.flush()
Example #16
0
def run_model(prototxt,
              weights,
              img0_p,
              img1_p,
              out_p,
              verbose=False,
              write_flow_image=True):

    if (not os.path.exists(weights)):
        raise BaseException('caffemodel does not exist: ' + weights)
    if (not os.path.exists(prototxt)):
        raise BaseException('deploy-proto does not exist: ' + prototxt)
    if (not os.path.exists(img0_p)):
        raise BaseException('img0 does not exist: ' + img0_p)
    if (not os.path.exists(img1_p)):
        raise BaseException('img1 does not exist: ' + img1_p)

    print("starting run_model")
    num_blobs = 2
    input_data = []
    img0 = misc.imread(img0_p)
    if len(img0.shape) < 3:
        input_data.append(img0[np.newaxis, np.newaxis, :, :])
    else:
        input_data.append(img0[np.newaxis, :, :, :].transpose(
            0, 3, 1, 2)[:, [2, 1, 0], :, :])
    img1 = misc.imread(img1_p)
    if len(img1.shape) < 3:
        input_data.append(img1[np.newaxis, np.newaxis, :, :])
    else:
        input_data.append(img1[np.newaxis, :, :, :].transpose(
            0, 3, 1, 2)[:, [2, 1, 0], :, :])

    width = input_data[0].shape[3]
    height = input_data[0].shape[2]
    vars = {}
    vars['TARGET_WIDTH'] = width
    vars['TARGET_HEIGHT'] = height
    print(width, height)

    divisor = 64.
    vars['ADAPTED_WIDTH'] = int(ceil(width / divisor) * divisor)
    vars['ADAPTED_HEIGHT'] = int(ceil(height / divisor) * divisor)

    vars['SCALE_WIDTH'] = width / float(vars['ADAPTED_WIDTH'])
    vars['SCALE_HEIGHT'] = height / float(vars['ADAPTED_HEIGHT'])

    tmp = tempfile.NamedTemporaryFile(mode='w', delete=True)

    proto = open(prototxt).readlines()
    for line in proto:
        for key, value in vars.items():
            tag = "$%s$" % key
            line = line.replace(tag, str(value))

        tmp.write(line)

    tmp.flush()

    if not verbose:
        caffe.set_logging_disabled()

    #caffe.set_device(args.gpu)

    caffe.set_mode_gpu()

    net = caffe.Net(tmp.name, weights, caffe.TEST)

    input_dict = {}
    for blob_idx in range(num_blobs):
        input_dict[net.inputs[blob_idx]] = input_data[blob_idx]

    #
    # There is some non-deterministic nan-bug in caffe
    # it seems to be a race-condition
    #
    print('Network forward pass using %s.' % weights)
    i = 1
    while i <= 5:
        i += 1

        net.forward(**input_dict)

        containsNaN = False
        for name in net.blobs:
            blob = net.blobs[name]
            has_nan = np.isnan(blob.data[...]).any()

            if has_nan:
                print('blob %s contains nan' % name)
                containsNaN = True

        if not containsNaN:
            print('Succeeded.')
            break
        else:
            print('**************** FOUND NANs, RETRYING ****************')

    blob = np.squeeze(net.blobs['predict_flow_final'].data).transpose(1, 2, 0)

    writeFlow(out_p, blob)

    if write_flow_image:
        print("saving flow image, and warp image")
        save_flow_image(blob, out_p + ".png")
        # save warp
        expected_img0 = apply_flow_reverse(img1, blob)
        misc.imsave(out_p + "_warp.png", expected_img0)
def motion_analysis_flownet2(args):
    data_base_dir = args.dir
    folder = data_base_dir + 'classifiedgood/'
    outputfolder = data_base_dir + 'opticalflow2/'
    if os.path.isdir(outputfolder):
        rmtree(outputfolder)
    os.mkdir(outputfolder)

    # sample two images to initialize the blob dimensions
    imglist = os.listdir(folder)
    imglist.sort()
    img0 = misc.imread(folder + imglist[0])
    img1 = misc.imread(folder + imglist[1])
    # prepare x axis and y axis for faster computation
    x_axis = np.arange(0, img0.shape[0], dtype=np.int)
    x_axis = repmat(x_axis[:, np.newaxis], 1, img0.shape[1])
    y_axis = np.arange(0, img0.shape[1], dtype=np.int)
    y_axis = repmat(y_axis, img0.shape[0], 1)

    def warp_NCC_opt(img0, img1, flow):
        nx = (x_axis + flow[:, :, 1]).astype(int)
        ny = (y_axis + flow[:, :, 0]).astype(int)
        pos0 = (nx <= img1.shape[0] - 1) & (nx >= 0) & (
            ny <= img1.shape[1] - 1) & (ny >= 0)
        patch0 = img0[pos0]
        pos1x = nx[pos0]
        pos1y = ny[pos0]
        nn = [pos1x, pos1y]
        patch1 = img1[nn]
        return correlation_coefficient(patch0, patch1)

    input_data = []
    if len(img0.shape) < 3:
        input_data.append(img0[np.newaxis, np.newaxis, :, :])
    else:
        input_data.append(img0[np.newaxis, :, :, :].transpose(
            0, 3, 1, 2)[:, [2, 1, 0], :, :])
    print('input blob 0 :'),
    print(input_data[0].shape)
    if len(img1.shape) < 3:
        input_data.append(img1[np.newaxis, np.newaxis, :, :])
    else:
        input_data.append(img1[np.newaxis, :, :, :].transpose(
            0, 3, 1, 2)[:, [2, 1, 0], :, :])
    print('input blob 1 :'),
    print(input_data[1].shape)

    width = input_data[0].shape[3]
    height = input_data[0].shape[2]
    vars = {}
    vars['TARGET_WIDTH'] = width
    vars['TARGET_HEIGHT'] = height

    divisor = 64.
    vars['ADAPTED_WIDTH'] = int(ceil(width / divisor) * divisor)
    vars['ADAPTED_HEIGHT'] = int(ceil(height / divisor) * divisor)

    vars['SCALE_WIDTH'] = width / float(vars['ADAPTED_WIDTH'])
    vars['SCALE_HEIGHT'] = height / float(vars['ADAPTED_HEIGHT'])

    tmp = tempfile.NamedTemporaryFile(mode='w', delete=True)

    proto = open(args.deployproto).readlines()
    for line in proto:
        for key, value in vars.items():
            tag = "$%s$" % key
            line = line.replace(tag, str(value))

        tmp.write(line)

    tmp.flush()

    if not args.verbose:
        caffe.set_logging_disabled()
    caffe.set_device(args.gpu)
    caffe.set_mode_gpu()
    net = caffe.Net(tmp.name, args.caffemodel, caffe.TEST)

    # calculate sharpness and sort by sharpness
    sharpness = []
    print('Calculating sharpness ...')
    dl = DoubleList()
    D = {}
    for f in imglist:
        dl.append(f)
        D[f] = dl.tail
    for i in range(len(imglist)):
        print('%d / %d' % (i, len(imglist)))
        img = misc.imread(folder + imglist[i], flatten=True)
        sobelx = cv2.Sobel(img, cv2.CV_64F, 1, 0, ksize=5)
        sobely = cv2.Sobel(img, cv2.CV_64F, 0, 1, ksize=5)
        sharpness.append((np.sum(sobelx**2) + np.sum(sobely**2)) / img.size)
    order = np.argsort(sharpness)
    ## load img0 and img1 iteratively from folder

    score = []
    keyframes = []
    for j in range(len(imglist)):
        print('Homography estimation: %d / %d   sharpness = %.2f' %
              (j, len(imglist), sharpness[order[j]]))
        if order[j] == 0 or order[j] == len(imglist) - 1:
            keyframes.append(imglist[order[j]])
            score.append(0.9)
            continue
        ID = order[j]
        PrevName = D[imglist[ID]].prev.data
        NextName = D[imglist[ID]].next.data
        Prev = misc.imread(folder + PrevName, mode='RGB')
        Next = misc.imread(folder + NextName, mode='RGB')
        input_dict = {}
        input_data = []
        if len(Prev.shape) < 3:
            input_data.append(Prev[np.newaxis, np.newaxis, :, :])
        else:
            input_data.append(Prev[np.newaxis, :, :, :].transpose(
                0, 3, 1, 2)[:, [2, 1, 0], :, :])
        if len(Next.shape) < 3:
            input_data.append(Next[np.newaxis, np.newaxis, :, :])
        else:
            input_data.append(Next[np.newaxis, :, :, :].transpose(
                0, 3, 1, 2)[:, [2, 1, 0], :, :])
        input_dict[net.inputs[0]] = input_data[0]
        input_dict[net.inputs[1]] = input_data[1]

        print('Network forward pass using %s and %s' % (PrevName, NextName))
        i = 1
        while i <= 5:
            i += 1

            net.forward(**input_dict)

            containsNaN = False
            for name in net.blobs:
                blob = net.blobs[name]
                has_nan = np.isnan(blob.data[...]).any()

                if has_nan:
                    print('blob %s contains nan' % name)
                    containsNaN = True

            if not containsNaN:
                print('Succeeded.')
                break
            else:
                print('**************** FOUND NANs, RETRYING ****************')

        flow = np.squeeze(net.blobs['predict_flow_final'].data).transpose(
            1, 2, 0)
        #s = warp_NCC(img0=rgb2gray(Prev), img1=rgb2gray(Next), flow=flow)
        s = warp_NCC_opt(img0=rgb2gray(Prev), img1=rgb2gray(Next), flow=flow)
        #warpped = warpFlow(rgb2gray(Next), flow)
        #s = correlation_coefficient(warpped, rgb2gray(Prev))
        print('correlation coefficient = %.4f' % s)
        if s > 0.98:
            dl.remove_byaddress(D[imglist[ID]])
            print('Redundant: ' + imglist[ID])
        else:
            keyframes.append(imglist[order[j]])
            score.append(s)
            print('Keyframe:  ' + imglist[ID])

    ordkeyframes = np.argsort(keyframes)
    keyframes.sort()
    score = np.asarray(score)
    score = score[np.asarray(ordkeyframes)]

    offilename = data_base_dir + 'opticalflowscore2.txt'
    if os.path.exists(offilename):
        os.remove(offilename)
    offile = open(offilename, 'w')
    for i in range(len(keyframes)):
        offile.write('%s %.4f\n' % (keyframes[i], score[i]))
        os.system('cp ' + folder + keyframes[i] + ' ' + outputfolder +
                  keyframes[i])

    analyze_score(data_base_dir=data_base_dir,
                  fname='opticalflowscore2',
                  imgnames=keyframes,
                  score=score)

    del net