Example #1
0
    def __init__(self, net, state, loop_sleep, pause_after_keys, heartbeat_required):
        CodependentThread.__init__(self, heartbeat_required)
        self.daemon = True
        self.net = net
        self.input_dims = self.net.blobs['data'].data.shape[2:4]    # e.g. (227,227)
        self.state = state
        self.frames_processed_fwd = 0
        self.frames_processed_back = 0
        self.loop_sleep = loop_sleep
        self.pause_after_keys = pause_after_keys
        self.debug_level = 0
        self.descriptor = None
        self.descriptor_layer_1 = 'conv5'
        self.descriptor_layer_2 = 'conv4'
        self.descriptor_layers = ['conv5','conv4']
        self.net_input_image = None
        self.descriptor_handler = DescriptorHandler(self.state.settings.ros_dir + '/models/memory/', self.descriptor_layers)
        self.data_handler = DataHandler(self.state.settings.ros_dir)
        self.available_layer = ['conv1', 'pool1', 'norm1', 'conv2', 'pool2', 'norm2', 'conv3', 'conv4', 'conv5', 'pool5', 'fc6', 'fc7', 'fc8', 'prob']
        #['conv1', 'conv2', 'conv3', 'conv4', 'conv5', 'fc6', 'fc7', 'fc8', 'prob']
        # print "layers ", list(self.net._layer_names)

        s = rospy.Service('get_cnn_state', GetState, self.handle_get_cnn_state)
Example #2
0
    def __init__(self, settings):
        print 'initialize'
        self.settings = settings
        self.descriptor_layers = ['conv5','conv4']

        self._data_mean = np.load(settings.caffevis_data_mean)
        # Crop center region (e.g. 227x227) if mean is larger (e.g. 256x256)
        excess_h = self._data_mean.shape[1] - self.settings.caffevis_data_hw[0]
        excess_w = self._data_mean.shape[2] - self.settings.caffevis_data_hw[1]
        assert excess_h >= 0 and excess_w >= 0, 'mean should be at least as large as %s' % repr(self.settings.caffevis_data_hw)
        self._data_mean = self._data_mean[:, excess_h:(excess_h+self.settings.caffevis_data_hw[0]),
                                          excess_w:(excess_w+self.settings.caffevis_data_hw[1])]
        self._net_channel_swap = (2,1,0)
        self._net_channel_swap_inv = tuple([self._net_channel_swap.index(ii) for ii in range(len(self._net_channel_swap))])
        self._range_scale = 1.0      # not needed; image comes in [0,255]



        if settings.caffevis_mode_gpu:
            caffe.set_mode_gpu()
            print 'CaffeVisApp mode: GPU'
        else:
            caffe.set_mode_cpu()
            print 'CaffeVisApp mode: CPU'

        self.net = caffe.Classifier(
            settings.caffevis_deploy_prototxt,
            settings.caffevis_network_weights,
            mean = self._data_mean,
            channel_swap = self._net_channel_swap,
            raw_scale = self._range_scale,
            #image_dims = (227,227),
        )
        self.input_dims = self.net.blobs['data'].data.shape[2:4]    # e.g. (227,227)
        self.img_file_list = []

        self.descriptor_handler = DescriptorHandler(self.settings.ros_dir + '/eric_data_set/model_seg/', self.descriptor_layers)
Example #3
0
class CaffeProcThread(CodependentThread):
    '''Runs Caffe in separate thread.'''

    def __init__(self, net, state, loop_sleep, pause_after_keys, heartbeat_required):
        CodependentThread.__init__(self, heartbeat_required)
        self.daemon = True
        self.net = net
        self.input_dims = self.net.blobs['data'].data.shape[2:4]    # e.g. (227,227)
        self.state = state
        self.frames_processed_fwd = 0
        self.frames_processed_back = 0
        self.loop_sleep = loop_sleep
        self.pause_after_keys = pause_after_keys
        self.debug_level = 0
        self.descriptor = None
        self.descriptor_layer_1 = 'conv5'
        self.descriptor_layer_2 = 'conv4'
        self.descriptor_layers = ['conv5','conv4']
        self.net_input_image = None
        self.descriptor_handler = DescriptorHandler(self.state.settings.ros_dir + '/models/memory/', self.descriptor_layers)
        self.data_handler = DataHandler(self.state.settings.ros_dir)
        self.available_layer = ['conv1', 'pool1', 'norm1', 'conv2', 'pool2', 'norm2', 'conv3', 'conv4', 'conv5', 'pool5', 'fc6', 'fc7', 'fc8', 'prob']
        #['conv1', 'conv2', 'conv3', 'conv4', 'conv5', 'fc6', 'fc7', 'fc8', 'prob']
        # print "layers ", list(self.net._layer_names)

        s = rospy.Service('get_cnn_state', GetState, self.handle_get_cnn_state)

    def handle_get_cnn_state(self, req):
        resp = GetStateResponse()
        print "get req", req
        state = perception_msgs.msg.State()
        state.type = req.type
        state.name = req.name
        if req.name == "None":
            state.name = self.data_handler.save_net(self.net)
        else:
            print "not implemented yet"
        resp.state = state
        resp.pose = Pose()
        print "state", resp

        return resp

    def mask_out(self, data, mask):
        # print "data shape", data.shape
        dim = data.shape

        for y in range(dim[2]):
            for x in range(dim[3]):
                if is_masked((dim[2],dim[3]),(x,y),mask):
                    data[:,:,y,x] = 0

        return data

    def net_preproc_forward(self, img):
        assert img.shape == (227,227,3), 'img is wrong size'
        #resized = caffe.io.resize_image(img, net.image_dims)   # e.g. (227, 227, 3)
        data_blob = self.net.transformer.preprocess('data', img)                # e.g. (3, 227, 227), mean subtracted and scaled to [0,255]
        data_blob = data_blob[np.newaxis,:,:,:]                   # e.g. (1, 3, 227, 227)
        output = self.net.forward(data=data_blob)
        return output

    def net_proc_forward_layer(self, img, mask):
        assert img.shape == (227,227,3), 'img is wrong size'
        #resized = caffe.io.resize_image(img, net.image_dims)   # e.g. (227, 227, 3)
        data_blob = self.net.transformer.preprocess('data', img)                # e.g. (3, 227, 227), mean subtracted and scaled to [0,255]
        data_blob = data_blob[np.newaxis,:,:,:]                   # e.g. (1, 3, 227, 227)
        # print "mask", mask.shape
        self.net.blobs['data'].data[...] = data_blob
        proc_layers = ['conv1', 'conv2', 'conv3', 'conv4', 'conv5', 'prob']


        mask_layers = ['']
        mode = 0
        if mode == 0:
            self.net.forward_from_to(start='conv1',end='relu1')
            self.mask_out(self.net.blobs['conv1'].data, mask)
            self.net.forward_from_to(start='relu1',end='prob')

        elif mode == 1:
            self.net.forward_from_to(start='conv1',end='relu1')
            self.mask_out(self.net.blobs['conv1'].data, mask)
            self.net.forward_from_to(start='relu1',end='conv2')

            self.net.forward_from_to(start='conv2',end='relu2')
            self.mask_out(self.net.blobs['conv2'].data, mask)
            self.net.forward_from_to(start='relu2',end='conv3')

            self.net.forward_from_to(start='conv3',end='relu3')
            self.mask_out(self.net.blobs['conv3'].data, mask)
            self.net.forward_from_to(start='relu3',end='conv4')

            self.net.forward_from_to(start='conv4',end='relu4')
            self.mask_out(self.net.blobs['conv4'].data, mask)
            self.net.forward_from_to(start='relu4',end='conv5')

            self.net.forward_from_to(start='conv5',end='relu5')
            self.mask_out(self.net.blobs['conv5'].data, mask)
            self.net.forward_from_to(start='relu5',end='prob')

        elif mode == 2:
            for idx in range(0,len(proc_layers)-1):
                output = self.net.forward_from_to(start=proc_layers[idx],end=proc_layers[idx+1])
                self.mask_out(self.net.blobs[proc_layers[idx]].data, mask)

        # for idx in range(len(self.available_layer)-1):
        #     output = self.net.forward(data=data_blob,start=self.available_layer[idx],end=self.available_layer[idx+1])
        #     if self.available_layer[idx].startswith("conv"):
        #         new_blob = self.net.blobs[self.available_layer[idx]].data
        #         new_blob.data = self.mask_out(self.net.blobs[self.available_layer[idx]].data, mask)
        #         self.net.blobs[self.available_layer[idx]] = new_blob
            # print output

        # return output

    def save_descriptor(self):
        print 'save descriptor'
        # print type(self.net.blobs[self.descriptor_layer_1])
        # descriptor_1 = np.array(caffe.io.blobproto_to_array(self.net.blobs[self.descriptor_layer_1]))
        # descriptor_2 = np.array(caffe.io.blobproto_to_array(self.net.blobs[self.descriptor_layer_2]))
        # print descriptor
        self.time_name = strftime("%d-%m-%Y-%H:%M:%S", gmtime())

        desc = self.descriptor_handler.gen_descriptor(self.time_name, self.net.blobs)
        self.descriptor_handler.save(self.state.settings.ros_dir + '/models/memory/', desc)

        # np.save(self.state.settings.ros_dir + '/models/memory/' + self.time_name + "_" + self.descriptor_layer_1 + '.npy', descriptor_1)
        # np.save(self.state.settings.ros_dir + '/models/memory/' + self.time_name + "_" + self.descriptor_layer_2 + '.npy', descriptor_2)
        cv2.imwrite(self.state.settings.ros_dir + '/models/memory/' + self.time_name + '.jpg', self.net_input_image[:,:,::-1])

        self.state.save_descriptor = False

    def run(self):
        print 'CaffeProcThread.run called'
        frame = None
        mask = None
        while not self.is_timed_out():
            with self.state.lock:
                if self.state.quit:
                    #print 'CaffeProcThread.run: quit is True'
                    #print self.state.quit
                    break

                #print 'CaffeProcThread.run: caffe_net_state is:', self.state.caffe_net_state

                #print 'CaffeProcThread.run loop: next_frame: %s, caffe_net_state: %s, back_enabled: %s' % (
                #    'None' if self.state.next_frame is None else 'Avail',
                #    self.state.caffe_net_state,
                #    self.state.back_enabled)

                frame = None
                mask = None
                run_fwd = False
                run_back = False
                if self.state.caffe_net_state == 'free' and time.time() - self.state.last_key_at > self.pause_after_keys:
                    frame = self.state.next_frame
                    mask = self.state.mask
                    self.state.next_frame = None
                    back_enabled = self.state.back_enabled
                    back_mode = self.state.back_mode
                    back_stale = self.state.back_stale
                    #state_layer = self.state.layer
                    #selected_unit = self.state.selected_unit
                    backprop_layer = self.state.backprop_layer
                    backprop_unit = self.state.backprop_unit

                    # Forward should be run for every new frame
                    run_fwd = (frame is not None)
                    # Backward should be run if back_enabled and (there was a new frame OR back is stale (new backprop layer/unit selected))
                    run_back = (back_enabled and (run_fwd or back_stale))
                    self.state.caffe_net_state = 'proc' if (run_fwd or run_back) else 'free'

            #print 'run_fwd,run_back =', run_fwd, run_back

            if run_fwd:
                #print 'TIMING:, processing frame'
                self.frames_processed_fwd += 1
                self.net_input_image = cv2.resize(frame, self.input_dims)
                with WithTimer('CaffeProcThread:forward', quiet = self.debug_level < 1):
                    print "run forward layer"
                    self.net_proc_forward_layer(self.net_input_image, mask)
                    # self.net_preproc_forward(self.net_input_image)

            if self.state.save_descriptor:
                self.save_descriptor()

            # switch descriptor for match and back prop
            if self.state.next_descriptor:
                    print 'load descriptor'
                    self.descriptor = self.descriptor_handler.get_next()
                    self.state.next_descriptor = False


            if self.state.compare_descriptor:
                # find
                print 'compare'
                desc_current = self.descriptor_handler.gen_descriptor('current', self.net.blobs)
                match_file = self.descriptor_handler.get_max_match(desc_current)
                print 'match: ' + match_file
                self.state.compare_descriptor = False

            if run_back:
                print "run backward"
                # Match to saved descriptor
                if self.state.match_descriptor:
                    print '*'
                    diffs = self.net.blobs[self.descriptor_layer_1].diff * 0

                    # zero all diffs if doesn't match
                    print "shape ", self.net.blobs[self.descriptor_layer_1].data.shape
                    for unit, response in enumerate(self.net.blobs[self.descriptor_layer_1].data[0]):
                        if response.max() > 0 and abs(response.max() - self.descriptor.get_sig_list()[0][0][unit].max())/response.max() < 0.2:
                            diffs[0][unit] = self.net.blobs[self.descriptor_layer_1].data[0][unit]

                    assert back_mode in ('grad', 'deconv')
                    if back_mode == 'grad':
                        with WithTimer('CaffeProcThread:backward', quiet = self.debug_level < 1):
                            #print '**** Doing backprop with %s diffs in [%s,%s]' % (backprop_layer, diffs.min(), diffs.max())
                            self.net.backward_from_layer(self.descriptor_layer_1, diffs, zero_higher = True)
                    else:
                        with WithTimer('CaffeProcThread:deconv', quiet = self.debug_level < 1):
                            #print '**** Doing deconv with %s diffs in [%s,%s]' % (backprop_layer, diffs.min(), diffs.max())
                            self.net.deconv_from_layer(self.descriptor_layer_1, diffs, zero_higher = True)

                    with self.state.lock:
                        self.state.back_stale = False

                # Filter when back propagating
                elif self.state.backprop_filter:
                    print "run_back"
                    # print backprop_layer
                    start_layer_idx = self.available_layer.index(backprop_layer)
                    idx = start_layer_idx
                    for current_layer in list(reversed(self.available_layer[0:start_layer_idx+1])):

                        diffs = self.net.blobs[current_layer].diff * 0
                        max_response = self.net.blobs[current_layer].data[0].max()
                        for unit, response in enumerate(self.net.blobs[current_layer].data[0]):
                            if response.max() > max_response * 0.6:
                                diffs[0][unit] = self.net.blobs[current_layer].data[0,unit]


                        assert back_mode in ('grad', 'deconv')
                        if back_mode == 'grad':
                            with WithTimer('CaffeProcThread:backward', quiet = self.debug_level < 1):
                                #print '**** Doing backprop with %s diffs in [%s,%s]' % (backprop_layer, diffs.min(), diffs.max())
                                self.net.backward_from_to_layer(current_layer, diffs, self.available_layer[idx-1], zero_higher = (idx == start_layer_idx))
                        # else:
                        #     with WithTimer('CaffeProcThread:deconv', quiet = self.debug_level < 1):
                        #         #print '**** Doing deconv with %s diffs in [%s,%s]' % (backprop_layer, diffs.min(), diffs.max())
                        #         self.net.deconv_from_layer(backprop_layer, diffs, zero_higher = True)
                        idx -= 1
                    with self.state.lock:
                        self.state.back_stale = False

                # original approach
                else:
                    diffs = self.net.blobs[backprop_layer].diff * 0
                    diffs[0][backprop_unit] = self.net.blobs[backprop_layer].data[0,backprop_unit]

                    assert back_mode in ('grad', 'deconv')
                    if back_mode == 'grad':
                        with WithTimer('CaffeProcThread:backward', quiet = self.debug_level < 1):
                            #print '**** Doing backprop with %s diffs in [%s,%s]' % (backprop_layer, diffs.min(), diffs.max())
                            self.net.backward_from_layer(backprop_layer, diffs, zero_higher = True)
                    else:
                        with WithTimer('CaffeProcThread:deconv', quiet = self.debug_level < 1):
                            #print '**** Doing deconv with %s diffs in [%s,%s]' % (backprop_layer, diffs.min(), diffs.max())
                            self.net.deconv_from_layer(backprop_layer, diffs, zero_higher = True)

                    with self.state.lock:
                        self.state.back_stale = False

            if run_fwd or run_back:
                with self.state.lock:
                    self.state.caffe_net_state = 'free'
                    self.state.drawing_stale = True
            else:
                time.sleep(self.loop_sleep)
            time.sleep(0.1)
        print 'CaffeProcThread.run: finished'
        print 'CaffeProcThread.run: processed %d frames fwd, %d frames back' % (self.frames_processed_fwd, self.frames_processed_back)
Example #4
0
class Tester:

    def __init__(self, settings):
        print 'initialize'
        self.settings = settings
        self.descriptor_layers = ['conv5','conv4']

        self._data_mean = np.load(settings.caffevis_data_mean)
        # Crop center region (e.g. 227x227) if mean is larger (e.g. 256x256)
        excess_h = self._data_mean.shape[1] - self.settings.caffevis_data_hw[0]
        excess_w = self._data_mean.shape[2] - self.settings.caffevis_data_hw[1]
        assert excess_h >= 0 and excess_w >= 0, 'mean should be at least as large as %s' % repr(self.settings.caffevis_data_hw)
        self._data_mean = self._data_mean[:, excess_h:(excess_h+self.settings.caffevis_data_hw[0]),
                                          excess_w:(excess_w+self.settings.caffevis_data_hw[1])]
        self._net_channel_swap = (2,1,0)
        self._net_channel_swap_inv = tuple([self._net_channel_swap.index(ii) for ii in range(len(self._net_channel_swap))])
        self._range_scale = 1.0      # not needed; image comes in [0,255]



        if settings.caffevis_mode_gpu:
            caffe.set_mode_gpu()
            print 'CaffeVisApp mode: GPU'
        else:
            caffe.set_mode_cpu()
            print 'CaffeVisApp mode: CPU'

        self.net = caffe.Classifier(
            settings.caffevis_deploy_prototxt,
            settings.caffevis_network_weights,
            mean = self._data_mean,
            channel_swap = self._net_channel_swap,
            raw_scale = self._range_scale,
            #image_dims = (227,227),
        )
        self.input_dims = self.net.blobs['data'].data.shape[2:4]    # e.g. (227,227)
        self.img_file_list = []

        self.descriptor_handler = DescriptorHandler(self.settings.ros_dir + '/eric_data_set/model_seg/', self.descriptor_layers)

    def run(self,dir):
        self.init_img_file_list(dir)
        for img_file in self.img_file_list:
            # print "test:  ", img_file
            match_file = self.classify(dir+img_file)
            print img_file +":"+ match_file
            # self.classify(dir+self.img_file_list[0])

        pass

    def net_preproc_forward(self, net, img):
        assert img.shape == (227,227,3), 'img is wrong size'
        #resized = caffe.io.resize_image(img, net.image_dims)   # e.g. (227, 227, 3)
        data_blob = net.transformer.preprocess('data', img)                # e.g. (3, 227, 227), mean subtracted and scaled to [0,255]
        data_blob = data_blob[np.newaxis,:,:,:]                   # e.g. (1, 3, 227, 227)
        output = net.forward(data=data_blob)
        return output

    def classify(self, img_file):

        image = cv2_read_file_rgb(img_file)
        image = crop_to_square(image)
        image = cv2.resize(image, self.input_dims)
        self.net_preproc_forward(self.net, image)
        desc_current = self.descriptor_handler.gen_descriptor('current', self.net.blobs)
        match_file = self.descriptor_handler.get_max_match(desc_current)
        return match_file

    def init_img_file_list(self,dir):
        match_flags = re.IGNORECASE
        for filename in os.listdir(dir):
            if re.match('.*\.(jpg|jpeg|png)$', filename, match_flags):
                self.img_file_list.append(filename)
        self.img_file_list = sorted(self.img_file_list)

    def create_desc_files(self,dir):
        self.init_img_file_list(dir)

        for file in self.img_file_list:
            print 'file ' + file
            image = cv2_read_file_rgb(dir+file)
            image = crop_to_square(image)
            image = cv2.resize(image, self.input_dims)
            self.net_preproc_forward(self.net, image)
            tag_name = file.split('.')[0]

            desc = self.descriptor_handler.gen_descriptor(tag_name, self.net.blobs)
            self.descriptor_handler.save(dir, desc)