def doExcitationBackprop(net, img, tagName): # load image, rescale minDim = min(img.shape[:2]) # newSize = (int(img.shape[0]*imgScale/float(minDim)), int(img.shape[1]*imgScale/float(minDim))) #print newSize newSize = (224, 224) imgS = transform.resize(img, newSize) # reshape net net.blobs['data'].reshape(1, 3, newSize[0], newSize[1]) transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape}) transformer.set_mean('data', np.array([103.939, 116.779, 123.68])) transformer.set_transpose('data', (2, 0, 1)) transformer.set_channel_swap('data', (2, 1, 0)) transformer.set_raw_scale('data', 255.0) # forward pass net.blobs['data'].data[...] = transformer.preprocess('data', imgS) out = net.forward(end=topLayerName) # switch to the excitation backprop mode caffe.set_mode_eb_gpu() tagID = tag2ID[tagName] net.blobs[topBlobName].diff[0][...] = 0 net.blobs[topBlobName].diff[0][tagID] = np.exp( net.blobs[topBlobName].data[0][tagID].copy()) net.blobs[topBlobName].diff[0][tagID] /= net.blobs[topBlobName].diff[0][ tagID].sum() # invert the top layer weights net.params[topLayerName][0].data[...] *= -1 out = net.backward(start=topLayerName, end=secondTopLayerName) buff = net.blobs[secondTopBlobName].diff.copy() # invert back net.params[topLayerName][0].data[...] *= -1 out = net.backward(start=topLayerName, end=secondTopLayerName) # compute the contrastive signal net.blobs[secondTopBlobName].diff[...] -= buff # get attention map out = net.backward(start=secondTopLayerName, end=outputLayerName) attMap = np.maximum(net.blobs[outputBlobName].diff[0].sum(0), 0) # resize back to original image size #attMap = transform.resize(attMap, (img.shape[:2]), order = 3, mode = 'nearest') return attMap
def doExcitationBackprop(net, img, tagName): # load image, rescale minDim = min(img.shape[:2]) newSize = (int(img.shape[0]*imgScale/float(minDim)), int(img.shape[1]*imgScale/float(minDim))) imgS = transform.resize(img, newSize) # reshape net net.blobs['data'].reshape(1,3,newSize[0],newSize[1]) transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape}) transformer.set_mean('data', np.array([103.939, 116.779, 123.68])) transformer.set_transpose('data', (2,0,1)) transformer.set_channel_swap('data', (2,1,0)) transformer.set_raw_scale('data', 255.0) # forward pass net.blobs['data'].data[...] = transformer.preprocess('data', imgS) out = net.forward(end = topLayerName) # switch to the excitation backprop mode caffe.set_mode_eb_gpu() tagID = tag2ID[tagName] net.blobs[topBlobName].diff[0][...] = 0 net.blobs[topBlobName].diff[0][tagID] = np.exp(net.blobs[topBlobName].data[0][tagID].copy()) net.blobs[topBlobName].diff[0][tagID] /= net.blobs[topBlobName].diff[0][tagID].sum() # invert the top layer weights net.params[topLayerName][0].data[...] *= -1 out = net.backward(start = topLayerName, end = secondTopLayerName) buff = net.blobs[secondTopBlobName].diff.copy() # invert back net.params[topLayerName][0].data[...] *= -1 out = net.backward(start = topLayerName, end = secondTopLayerName) # compute the contrastive signal net.blobs[secondTopBlobName].diff[...] -= buff # get attention map out = net.backward(start = secondTopLayerName, end = outputLayerName) attMap = np.maximum(net.blobs[outputBlobName].diff[0].sum(0), 0) # resize back to original image size attMap = transform.resize(attMap, (img.shape[:2]), order = 3, mode = 'nearest') return attMap
def backprop(self, neuron_id, contrastive=True): # eb = excitation backprop caffe.set_mode_eb_gpu() net = self.net top = net.blobs[self.top_blob_name].data[0][neuron_id].copy() if not np.any(top): print("WARNING: top data is empty") top = np.exp(1.) net.blobs[self.top_blob_name].diff[0][...] = 0 net.blobs[self.top_blob_name].diff[0][neuron_id] = top #np.exp(net.blobs[self.top_blob_name].data[0][neuron_id].copy()) net.blobs[self.top_blob_name].diff[0][neuron_id] /= \ net.blobs[self.top_blob_name].diff[0][neuron_id].sum() if contrastive: # invert the top layer weights net.params[self.top_layer_name][0].data[...] *= -1 net.backward(start=self.top_layer_name, end=self.second_top_layer_name) # Grab the signal when all uninteresting neurons are set buff = net.blobs[self.second_top_blob_name].diff.copy() # restore layer: make all uninteresting neurons uninteresting again net.params[self.top_layer_name][0].data[...] *= -1 # Grab the signal when only the interesting neuron is set net.backward(start=self.top_layer_name, end=self.second_top_layer_name) # Combine results of inverted and non inverted backprop to compute contrastive signal net.blobs[self.second_top_blob_name].diff[...] -= buff # compute the contrastive signal net.backward(start=self.second_top_layer_name, end=self.output_layer_name) else: net.backward(start=self.top_layer_name, end=self.output_layer_name) attention_map = np.maximum( net.blobs[self.output_blob_name].diff[0].sum(0), 0 ) return attention_map
#------------------- original_data.append(shift_data_in) data.append(processed_image) #----FWD caffe.set_device(device_net_conventional) net_conventional.blobs['data'].reshape(end_point - v, 3, image_dim_conventional, image_dim_conventional) net_conventional.blobs['data'].data[...] = data[0:end_point - v] net_conventional.forward() #---- #----BWD using EB caffe.set_mode_eb_gpu() attMaps = [] tagID_top = np.zeros( (top_classes, net_conventional.blobs['probs'].data.shape[0]), dtype=int) for cntr in range(top_classes): net_conventional.blobs[topBlobName].diff[...] = 0 for j in range(net_conventional.blobs['probs'].data.shape[0]): tagScore = util.getTagScore( net_conventional.blobs[topLayerName].data[j, :], tags, tag2ID) tagScore.sort(key=operator.itemgetter(1), reverse=True) if cntr == 0: predict_labels.append( np.argmax( net_conventional.blobs['probs'].data[j, :].copy()))
def compute_heatmap(net, transformer, paths, labels, heatmap_type, topBlobName, topLayerName, outputBlobName='data', outputLayerName='data', secondTopBlobName='pool5/7x7_s1', secondTopLayerName='pool5/7x7_s1', norm_deg=np.inf, gpu=None): if gpu == None: if heatmap_type == 'saliency' or heatmap_type == 'grad_cam': caffe.set_mode_cpu() elif heatmap_type == 'guided_backprop': caffe.set_mode_dc_cpu() elif heatmap_type == 'excitation_backprop' or heatmap_type == 'contrast_excitation_backprop': caffe.set_mode_eb_cpu() else: print 'heatmap_type %s is not supported' % heatmap_type return else: caffe.set_device(gpu) if heatmap_type == 'saliency' or heatmap_type == 'grad_cam': caffe.set_mode_gpu() elif heatmap_type == 'guided_backprop': caffe.set_mode_dc_gpu() elif heatmap_type == 'excitation_backprop' or heatmap_type == 'contrast_excitation_backprop': caffe.set_mode_eb_gpu() else: print 'heatmap_type %s is not supported' % heatmap_type return if isinstance(paths, basestring): num_imgs = 1 assert (isinstance(labels, int)) else: num_imgs = len(paths) assert (num_imgs == len(labels)) net.blobs['data'].reshape(num_imgs, 3, net.blobs['data'].data.shape[2], net.blobs['data'].data.shape[3]) if num_imgs == 1: net.blobs['data'].data[...] = transformer.preprocess( 'data', caffe.io.load_image(paths)) net.forward() net.blobs[topBlobName].diff[0][...] = 0 net.blobs[topBlobName].diff[0][labels] = 1 else: for i in range(num_imgs): net.blobs['data'].data[i, ...] = transformer.preprocess( 'data', caffe.io.load_image(paths[i])) net.forward() for i in range(num_imgs): net.blobs[topBlobName].diff[i][...] = 0 net.blobs[topBlobName].diff[i][labels[i]] = 1 if heatmap_type == 'contrast_excitation_backprop': # invert top layer weights net.params[topLayerName][0].data[...] *= -1 out = net.backward(start=topLayerName, end=secondTopLayerName) buff = net.blobs[secondTopBlobName].diff.copy() # invert back net.params[topLayerName][0].data[...] *= -1 out = net.backward(start=topLayerName, end=secondTopLayerName) # compute the contrastive signal net.blobs[secondTopBlobName].diff[...] -= buff net.backward(start=secondTopLayerName, end=outputLayerName) elif heatmap_type == 'grad_cam': net.backward(start=topLayerName, end=outputLayerName) activations = net.blobs[outputBlobName].data.copy( ) # TODO: check if copy is needed gradient = net.blobs[outputBlobName].diff.copy() alphas = np.mean(gradient, (2, 3)) attMaps = np.squeeze( np.maximum( np.sum( activations * np.broadcast_to( np.expand_dims(np.expand_dims(alphas, 2), 3), activations.shape), 1), 0)) return attMaps else: try: net.backward(start=topLayerName, end=outputLayerName) except: assert (outputLayerName == 'data') net.backward(start=topLayerName) if np.isinf(norm_deg): if norm_deg == np.inf: attMaps = np.squeeze(np.abs(net.blobs[outputBlobName].diff).max(1)) else: attMaps = np.squeeze(np.abs(net.blobs[outputBlobName].diff).min(1)) else: if norm_deg == 0: attMaps = np.squeeze(net.blobs[outputBlobName.diff]) elif norm_deg == -1: attMaps = np.squeeze(net.blobs[outputBlobName].diff.sum(1)) elif norm_deg == -2: attMaps = np.squeeze( np.maximum(net.blobs[outputBlobName].diff.sum(1), 0)) else: attMaps = np.squeeze( ((np.abs(net.blobs[outputBlobName].diff)**norm_deg).sum(1))**( 1 / float(norm_deg))) # TODO: test this case return attMaps