def doExcitationBackprop(net, img, tagName):
# load image, rescale
minDim = min(img.shape[:2])
# newSize = (int(img.shape[0]*imgScale/float(minDim)), int(img.shape[1]*imgScale/float(minDim)))
#print newSize
newSize = (224, 224)
imgS = transform.resize(img, newSize)
# reshape net
net.blobs['data'].reshape(1, 3, newSize[0], newSize[1])
transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape})
transformer.set_mean('data', np.array([103.939, 116.779, 123.68]))
transformer.set_transpose('data', (2, 0, 1))
transformer.set_channel_swap('data', (2, 1, 0))
transformer.set_raw_scale('data', 255.0)
# forward pass
net.blobs['data'].data[...] = transformer.preprocess('data', imgS)
out = net.forward(end=topLayerName)
# switch to the excitation backprop mode
caffe.set_mode_eb_gpu()
tagID = tag2ID[tagName]
net.blobs[topBlobName].diff[0][...] = 0
net.blobs[topBlobName].diff[0][tagID] = np.exp(
net.blobs[topBlobName].data[0][tagID].copy())
net.blobs[topBlobName].diff[0][tagID] /= net.blobs[topBlobName].diff[0][
tagID].sum()
# invert the top layer weights
net.params[topLayerName][0].data[...] *= -1
out = net.backward(start=topLayerName, end=secondTopLayerName)
buff = net.blobs[secondTopBlobName].diff.copy()
# invert back
net.params[topLayerName][0].data[...] *= -1
out = net.backward(start=topLayerName, end=secondTopLayerName)
# compute the contrastive signal
net.blobs[secondTopBlobName].diff[...] -= buff
# get attention map
out = net.backward(start=secondTopLayerName, end=outputLayerName)
attMap = np.maximum(net.blobs[outputBlobName].diff[0].sum(0), 0)
# resize back to original image size
#attMap = transform.resize(attMap, (img.shape[:2]), order = 3, mode = 'nearest')
return attMap
def doExcitationBackprop(net, img, tagName):
# load image, rescale
minDim = min(img.shape[:2])
newSize = (int(img.shape[0]*imgScale/float(minDim)), int(img.shape[1]*imgScale/float(minDim)))
imgS = transform.resize(img, newSize)
# reshape net
net.blobs['data'].reshape(1,3,newSize[0],newSize[1])
transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape})
transformer.set_mean('data', np.array([103.939, 116.779, 123.68]))
transformer.set_transpose('data', (2,0,1))
transformer.set_channel_swap('data', (2,1,0))
transformer.set_raw_scale('data', 255.0)
# forward pass
net.blobs['data'].data[...] = transformer.preprocess('data', imgS)
out = net.forward(end = topLayerName)
# switch to the excitation backprop mode
caffe.set_mode_eb_gpu()
tagID = tag2ID[tagName]
net.blobs[topBlobName].diff[0][...] = 0
net.blobs[topBlobName].diff[0][tagID] = np.exp(net.blobs[topBlobName].data[0][tagID].copy())
net.blobs[topBlobName].diff[0][tagID] /= net.blobs[topBlobName].diff[0][tagID].sum()
# invert the top layer weights
net.params[topLayerName][0].data[...] *= -1
out = net.backward(start = topLayerName, end = secondTopLayerName)
buff = net.blobs[secondTopBlobName].diff.copy()
# invert back
net.params[topLayerName][0].data[...] *= -1
out = net.backward(start = topLayerName, end = secondTopLayerName)
# compute the contrastive signal
net.blobs[secondTopBlobName].diff[...] -= buff
# get attention map
out = net.backward(start = secondTopLayerName, end = outputLayerName)
attMap = np.maximum(net.blobs[outputBlobName].diff[0].sum(0), 0)
# resize back to original image size
attMap = transform.resize(attMap, (img.shape[:2]), order = 3, mode = 'nearest')
return attMap
def backprop(self, neuron_id, contrastive=True):
# eb = excitation backprop
caffe.set_mode_eb_gpu()
net = self.net
top = net.blobs[self.top_blob_name].data[0][neuron_id].copy()
if not np.any(top):
print("WARNING: top data is empty")
top = np.exp(1.)
net.blobs[self.top_blob_name].diff[0][...] = 0
net.blobs[self.top_blob_name].diff[0][neuron_id] = top
#np.exp(net.blobs[self.top_blob_name].data[0][neuron_id].copy())
net.blobs[self.top_blob_name].diff[0][neuron_id] /= \
net.blobs[self.top_blob_name].diff[0][neuron_id].sum()
if contrastive:
# invert the top layer weights
net.params[self.top_layer_name][0].data[...] *= -1
net.backward(start=self.top_layer_name, end=self.second_top_layer_name)
# Grab the signal when all uninteresting neurons are set
buff = net.blobs[self.second_top_blob_name].diff.copy()
# restore layer: make all uninteresting neurons uninteresting again
net.params[self.top_layer_name][0].data[...] *= -1
# Grab the signal when only the interesting neuron is set
net.backward(start=self.top_layer_name, end=self.second_top_layer_name)
# Combine results of inverted and non inverted backprop to compute contrastive signal
net.blobs[self.second_top_blob_name].diff[...] -= buff
# compute the contrastive signal
net.backward(start=self.second_top_layer_name, end=self.output_layer_name)
else:
net.backward(start=self.top_layer_name, end=self.output_layer_name)
attention_map = np.maximum(
net.blobs[self.output_blob_name].diff[0].sum(0),
0
)
return attention_map
#-------------------
original_data.append(shift_data_in)
data.append(processed_image)
#----FWD
caffe.set_device(device_net_conventional)
net_conventional.blobs['data'].reshape(end_point - v, 3,
image_dim_conventional,
image_dim_conventional)
net_conventional.blobs['data'].data[...] = data[0:end_point - v]
net_conventional.forward()
#----
#----BWD using EB
caffe.set_mode_eb_gpu()
attMaps = []
tagID_top = np.zeros(
(top_classes, net_conventional.blobs['probs'].data.shape[0]),
dtype=int)
for cntr in range(top_classes):
net_conventional.blobs[topBlobName].diff[...] = 0
for j in range(net_conventional.blobs['probs'].data.shape[0]):
tagScore = util.getTagScore(
net_conventional.blobs[topLayerName].data[j, :], tags, tag2ID)
tagScore.sort(key=operator.itemgetter(1), reverse=True)
if cntr == 0:
predict_labels.append(
np.argmax(
net_conventional.blobs['probs'].data[j, :].copy()))
def compute_heatmap(net,
transformer,
paths,
labels,
heatmap_type,
topBlobName,
topLayerName,
outputBlobName='data',
outputLayerName='data',
secondTopBlobName='pool5/7x7_s1',
secondTopLayerName='pool5/7x7_s1',
norm_deg=np.inf,
gpu=None):
if gpu == None:
if heatmap_type == 'saliency' or heatmap_type == 'grad_cam':
caffe.set_mode_cpu()
elif heatmap_type == 'guided_backprop':
caffe.set_mode_dc_cpu()
elif heatmap_type == 'excitation_backprop' or heatmap_type == 'contrast_excitation_backprop':
caffe.set_mode_eb_cpu()
else:
print 'heatmap_type %s is not supported' % heatmap_type
return
else:
caffe.set_device(gpu)
if heatmap_type == 'saliency' or heatmap_type == 'grad_cam':
caffe.set_mode_gpu()
elif heatmap_type == 'guided_backprop':
caffe.set_mode_dc_gpu()
elif heatmap_type == 'excitation_backprop' or heatmap_type == 'contrast_excitation_backprop':
caffe.set_mode_eb_gpu()
else:
print 'heatmap_type %s is not supported' % heatmap_type
return
if isinstance(paths, basestring):
num_imgs = 1
assert (isinstance(labels, int))
else:
num_imgs = len(paths)
assert (num_imgs == len(labels))
net.blobs['data'].reshape(num_imgs, 3, net.blobs['data'].data.shape[2],
net.blobs['data'].data.shape[3])
if num_imgs == 1:
net.blobs['data'].data[...] = transformer.preprocess(
'data', caffe.io.load_image(paths))
net.forward()
net.blobs[topBlobName].diff[0][...] = 0
net.blobs[topBlobName].diff[0][labels] = 1
else:
for i in range(num_imgs):
net.blobs['data'].data[i, ...] = transformer.preprocess(
'data', caffe.io.load_image(paths[i]))
net.forward()
for i in range(num_imgs):
net.blobs[topBlobName].diff[i][...] = 0
net.blobs[topBlobName].diff[i][labels[i]] = 1
if heatmap_type == 'contrast_excitation_backprop':
# invert top layer weights
net.params[topLayerName][0].data[...] *= -1
out = net.backward(start=topLayerName, end=secondTopLayerName)
buff = net.blobs[secondTopBlobName].diff.copy()
# invert back
net.params[topLayerName][0].data[...] *= -1
out = net.backward(start=topLayerName, end=secondTopLayerName)
# compute the contrastive signal
net.blobs[secondTopBlobName].diff[...] -= buff
net.backward(start=secondTopLayerName, end=outputLayerName)
elif heatmap_type == 'grad_cam':
net.backward(start=topLayerName, end=outputLayerName)
activations = net.blobs[outputBlobName].data.copy(
) # TODO: check if copy is needed
gradient = net.blobs[outputBlobName].diff.copy()
alphas = np.mean(gradient, (2, 3))
attMaps = np.squeeze(
np.maximum(
np.sum(
activations * np.broadcast_to(
np.expand_dims(np.expand_dims(alphas, 2), 3),
activations.shape), 1), 0))
return attMaps
else:
try:
net.backward(start=topLayerName, end=outputLayerName)
except:
assert (outputLayerName == 'data')
net.backward(start=topLayerName)
if np.isinf(norm_deg):
if norm_deg == np.inf:
attMaps = np.squeeze(np.abs(net.blobs[outputBlobName].diff).max(1))
else:
attMaps = np.squeeze(np.abs(net.blobs[outputBlobName].diff).min(1))
else:
if norm_deg == 0:
attMaps = np.squeeze(net.blobs[outputBlobName.diff])
elif norm_deg == -1:
attMaps = np.squeeze(net.blobs[outputBlobName].diff.sum(1))
elif norm_deg == -2:
attMaps = np.squeeze(
np.maximum(net.blobs[outputBlobName].diff.sum(1), 0))
else:
attMaps = np.squeeze(
((np.abs(net.blobs[outputBlobName].diff)**norm_deg).sum(1))**(
1 / float(norm_deg)))
# TODO: test this case
return attMaps