def test_process(self,
interp_fn,
min_scale,
max_scale=None,
resize_to_original=None):
tf.reset_default_graph()
inputs_np = np.random.rand(20, 30, 3).astype(np.float32)
augmentation = ImageRandomCrop(
min_scale=min_scale,
max_scale=max_scale,
resize_to_original=resize_to_original,
augmentation_probability=1.0,
).build()
result = augmentation.process(images=tf.constant(inputs_np))
self.assertSetEqual(set(augmentation.generated_keys_all), set(result))
result_eval = self.evaluate(result)
offset_eval = np.floor(result_eval["crop_offset"] *
np.array([20, 30])).astype(np.int32)
size_eval = np.floor(result_eval["crop_scale"] *
np.array([20, 30])).astype(np.int32)
result_must = inputs_np[offset_eval[0]:offset_eval[0] + size_eval[0],
offset_eval[1]:offset_eval[1] + size_eval[1]]
if resize_to_original:
result_must = skimage_resize(result_must, [20, 30],
0,
anti_aliasing=False)
self.assertAllClose(result_must, result_eval["images"])
def resize_array(images, dim=None):
"""Rescales array of images to specified dimensions."""
size = images.shape[0]
imgs = np.zeros((size, dim, dim))
for i in range(size):
imgs[i, :, :] = skimage_resize(images[i, :, :], (dim, dim))
return imgs
def generate_gif(gif_save_path, model_name, frames_for_gif, reward):
"""
Args:
frames_for_gif: A sequence of (210, 160, 3) frames of an Atari game in RGB
reward: Integer, Total reward of the episode that es ouputted as a gif
"""
if not os.path.exists(gif_save_path):
os.makedirs(gif_save_path)
for idx, frame_idx in enumerate(frames_for_gif):
frames_for_gif[idx] = skimage_resize(frame_idx, (420, 320, 3),
preserve_range=True,
order=0).astype(np.uint8)
fname = gif_save_path + model_name + "-EvalReward_{}.gif".format(reward)
imageio.mimsave(fname, frames_for_gif, duration=1 / 30)
def smaller(arr, scale=2.):
"""shrink image in np.array form
shrink image in nparray form by scale, default is shrink by factor of 2
Args:
arr: image as np.array
scale: default 2.
Returns:
image as np.array
"""
if hasattr(arr, 'numpy') and callable(arr.numpy):
arr = arr.numpy()
h,w,c = arr.shape
dim = (np.array([h,w])//scale)
return skimage_resize( arr, dim , anti_aliasing=True).astype('float32')
def resize_img(img, resize=512):
"""
Resize short side to resize
@img: np.ndarray The image numpy.array to resize
@resize: Int. The desired lenght of the short side of the image.
"""
height, width = img.shape[0], img.shape[1]
img = preprocess_input(img)
if height < width:
ratio = height / resize
long_side = round(width / ratio)
resize_shape = (resize, long_side, 3)
else: #height > width
ratio = width / resize
long_side = round(height / ratio)
resize_shape = (long_side, resize, 3)
return skimage_resize(img, resize_shape, mode='reflect')
def proc_subimage(self, x):
'''
Takes the gradient of the measured data, applies denoising if specified,
normalizes, autoflips if specified,
and then adjusts the size (if necessary)
Args:
x = an array with data
'''
# take gradient
x = gradient(x)
# apply thresholding
if 'threshold' in self.denoising:
if self.threshold_val is not None:
grad_x = apply_threshold(x, self.threshold_val)
else:
grad_x = apply_threshold(x)
# apply clipping
if 'clip' in self.denoising:
if self.clip_val is not None:
grad_x = apply_clipping(grad_x, self.clip_val)
else:
grad_x = apply_clipping(grad_x)
# normalize with zscore normalization
x = zscore_norm(x)
# autoflip by skew of image gradient
if self.autoflip:
x = autoflip_skew(x)
target_shape = (config.SUB_SIZE, config.SUB_SIZE, 1)
if x.shape != target_shape:
x = skimage_resize(x, target_shape)
return x
def process(self, numpy_data):
output_size = self._parameters['output_size']
log.debug('resizing data to {}'.format(output_size))
return skimage_resize(numpy_data, output_size)
def __call__(self, img):
img = skimage_resize(img, self.output_size, mode='constant')
return img
def gradcam(self, sess, input_data, target_index=None, feed_options=dict()):
""" Calculate Grad-CAM (class activation map) and Guided Grad-CAM for given input on target class
Parameters
----------
sess: tf.Session
Tensorflow session
input_data : numpy.ndarray
A single input instance
target_index : int
Target class index
If None, predicted class index is used
feed_options : dict
Optional parameters to graph
Returns
-------
dict
Note
----
Keys in return:
* gradcam_img: Heatmap overlayed on input
* guided_gradcam_img: Guided Grad-CAM result
* heatmap: Heatmap of input on the target class
* guided_backprop: Guided backprop result
"""
input_feed = np.expand_dims(input_data, axis=0)
if input_data.ndim == 3:
is_image = True
image_height, image_width = input_data.shape[:2]
if input_data.ndim == 1:
is_image = False
input_length = input_data.shape[0]
if target_index is not None:
feed_dict = {self._x_placeholder: input_feed, self._class_idx: target_index}
feed_dict.update(feed_options)
conv_out_eval, grad_eval = sess.run([self._target_ts, self._grad_by_idx], feed_dict=feed_dict)
else:
feed_dict = {self._x_placeholder: input_feed}
feed_dict.update(feed_options)
conv_out_eval, grad_eval = sess.run([self._target_ts, self._grad_by_top1], feed_dict=feed_dict)
weights = np.mean(grad_eval, axis=(0, 1, 2))
conv_out_eval = np.squeeze(conv_out_eval, axis=0)
cam = np.zeros(conv_out_eval.shape[:2], dtype=np.float32)
for i, w in enumerate(weights):
cam += w * conv_out_eval[:, :, i]
if is_image:
cam += 1
cam = cv2.resize(cam, (image_height, image_width))
saliency_val = sess.run(self._saliency_map, feed_dict={self._x_placeholder: input_feed})
saliency_val = np.squeeze(saliency_val, axis=0)
else:
cam = skimage_resize(cam, (input_length, 1), preserve_range=True, mode='reflect')
cam = np.transpose(cam)
cam = np.maximum(cam, 0)
heatmap = cam / np.max(cam)
ret = {'heatmap': heatmap}
if is_image:
ret.update({
'gradcam_img': self.overlay_gradcam(input_data, heatmap),
'guided_gradcam_img': _deprocess_image(saliency_val * heatmap[..., None]),
'guided_backprop': saliency_val
})
return ret
def resize_image(image, size):
return skimage_resize(image, (size, size),
mode='constant',
anti_aliasing=True)
def gradcam(self,
sess,
input_data,
target_index=None,
feed_options=dict()):
""" Calculate Grad-CAM (class activation map) and Guided Grad-CAM for given input on target class
Parameters
----------
sess: tf.Session
Tensorflow session
input_data : numpy.ndarray
A single input instance
target_index : int
Target class index
If None, predicted class index is used
feed_options : dict
Optional parameters to graph
Returns
-------
dict
Note
----
Keys in return:
* gradcam_img: Heatmap overlayed on input
* guided_gradcam_img: Guided Grad-CAM result
* heatmap: Heatmap of input on the target class
* guided_backprop: Guided backprop result
"""
input_feed = np.expand_dims(input_data, axis=0)
if input_data.ndim == 3:
is_image = True
image_height, image_width = input_data.shape[:2]
if input_data.ndim == 1:
is_image = False
input_length = input_data.shape[0]
if target_index is not None:
feed_dict = {
self._x_placeholder: input_feed,
self._class_idx: target_index
}
feed_dict.update(feed_options)
conv_out_eval, grad_eval = sess.run(
[self._target_ts, self._grad_by_idx], feed_dict=feed_dict)
else:
feed_dict = {self._x_placeholder: input_feed}
feed_dict.update(feed_options)
conv_out_eval, grad_eval = sess.run(
[self._target_ts, self._grad_by_top1], feed_dict=feed_dict)
weights = np.mean(grad_eval, axis=(0, 1, 2))
conv_out_eval = np.squeeze(conv_out_eval, axis=0)
cam = np.zeros(conv_out_eval.shape[:2], dtype=np.float32)
for i, w in enumerate(weights):
cam += w * conv_out_eval[:, :, i]
if is_image:
cam += 1
cam = cv2.resize(cam, (image_height, image_width))
saliency_val = sess.run(
self._saliency_map,
feed_dict={self._x_placeholder: input_feed})
saliency_val = np.squeeze(saliency_val, axis=0)
else:
cam = skimage_resize(cam, (input_length, 1),
preserve_range=True,
mode='reflect')
cam = np.transpose(cam)
cam = np.maximum(cam, 0)
heatmap = cam / np.max(cam)
ret = {'heatmap': heatmap}
if is_image:
ret.update({
'gradcam_img':
self.overlay_gradcam(input_data, heatmap),
'guided_gradcam_img':
_deprocess_image(saliency_val * heatmap[..., None]),
'guided_backprop':
saliency_val
})
return ret