def tile_pil_image(image: Image.Image,
tile_factor: int = 0,
shade: bool = True) -> Image.Image:
"""
Tile PIL Image.
:param image: PIL Image object
:param tile_factor: number of tiles, if tile_factor=0, this
function returns a copy of the input image
:param shade: make non-central tiles gray scaled.
:return: Image of shape [W*n, H*n, C], where n=2 * tile_factor + 1
"""
shape = ImageShape(*image.size)
new_img = image.copy()
if tile_factor > 0:
tf = 2 * tile_factor + 1
new_img = Image.new("RGB", (shape.w * tf, shape.h * tf))
new_shape = ImageShape(*new_img.size)
gray_img = image.convert("LA").convert("RGB")
for n, i in enumerate(range(0, new_shape.w, shape.w)):
for m, j in enumerate(range(0, new_shape.h, shape.h)):
distance = (abs(m - tile_factor) + abs(n - tile_factor)) / tf
alpha = distance > 0 if shade else 0
# place image at position (i, j)
new_img.paste(Image.blend(image, gray_img, alpha), (i, j))
return new_img
def test_compare_apis():
image = get_test_image(tile_factor=0)
rf_api0 = TFFeatureMapsReceptiveField(model_fm_build_func)
rf_params_api0 = rf_api0.compute(input_shape=ImageShape(*image.shape))
rf_api1 = TFReceptiveField(model_build_func)
rf_params_api1 = rf_api1.compute(
input_shape=ImageShape(*image.shape),
input_tensor="input_image",
output_tensors=["feature_map0", "feature_map1"],
)
for api0, api1 in zip(rf_params_api0, rf_params_api1):
assert_allclose(api0.rf, api1.rf)
assert_allclose(api0.size, api1.size)
def test_tensorflow():
image = get_test_image(tile_factor=0)
rf = TFReceptiveField(model_build_func)
rf_params0 = rf.compute(input_shape=ImageShape(*image.shape),
input_layer='input_image',
output_layer='feature_map')
image = get_test_image(tile_factor=1)
rf = TFReceptiveField(model_build_func)
rf_params1 = rf.compute(input_shape=ImageShape(*image.shape),
input_layer='input_image',
output_layer='feature_map')
assert_allclose(rf_params0, rf_params1)
def _prepare_gradient_func(
self, input_shape: ImageShape, input_layer: str,
output_layers: List[str]
) -> Tuple[Callable, GridShape, List[GridShape]]:
"""
Computes gradient function and additional parameters. Note
that the receptive field parameters like stride or size, do not
depend on input image shape. However, if the RF of original network
is bigger than input_shape this method will fail. Hence it is
recommended to increase the input shape.
:param input_shape: shape of the input image. Used in @model_func.
:param input_layer: name of the input layer.
:param output_layers: a list of names of the target feature map layers.
:returns
gradient_function: a function which returns gradient w.r.t. to
the input image
input_shape: a shape of the input image tensor
output_shape: a shapes of the output feature map tensors
"""
model = self._model_func(ImageShape(*input_shape))
if self.init_weights:
setup_model_weights(model)
gradient_function, input_shape, output_shapes = _define_receptive_field_func(
model, input_layer, output_layers)
return (
gradient_function,
GridShape(*input_shape),
[GridShape(*output_shape) for output_shape in output_shapes],
)
def get_default_image(
shape: ImageShape,
tile_factor: int = 0,
shade: bool = True,
as_image: bool = False,
name: str = "lena",
) -> Union[numpy.ndarray, Image.Image]:
"""
Loads default image from resources and reshape it to size
shape.
:param shape: [width, height]
:param tile_factor: tile image, if 0 the resulting image shape is
[width, height], otherwise the output size is defined by number of
tiles. tile_factor is a non-negative integer number.
:param shade: if True and tile_factor > 0 it makes tiles gray scale
:param as_image: if True, function returns PIL Image object, else
numpy array.
:param name: name of the default image to be loaded. Call
get_default_images() to list available images names (default - lena).
:return: numpy array of shape [width, height, 3] if as_image=False,
otherwise PIL Image object
"""
shape = ImageShape(*shape)
tile_factor = int(tile_factor)
img = Image.open(_get_default_image_path(name=name), mode="r")
img = img.resize((shape.w, shape.h), Image.ANTIALIAS)
img = tile_pil_image(img, tile_factor=tile_factor, shade=shade)
if as_image:
return img
return numpy.array(img)
def _prepare_gradient_func(
self, input_shape: ImageShape
) -> Tuple[Callable, GridShape, List[GridShape]]:
"""
Computes gradient function and additional parameters. Note
that the receptive field parameters like stride or size, do not
depend on input image shape. However, if the RF of original network
is bigger than input_shape this method will fail. Hence it is
recommended to increase the input shape.
:param input_shape: shape of the input image, which is feed to the
Pytorch module.
:returns
gradient_function: a function which returns gradient w.r.t. to
the input image.
input_shape: a shape of the input image tensor.
output_shapes: a list shapes of the output feature map tensors.
"""
input_shape = ImageShape(*input_shape)
input_shape = GridShape(
n=1,
c=input_shape.c,
h=input_shape.h,
w=input_shape.w,
)
gradient_function, input_shape, output_shapes = \
_define_receptive_field_func(self._model_func, input_shape)
return gradient_function, input_shape, output_shapes
def test_same():
image = get_test_image(tile_factor=0)
rf = KerasReceptiveField(get_build_func(padding='same'), init_weights=True)
rf_params0 = rf.compute(input_shape=ImageShape(*image.shape),
input_layer='input_image',
output_layer='conv')
print(rf_params0)
image = get_test_image(tile_factor=1)
rf = KerasReceptiveField(get_build_func(padding='same'), init_weights=True)
rf_params1 = rf.compute(input_shape=ImageShape(*image.shape),
input_layer='input_image',
output_layer='conv')
print(rf_params1)
assert_allclose(rf_params0, rf_params1)
def test_tensorflow():
image = get_test_image(tile_factor=0)
rf = TFReceptiveField(model_build_func)
rf_params0 = rf.compute(
input_shape=ImageShape(*image.shape),
input_tensor="input_image",
output_tensors=["feature_map0"],
)[0]
image = get_test_image(tile_factor=1)
rf = TFReceptiveField(model_build_func)
rf_params1 = rf.compute(
input_shape=ImageShape(*image.shape),
input_tensor="input_image",
output_tensors=["feature_map0"],
)[0]
assert_allclose(rf_params0.rf, rf_params1.rf)
def get_default_image(
shape: ImageShape,
tile_factor: int = 0,
shade: bool = True,
as_image: bool = False) -> Union[numpy.ndarray, Image.Image]:
"""
Loads default image from resources and reshape it to size
shape.
:param shape: [width, height]
:param tile_factor: tile image, if 0 the resulting image shape is
[width, height], otherwise the output size is defined by number of
tiles. tile_factor is a non-negative integer number.
:param shade: if True and tile_factor > 0 it makes tiles gray scale
:param as_image: if True, function returns PIL Image object, else
numpy array.
:return: numpy array of shape [width, height, 3] if as_image=False,
wise PIL Image object
"""
shape = ImageShape(*shape)
tile_factor = int(tile_factor)
img = Image.open(os.path.join(dir_path, 'resources/lena.jpg'), mode='r')
img = img.resize((shape.w, shape.h), Image.ANTIALIAS)
if tile_factor > 0:
tf = 2 * tile_factor + 1
new_img = Image.new('RGB', (shape.w * tf, shape.h * tf))
new_shape = ImageShape(*new_img.size)
gray_img = img.convert('LA').convert('RGB')
for n, i in enumerate(range(0, new_shape.w, shape.w)):
for m, j in enumerate(range(0, new_shape.h, shape.h)):
distance = (abs(m - tile_factor) + abs(n - tile_factor)) / tf
alpha = distance > 0 if shade else 0
# place image at position (i, j)
new_img.paste(Image.blend(img, gray_img, alpha), (i, j))
img = new_img
if as_image:
return img
return numpy.array(img)
def test_same():
image = get_test_image(tile_factor=0)
rf = KerasReceptiveField(get_build_func(padding="same"), init_weights=True)
rf_params0 = rf.compute(
input_shape=ImageShape(*image.shape),
input_layer="input_image",
output_layers=["conv1"],
)
print(rf_params0)
image = get_test_image(tile_factor=1)
rf = KerasReceptiveField(get_build_func(padding="same"), init_weights=True)
rf_params1 = rf.compute(
input_shape=ImageShape(*image.shape),
input_layer="input_image",
output_layers=["conv1"],
)
print(rf_params1)
assert_allclose(rf_params0[0].rf, rf_params1[0].rf)
def test_multiple_feature_maps_secondary_api():
image = get_test_image(tile_factor=0)
rf = TFFeatureMapsReceptiveField(model_fm_build_func)
rf_params = rf.compute(input_shape=ImageShape(*image.shape))
rfs = 3 + 2 + 1 + 2 * 2 + 2 * 2
assert_allclose(rf_params[0].rf.size, (rfs, rfs))
assert_allclose(rf_params[0].rf.stride, (2, 2))
rfs = 3 + 2 + 1 + 2 * 2 + 2 * 2 + 1 * 2 + 2 * 4 + 2 * 4 + 2 * 4
assert_allclose(rf_params[1].rf.size, (rfs, rfs))
assert_allclose(rf_params[1].rf.stride, (4, 4))
def test_expected_values():
image = get_test_image(tile_factor=0)
rf = KerasReceptiveField(get_build_func(padding="valid"),
init_weights=True)
rf_params0 = rf.compute(
input_shape=ImageShape(*image.shape),
input_layer="input_image",
output_layers=["conv1"],
)[0]
assert_allclose(rf_params0.rf.stride, (2, 2))
assert_allclose(rf_params0.rf.size,
(((2 + 1) * 2 + 2) * 2, ((2 + 1) * 2 + 2) * 2))
def _prepare_gradient_func(
self, input_shape: ImageShape, input_layer: str,
output_layer: str) -> Tuple[Callable, GridShape, GridShape]:
model = self.model_func(ImageShape(*input_shape))
if self.init_weights:
setup_model_weights(model)
gradient_function, input_shape, output_shape = \
_define_receptive_field_func(model, input_layer, output_layer)
return gradient_function, \
GridShape(*input_shape), \
GridShape(*output_shape)
def test_example_network():
input_shape = [96, 96, 3]
rf = PytorchReceptiveField(model_fn)
rf_params = rf.compute(input_shape=ImageShape(*input_shape))
assert_allclose(rf_params[0].rf.size, (6, 6))
assert_allclose(rf_params[0].rf.stride, (2, 2))
rs = 6 + (2 + 2 + 1) * 2
assert_allclose(rf_params[1].rf.size, (rs, rs))
assert_allclose(rf_params[1].rf.stride, (4, 4))
rs = 6 + (2 + 2 + 1) * 2 + (2 + 2 + 1) * 4
assert_allclose(rf_params[2].rf.size, (rs, rs))
assert_allclose(rf_params[2].rf.stride, (8, 8))
def test_multiple_feature_maps():
image = get_test_image(tile_factor=0)
rf = TFReceptiveField(model_build_func)
rf_params = rf.compute(
input_shape=ImageShape(*image.shape),
input_tensor="input_image",
output_tensors=["feature_map0", "feature_map1"],
)
rfs = 3 + 2 + 1 + 2 * 2 + 2 * 2
assert_allclose(rf_params[0].rf.size, (rfs, rfs))
assert_allclose(rf_params[0].rf.stride, (2, 2))
rfs = 3 + 2 + 1 + 2 * 2 + 2 * 2 + 1 * 2 + 2 * 4 + 2 * 4 + 2 * 4
assert_allclose(rf_params[1].rf.size, (rfs, rfs))
assert_allclose(rf_params[1].rf.stride, (4, 4))
def _prepare_gradient_func(
self,
input_shape: ImageShape,
input_layer: str,
output_layer: str
) -> Tuple[Callable, GridShape, GridShape]:
# this function will create default graph
_ = self.model_func(ImageShape(*input_shape))
default_graph = tf.get_default_graph()
# get graph tensors by names
input_tensor = default_graph \
.get_operation_by_name(input_layer).outputs[0]
output_tensor = default_graph \
.get_operation_by_name(output_layer).outputs[0]
# get their shapes
output_shape = _get_tensor_shape(output_tensor)
input_shape = _get_tensor_shape(input_tensor)
# define loss function
output_shape = (1, output_shape[1], output_shape[2], 1)
receptive_field_mask = tf.placeholder(
tf.float32, shape=output_shape, name='grid'
)
x = tf.reduce_mean(output_tensor, -1, keep_dims=True)
fake_loss = x * receptive_field_mask
fake_loss = tf.reduce_mean(fake_loss)
grads = tf.gradients(fake_loss, input_tensor)
# here we use Keras API to define gradient function which is simpler
# than native tf
gradient_function = keras.backend.function(
inputs=[receptive_field_mask, input_tensor, keras.backend.learning_phase()],
outputs=grads
)
return gradient_function, \
GridShape(*input_shape), \
GridShape(*output_shape)
def test_example_network():
input_shape = [120, 120, 3]
rf = PytorchReceptiveField(model_fn)
rf_params = rf.compute(input_shape=ImageShape(*input_shape))
assert_allclose(rf_params[0].rf.size, (6, 6))
assert_allclose(rf_params[0].rf.stride, (2, 2))
rs = 6 + (2 + 2 + 1) * 2
assert_allclose(rf_params[1].rf.size, (rs, rs))
assert_allclose(rf_params[1].rf.stride, (4, 4))
rs = 6 + (2 + 2 + 1) * 2 + (2 + 2 + 1) * 4
assert_allclose(rf_params[2].rf.size, (rs, rs))
assert_allclose(rf_params[2].rf.stride, (8, 8))
#rf.plot_gradient_at(fm_id=1, point=(9, 9))
rf.plot_rf_grids(get_default_image(input_shape, name='cat'), plot_naive_rf=False, figsize=(6, 6))
plt.show()
def _prepare_gradient_func(
self, input_shape: ImageShape, input_tensor: str, output_tensors: List[str]
) -> Tuple[Callable, GridShape, List[GridShape]]:
"""
Computes gradient function and additional parameters. Note
that the receptive field parameters like stride or size, do not
depend on input image shape. However, if the RF of original network
is bigger than input_shape this method will fail. Hence it is
recommended to increase the input shape.
:param input_shape: shape of the input image. Used in @model_func.
:param input_tensor: name of the input image tensor.
:param output_tensors: a list of names of the target
feature map tensors.
:returns
gradient_function: a function which returns gradient w.r.t. to
the input image.
input_shape: a shape of the input image tensor.
output_shape: a list shapes of the output feature map tensors.
"""
if self._session is not None:
tf.reset_default_graph()
self._session.close()
with tf.Graph().as_default() as graph:
with tf.variable_scope("", reuse=tf.AUTO_REUSE):
# this function will create default graph
_ = self._model_func(ImageShape(*input_shape))
# default_graph = tf.get_default_graph()
# get graph tensors by names
input_tensor = graph.get_operation_by_name(input_tensor).outputs[0]
input_shape = _get_tensor_shape(input_tensor)
grads = []
receptive_field_masks = []
output_shapes = []
for output_tensor in output_tensors:
output_tensor = graph.get_operation_by_name(output_tensor).outputs[0]
# shapes
output_shape = _get_tensor_shape(output_tensor)
output_shape = (1, output_shape[1], output_shape[2], 1)
output_shapes.append(output_shape)
# define loss function
receptive_field_mask = tf.placeholder(
tf.float32, shape=output_shape, name="grid"
)
grad = _define_fm_gradient(
input_tensor, output_tensor, receptive_field_mask
)
grads.append(grad)
receptive_field_masks.append(receptive_field_mask)
_logger.info(f"Feature maps shape: {output_shapes}")
_logger.info(f"Input shape : {input_shape}")
self._session = tf.Session(graph=graph)
self._session.run(tf.global_variables_initializer())
def gradient_fn(fm_masks, input_image):
fetch_dict = {
mask_t: mask_np
for mask_t, mask_np in zip(receptive_field_masks, fm_masks)
}
fetch_dict[input_tensor] = input_image
return self._session.run(grads, feed_dict=fetch_dict)
return (
gradient_fn,
GridShape(*input_shape),
[GridShape(*output_shape) for output_shape in output_shapes],
)
def plot_receptive_grid(input_shape: GridShape,
output_shape: GridShape,
rf_params: ReceptiveFieldDescription,
custom_image: np.ndarray = None,
plot_naive_rf: bool = False,
**plot_params) -> None:
if custom_image is None:
img = get_default_image(shape=ImageShape(input_shape.h, input_shape.w))
else:
img = custom_image
figsize = plot_params.get("figsize", (10, 10))
# plot image
plt.figure(figsize=figsize)
ax = plt.subplot(111)
plt.imshow(img)
# plot naive receptive field grid
if plot_naive_rf:
dw = input_shape.w / output_shape.w
dh = input_shape.h / output_shape.h
for i, j in itertools.product(range(output_shape.w),
range(output_shape.h)):
x0, x1 = i * dw, (i + 1) * dw
y0, y1 = j * dh, (j + 1) * dh
ax.add_patch(
patches.Rectangle((y0, x0),
dh,
dw,
alpha=0.9,
fill=False,
edgecolor="gray",
linewidth=1))
rf_offset = rf_params.offset
rf_size = rf_params.size
rf_stride = rf_params.stride
# map from output grid space to input image
def map_point(i: int, j: int):
return np.array(rf_offset) + np.array([i, j]) * np.array(rf_stride)
# plot RF grid based on rf params
points = [
map_point(i, j) for i, j in itertools.product(range(output_shape.w),
range(output_shape.h))
]
points = np.array(points)
plt.scatter(points[:, 1],
points[:, 0],
marker="o",
c=(0.2, 0.9, 0.1, 0.9),
s=10)
# plot receptive field from corner point
_plot_rect(ax,
rect=to_rf_rect(rf_offset, rf_size),
color=(0.9, 0.3, 0.2),
linewidth=5,
size=90)
center_point = map_point(output_shape.w // 2, output_shape.h // 2)
_plot_rect(ax,
rect=to_rf_rect(GridPoint(center_point[0], center_point[1]),
rf_size),
color=(0.1, 0.3, 0.9),
linewidth=5,
size=90)
last_point = map_point(output_shape.w - 1, output_shape.h - 1)
_plot_rect(ax,
rect=to_rf_rect(GridPoint(last_point[0], last_point[1]),
rf_size),
color=(0.1, 0.9, 0.3),
linewidth=5,
size=90)
ax.set_aspect('equal')
def test_example_network():
input_shape = [224, 288, 4]
rf = PytorchReceptiveField(model_fn)
rf_params = rf.compute(input_shape=ImageShape(*input_shape))
'''
def plot_receptive_grid(input_shape: GridShape,
output_shape: GridShape,
rf_params: ReceptiveFieldDescription,
custom_image: Optional[np.ndarray] = None,
plot_naive_rf: bool = False,
axis: Optional[Any] = None,
**plot_params) -> None:
"""
Visualize receptive field grid.
:param input_shape: an input image shape as an instance of GridShape
:param output_shape: an output feature map shape
:param rf_params: an instance of ReceptiveFieldDescription computed for
this feature map.
:param custom_image: optional image [height, width, 3] to be plotted as
a background.
:param plot_naive_rf: plot naive version of the receptive field. Naive
version of RF does not take strides, and offsets into considerations,
it is a simple linear mapping from N points in feature map to pixels
in the image.
:param axis: a matplotlib axis object as returned by the e.g. plt.subplot
function. If not None then axis is used for visualizations otherwise
default figure is created.
:param plot_params: additional plot params: figsize=(5, 5)
"""
if custom_image is None:
img = get_default_image(shape=ImageShape(input_shape.h, input_shape.w))
else:
img = custom_image
figsize = plot_params.get("figsize", (10, 10))
# plot image
if axis is None:
plt.figure(figsize=figsize)
axis = plt.subplot(111)
axis.imshow(img)
# plot naive receptive field grid
if plot_naive_rf:
dw = input_shape.w / output_shape.w
dh = input_shape.h / output_shape.h
for i, j in itertools.product(range(output_shape.w),
range(output_shape.h)):
x0, x1 = i * dw, (i + 1) * dw
y0, y1 = j * dh, (j + 1) * dh
axis.add_patch(
patches.Rectangle(
(y0, x0),
dh,
dw,
alpha=0.9,
fill=False,
edgecolor="gray",
linewidth=1,
))
rf_offset = rf_params.offset
rf_size = rf_params.size
rf_stride = rf_params.stride
# map from output grid space to input image
def map_point(i: int, j: int):
return np.array(rf_offset) + np.array([i, j]) * np.array(rf_stride)
# plot RF grid based on rf params
'''
points = [
map_point(i, j)
for i, j in itertools.product(range(output_shape.w), range(output_shape.h))
]
points = np.array(points)
axis.scatter(points[:, 1], points[:, 0], marker="o", c=(0.2, 0.9, 0.1, 0.9), s=10)
'''
# plot receptive field from corner point
_plot_rect(
axis,
rect=to_rf_rect(rf_offset, rf_size),
color=(0.9, 0.3, 0.2),
linewidth=2,
size=10,
)
center_point = map_point(output_shape.w // 2, output_shape.h // 2)
_plot_rect(
axis,
rect=to_rf_rect(GridPoint(center_point[0], center_point[1]), rf_size),
color=(0.1, 0.3, 0.9),
linewidth=2,
size=10,
)
last_point = map_point(output_shape.w - 1, output_shape.h - 1)
_plot_rect(
axis,
rect=to_rf_rect(GridPoint(last_point[0], last_point[1]), rf_size),
color=(0.1, 0.9, 0.3),
linewidth=2,
size=10,
)
axis.set_aspect("equal")
def input_shape(self) -> ImageShape:
"""Return input shape of the feature extractor"""
self._check()
return ImageShape(w=self._input_shape.w,
h=self._input_shape.h,
c=self._input_shape.c)
