def plot_gradient_at(
self,
fm_id: int,
point: GridPoint,
image: Optional[np.ndarray] = None,
**plot_params,
) -> None:
"""
Plot gradient map generated by certain `point` at selected feature map.
:param fm_id: an index of feature map for which gradient map
will be plotted.
:param point: a GridPoint on feature map selected by fm_id.
:param image: (optional) image of shape [W, H, 3]
:param plot_params: (optional) additional plot parameters:
figsize: tuple of int (5, 5)
axis: a matplotlib axis object as returned by e.g. plt.subplot
function. If not None then provided axis is used for
visualizations otherwise default figure is created, with
optional figsize.
"""
points = [GridPoint(0, 0)] * self.num_feature_maps
points[fm_id] = GridPoint(*point)
receptive_field_grads = self._get_gradient_from_grid_points(
points=points)
plot_gradient_field(
receptive_field_grad=receptive_field_grads[fm_id],
image=image,
**plot_params,
)
def compute(self, input_shape: ImageShape, input_layer: str,
output_layer: str) -> ReceptiveFieldDescription:
"""
Compute ReceptiveFieldDescription of given model for image of
shape input_shape [W, H, C]. If receptive field of the network
is bigger thant input_shape this method will raise exception.
In order to solve with problem try to increase input_shape.
:param input_shape: shape of the input image e.g. (224, 224, 3)
:param input_layer: name of the input layer
:param output_layer: name of the target layer
:return: estimated ReceptiveFieldDescription
"""
# define gradient function
self.build_gradient_func(input_shape=input_shape,
input_layer=input_layer,
output_layer=output_layer)
# receptive field at map center
rf_grad00 = self._get_gradient_activation_at_map_center(
self.get_gradient_func(),
self.get_input_shape(),
self.get_output_shape(),
center_offset=GridPoint(0, 0))
rf_at00 = _estimate_rf_from_gradients(rf_grad00)
# receptive field at map center with offset (1, 1)
rf_grad11 = self._get_gradient_activation_at_map_center(
self.get_gradient_func(),
self.get_input_shape(),
self.get_output_shape(),
center_offset=GridPoint(1, 1))
rf_at11 = _estimate_rf_from_gradients(rf_grad11)
# receptive field at feature map grid start x=0, y=0
rf_grad_point00 = self._get_gradient_from_grid_point(
self.get_gradient_func(),
self.get_input_shape(),
self.get_output_shape(),
point=GridPoint(0, 0))
rf_at_point00 = _estimate_rf_from_gradients(rf_grad_point00)
# compute position of the first anchor, center point of rect
x0 = rf_at_point00.w - rf_at00.w / 2
y0 = rf_at_point00.h - rf_at00.h / 2
# compute feature map/input image offsets
dx = rf_at11.x - rf_at00.x
dy = rf_at11.y - rf_at00.y
# compute receptive field size
size = Size(rf_at00.w, rf_at00.h)
rf_params = ReceptiveFieldDescription(offset=(x0, y0),
stride=(dx, dy),
size=size)
self.rf_params = rf_params
return rf_params
def compute(self, *args, **kwargs) -> List[FeatureMapDescription]:
"""
Compute ReceptiveFieldDescription of given model for image of
shape input_shape [H, W, C]. If receptive field of the network
is bigger thant input_shape this method will raise exception.
In order to solve with problem try to increase input_shape.
:param args: a list of arguments which depend on the API
:param kwargs: keywords which depend on the API
:return a list of estimated FeatureMapDescription for each feature
map.
"""
# define gradient function
self._build_gradient_func(*args, **kwargs)
# receptive field at map center
rf_grads00 = self._get_gradient_activation_at_map_center(
center_offsets=[GridPoint(0, 0)] * self.num_feature_maps)
rfs_at00 = estimate_rf_from_gradients(rf_grads00)
# receptive field at map center with offset (1, 1)
rf_grads11 = self._get_gradient_activation_at_map_center(
center_offsets=[GridPoint(1, 1)] * self.num_feature_maps)
rfs_at11 = estimate_rf_from_gradients(rf_grads11)
# receptive field at feature map grid start x=0, y=0
rf_grads_point00 = self._get_gradient_from_grid_points(
points=[GridPoint(0, 0)] * self.num_feature_maps)
rfs_at_point00 = estimate_rf_from_gradients(rf_grads_point00)
self._rf_params = []
for fm, (rf_at_point00, rf_at00,
rf_at11) in enumerate(zip(rfs_at_point00, rfs_at00,
rfs_at11)):
# compute position of the first anchor, center point of rect
x0 = rf_at_point00.w - rf_at00.w / 2
y0 = rf_at_point00.h - rf_at00.h / 2
# compute feature map/input image offsets
dx = rf_at11.x - rf_at00.x
dy = rf_at11.y - rf_at00.y
# compute receptive field size
size = Size(rf_at00.w, rf_at00.h)
rf_params = ReceptiveFieldDescription(offset=(x0, y0),
stride=(dx, dy),
size=size)
_logger.info(
f"Estimated receptive field for feature map [{fm}]: {rf_params}"
)
self._rf_params.append(rf_params)
return self.feature_maps_desc
def _get_gradient_activation_at_map_center(self,
gradient_function: Callable,
input_shape: GridShape,
output_shape: GridShape,
center_offset: GridPoint,
intensity: float = 1):
_logger.debug(f"Computing receptive field at center "
f"({output_shape.w//2}, {output_shape.h//2}) "
f"with offset {center_offset}")
# compute grid center
cx = output_shape.w // 2 - 1 \
if output_shape.w % 2 == 0 else output_shape.w // 2
cy = output_shape.h // 2 - 1 \
if output_shape.h % 2 == 0 else output_shape.h // 2
cx += center_offset.x
cy += center_offset.y
return self._get_gradient_from_grid_point(
gradient_function=gradient_function,
input_shape=input_shape,
output_shape=output_shape,
point=GridPoint(x=cx, y=cy),
intensity=intensity)
def _get_gradient_activation_at_map_center(
self,
center_offset: GridPoint,
intensity: float = 1
):
_logger.debug(
f"Computing receptive field at center "
f"({self.output_shape.w//2}, {self.output_shape.h//2}) "
f"with offset {center_offset}")
# compute grid center
w = self.output_shape.w
h = self.output_shape.h
cx = w // 2 - 1 \
if w % 2 == 0 else w // 2
cy = h // 2 - 1 \
if h % 2 == 0 else h // 2
cx += center_offset.x
cy += center_offset.y
return self._get_gradient_from_grid_point(
point=GridPoint(x=cx, y=cy),
intensity=intensity
)
def plot_gradients_at(
self,
points: List[GridPoint],
image: Optional[np.ndarray] = None,
layout: Optional[Tuple[int, int]] = None,
figsize: Optional[Tuple[int, int]] = None,
) -> None:
"""
Plot gradient map generated by certain `point` at feature map.
:param points: a list of GridPoint on feature maps.
:param image: (optional) image of shape [W, H, 3]
:param layout: (optional) a matplotlib subplot layout given by
tuple of form (num_rows, num_cols). If None the layout is set
to (num_feature_maps, 1).
:param figsize: (optional) a matplotlib figsize. If None the default
figure is used.
"""
if layout is None:
layout = (self.num_feature_maps, 1)
if figsize is not None:
plt.figure(figsize=figsize)
receptive_field_grads = self._get_gradient_from_grid_points(
points=[GridPoint(*point) for point in points])
for fm in range(self.num_feature_maps):
axis = plt.subplot(layout[0], layout[1], fm + 1)
plot_gradient_field(receptive_field_grad=receptive_field_grads[fm],
image=image,
axis=axis)
def _get_gradient_activation_at_map_center(
self,
center_offsets: List[GridPoint],
intensity: float = 1) -> List[np.ndarray]:
points = []
for fm in range(self.num_feature_maps):
output_shape = self._output_shapes[fm]
center_offset = center_offsets[fm]
_logger.debug(
f"Computing receptive field for feature map [{fm}] at center "
f"({output_shape.w//2}, {output_shape.h//2}) "
f"with offset {center_offset}")
# compute grid center
w = output_shape.w
h = output_shape.h
cx = w // 2 - 1 if w % 2 == 0 else w // 2
cy = h // 2 - 1 if h % 2 == 0 else h // 2
cx += center_offset.x
cy += center_offset.y
points.append(GridPoint(x=cx, y=cy))
return self._get_gradient_from_grid_points(points=points,
intensity=intensity)
def plot_gradient_at(self,
point: GridPoint,
image: np.ndarray = None,
**plot_params):
receptive_field_grad = self._get_gradient_from_grid_point(
self.get_gradient_func(),
self.get_input_shape(),
self.get_output_shape(),
point=GridPoint(*point))
plot_gradient_field(receptive_field_grad=receptive_field_grad,
image=image,
**plot_params)
def plot_receptive_grid(input_shape: GridShape,
output_shape: GridShape,
rf_params: ReceptiveFieldDescription,
custom_image: Optional[np.ndarray] = None,
plot_naive_rf: bool = False,
axis: Optional[Any] = None,
**plot_params) -> None:
"""
Visualize receptive field grid.
:param input_shape: an input image shape as an instance of GridShape
:param output_shape: an output feature map shape
:param rf_params: an instance of ReceptiveFieldDescription computed for
this feature map.
:param custom_image: optional image [height, width, 3] to be plotted as
a background.
:param plot_naive_rf: plot naive version of the receptive field. Naive
version of RF does not take strides, and offsets into considerations,
it is a simple linear mapping from N points in feature map to pixels
in the image.
:param axis: a matplotlib axis object as returned by the e.g. plt.subplot
function. If not None then axis is used for visualizations otherwise
default figure is created.
:param plot_params: additional plot params: figsize=(5, 5)
"""
if custom_image is None:
img = get_default_image(shape=ImageShape(input_shape.h, input_shape.w))
else:
img = custom_image
figsize = plot_params.get("figsize", (10, 10))
# plot image
if axis is None:
plt.figure(figsize=figsize)
axis = plt.subplot(111)
axis.imshow(img)
# plot naive receptive field grid
if plot_naive_rf:
dw = input_shape.w / output_shape.w
dh = input_shape.h / output_shape.h
for i, j in itertools.product(range(output_shape.w),
range(output_shape.h)):
x0, x1 = i * dw, (i + 1) * dw
y0, y1 = j * dh, (j + 1) * dh
axis.add_patch(
patches.Rectangle(
(y0, x0),
dh,
dw,
alpha=0.9,
fill=False,
edgecolor="gray",
linewidth=1,
))
rf_offset = rf_params.offset
rf_size = rf_params.size
rf_stride = rf_params.stride
# map from output grid space to input image
def map_point(i: int, j: int):
return np.array(rf_offset) + np.array([i, j]) * np.array(rf_stride)
# plot RF grid based on rf params
'''
points = [
map_point(i, j)
for i, j in itertools.product(range(output_shape.w), range(output_shape.h))
]
points = np.array(points)
axis.scatter(points[:, 1], points[:, 0], marker="o", c=(0.2, 0.9, 0.1, 0.9), s=10)
'''
# plot receptive field from corner point
_plot_rect(
axis,
rect=to_rf_rect(rf_offset, rf_size),
color=(0.9, 0.3, 0.2),
linewidth=2,
size=10,
)
center_point = map_point(output_shape.w // 2, output_shape.h // 2)
_plot_rect(
axis,
rect=to_rf_rect(GridPoint(center_point[0], center_point[1]), rf_size),
color=(0.1, 0.3, 0.9),
linewidth=2,
size=10,
)
last_point = map_point(output_shape.w - 1, output_shape.h - 1)
_plot_rect(
axis,
rect=to_rf_rect(GridPoint(last_point[0], last_point[1]), rf_size),
color=(0.1, 0.9, 0.3),
linewidth=2,
size=10,
)
axis.set_aspect("equal")
def plot_receptive_grid(input_shape: GridShape,
output_shape: GridShape,
rf_params: ReceptiveFieldDescription,
custom_image: np.ndarray = None,
plot_naive_rf: bool = False,
**plot_params) -> None:
if custom_image is None:
img = get_default_image(shape=ImageShape(input_shape.h, input_shape.w))
else:
img = custom_image
figsize = plot_params.get("figsize", (10, 10))
# plot image
plt.figure(figsize=figsize)
ax = plt.subplot(111)
plt.imshow(img)
# plot naive receptive field grid
if plot_naive_rf:
dw = input_shape.w / output_shape.w
dh = input_shape.h / output_shape.h
for i, j in itertools.product(range(output_shape.w),
range(output_shape.h)):
x0, x1 = i * dw, (i + 1) * dw
y0, y1 = j * dh, (j + 1) * dh
ax.add_patch(
patches.Rectangle((y0, x0),
dh,
dw,
alpha=0.9,
fill=False,
edgecolor="gray",
linewidth=1))
rf_offset = rf_params.offset
rf_size = rf_params.size
rf_stride = rf_params.stride
# map from output grid space to input image
def map_point(i: int, j: int):
return np.array(rf_offset) + np.array([i, j]) * np.array(rf_stride)
# plot RF grid based on rf params
points = [
map_point(i, j) for i, j in itertools.product(range(output_shape.w),
range(output_shape.h))
]
points = np.array(points)
plt.scatter(points[:, 1],
points[:, 0],
marker="o",
c=(0.2, 0.9, 0.1, 0.9),
s=10)
# plot receptive field from corner point
_plot_rect(ax,
rect=to_rf_rect(rf_offset, rf_size),
color=(0.9, 0.3, 0.2),
linewidth=5,
size=90)
center_point = map_point(output_shape.w // 2, output_shape.h // 2)
_plot_rect(ax,
rect=to_rf_rect(GridPoint(center_point[0], center_point[1]),
rf_size),
color=(0.1, 0.3, 0.9),
linewidth=5,
size=90)
last_point = map_point(output_shape.w - 1, output_shape.h - 1)
_plot_rect(ax,
rect=to_rf_rect(GridPoint(last_point[0], last_point[1]),
rf_size),
color=(0.1, 0.9, 0.3),
linewidth=5,
size=90)
ax.set_aspect('equal')