def test_input_validation(args: dict, data: st.DataObject):
kwargs = dict(arr=np.arange(36).reshape(6, 6),
window_shape=(1, 1),
step=1,
dilation=None)
kwargs.update(
(k, (data.draw(v, label=k)) if isinstance(v, st.SearchStrategy) else v)
for k, v in args.items())
with pytest.raises((ValueError, TypeError)):
sliding_window_view(**kwargs)
def test_sliding_window(data, x):
""" Test variations of window-shape, step, and dilation for sliding window
view of N-dimensional array."""
win_dim = data.draw(st.integers(1, x.ndim), label="win_dim")
win_shape = data.draw(st.tuples(*(st.integers(1, s)
for s in x.shape[-win_dim:])),
label="win_shape")
step = data.draw(st.tuples(*(st.integers(1, s)
for s in x.shape[-win_dim:])),
label="step")
max_dilation = np.array(x.shape[-win_dim:]) // win_shape
dilation = data.draw(st.one_of(
st.none(), st.tuples(*(st.integers(1, s) for s in max_dilation))),
label="dilation")
y = sliding_window_view(x,
window_shape=win_shape,
step=step,
dilation=dilation)
if dilation is None:
dilation = np.ones((len(win_shape), ), dtype=int)
for ind in np.ndindex(*y.shape[:win_dim]):
slices = tuple(
slice(i * s, i * s + w * d, d)
for i, w, s, d in zip(ind, win_shape, step, dilation))
assert_allclose(actual=y[tuple([*ind])], desired=x[(..., *slices)])
def backward_var(self, grad, index, **kwargs):
""" Computes dX, where X is the data batch
Parameters
----------
grad : numpy.ndarray, shape=(N, F, G0, ...)"""
x, w = (i.data for i in self.variables)
num_conv_channels = grad.ndim - 2
if index == 0: # backprop through x
x_shape = x.shape[:2] + tuple(
i + 2 * p
for i, p in zip(x.shape[-num_conv_channels:], self.padding))
dx = np.zeros(x_shape, dtype=x.dtype) # (N, C, X0, ...)
# `gp` stores all of the various broadcast multiplications of each grad
# element against the conv filter.
# (N, F, G0, ...) -tdot- (F, C, W0, ...) --> (N, G0, ..., C, W0, ...)
gp = np.tensordot(grad, w, axes=[[1], [0]])
for ind in np.ndindex(grad.shape[-num_conv_channels:]):
# ind: (g0, ...) - grid-position of filter placement
slices = tuple(
slice(i * s, i * s + w * d, d) for i, w, s, d in zip(
ind, w.shape[2:], self.stride, self.dilation))
# Add (grad-element * filter) to each appropriate window position in `dx`
# dx[N, C, g0*s0 : g0*s0 + w0*d0 : d0, (...)] += gp[N, g0, (...), C, W0, (...)]
dx[(..., *slices)] += gp[(slice(None), *ind, ...)]
# remove padding from dx
if sum(self.padding):
no_pads = tuple(
slice(p, -p if p else None) for p in self.padding)
dx = dx[(..., *no_pads)]
return dx
else: # backprop through w
# backprop into f
# symmetric 0-padding for H, W dimensions
axis_pad = tuple((i, i) for i in (0, 0, *self.padding))
x = np.pad(x, axis_pad, mode='constant') if sum(
self.padding) else x
# (G0, ...) is the tuple of grid-indices for placing each window (not including stride)
# (N, C, X0, ...) -> (G0, ..., N, C, W0, ...)
windowed_data = sliding_window_view(x,
window_shape=w.shape[2:],
step=self.stride,
dilation=self.dilation)
# (N, F, G0, ...) -tdot- (G0, ..., N, C, W0, ...) --> (F, C, W0, ...)
grad_axes = list(range(
2, num_conv_channels + 2)) + [0] # (G0, ..., N)
window_axes = list(range(num_conv_channels + 1)) # (G0, ..., N)
return np.tensordot(grad,
windowed_data,
axes=[grad_axes, window_axes])
def test_memory_details(dtype):
""" Ensure that:
- function handles non C-contiguous layouts correctly
- output is view of input
- output is not writeable"""
x = np.arange(20).reshape(2, 10).astype(dtype)
x = np.asfortranarray(x)
y = sliding_window_view(x, (5, ), 5)
soln = np.array([[[0, 1, 2, 3, 4], [10, 11, 12, 13, 14]],
[[5, 6, 7, 8, 9], [15, 16, 17, 18, 19]]])
assert not y.flags["WRITEABLE"]
assert_allclose(actual=y, desired=soln)
x = np.arange(20).reshape(2, 10)
x = np.ascontiguousarray(x)
y = sliding_window_view(x, (5, ), 5)
assert not y.flags["WRITEABLE"]
assert np.shares_memory(x, y)
assert_allclose(actual=y, desired=soln)
def backward_var(self, grad, index, **kwargs):
""" Parameters
----------
grad : numpy.ndarray, shape=((N0, ...), G0, ...),
index : int"""
var = self.variables[index]
x = var.data
num_pool = len(self.pool)
sl = sliding_window_view(x, self.pool, self.stride)
grid_shape = sl.shape
maxed = sl.reshape(*sl.shape[:-num_pool], -1).argmax(-1)
axes = tuple(range(maxed.ndim))
# argmax within a given flat-window
maxed = maxed.transpose(axes[num_pool:] +
axes[:num_pool]) # ((N0, ...), G0, ...)
# flat-index offset associated with reshaped window within `x`
row_major_offset = tuple(np.cumprod(
x.shape[-num_pool:][:0:-1])[::-1]) + (1, )
# flat index of argmax, updated based on position within window, according to shape of `x`
in_window_offset = sum(ind * off for ind, off in zip(
np.unravel_index(maxed, self.pool), row_major_offset))
# flat-index of strided window placement, relative to `x`
window_offset = sum(ind * s * off for ind, s, off in zip(
np.indices(grid_shape[:num_pool]), self.stride, row_major_offset))
# indices required to traverse pool-axis-flattened array
# ((N0, ...) G0*...)
flat_grid_shape = (*maxed.shape[:-num_pool],
np.prod(maxed.shape[-num_pool:]))
index = np.indices(flat_grid_shape)
# update trailing indices to traverse location of max entries within pooled axes
index[-1] = (in_window_offset + window_offset).reshape(
*flat_grid_shape[:-1], -1)
# accumulate gradient within pool-axis-flattened dx, then reshape to match shape of `x`
dx = np.zeros(x.shape[:-num_pool] + (np.prod(x.shape[-num_pool:]), ))
np.add.at(dx, tuple(index), grad.reshape(*x.shape[:-num_pool], -1))
return dx.reshape(x.shape)
def __call__(self, x, w, *, stride, padding=0, dilation=1):
self.variables = (x, w)
# x ... data: (N, C, X0, X1, ...)
# w ... filters: (F, C, W0, W1, ...)
x = x.data
w = w.data
assert x.ndim > 2
assert x.ndim == w.ndim
assert w.shape[1] == x.shape[
1], "The channel-depth of the batch and filters must agree"
num_conv_channels = w.ndim - 2
x_shape = np.array(
x.shape[2:]
) # (X0, ...): shape of the channels being convolved over
w_shape = np.array(w.shape[2:]) # (W0, ...): shape of each conv filter
dilation = np.array((dilation, ) * num_conv_channels) if isinstance(
dilation, Integral) else np.array(dilation, dtype=int)
assert len(dilation) == num_conv_channels and all(
d >= 1 and isinstance(d, Integral) for d in dilation)
padding = np.array((padding, ) * num_conv_channels) if isinstance(
padding, Integral) else np.array(padding, dtype=int)
assert len(padding) == num_conv_channels and all(
p >= 0 and isinstance(p, Integral) for p in padding)
stride = np.array((stride, ) * num_conv_channels) if isinstance(
stride, Integral) else np.asarray(stride, dtype=int)
assert len(stride) == num_conv_channels and all(
s >= 1 and isinstance(s, Integral) for s in stride)
out_shape = (x_shape + 2 * padding -
((w_shape - 1) * dilation + 1)) / stride + 1
if not all(i.is_integer() and i > 0 for i in out_shape):
msg = "Stride and kernel dimensions are incompatible: \n"
msg += "Input dimensions: {}\n".format(tuple(x_shape))
msg += "Stride dimensions: {}\n".format(tuple(stride))
msg += "Kernel dimensions: {}\n".format(tuple(w_shape))
msg += "Padding dimensions: {}\n".format(tuple(padding))
msg += "Dilation dimensions: {}\n".format(tuple(dilation))
raise ValueError(msg)
self.padding = padding
self.stride = stride
self.dilation = dilation
# symmetric 0-padding for X0, X1, ... dimensions
axis_pad = tuple((i, i) for i in (0, 0, *padding))
x = np.pad(x, axis_pad, mode='constant') if sum(padding) else x
# (G0, ...) is the tuple of grid-positions for placing each window (not including stride)
# (N, C, X0, ...) -> (G0, ..., N, C, W0, ...)
windowed_data = sliding_window_view(x,
window_shape=w_shape,
step=self.stride,
dilation=self.dilation)
w_conv_channels = list(range(1, num_conv_channels + 2)) # C, W0, ...
window_conv_channels = [
i + 1 + num_conv_channels # C, W0, ...
for i in range(num_conv_channels + 1)
]
# (F, C, W0, ...) ⋆ (G0, ..., N, C, W0, ...) -> (F, G0, ..., N)
conv_out = np.tensordot(w,
windowed_data,
axes=[w_conv_channels, window_conv_channels])
# (F, G0, ..., N) -> (N, F, G0, ...)
out = np.moveaxis(conv_out, source=-1, destination=0)
return out if out.flags['C_CONTIGUOUS'] else np.ascontiguousarray(out)
def __call__(self, x, pool, stride):
""" Perform max-pooling over the last N dimensions of a data batch.
Extended Summary
----------------
The data consists of N trailing axes to be pooled over, denoted by ``C0, ...``. These
can be preceded, optionally, by un-pooled axes, denoted by ``(N0, ...)``. The dimensions
of the window over which pooling is performed is denoted by ``P0, ...``. The window
is placed with stride values ``S0, ...``.
Ultimately the pooled channels have a shape ``G0, ...``.
Parameters
----------
x : mygrad.Tensor, shape=([...], C0, ...)
The data batch; to be pooled along the trailing axes denoted by ``C0, ...``.
pool : Tuple[Integral, ...], (P0, ...)
The extent of the pooling window along the ``(C0, ...)`` axes, respectively. The
length of `pool` determines ``N`` - the number of trailing dimensions to pool over.
stride : Union[Integral, Tuple[Integral, ...]], (S0, ...)
The spacing used to place the pooling window, along ``(P0, ...)`` axes, respectively.
If a single value is provided, it is used for all N pooling axes.
Returns
-------
numpy.ndarray, shape=([...], G0, ...)
The pooled data batch.
Notes
-----
Only 'valid' placements of the pooling window are permitted - the pooling
window cannot extend passed the "boundaries" of the data
dimensions.
"""
self.variables = (x, ) # data: ((N0, ...), C0, ...)
x = x.data
assert isinstance(pool, (tuple, list, np.ndarray)) and all(
i >= 0 and isinstance(i, Integral) for i in pool)
pool = np.asarray(pool, dtype=int)
assert all(i > 0 for i in pool)
assert x.ndim >= len(
pool
), "The number of pooled dimensions cannot exceed the dimensionality of the data."
stride = (np.array([stride] * len(pool)) if isinstance(
stride, Integral) else np.asarray(stride, dtype=int))
assert len(stride) == len(pool) and all(
s >= 1 and isinstance(s, Integral) for s in stride)
self.pool = pool # (P0, ...)
self.stride = stride # (S0, ...)
num_pool = len(pool)
num_no_pool = x.ndim - num_pool
x_shape = np.array(x.shape[num_no_pool:])
w_shape = pool
out_shape = (x_shape - w_shape) / stride + 1
if not all(i.is_integer() and i > 0 for i in out_shape):
msg = f"Stride and kernel dimensions are incompatible: \n"
msg += f"Input dimensions: {(tuple(x_shape))}\n"
msg += f"Stride dimensions: {(tuple(stride))}\n"
msg += f"Pooling dimensions: {(tuple(w_shape))}\n"
raise ValueError(msg)
pool_axes = tuple(-(i + 1) for i in range(num_pool))
# (G0, ...) is the tuple of grid-positions for placing each window (not including stride)
# sliding_window_view(x): ((N0, ...), C0, ...) -> (G0, ..., (N0, ...), P0, ...)
# max-pool: (G0, ..., (N0, ...), P0, ...) -> (G0, ..., (N0, ...))
maxed = sliding_window_view(x, self.pool,
self.stride).max(axis=pool_axes)
axes = tuple(range(maxed.ndim))
# (G0, ..., (N0, ...)) -> ((N0, ...), G0, ...)
out = maxed.transpose(axes[-num_no_pool:] + axes[:-num_no_pool])
return out if out.flags["C_CONTIGUOUS"] else np.ascontiguousarray(out)