def test_input_validation(args: dict, data: st.DataObject): kwargs = dict(arr=np.arange(36).reshape(6, 6), window_shape=(1, 1), step=1, dilation=None) kwargs.update( (k, (data.draw(v, label=k)) if isinstance(v, st.SearchStrategy) else v) for k, v in args.items()) with pytest.raises((ValueError, TypeError)): sliding_window_view(**kwargs)
def test_sliding_window(data, x): """ Test variations of window-shape, step, and dilation for sliding window view of N-dimensional array.""" win_dim = data.draw(st.integers(1, x.ndim), label="win_dim") win_shape = data.draw(st.tuples(*(st.integers(1, s) for s in x.shape[-win_dim:])), label="win_shape") step = data.draw(st.tuples(*(st.integers(1, s) for s in x.shape[-win_dim:])), label="step") max_dilation = np.array(x.shape[-win_dim:]) // win_shape dilation = data.draw(st.one_of( st.none(), st.tuples(*(st.integers(1, s) for s in max_dilation))), label="dilation") y = sliding_window_view(x, window_shape=win_shape, step=step, dilation=dilation) if dilation is None: dilation = np.ones((len(win_shape), ), dtype=int) for ind in np.ndindex(*y.shape[:win_dim]): slices = tuple( slice(i * s, i * s + w * d, d) for i, w, s, d in zip(ind, win_shape, step, dilation)) assert_allclose(actual=y[tuple([*ind])], desired=x[(..., *slices)])
def backward_var(self, grad, index, **kwargs): """ Computes dX, where X is the data batch Parameters ---------- grad : numpy.ndarray, shape=(N, F, G0, ...)""" x, w = (i.data for i in self.variables) num_conv_channels = grad.ndim - 2 if index == 0: # backprop through x x_shape = x.shape[:2] + tuple( i + 2 * p for i, p in zip(x.shape[-num_conv_channels:], self.padding)) dx = np.zeros(x_shape, dtype=x.dtype) # (N, C, X0, ...) # `gp` stores all of the various broadcast multiplications of each grad # element against the conv filter. # (N, F, G0, ...) -tdot- (F, C, W0, ...) --> (N, G0, ..., C, W0, ...) gp = np.tensordot(grad, w, axes=[[1], [0]]) for ind in np.ndindex(grad.shape[-num_conv_channels:]): # ind: (g0, ...) - grid-position of filter placement slices = tuple( slice(i * s, i * s + w * d, d) for i, w, s, d in zip( ind, w.shape[2:], self.stride, self.dilation)) # Add (grad-element * filter) to each appropriate window position in `dx` # dx[N, C, g0*s0 : g0*s0 + w0*d0 : d0, (...)] += gp[N, g0, (...), C, W0, (...)] dx[(..., *slices)] += gp[(slice(None), *ind, ...)] # remove padding from dx if sum(self.padding): no_pads = tuple( slice(p, -p if p else None) for p in self.padding) dx = dx[(..., *no_pads)] return dx else: # backprop through w # backprop into f # symmetric 0-padding for H, W dimensions axis_pad = tuple((i, i) for i in (0, 0, *self.padding)) x = np.pad(x, axis_pad, mode='constant') if sum( self.padding) else x # (G0, ...) is the tuple of grid-indices for placing each window (not including stride) # (N, C, X0, ...) -> (G0, ..., N, C, W0, ...) windowed_data = sliding_window_view(x, window_shape=w.shape[2:], step=self.stride, dilation=self.dilation) # (N, F, G0, ...) -tdot- (G0, ..., N, C, W0, ...) --> (F, C, W0, ...) grad_axes = list(range( 2, num_conv_channels + 2)) + [0] # (G0, ..., N) window_axes = list(range(num_conv_channels + 1)) # (G0, ..., N) return np.tensordot(grad, windowed_data, axes=[grad_axes, window_axes])
def test_memory_details(dtype): """ Ensure that: - function handles non C-contiguous layouts correctly - output is view of input - output is not writeable""" x = np.arange(20).reshape(2, 10).astype(dtype) x = np.asfortranarray(x) y = sliding_window_view(x, (5, ), 5) soln = np.array([[[0, 1, 2, 3, 4], [10, 11, 12, 13, 14]], [[5, 6, 7, 8, 9], [15, 16, 17, 18, 19]]]) assert not y.flags["WRITEABLE"] assert_allclose(actual=y, desired=soln) x = np.arange(20).reshape(2, 10) x = np.ascontiguousarray(x) y = sliding_window_view(x, (5, ), 5) assert not y.flags["WRITEABLE"] assert np.shares_memory(x, y) assert_allclose(actual=y, desired=soln)
def backward_var(self, grad, index, **kwargs): """ Parameters ---------- grad : numpy.ndarray, shape=((N0, ...), G0, ...), index : int""" var = self.variables[index] x = var.data num_pool = len(self.pool) sl = sliding_window_view(x, self.pool, self.stride) grid_shape = sl.shape maxed = sl.reshape(*sl.shape[:-num_pool], -1).argmax(-1) axes = tuple(range(maxed.ndim)) # argmax within a given flat-window maxed = maxed.transpose(axes[num_pool:] + axes[:num_pool]) # ((N0, ...), G0, ...) # flat-index offset associated with reshaped window within `x` row_major_offset = tuple(np.cumprod( x.shape[-num_pool:][:0:-1])[::-1]) + (1, ) # flat index of argmax, updated based on position within window, according to shape of `x` in_window_offset = sum(ind * off for ind, off in zip( np.unravel_index(maxed, self.pool), row_major_offset)) # flat-index of strided window placement, relative to `x` window_offset = sum(ind * s * off for ind, s, off in zip( np.indices(grid_shape[:num_pool]), self.stride, row_major_offset)) # indices required to traverse pool-axis-flattened array # ((N0, ...) G0*...) flat_grid_shape = (*maxed.shape[:-num_pool], np.prod(maxed.shape[-num_pool:])) index = np.indices(flat_grid_shape) # update trailing indices to traverse location of max entries within pooled axes index[-1] = (in_window_offset + window_offset).reshape( *flat_grid_shape[:-1], -1) # accumulate gradient within pool-axis-flattened dx, then reshape to match shape of `x` dx = np.zeros(x.shape[:-num_pool] + (np.prod(x.shape[-num_pool:]), )) np.add.at(dx, tuple(index), grad.reshape(*x.shape[:-num_pool], -1)) return dx.reshape(x.shape)
def __call__(self, x, w, *, stride, padding=0, dilation=1): self.variables = (x, w) # x ... data: (N, C, X0, X1, ...) # w ... filters: (F, C, W0, W1, ...) x = x.data w = w.data assert x.ndim > 2 assert x.ndim == w.ndim assert w.shape[1] == x.shape[ 1], "The channel-depth of the batch and filters must agree" num_conv_channels = w.ndim - 2 x_shape = np.array( x.shape[2:] ) # (X0, ...): shape of the channels being convolved over w_shape = np.array(w.shape[2:]) # (W0, ...): shape of each conv filter dilation = np.array((dilation, ) * num_conv_channels) if isinstance( dilation, Integral) else np.array(dilation, dtype=int) assert len(dilation) == num_conv_channels and all( d >= 1 and isinstance(d, Integral) for d in dilation) padding = np.array((padding, ) * num_conv_channels) if isinstance( padding, Integral) else np.array(padding, dtype=int) assert len(padding) == num_conv_channels and all( p >= 0 and isinstance(p, Integral) for p in padding) stride = np.array((stride, ) * num_conv_channels) if isinstance( stride, Integral) else np.asarray(stride, dtype=int) assert len(stride) == num_conv_channels and all( s >= 1 and isinstance(s, Integral) for s in stride) out_shape = (x_shape + 2 * padding - ((w_shape - 1) * dilation + 1)) / stride + 1 if not all(i.is_integer() and i > 0 for i in out_shape): msg = "Stride and kernel dimensions are incompatible: \n" msg += "Input dimensions: {}\n".format(tuple(x_shape)) msg += "Stride dimensions: {}\n".format(tuple(stride)) msg += "Kernel dimensions: {}\n".format(tuple(w_shape)) msg += "Padding dimensions: {}\n".format(tuple(padding)) msg += "Dilation dimensions: {}\n".format(tuple(dilation)) raise ValueError(msg) self.padding = padding self.stride = stride self.dilation = dilation # symmetric 0-padding for X0, X1, ... dimensions axis_pad = tuple((i, i) for i in (0, 0, *padding)) x = np.pad(x, axis_pad, mode='constant') if sum(padding) else x # (G0, ...) is the tuple of grid-positions for placing each window (not including stride) # (N, C, X0, ...) -> (G0, ..., N, C, W0, ...) windowed_data = sliding_window_view(x, window_shape=w_shape, step=self.stride, dilation=self.dilation) w_conv_channels = list(range(1, num_conv_channels + 2)) # C, W0, ... window_conv_channels = [ i + 1 + num_conv_channels # C, W0, ... for i in range(num_conv_channels + 1) ] # (F, C, W0, ...) ⋆ (G0, ..., N, C, W0, ...) -> (F, G0, ..., N) conv_out = np.tensordot(w, windowed_data, axes=[w_conv_channels, window_conv_channels]) # (F, G0, ..., N) -> (N, F, G0, ...) out = np.moveaxis(conv_out, source=-1, destination=0) return out if out.flags['C_CONTIGUOUS'] else np.ascontiguousarray(out)
def __call__(self, x, pool, stride): """ Perform max-pooling over the last N dimensions of a data batch. Extended Summary ---------------- The data consists of N trailing axes to be pooled over, denoted by ``C0, ...``. These can be preceded, optionally, by un-pooled axes, denoted by ``(N0, ...)``. The dimensions of the window over which pooling is performed is denoted by ``P0, ...``. The window is placed with stride values ``S0, ...``. Ultimately the pooled channels have a shape ``G0, ...``. Parameters ---------- x : mygrad.Tensor, shape=([...], C0, ...) The data batch; to be pooled along the trailing axes denoted by ``C0, ...``. pool : Tuple[Integral, ...], (P0, ...) The extent of the pooling window along the ``(C0, ...)`` axes, respectively. The length of `pool` determines ``N`` - the number of trailing dimensions to pool over. stride : Union[Integral, Tuple[Integral, ...]], (S0, ...) The spacing used to place the pooling window, along ``(P0, ...)`` axes, respectively. If a single value is provided, it is used for all N pooling axes. Returns ------- numpy.ndarray, shape=([...], G0, ...) The pooled data batch. Notes ----- Only 'valid' placements of the pooling window are permitted - the pooling window cannot extend passed the "boundaries" of the data dimensions. """ self.variables = (x, ) # data: ((N0, ...), C0, ...) x = x.data assert isinstance(pool, (tuple, list, np.ndarray)) and all( i >= 0 and isinstance(i, Integral) for i in pool) pool = np.asarray(pool, dtype=int) assert all(i > 0 for i in pool) assert x.ndim >= len( pool ), "The number of pooled dimensions cannot exceed the dimensionality of the data." stride = (np.array([stride] * len(pool)) if isinstance( stride, Integral) else np.asarray(stride, dtype=int)) assert len(stride) == len(pool) and all( s >= 1 and isinstance(s, Integral) for s in stride) self.pool = pool # (P0, ...) self.stride = stride # (S0, ...) num_pool = len(pool) num_no_pool = x.ndim - num_pool x_shape = np.array(x.shape[num_no_pool:]) w_shape = pool out_shape = (x_shape - w_shape) / stride + 1 if not all(i.is_integer() and i > 0 for i in out_shape): msg = f"Stride and kernel dimensions are incompatible: \n" msg += f"Input dimensions: {(tuple(x_shape))}\n" msg += f"Stride dimensions: {(tuple(stride))}\n" msg += f"Pooling dimensions: {(tuple(w_shape))}\n" raise ValueError(msg) pool_axes = tuple(-(i + 1) for i in range(num_pool)) # (G0, ...) is the tuple of grid-positions for placing each window (not including stride) # sliding_window_view(x): ((N0, ...), C0, ...) -> (G0, ..., (N0, ...), P0, ...) # max-pool: (G0, ..., (N0, ...), P0, ...) -> (G0, ..., (N0, ...)) maxed = sliding_window_view(x, self.pool, self.stride).max(axis=pool_axes) axes = tuple(range(maxed.ndim)) # (G0, ..., (N0, ...)) -> ((N0, ...), G0, ...) out = maxed.transpose(axes[-num_no_pool:] + axes[:-num_no_pool]) return out if out.flags["C_CONTIGUOUS"] else np.ascontiguousarray(out)