def test_dihedral(self): N = 8 g = FlipRot2dOnR2(N, axis=np.pi / 3) r1 = FieldType(g, list(g.representations.values())) r2 = FieldType(g, list(g.representations.values())) # r1 = FieldType(g, [g.trivial_repr]) # r2 = FieldType(g, [g.fibergroup.irrep(1, 0)]) # r2 = FieldType(g, [irr for irr in g.fibergroup.irreps.values() if irr.size == 1]) # r2 = FieldType(g, [g.regular_repr]) s = 7 # sigma = 0.6 # fco = lambda r: 1. * r * np.pi # fco = lambda r: 2 * r sigma = None fco = None cl = R2Conv(r1, r2, s, basisexpansion='blocks', sigma=sigma, frequencies_cutoff=fco, bias=True) for _ in range(8): # cl.basisexpansion._init_weights() init.generalized_he_init(cl.weights.data, cl.basisexpansion) cl.eval() cl.check_equivariance()
def test_flip(self): # g = Flip2dOnR2(axis=np.pi/3) g = Flip2dOnR2(axis=np.pi / 2) r1 = FieldType(g, list(g.representations.values())) r2 = FieldType(g, list(g.representations.values()) * 3).sorted() s = 9 # sigma = 0.6 # fco = lambda r: 1. * r * np.pi # fco = lambda r: 2 * r sigma = None fco = None cl = R2Conv(r1, r2, s, basisexpansion='blocks', sigma=sigma, frequencies_cutoff=fco, bias=True) for _ in range(32): # cl.basisexpansion._init_weights() init.generalized_he_init(cl.weights.data, cl.basisexpansion) cl.eval() cl.check_equivariance()
def test_so2(self): N = 7 g = Rot2dOnR2(-1, N) r1 = FieldType(g, list(g.representations.values())) r2 = FieldType(g, list(g.representations.values())) s = 7 # sigma = 0.6 # fco = lambda r: 1. * r * np.pi # fco = lambda r: 2 * r sigma = None fco = None cl = R2Conv(r1, r2, s, basisexpansion='blocks', sigma=sigma, frequencies_cutoff=fco, bias=True) for _ in range(8): # cl.basisexpansion._init_weights() init.generalized_he_init(cl.weights.data, cl.basisexpansion) cl.eval() cl.check_equivariance()
def test_padding_mode_reflect(self): g = Flip2dOnR2(axis=np.pi / 2) r1 = FieldType(g, [g.trivial_repr]) r2 = FieldType(g, [g.regular_repr]) s = 3 cl = R2Conv(r1, r2, s, bias=True, padding=1, padding_mode='reflect', initialize=False) for _ in range(32): init.generalized_he_init(cl.weights.data, cl.basisexpansion) cl.eval() cl.check_equivariance()
def check(self, r1: FieldType, r2: FieldType): np.set_printoptions(precision=7, threshold=60000, suppress=True) assert r1.gspace == r2.gspace s = 7 cl = R2Conv(r1, r2, s, basisexpansion='blocks', sigma=[0.01] + [0.6] * int(s // 2), frequencies_cutoff=3.) ys = [] for _ in range(1000): init.generalized_he_init(cl.weights.data, cl.basisexpansion) cl.eval() x = torch.randn(10, r1.size, s, s) xg = GeometricTensor(x, r1) y = cl(xg).tensor del xg del x ys.append(y.transpose(0, 1).reshape(r2.size, -1)) ys = torch.cat(ys, dim=1) mean = ys.mean(1) std = ys.std(1) print(mean) print(std) self.assertTrue( torch.allclose(torch.zeros_like(mean), mean, rtol=2e-2, atol=5e-2)) self.assertTrue( torch.allclose(torch.ones_like(std), std, rtol=1e-1, atol=6e-2))
def test_cyclic(self): N = 8 g = Rot2dOnR2(N) r1 = FieldType(g, list(g.representations.values())) r2 = FieldType(g, list(g.representations.values()) * 2) # r1 = FieldType(g, [g.trivial_repr]) # r2 = FieldType(g, [g.regular_repr]) s = 7 sigma = None # fco = lambda r: 1. * r * np.pi fco = None cl = R2Conv(r1, r2, s, basisexpansion='blocks', sigma=sigma, frequencies_cutoff=fco, bias=True) cl.bias.data = 20 * torch.randn_like(cl.bias.data) for _ in range(1): init.generalized_he_init(cl.weights.data, cl.basisexpansion) cl.eval() cl.check_equivariance() cl.train() for _ in range(1): cl.check_equivariance() cl.eval() for _ in range(5): init.generalized_he_init(cl.weights.data, cl.basisexpansion) cl.eval() filter = cl.filter.clone() cl.check_equivariance() self.assertTrue(torch.allclose(filter, cl.filter))
def test_padding_mode_circular(self): g = FlipRot2dOnR2(4, axis=np.pi / 2) r1 = FieldType(g, [g.trivial_repr]) r2 = FieldType(g, [g.regular_repr]) for mode in ['circular', 'reflect', 'replicate']: for s in [3, 5, 7]: padding = s // 2 cl = R2Conv(r1, r2, s, bias=True, padding=padding, padding_mode=mode, initialize=False) for _ in range(10): init.generalized_he_init(cl.weights.data, cl.basisexpansion) cl.eval() cl.check_equivariance()
def __init__( self, in_type: FieldType, out_type: FieldType, kernel_size: int, padding: int = 0, stride: int = 1, dilation: int = 1, groups: int = 1, bias: bool = True, basisexpansion: str = 'blocks', sigma: Union[List[float], float] = None, frequencies_cutoff: Union[float, Callable[[float], int]] = None, rings: List[float] = None, maximum_offset: int = None, recompute: bool = False, basis_filter: Callable[[dict], bool] = None, ): r""" G-steerable planar convolution mapping between the input and output :class:`~e2cnn.nn.FieldType` s specified by the parameters ``in_type`` and ``out_type``. This operation is equivariant under the action of :math:`\R^2\rtimes G` where :math:`G` is the :attr:`e2cnn.nn.FieldType.fibergroup` of ``in_type`` and ``out_type``. Specifically, let :math:`\rho_\text{in}: G \to \GL{\R^{c_\text{in}}}` and :math:`\rho_\text{out}: G \to \GL{\R^{c_\text{out}}}` be the representations specified by the input and output field types. Then :class:`~e2cnn.nn.R2Conv` guarantees an equivariant mapping .. math:: \kappa \star [\mathcal{T}^\text{in}_{g,u} . f] = \mathcal{T}^\text{out}_{g,u} . [\kappa \star f] \qquad\qquad \forall g \in G, u \in \R^2 where the transformation of the input and output fields are given by .. math:: [\mathcal{T}^\text{in}_{g,u} . f](x) &= \rho_\text{in}(g)f(g^{-1} (x - u)) \\ [\mathcal{T}^\text{out}_{g,u} . f](x) &= \rho_\text{out}(g)f(g^{-1} (x - u)) \\ The equivariance of G-steerable convolutions is guaranteed by restricting the space of convolution kernels to an equivariant subspace. As proven in `3D Steerable CNNs <https://arxiv.org/abs/1807.02547>`_, this parametrizes the *most general equivariant convolutional map* between the input and output fields. For feature fields on :math:`\R^2` (e.g. images), the complete G-steerable kernel spaces for :math:`G \leq \O2` is derived in `General E(2)-Equivariant Steerable CNNs <https://arxiv.org/abs/1911.08251>`_. During training, in each forward pass the module expand the basis of G-steerable kernels with learned weights before calling :func:`torch.nn.functional.conv2d`. When :meth:`~troch.nn.Module.eval()` is called, the filter is built with the current trained weights and stored for future reuse such that no overhead of expanding the kernel remains. The parameters ``basisexpansion``, ``sigma``, ``frequencies_cutoff``, ``rings`` and ``maximum_offset`` are optional parameters used to control how the basis for the filters is built, how it is sampled on the filter grid and how it is expanded to build the filter. We suggest to keep these default values. Args: in_type (FieldType): the type of the input field, specifying its transformation law out_type (FieldType): the type of the output field, specifying its transformation law kernel_size (int): the size of the (square) filter padding(int, optional): implicit zero paddings on both sides of the input. Default: ``0`` stride(int, optional): the stride of the kernel. Default: ``1`` dilation(int, optional): the spacing between kernel elements. Default: ``1`` groups (int, optional): number of blocked connections from input channels to output channels. It allows depthwise convolution. When used, the input and output types need to be divisible in ``groups`` groups, all equal to each other. Default: ``1``. bias (bool, optional): Whether to add a bias to the output (only to fields which contain a trivial irrep) or not. Default ``True`` basisexpansion (str, optional): the basis expansion algorithm to use sigma (list or float, optional): width of each ring where the bases are sampled. If only one scalar is passed, it is used for all rings. frequencies_cutoff (callable or float, optional): function mapping the radii of the basis elements to the maximum frequency accepted. If a float values is passed, the maximum frequency is equal to the radius times this factor. By default (``None``), a more complex policy is used. rings (list, optional): radii of the rings where to sample the bases maximum_offset (int, optional): number of additional (aliased) frequencies in the intertwiners for finite groups. By default (``None``), all additional frequencies allowed by the frequencies cut-off are used. recompute (bool, optional): if ``True``, recomputes a new basis for the equivariant kernels. By Default (``False``), it caches the basis built or reuse a cached one, if it is found. basis_filter (callable, optional): function which takes as input a descriptor of a basis element (as a dictionary) and returns a boolean value: whether to preserve (``True``) or discard (``False``) the basis element. By default (``None``), no filtering is applied. Attributes: ~.weights (torch.Tensor): the learnable parameters which are used to expand the kernel ~.filter (torch.Tensor): the convolutional kernel obtained by expanding the parameters in :attr:`~e2cnn.nn.R2Conv.weights` ~.bias (torch.Tensor): the learnable parameters which are used to expand the bias, if ``bias=True`` ~.expanded_bias (torch.Tensor): the equivariant bias which is summed to the output, obtained by expanding the parameters in :attr:`~e2cnn.nn.R2Conv.bias` """ assert in_type.gspace == out_type.gspace assert isinstance(in_type.gspace, GeneralOnR2) super(R2Conv, self).__init__() self.space = in_type.gspace self.in_type = in_type self.out_type = out_type self.kernel_size = kernel_size self.stride = stride self.dilation = dilation self.padding = padding self.groups = groups if groups > 1: # Check the input and output classes can be split in `groups` groups, all equal to each other # first, check that the number of fields is divisible by `groups` assert len(in_type) % groups == 0 assert len(out_type) % groups == 0 in_size = len(in_type) // groups out_size = len(out_type) // groups # then, check that all groups are equal to each other, i.e. have the same types in the same order assert all( in_type.representations[i] == in_type.representations[i % in_size] for i in range(len(in_type))) assert all(out_type.representations[i] == out_type.representations[ i % out_size] for i in range(len(out_type))) # finally, retrieve the type associated to a single group in input. # this type will be used to build a smaller kernel basis and a smaller filter # as in PyTorch, to build a filter for grouped convolution, we build a filter which maps from one input # group to all output groups. Then, PyTorch's standard convolution routine interpret this filter as `groups` # different filters, each mapping an input group to an output group. in_type = in_type.index_select(list(range(in_size))) if bias: # bias can be applied only to trivial irreps inside the representation # to apply bias to a field we learn a bias for each trivial irreps it contains # and, then, we transform it with the change of basis matrix to be able to apply it to the whole field # this is equivalent to transform the field to its irreps through the inverse change of basis, # sum the bias only to the trivial irrep and then map it back with the change of basis # count the number of trivial irreps trivials = 0 for r in self.out_type: for irr in r.irreps: if self.out_type.fibergroup.irreps[irr].is_trivial(): trivials += 1 # if there is at least 1 trivial irrep if trivials > 0: # matrix containing the columns of the change of basis which map from the trivial irreps to the # field representations. This matrix allows us to map the bias defined only over the trivial irreps # to a bias for the whole field more efficiently bias_expansion = torch.zeros(self.out_type.size, trivials) p, c = 0, 0 for r in self.out_type: pi = 0 for irr in r.irreps: irr = self.out_type.fibergroup.irreps[irr] if irr.is_trivial(): bias_expansion[p:p + r.size, c] = torch.tensor( r.change_of_basis[:, pi]) c += 1 pi += irr.size p += r.size self.register_buffer("bias_expansion", bias_expansion) self.bias = Parameter(torch.zeros(trivials), requires_grad=True) self.register_buffer("expanded_bias", torch.zeros(out_type.size)) else: self.bias = None self.expanded_bias = None else: self.bias = None self.expanded_bias = None grid, basis_filter, rings, sigma, maximum_frequency = compute_basis_params( kernel_size, frequencies_cutoff, rings, sigma, dilation, basis_filter) # BasisExpansion: submodule which takes care of building the filter self._basisexpansion = None # notice that `in_type` is used instead of `self.in_type` such that it works also when `groups > 1` if basisexpansion == 'blocks': self._basisexpansion = BlocksBasisExpansion( in_type, out_type, grid, sigma=sigma, rings=rings, maximum_offset=maximum_offset, maximum_frequency=maximum_frequency, basis_filter=basis_filter, recompute=recompute) else: raise ValueError('Basis Expansion algorithm "%s" not recognized' % basisexpansion) self.weights = Parameter(torch.zeros(self.basisexpansion.dimension()), requires_grad=True) self.register_buffer( "filter", torch.zeros(out_type.size, in_type.size, kernel_size, kernel_size)) # by default, the weights are initialized with a generalized form of He's weight initialization init.generalized_he_init(self.weights.data, self.basisexpansion)
def __init__( self, in_type: FieldType, out_type: FieldType, kernel_size: int, padding: int = 0, output_padding: int = 0, stride: int = 1, dilation: int = 1, groups: int = 1, bias: bool = True, basisexpansion: str = 'blocks', sigma: Union[List[float], float] = None, frequencies_cutoff: Union[float, Callable[[float], int]] = None, rings: List[float] = None, maximum_offset: int = None, recompute: bool = False, basis_filter: Callable[[dict], bool] = None, initialize: bool = True, ): r""" Transposed G-steerable planar convolution layer. .. warning :: Transposed convolution can produce artifacts which can harm the overall equivariance of the model. We suggest using :class:`~e2cnn.nn.R2Upsampling` combined with :class:`~e2cnn.nn.R2Conv` to perform upsampling. .. seealso :: For additional information about the parameters and the methods of this class, see :class:`e2cnn.nn.R2Conv`. The two modules are essentially the same, except for the type of convolution used. Args: in_type (FieldType): the type of the input field out_type (FieldType): the type of the output field kernel_size (int): the size of the filter padding(int, optional): implicit zero paddings on both sides of the input. Default: ``0`` output_padding(int, optional): implicit zero paddings on both sides of the input. Default: ``0`` stride(int, optional): the stride of the convolving kernel. Default: ``1`` dilation(int, optional): the spacing between kernel elements. Default: ``1`` groups (int, optional): number of blocked connections from input channels to output channels. Default: ``1``. bias (bool, optional): Whether to add a bias to the output (only to fields which contain a trivial irrep) or not. Default ``True`` initialize (bool, optional): initialize the weights of the model. Default: ``True`` """ assert in_type.gspace == out_type.gspace assert isinstance(in_type.gspace, GeneralOnR2) super(R2ConvTransposed, self).__init__() self.space = in_type.gspace self.in_type = in_type self.out_type = out_type self.kernel_size = kernel_size self.stride = stride self.dilation = dilation self.padding = padding self.output_padding = output_padding self.groups = groups if groups > 1: # Check the input and output classes can be split in `groups` groups, all equal to each other # first, check that the number of fields is divisible by `groups` assert len(in_type) % groups == 0 assert len(out_type) % groups == 0 in_size = len(in_type) // groups out_size = len(out_type) // groups # then, check that all groups are equal to each other, i.e. have the same types in the same order assert all( in_type.representations[i] == in_type.representations[i % in_size] for i in range(len(in_type))) assert all(out_type.representations[i] == out_type.representations[ i % out_size] for i in range(len(out_type))) # finally, retrieve the type associated to a single group in output. # this type will be used to build a smaller kernel basis and a smaller filter # as in PyTorch, to build a filter for grouped convolution, we build a filter which maps from all input # groups to one output group. Then, PyTorch's standard convolution routine interpret this filter as `groups` # different filters, each mapping an input group to an output group. out_type = out_type.index_select(list(range(out_size))) if bias: # bias can be applied only to trivial irreps inside the representation # to apply bias to a field we learn a bias for each trivial irreps it contains # and, then, we transform it with the change of basis matrix to be able to apply it to the whole field # this is equivalent to transform the field to its irreps through the inverse change of basis, # sum the bias only to the trivial irrep and then map it back with the change of basis # count the number of trivial irreps trivials = 0 for r in self.out_type: for irr in r.irreps: if self.out_type.fibergroup.irreps[irr].is_trivial(): trivials += 1 # if there is at least 1 trivial irrep if trivials > 0: # matrix containing the columns of the change of basis which map from the trivial irreps to the # field representations. This matrix allows us to map the bias defined only over the trivial irreps # to a bias for the whole field more efficiently bias_expansion = torch.zeros(self.out_type.size, trivials) p, c = 0, 0 for r in self.out_type: pi = 0 for irr in r.irreps: irr = self.out_type.fibergroup.irreps[irr] if irr.is_trivial(): bias_expansion[p:p + r.size, c] = torch.tensor( r.change_of_basis[:, pi]) c += 1 pi += irr.size p += r.size self.register_buffer("bias_expansion", bias_expansion) self.bias = Parameter(torch.zeros(trivials), requires_grad=True) self.register_buffer("expanded_bias", torch.zeros(out_type.size)) else: self.bias = None self.expanded_bias = None else: self.bias = None self.expanded_bias = None grid, basis_filter, rings, sigma, maximum_frequency = compute_basis_params( kernel_size, frequencies_cutoff, rings, sigma, dilation, basis_filter) # BasisExpansion: submodule which takes care of building the filter self._basisexpansion = None # notice that `out_type` is used instead of `self.out_type` such that it works also when `groups > 1` if basisexpansion == 'blocks': self._basisexpansion = BlocksBasisExpansion( in_type, out_type, grid, sigma=sigma, rings=rings, maximum_offset=maximum_offset, maximum_frequency=maximum_frequency, basis_filter=basis_filter, recompute=recompute) else: raise ValueError('Basis Expansion algorithm "%s" not recognized' % basisexpansion) self.weights = Parameter(torch.zeros(self.basisexpansion.dimension()), requires_grad=True) self.register_buffer( "filter", torch.zeros(in_type.size, out_type.size, kernel_size, kernel_size)) if initialize: # by default, the weights are initialized with a generalized form of he's weight initialization init.generalized_he_init(self.weights.data, self.basisexpansion)