def WideResnet(n_blocks=3, widen_factor=1, n_output_classes=10, bn_momentum=0.9, mode='train'): """WideResnet from https://arxiv.org/pdf/1605.07146.pdf. Args: n_blocks: int, number of blocks in a group. total layers = 6n + 4. widen_factor: int, widening factor of each group. k=1 is vanilla resnet. n_output_classes: int, number of distinct output classes. bn_momentum: float, momentum in BatchNorm. mode: Whether we are training or evaluating or doing inference. Returns: The list of layers comprising a WideResnet model with the given parameters. """ return tl.Serial( tl.ToFloat(), tl.Conv(16, (3, 3), padding='SAME'), WideResnetGroup(n_blocks, 16 * widen_factor, bn_momentum=bn_momentum, mode=mode), WideResnetGroup(n_blocks, 32 * widen_factor, (2, 2), bn_momentum=bn_momentum, mode=mode), WideResnetGroup(n_blocks, 64 * widen_factor, (2, 2), bn_momentum=bn_momentum, mode=mode), tl.BatchNorm(momentum=bn_momentum, mode=mode), tl.Relu(), tl.AvgPool(pool_size=(8, 8)), tl.Flatten(), tl.Dense(n_output_classes), tl.LogSoftmax(), )
def test_forward(self): layer = tl.AvgPool(pool_size=(2, 2), strides=(2, 2)) x = np.array([[ [[1, 2, 3], [4, 5, 6], [10, 20, 30], [40, 50, 60]], [[4, 2, 3], [7, 1, 2], [40, 20, 30], [70, 10, 20]], ]]) y = layer(x) self.assertEqual(tl.to_list(y), [[[[4.0, 2.5, 3.5], [40, 25, 35]]]])
def Resnet50(d_hidden=64, n_output_classes=1001, mode='train', norm=tl.BatchNorm, non_linearity=tl.Relu): """ResNet. Args: d_hidden: Dimensionality of the first hidden layer (multiplied later). n_output_classes: Number of distinct output classes. mode: Whether we are training or evaluating or doing inference. norm: `Layer` used for normalization, Ex: BatchNorm or FilterResponseNorm. non_linearity: `Layer` used as a non-linearity, Ex: If norm is BatchNorm then this is a Relu, otherwise for FilterResponseNorm this should be ThresholdedLinearUnit. Returns: The list of layers comprising a ResNet model with the given parameters. """ # A ConvBlock configured with the given norm, non-linearity and mode. def Resnet50ConvBlock(filter_multiplier=1, strides=(2, 2)): filters = ([ filter_multiplier * dim for dim in [d_hidden, d_hidden, 4 * d_hidden] ]) return ConvBlock(3, filters, strides, norm, non_linearity, mode) # Same as above for IdentityBlock. def Resnet50IdentityBlock(filter_multiplier=1): filters = ([ filter_multiplier * dim for dim in [d_hidden, d_hidden, 4 * d_hidden] ]) return IdentityBlock(3, filters, norm, non_linearity, mode) return tl.Serial( tl.ToFloat(), tl.Conv(d_hidden, (7, 7), (2, 2), 'SAME'), norm(mode=mode), non_linearity(), tl.MaxPool(pool_size=(3, 3), strides=(2, 2)), Resnet50ConvBlock(strides=(1, 1)), [Resnet50IdentityBlock() for _ in range(2)], Resnet50ConvBlock(2), [Resnet50IdentityBlock(2) for _ in range(3)], Resnet50ConvBlock(4), [Resnet50IdentityBlock(4) for _ in range(5)], Resnet50ConvBlock(8), [Resnet50IdentityBlock(8) for _ in range(2)], tl.AvgPool(pool_size=(7, 7)), tl.Flatten(), tl.Dense(n_output_classes), tl.LogSoftmax(), )
def test_padding_same(self): layer = tl.AvgPool(pool_size=(3, ), strides=(3, ), padding='SAME') # One padding position needed; add at end. x = np.array([[[0, 9], [1, 8], [2, 7], [3, 6], [4, 5]]]) y = layer(x) self.assertEqual(tl.to_list(y), [[[1, 8], [3.5, 5.5]]]) # Two padding positions needed; add one at end and one at start. x = np.array([[[0, 9], [1, 8], [2, 7], [3, 6]]]) y = layer(x) self.assertEqual(tl.to_list(y), [[[.5, 8.5], [2.5, 6.5]]])
def Resnet50(d_hidden=64, n_output_classes=1001, mode='train'): """ResNet. Args: d_hidden: Dimensionality of the first hidden layer (multiplied later). n_output_classes: Number of distinct output classes. mode: Whether we are training or evaluating or doing inference. Returns: The list of layers comprising a ResNet model with the given parameters. """ return tl.Model( tl.ToFloat(), tl.Conv(d_hidden, (7, 7), (2, 2), 'SAME'), tl.BatchNorm(mode=mode), tl.Relu(), tl.MaxPool(pool_size=(3, 3), strides=(2, 2)), ConvBlock(3, [d_hidden, d_hidden, 4 * d_hidden], (1, 1), mode=mode), IdentityBlock(3, [d_hidden, d_hidden, 4 * d_hidden], mode=mode), IdentityBlock(3, [d_hidden, d_hidden, 4 * d_hidden], mode=mode), ConvBlock(3, [2 * d_hidden, 2 * d_hidden, 8 * d_hidden], (2, 2), mode=mode), IdentityBlock(3, [2 * d_hidden, 2 * d_hidden, 8 * d_hidden], mode=mode), IdentityBlock(3, [2 * d_hidden, 2 * d_hidden, 8 * d_hidden], mode=mode), IdentityBlock(3, [2 * d_hidden, 2 * d_hidden, 8 * d_hidden], mode=mode), ConvBlock(3, [4 * d_hidden, 4 * d_hidden, 16 * d_hidden], (2, 2), mode=mode), IdentityBlock(3, [4 * d_hidden, 4 * d_hidden, 16 * d_hidden], mode=mode), IdentityBlock(3, [4 * d_hidden, 4 * d_hidden, 16 * d_hidden], mode=mode), IdentityBlock(3, [4 * d_hidden, 4 * d_hidden, 16 * d_hidden], mode=mode), IdentityBlock(3, [4 * d_hidden, 4 * d_hidden, 16 * d_hidden], mode=mode), IdentityBlock(3, [4 * d_hidden, 4 * d_hidden, 16 * d_hidden], mode=mode), ConvBlock(3, [8 * d_hidden, 8 * d_hidden, 32 * d_hidden], (2, 2), mode=mode), IdentityBlock(3, [8 * d_hidden, 8 * d_hidden, 32 * d_hidden], mode=mode), IdentityBlock(3, [8 * d_hidden, 8 * d_hidden, 32 * d_hidden], mode=mode), tl.AvgPool(pool_size=(7, 7)), tl.Flatten(), tl.Dense(n_output_classes), tl.LogSoftmax(), )
def test_forward_shape(self): layer = tl.AvgPool(pool_size=(2, 2), strides=(1, 2)) x = np.ones((11, 6, 4, 17)) y = layer(x) self.assertEqual(y.shape, (11, 5, 2, 17))