def __init__(self, in_planes, planes, stride=1, p=0.2, keep_variance_fn=None): super(Bottleneck, self).__init__() self.keep_variance_fn = keep_variance_fn self.conv1 = adf.Conv2d(in_planes, planes, kernel_size=1, bias=False, keep_variance_fn=self.keep_variance_fn) self.bn1 = adf.BatchNorm2d(planes, keep_variance_fn=self.keep_variance_fn) self.conv2 = adf.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False, keep_variance_fn=self.keep_variance_fn) self.bn2 = adf.BatchNorm2d(planes, keep_variance_fn=self.keep_variance_fn) self.conv3 = adf.Conv2d(planes, self.expansion * planes, kernel_size=1, bias=False, keep_variance_fn=self.keep_variance_fn) self.bn3 = adf.BatchNorm2d(self.expansion * planes, keep_variance_fn=self.keep_variance_fn) self.ReLU = adf.ReLU(keep_variance_fn=self.keep_variance_fn) self.shortcut = adf.Sequential() if stride != 1 or in_planes != self.expansion * planes: self.shortcut = adf.Sequential( adf.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False, keep_variance_fn=self.keep_variance_fn), adf.BatchNorm2d(self.expansion * planes, keep_variance_fn=self.keep_variance_fn)) self.dropout = adf.Dropout(p=p, keep_variance_fn=self.keep_variance_fn)
def make_conv(inchannels, outchannels, kernel_size, stride, nonlinear=True, keep_variance_fn=None): padding = kernel_size // 2 if nonlinear: return adf.Sequential( adf.Conv2d(inchannels, outchannels, kernel_size=kernel_size, padding=padding, stride=stride, bias=True, keep_variance_fn=keep_variance_fn), adf.ReLU(keep_variance_fn=keep_variance_fn)) else: return adf.Conv2d(inchannels, outchannels, kernel_size=kernel_size, padding=padding, stride=stride, bias=True, keep_variance_fn=keep_variance_fn)
def deconv(in_planes, out_planes, kernel_size, stride, pad, nonlinear, bias, keep_variance_fn=None): if nonlinear: return adf.Sequential( adf.ConvTranspose2d( in_planes, out_planes, kernel_size=kernel_size, stride=stride, padding=pad, bias=bias, keep_variance_fn=keep_variance_fn), adf.LeakyReLU(0.1, keep_variance_fn=keep_variance_fn) ) else: return adf.ConvTranspose2d( in_planes, out_planes, kernel_size=kernel_size, stride=stride, padding=pad, bias=bias, keep_variance_fn=keep_variance_fn)
def _make_layer(self, block, planes, num_blocks, stride, keep_variance_fn=None): strides = [stride] + [1] * (num_blocks - 1) layers = [] for stride in strides: layers.append( block(self.in_planes, planes, stride, keep_variance_fn=self.keep_variance_fn)) self.in_planes = planes * block.expansion return adf.Sequential(*layers)
def __init__(self, img_channels, output_dim, noise_variance=1e-3, min_variance=1e-3, initialize_msra=False): super(Resnet8_MCDO_adf, self).__init__() p = FLAGS.dropout self._keep_variance_fn = lambda x: keep_variance(x, min_variance=min_variance) self._noise_variance = noise_variance self.layer1 = adf.Sequential( adf.Conv2d( in_channels=img_channels, out_channels=32, kernel_size=5, padding=5//2, stride=2, bias=True, keep_variance_fn=self._keep_variance_fn), adf.Dropout(p, keep_variance_fn=self._keep_variance_fn), adf.MaxPool2d(keep_variance_fn=self._keep_variance_fn)) self.residual_block_1a = adf.Sequential( adf.BatchNorm2d(32), adf.ReLU(), adf.Conv2d( in_channels=32, out_channels=32, kernel_size=3, padding=3//2, stride=2, bias=True, keep_variance_fn=self._keep_variance_fn), adf.Dropout(p, keep_variance_fn=self._keep_variance_fn), adf.BatchNorm2d(32), adf.ReLU(), adf.Conv2d( in_channels=32, out_channels=32, kernel_size=3, padding=3//2, bias=True, keep_variance_fn=self._keep_variance_fn), adf.Dropout(p, keep_variance_fn=self._keep_variance_fn)) self.parallel_conv_1 = adf.Sequential( adf.Conv2d( in_channels=32, out_channels=32, kernel_size=1, padding=1//2, stride=2, bias=True, keep_variance_fn=self._keep_variance_fn), adf.Dropout(p, keep_variance_fn=self._keep_variance_fn)) self.residual_block_2a = adf.Sequential( adf.BatchNorm2d(32), adf.ReLU(), adf.Conv2d( in_channels=32, out_channels=64, kernel_size=3, padding=3//2, stride=2, bias=True, keep_variance_fn=self._keep_variance_fn), adf.Dropout(p, keep_variance_fn=self._keep_variance_fn), adf.BatchNorm2d(64), adf.ReLU(), adf.Conv2d( in_channels=64, out_channels=64, kernel_size=3, padding=3//2, bias=True, keep_variance_fn=self._keep_variance_fn), adf.Dropout(p, keep_variance_fn=self._keep_variance_fn)) self.parallel_conv_2 = adf.Sequential( adf.Conv2d( in_channels=32, out_channels=64, kernel_size=1, padding=1//2, stride=2, bias=True, keep_variance_fn=self._keep_variance_fn), adf.Dropout(p, keep_variance_fn=self._keep_variance_fn)) self.residual_block_3a = adf.Sequential( adf.BatchNorm2d(64), adf.ReLU(), adf.Conv2d( in_channels=64, out_channels=128, kernel_size=3, padding=3//2, stride=2, bias=True, keep_variance_fn=self._keep_variance_fn), adf.Dropout(p, keep_variance_fn=self._keep_variance_fn), adf.BatchNorm2d(128), adf.ReLU(), adf.Conv2d( in_channels=128, out_channels=128, kernel_size=3, padding=3//2, bias=True, keep_variance_fn=self._keep_variance_fn), adf.Dropout(p, keep_variance_fn=self._keep_variance_fn)) self.parallel_conv_3 = adf.Sequential( adf.Conv2d( in_channels=64, out_channels=128, kernel_size=1, padding=1//2, stride=2, bias=True, keep_variance_fn=self._keep_variance_fn), adf.Dropout(p, keep_variance_fn=self._keep_variance_fn)) self.output_dim = output_dim self.last_block = adf.Sequential( adf.ReLU(), adf.Linear(6272,self.output_dim)) # Initialize layers exactly as in Keras for layer in self.modules(): if isinstance(layer, adf.Conv2d) or isinstance(layer, adf.Linear): # convolution: bias=0, weight=msra nn.init.xavier_uniform_(layer.weight, gain=nn.init.calculate_gain('relu')) if layer.bias is not None: nn.init.constant_(layer.bias, 0) elif isinstance(layer, adf.BatchNorm2d): nn.init.constant_(layer.weight, 1) nn.init.constant_(layer.bias, 0)