def __init__(self,
in_planes,
planes,
stride=1,
p=0.2,
keep_variance_fn=None):
super(Bottleneck, self).__init__()
self.keep_variance_fn = keep_variance_fn
self.conv1 = adf.Conv2d(in_planes,
planes,
kernel_size=1,
bias=False,
keep_variance_fn=self.keep_variance_fn)
self.bn1 = adf.BatchNorm2d(planes,
keep_variance_fn=self.keep_variance_fn)
self.conv2 = adf.Conv2d(planes,
planes,
kernel_size=3,
stride=stride,
padding=1,
bias=False,
keep_variance_fn=self.keep_variance_fn)
self.bn2 = adf.BatchNorm2d(planes,
keep_variance_fn=self.keep_variance_fn)
self.conv3 = adf.Conv2d(planes,
self.expansion * planes,
kernel_size=1,
bias=False,
keep_variance_fn=self.keep_variance_fn)
self.bn3 = adf.BatchNorm2d(self.expansion * planes,
keep_variance_fn=self.keep_variance_fn)
self.ReLU = adf.ReLU(keep_variance_fn=self.keep_variance_fn)
self.shortcut = adf.Sequential()
if stride != 1 or in_planes != self.expansion * planes:
self.shortcut = adf.Sequential(
adf.Conv2d(in_planes,
self.expansion * planes,
kernel_size=1,
stride=stride,
bias=False,
keep_variance_fn=self.keep_variance_fn),
adf.BatchNorm2d(self.expansion * planes,
keep_variance_fn=self.keep_variance_fn))
self.dropout = adf.Dropout(p=p, keep_variance_fn=self.keep_variance_fn)
def make_conv(inchannels,
outchannels,
kernel_size,
stride,
nonlinear=True,
keep_variance_fn=None):
padding = kernel_size // 2
if nonlinear:
return adf.Sequential(
adf.Conv2d(inchannels,
outchannels,
kernel_size=kernel_size,
padding=padding,
stride=stride,
bias=True,
keep_variance_fn=keep_variance_fn),
adf.ReLU(keep_variance_fn=keep_variance_fn))
else:
return adf.Conv2d(inchannels,
outchannels,
kernel_size=kernel_size,
padding=padding,
stride=stride,
bias=True,
keep_variance_fn=keep_variance_fn)
def deconv(in_planes, out_planes, kernel_size, stride, pad, nonlinear, bias, keep_variance_fn=None):
if nonlinear:
return adf.Sequential(
adf.ConvTranspose2d(
in_planes, out_planes, kernel_size=kernel_size,
stride=stride, padding=pad, bias=bias, keep_variance_fn=keep_variance_fn),
adf.LeakyReLU(0.1, keep_variance_fn=keep_variance_fn)
)
else:
return adf.ConvTranspose2d(
in_planes, out_planes, kernel_size=kernel_size, stride=stride, padding=pad, bias=bias,
keep_variance_fn=keep_variance_fn)
def _make_layer(self,
block,
planes,
num_blocks,
stride,
keep_variance_fn=None):
strides = [stride] + [1] * (num_blocks - 1)
layers = []
for stride in strides:
layers.append(
block(self.in_planes,
planes,
stride,
keep_variance_fn=self.keep_variance_fn))
self.in_planes = planes * block.expansion
return adf.Sequential(*layers)
def __init__(self, img_channels, output_dim, noise_variance=1e-3, min_variance=1e-3, initialize_msra=False):
super(Resnet8_MCDO_adf, self).__init__()
p = FLAGS.dropout
self._keep_variance_fn = lambda x: keep_variance(x, min_variance=min_variance)
self._noise_variance = noise_variance
self.layer1 = adf.Sequential(
adf.Conv2d(
in_channels=img_channels, out_channels=32, kernel_size=5, padding=5//2,
stride=2, bias=True, keep_variance_fn=self._keep_variance_fn),
adf.Dropout(p, keep_variance_fn=self._keep_variance_fn),
adf.MaxPool2d(keep_variance_fn=self._keep_variance_fn))
self.residual_block_1a = adf.Sequential(
adf.BatchNorm2d(32),
adf.ReLU(),
adf.Conv2d(
in_channels=32, out_channels=32, kernel_size=3, padding=3//2,
stride=2, bias=True, keep_variance_fn=self._keep_variance_fn),
adf.Dropout(p, keep_variance_fn=self._keep_variance_fn),
adf.BatchNorm2d(32),
adf.ReLU(),
adf.Conv2d(
in_channels=32, out_channels=32, kernel_size=3, padding=3//2,
bias=True, keep_variance_fn=self._keep_variance_fn),
adf.Dropout(p, keep_variance_fn=self._keep_variance_fn))
self.parallel_conv_1 = adf.Sequential(
adf.Conv2d(
in_channels=32, out_channels=32, kernel_size=1, padding=1//2,
stride=2, bias=True, keep_variance_fn=self._keep_variance_fn),
adf.Dropout(p, keep_variance_fn=self._keep_variance_fn))
self.residual_block_2a = adf.Sequential(
adf.BatchNorm2d(32),
adf.ReLU(),
adf.Conv2d(
in_channels=32, out_channels=64, kernel_size=3, padding=3//2,
stride=2, bias=True, keep_variance_fn=self._keep_variance_fn),
adf.Dropout(p, keep_variance_fn=self._keep_variance_fn),
adf.BatchNorm2d(64),
adf.ReLU(),
adf.Conv2d(
in_channels=64, out_channels=64, kernel_size=3, padding=3//2,
bias=True, keep_variance_fn=self._keep_variance_fn),
adf.Dropout(p, keep_variance_fn=self._keep_variance_fn))
self.parallel_conv_2 = adf.Sequential(
adf.Conv2d(
in_channels=32, out_channels=64, kernel_size=1, padding=1//2,
stride=2, bias=True, keep_variance_fn=self._keep_variance_fn),
adf.Dropout(p, keep_variance_fn=self._keep_variance_fn))
self.residual_block_3a = adf.Sequential(
adf.BatchNorm2d(64),
adf.ReLU(),
adf.Conv2d(
in_channels=64, out_channels=128, kernel_size=3, padding=3//2,
stride=2, bias=True, keep_variance_fn=self._keep_variance_fn),
adf.Dropout(p, keep_variance_fn=self._keep_variance_fn),
adf.BatchNorm2d(128),
adf.ReLU(),
adf.Conv2d(
in_channels=128, out_channels=128, kernel_size=3, padding=3//2,
bias=True, keep_variance_fn=self._keep_variance_fn),
adf.Dropout(p, keep_variance_fn=self._keep_variance_fn))
self.parallel_conv_3 = adf.Sequential(
adf.Conv2d(
in_channels=64, out_channels=128, kernel_size=1, padding=1//2,
stride=2, bias=True, keep_variance_fn=self._keep_variance_fn),
adf.Dropout(p, keep_variance_fn=self._keep_variance_fn))
self.output_dim = output_dim
self.last_block = adf.Sequential(
adf.ReLU(),
adf.Linear(6272,self.output_dim))
# Initialize layers exactly as in Keras
for layer in self.modules():
if isinstance(layer, adf.Conv2d) or isinstance(layer, adf.Linear): # convolution: bias=0, weight=msra
nn.init.xavier_uniform_(layer.weight, gain=nn.init.calculate_gain('relu'))
if layer.bias is not None:
nn.init.constant_(layer.bias, 0)
elif isinstance(layer, adf.BatchNorm2d):
nn.init.constant_(layer.weight, 1)
nn.init.constant_(layer.bias, 0)