def __init__(self,
args,
noise_variance=1e-3,
min_variance=1e-3,
initialize_msra=False):
super(LeNetADF, self).__init__()
self._keep_variance_fn = lambda x: keep_variance(
x, min_variance=min_variance)
self._noise_variance = noise_variance
self.conv1 = adf.Conv2d(1,
32,
kernel_size=5,
keep_variance_fn=self._keep_variance_fn)
self.relu1 = adf.ReLU(keep_variance_fn=self._keep_variance_fn)
self.maxpool1 = adf.MaxPool2d(keep_variance_fn=self._keep_variance_fn)
self.conv2 = adf.Conv2d(32,
64,
kernel_size=5,
keep_variance_fn=self._keep_variance_fn)
self.relu2 = adf.ReLU(keep_variance_fn=self._keep_variance_fn)
self.maxpool2 = adf.MaxPool2d(keep_variance_fn=self._keep_variance_fn)
self.fc1 = adf.Linear(1024,
1024,
keep_variance_fn=self._keep_variance_fn)
self.fc2 = adf.Linear(1024,
10,
keep_variance_fn=self._keep_variance_fn)
self.relu3 = adf.ReLU(keep_variance_fn=self._keep_variance_fn)
if initialize_msra:
finitialize_msra(self.modules())
# finitialize_msra([self.fc2], small=True)
else:
finitialize_xavier(self.modules())
def make_conv(inchannels,
outchannels,
kernel_size,
stride,
nonlinear=True,
keep_variance_fn=None):
padding = kernel_size // 2
if nonlinear:
return adf.Sequential(
adf.Conv2d(inchannels,
outchannels,
kernel_size=kernel_size,
padding=padding,
stride=stride,
bias=True,
keep_variance_fn=keep_variance_fn),
adf.ReLU(keep_variance_fn=keep_variance_fn))
else:
return adf.Conv2d(inchannels,
outchannels,
kernel_size=kernel_size,
padding=padding,
stride=stride,
bias=True,
keep_variance_fn=keep_variance_fn)
def conv(in_planes, out_planes, kernel_size, stride, pad, nonlinear, bias, keep_variance_fn=None):
if nonlinear:
return adf.Sequential(
adf.Conv2d(
in_planes, out_planes, kernel_size=kernel_size,
stride=stride, padding=pad, bias=bias, keep_variance_fn=keep_variance_fn),
adf.LeakyReLU(0.1, keep_variance_fn=keep_variance_fn)
)
else:
return adf.Conv2d(
in_planes, out_planes, kernel_size=kernel_size, stride=stride, padding=pad, bias=bias,
keep_variance_fn=keep_variance_fn)
def __init__(self,
in_planes,
planes,
stride=1,
p=0.2,
keep_variance_fn=None):
super(Bottleneck, self).__init__()
self.keep_variance_fn = keep_variance_fn
self.conv1 = adf.Conv2d(in_planes,
planes,
kernel_size=1,
bias=False,
keep_variance_fn=self.keep_variance_fn)
self.bn1 = adf.BatchNorm2d(planes,
keep_variance_fn=self.keep_variance_fn)
self.conv2 = adf.Conv2d(planes,
planes,
kernel_size=3,
stride=stride,
padding=1,
bias=False,
keep_variance_fn=self.keep_variance_fn)
self.bn2 = adf.BatchNorm2d(planes,
keep_variance_fn=self.keep_variance_fn)
self.conv3 = adf.Conv2d(planes,
self.expansion * planes,
kernel_size=1,
bias=False,
keep_variance_fn=self.keep_variance_fn)
self.bn3 = adf.BatchNorm2d(self.expansion * planes,
keep_variance_fn=self.keep_variance_fn)
self.ReLU = adf.ReLU(keep_variance_fn=self.keep_variance_fn)
self.shortcut = adf.Sequential()
if stride != 1 or in_planes != self.expansion * planes:
self.shortcut = adf.Sequential(
adf.Conv2d(in_planes,
self.expansion * planes,
kernel_size=1,
stride=stride,
bias=False,
keep_variance_fn=self.keep_variance_fn),
adf.BatchNorm2d(self.expansion * planes,
keep_variance_fn=self.keep_variance_fn))
self.dropout = adf.Dropout(p=p, keep_variance_fn=self.keep_variance_fn)
def __init__(self,
block,
num_blocks,
num_classes=10,
p=0.2,
noise_variance=1e-3,
min_variance=1e-3,
initialize_msra=False):
super(ResNetADFDropout, self).__init__()
self.keep_variance_fn = lambda x: keep_variance(
x, min_variance=min_variance)
self._noise_variance = noise_variance
self.in_planes = 64
self.conv1 = adf.Conv2d(3,
64,
kernel_size=3,
stride=1,
padding=1,
bias=False,
keep_variance_fn=self.keep_variance_fn)
self.bn1 = adf.BatchNorm2d(64, keep_variance_fn=self.keep_variance_fn)
self.ReLU = adf.ReLU(keep_variance_fn=self.keep_variance_fn)
self.layer1 = self._make_layer(block,
64,
num_blocks[0],
stride=1,
p=p,
keep_variance_fn=self.keep_variance_fn)
self.layer2 = self._make_layer(block,
128,
num_blocks[1],
stride=2,
p=p,
keep_variance_fn=self.keep_variance_fn)
self.layer3 = self._make_layer(block,
256,
num_blocks[2],
stride=2,
p=p,
keep_variance_fn=self.keep_variance_fn)
self.layer4 = self._make_layer(block,
512,
num_blocks[3],
stride=2,
p=p,
keep_variance_fn=self.keep_variance_fn)
self.linear = adf.Linear(512 * block.expansion,
num_classes,
keep_variance_fn=self.keep_variance_fn)
self.AvgPool2d = adf.AvgPool2d(keep_variance_fn=self.keep_variance_fn)
self.dropout = adf.Dropout(p=p, keep_variance_fn=self.keep_variance_fn)
def __init__(self, img_channels, output_dim, noise_variance=1e-3, min_variance=1e-3, initialize_msra=False):
super(Resnet8_MCDO_adf, self).__init__()
p = FLAGS.dropout
self._keep_variance_fn = lambda x: keep_variance(x, min_variance=min_variance)
self._noise_variance = noise_variance
self.layer1 = adf.Sequential(
adf.Conv2d(
in_channels=img_channels, out_channels=32, kernel_size=5, padding=5//2,
stride=2, bias=True, keep_variance_fn=self._keep_variance_fn),
adf.Dropout(p, keep_variance_fn=self._keep_variance_fn),
adf.MaxPool2d(keep_variance_fn=self._keep_variance_fn))
self.residual_block_1a = adf.Sequential(
adf.BatchNorm2d(32),
adf.ReLU(),
adf.Conv2d(
in_channels=32, out_channels=32, kernel_size=3, padding=3//2,
stride=2, bias=True, keep_variance_fn=self._keep_variance_fn),
adf.Dropout(p, keep_variance_fn=self._keep_variance_fn),
adf.BatchNorm2d(32),
adf.ReLU(),
adf.Conv2d(
in_channels=32, out_channels=32, kernel_size=3, padding=3//2,
bias=True, keep_variance_fn=self._keep_variance_fn),
adf.Dropout(p, keep_variance_fn=self._keep_variance_fn))
self.parallel_conv_1 = adf.Sequential(
adf.Conv2d(
in_channels=32, out_channels=32, kernel_size=1, padding=1//2,
stride=2, bias=True, keep_variance_fn=self._keep_variance_fn),
adf.Dropout(p, keep_variance_fn=self._keep_variance_fn))
self.residual_block_2a = adf.Sequential(
adf.BatchNorm2d(32),
adf.ReLU(),
adf.Conv2d(
in_channels=32, out_channels=64, kernel_size=3, padding=3//2,
stride=2, bias=True, keep_variance_fn=self._keep_variance_fn),
adf.Dropout(p, keep_variance_fn=self._keep_variance_fn),
adf.BatchNorm2d(64),
adf.ReLU(),
adf.Conv2d(
in_channels=64, out_channels=64, kernel_size=3, padding=3//2,
bias=True, keep_variance_fn=self._keep_variance_fn),
adf.Dropout(p, keep_variance_fn=self._keep_variance_fn))
self.parallel_conv_2 = adf.Sequential(
adf.Conv2d(
in_channels=32, out_channels=64, kernel_size=1, padding=1//2,
stride=2, bias=True, keep_variance_fn=self._keep_variance_fn),
adf.Dropout(p, keep_variance_fn=self._keep_variance_fn))
self.residual_block_3a = adf.Sequential(
adf.BatchNorm2d(64),
adf.ReLU(),
adf.Conv2d(
in_channels=64, out_channels=128, kernel_size=3, padding=3//2,
stride=2, bias=True, keep_variance_fn=self._keep_variance_fn),
adf.Dropout(p, keep_variance_fn=self._keep_variance_fn),
adf.BatchNorm2d(128),
adf.ReLU(),
adf.Conv2d(
in_channels=128, out_channels=128, kernel_size=3, padding=3//2,
bias=True, keep_variance_fn=self._keep_variance_fn),
adf.Dropout(p, keep_variance_fn=self._keep_variance_fn))
self.parallel_conv_3 = adf.Sequential(
adf.Conv2d(
in_channels=64, out_channels=128, kernel_size=1, padding=1//2,
stride=2, bias=True, keep_variance_fn=self._keep_variance_fn),
adf.Dropout(p, keep_variance_fn=self._keep_variance_fn))
self.output_dim = output_dim
self.last_block = adf.Sequential(
adf.ReLU(),
adf.Linear(6272,self.output_dim))
# Initialize layers exactly as in Keras
for layer in self.modules():
if isinstance(layer, adf.Conv2d) or isinstance(layer, adf.Linear): # convolution: bias=0, weight=msra
nn.init.xavier_uniform_(layer.weight, gain=nn.init.calculate_gain('relu'))
if layer.bias is not None:
nn.init.constant_(layer.bias, 0)
elif isinstance(layer, adf.BatchNorm2d):
nn.init.constant_(layer.weight, 1)
nn.init.constant_(layer.bias, 0)
