def identity_block(X, filters, stage, block, s):
conv_name_base = "res" + str(stage) + block + "_branch"
bn_name_base = "bn" + str(stage) + block + "_branch"
F1, F2 = filters
X_shortcut = X
X = Conv2D(out_channels=F1,
kernel_size=(3, 3),
stride=(s, s),
padding=1,
name=conv_name_base + "2a")(X)
X = BatchNormalization()(X)
X = Activation("relu")(X)
X = Conv2D(out_channels=F2,
kernel_size=(3, 3),
stride=(1, 1),
padding=1,
name=conv_name_base + "2b")(X)
X = BatchNormalization()(X)
X = Add()([X, X_shortcut])
X = Activation("relu")(X)
return X
def conv_block(X, filters, stage, block, s=2):
global i
conv_name_base = "res" + str(stage) + block + "_branch"
bn_name_base = "bn" + str(stage) + block + "_branch"
F1, F2, F3 = filters
X_shortcut = X
# 1
X = Conv2D(out_channels=F1,
kernel_size=(1, 1),
stride=(s, s),
padding=0,
name=i)(X)
i += 1
X = BatchNormalization(name=i)(X)
i += 1
X = Activation('relu', name=i)(X)
i += 1
# 2
X = Conv2D(out_channels=F2,
kernel_size=(3, 3),
stride=(1, 1),
padding=1,
name=i)(X)
i += 1
X = BatchNormalization(name=i)(X)
i += 1
X = Activation('relu', name=i)(X)
i += 1
# 3
X = Conv2D(out_channels=F3,
kernel_size=(1, 1),
stride=(1, 1),
padding=0,
name=i)(X)
i += 1
X = BatchNormalization(name=i)(X)
i += 1
X_shortcut = Conv2D(out_channels=F3,
kernel_size=(1, 1),
stride=(s, s),
name=i)(X_shortcut)
i += 1
X_shortcut = BatchNormalization(name=i)(X_shortcut)
i += 1
X = Add(name=i)([X, X_shortcut])
i += 1
X = Activation('relu', name=i)(X)
i += 1
return X
def __init__(self, in_channels, out_channels, stride):
super().__init__()
self.conv1 = nn.Sequential(
Conv2D(out_channels, 3, stride=stride, padding=1),
BatchNormalization(),
ReLU(True),
Conv2D(out_channels, 3, stride=1, padding=1),
BatchNormalization(),
)
if stride != 1 or in_channels != out_channels:
self.shortcut = nn.Sequential(
Conv2D(out_channels, 1, stride=stride), BatchNormalization())
else:
self.shortcut = nn.Sequential()
self.relu = ReLU(inplace=True)
def ResNet50(input_shape=(3, 64, 64), classes=100):
global i
X_input = Input(input_shape, name=i)
i += 1
# 进行填充
X = ZeroPadding2D((3, 3), name=i)(X_input)
i += 1
# stage 1
# 卷积
X = Conv2D(64, (7, 7), stride=(2, 2), name=i)(X)
i += 1
# 归一化
X = BatchNormalization(name=i)(X)
i += 1
# relu
X = Activation('relu', name=i)(X)
i += 1
# 最大池化
X = MaxPooling2D(3, 2, name=i)(X)
i += 1
# stage 2
X = conv_block(X, filters=[64, 64, 256], stage=2, block='a', s=1)
X = ide_block(X, filters=[64, 64, 256], stage=2, block='b')
X = ide_block(X, filters=[64, 64, 256], stage=2, block='c')
# stage 3
X = conv_block(X, filters=[128, 128, 512], stage=3, block='a', s=2)
X = ide_block(X, filters=[128, 128, 512], stage=3, block='b')
X = ide_block(X, filters=[128, 128, 512], stage=3, block='c')
X = ide_block(X, filters=[128, 128, 512], stage=3, block='d')
# stage 4
X = conv_block(X, filters=[256, 256, 1024], stage=4, block='a', s=2)
X = ide_block(X, filters=[256, 256, 1024], stage=4, block='b')
X = ide_block(X, filters=[256, 256, 1024], stage=4, block='c')
X = ide_block(X, filters=[256, 256, 1024], stage=4, block='d')
X = ide_block(X, filters=[256, 256, 1024], stage=4, block='e')
X = ide_block(X, filters=[256, 256, 1024], stage=4, block='f')
# stage 5
X = conv_block(X, filters=[512, 512, 2048], stage=5, block='a', s=2)
X = ide_block(X, filters=[512, 512, 2048], stage=5, block='b')
X = ide_block(X, filters=[512, 512, 2048], stage=5, block='c')
# 均值池化
X = AvgPooling2D(2, 1, name=i)(X)
i += 1
# 全连接层
X = Flatten(name=i)(X)
i += 1
X = Dense(classes, name=i)(X)
i += 1
model = Model(inputs=X_input, outputs=X)
return model
def ConvNet(n_classes=10):
return nn.Sequential(
Reshape((1, 28, 28), input_shape=(784, )),
Conv2D(8, 3),
BatchNormalization(),
ReLU(True),
MaxPooling2D(),
Conv2D(16, 3),
BatchNormalization(),
ReLU(True),
MaxPooling2D(),
Conv2D(32, 3),
BatchNormalization(),
ReLU(True),
Flatten(),
# Dense(100, activation='relu'),
Dense(100),
ReLU(True),
Dense(n_classes))
def __init__(self):
super().__init__()
self.conv1 = nn.Sequential(
Conv2D(out_channels=64, kernel_size=7, stride=2, padding=3),
BatchNormalization(), ReLU(inplace=True))
self.pool1 = MaxPooling2D(kernel_size=3, stride=2, padding=1)
self.in_channels = 64
self.layer1 = self.make_layer(BasicBlock, 64, 2, 1)
self.layer2 = self.make_layer(BasicBlock, 128, 2, 2)
self.layer3 = self.make_layer(BasicBlock, 256, 2, 2)
self.layer4 = self.make_layer(BasicBlock, 512, 2, 2)
self.pool2 = AvgPooling2D(2)
self.flat = Flatten()
self.fc = Dense(100)
def ResNet18(input_shape=(3, 56, 56), classes=100):
X_input = Input(input_shape)
# stage1
X = Conv2D(out_channels=64,
kernel_size=(7, 7),
stride=(2, 2),
name="conv1",
padding=3)(X_input)
X = BatchNormalization(name="bn1")(X)
X = Activation("relu")(X)
X = MaxPooling2D(kernel_size=3, stride=2, padding=1)(X)
# stage2
X = identity_block(X, filters=[64, 64], stage=2, block="b", s=1)
X = identity_block(X, filters=[64, 64], stage=2, block="c", s=1)
# stage3
X = convolutional_block(X, filters=[128, 128], stage=3, block="a", s=2)
X = identity_block(X, filters=[128, 128], stage=3, block="b", s=1)
# stage4
X = convolutional_block(X, filters=[256, 256], stage=4, block="a", s=2)
X = identity_block(X, filters=[256, 256], stage=4, block="b", s=1)
# stage5
X = convolutional_block(X, filters=[512, 512], stage=5, block="a", s=2)
X = identity_block(X, filters=[512, 512], stage=5, block="b", s=1)
X = AvgPooling2D(2)(X)
X = Flatten()(X)
X = Dense(
classes,
name="fc" + str(classes),
)(X)
model = Model(inputs=X_input, outputs=X)
return model
def xs_network(weight1, bias1, bn_weight1, bn_bias1, weight2, bias2):
weight1 = xs.nn.Parameter(weight1)
bias1 = xs.nn.Parameter(bias1)
bn_weight1 = xs.nn.Parameter(bn_weight1)
bn_bias1 = xs.nn.Parameter(bn_bias1)
weight2 = xs.nn.Parameter(weight2)
bias2 = xs.nn.Parameter(bias2)
l1 = Conv2D(out_channels=3, kernel_size=3)
bn1 = BatchNormalization(epsilon=1e-5, momentum=0.9)
a1 = ReLU()
l2 = Conv2D(out_channels=5, kernel_size=3)
l1.parameters([weight1, bias1])
l2.parameters([weight2, bias2])
bn1.parameters([bn_weight1, bn_bias1])
bn1.moving_mean = xs.zeros(3)
bn1.moving_variance = xs.ones(3)
net = xs.nn.Sequential(l1, bn1, a1, l2)
return net
from xs.layers import BatchNormalization, Flatten
from xs.utils.toolkit import gradient_check
# set seed
xs.manual_seed_all(0)
# random generate data
x = xs.randn(1, 3, 5, 5, requires_grad=True)
y = xs.randn(1, 75)
# set criterion
criterion = nn.MSELoss()
bn = nn.Sequential(
# declare BatchNormalization layer
BatchNormalization(),
Flatten())
out = bn(x)
loss = criterion(out, y)
loss.backward()
# get weight, bias from convolutional layer
gamma, beta = bn.parameters()
print("1.====================> BN Backward")
print("weight grad: \n", gamma.grad)
print("bias grad: \n", beta.grad)
print("2.====================> Gradient Check")
mathematical_weight_grad, mathematical_bias_grad = gradient_check(
x, y, bn, criterion)