def res_block(input,
filters,
kernel_size=(3, 3),
strides=(1, 1),
use_dropout=False):
x = KL.Conv2D(
filters=filters, #使用步长为1的卷积，保持大小不变
kernel_size=kernel_size,
strides=strides,
padding='same')(input)
x = SwitchNormalization()(x)
x = KL.Activation('relu')(x)
if use_dropout: #使用dropout
x = KL.Dropout(0.5)(x)
x = KL.Conv2D(
filters=filters, #再来一次步长为1的卷积
kernel_size=kernel_size,
strides=strides,
padding='same')(x)
x = SwitchNormalization()(x)
#将卷积后的结果与原始输入相加
merged = KL.Add()([input, x]) #残差层
return merged
def test_switchnorm_mode_twice():
# This is a regression test for issue #4881 with the old
# switch normalization functions in the Theano backend.
model = Sequential()
model.add(SwitchNormalization(input_shape=(10, 5, 5), axis=1))
model.add(SwitchNormalization(input_shape=(10, 5, 5), axis=1))
model.compile(loss='mse', optimizer='sgd')
x = np.random.normal(loc=5.0, scale=10.0, size=(20, 10, 5, 5))
model.fit(x, x, epochs=1, verbose=0)
model.predict(x)
def res_block(input,
filters,
kernel_size=(3, 3),
strides=(1, 1),
normlization='instance'):
initializer = tf.random_normal_initializer(0., 0.02)
#使用步长为1的卷积，保持大小不变
x = keras.layers.Conv2D(filters=filters,
kernel_size=kernel_size,
strides=strides,
padding='same',
kernel_initializer=initializer,
use_bias=False)(input)
if normlization == 'instance':
x = tfa.layers.InstanceNormalization(
axis=3,
center=True,
scale=True,
beta_initializer="random_uniform",
gamma_initializer="random_uniform")(x)
else:
x = SwitchNormalization(axis=-1)(x)
x = keras.layers.Activation('relu')(x)
x = keras.layers.Dropout(0.5)(x)
#再来一次步长为1的卷积
x = keras.layers.Conv2D(filters=filters,
kernel_size=kernel_size,
strides=strides,
padding='same',
kernel_initializer=initializer,
use_bias=False)(x)
if normlization == 'instance':
x = tfa.layers.InstanceNormalization(
axis=3,
center=True,
scale=True,
beta_initializer="random_uniform",
gamma_initializer="random_uniform")(x)
else:
x = SwitchNormalization(axis=-1)(x)
#将卷积后的结果与原始输入相加
out = keras.layers.Add()([input, x]) #残差层
return out
def test_shared_switchnorm():
'''Test that a BN layer can be shared
across different data streams.
'''
# Test single layer reuse
bn = SwitchNormalization(input_shape=(10, ))
x1 = Input(shape=(10, ))
bn(x1)
x2 = Input(shape=(10, ))
y2 = bn(x2)
x = np.random.normal(loc=5.0, scale=10.0, size=(2, 10))
model = Model(x2, y2)
assert len(model.updates) == 2
model.compile('sgd', 'mse')
model.train_on_batch(x, x)
# Test model-level reuse
x3 = Input(shape=(10, ))
y3 = model(x3)
new_model = Model(x3, y3)
assert len(model.updates) == 2
new_model.compile('sgd', 'mse')
new_model.train_on_batch(x, x)
def generator_model(image_shape, istrain=True): #构建生成器模型
#构建输入层（与动态图不兼容）
inputs = KL.Input(shape=(image_shape[0], image_shape[1], input_nc))
#步长为1的卷积操作，尺寸不变
x = KL.Conv2D(filters=ngf, kernel_size=(7, 7), padding='same')(inputs)
x = SwitchNormalization()(x)
x = KL.Activation('relu')(x)
n_downsampling = 2
for i in range(n_downsampling): #两次下采样
mult = 2**i
x = KL.Conv2D(filters=ngf * mult * 2,
kernel_size=(3, 3),
strides=2,
padding='same')(x)
x = SwitchNormalization()(x)
x = KL.Activation('relu')(x)
mult = 2**n_downsampling
for i in range(n_blocks_gen): #定义多个残差层
x = res_block(x, ngf * mult, use_dropout=istrain)
for i in range(n_downsampling): #两次上采样
mult = 2**(n_downsampling - i)
x = KL.UpSampling2D()(x)
x = KL.Conv2D(filters=int(ngf * mult / 2),
kernel_size=(3, 3),
padding='same')(x)
x = SwitchNormalization()(x)
x = KL.Activation('relu')(x)
#步长为1的卷积操作
x = KL.Conv2D(filters=output_nc, kernel_size=(7, 7), padding='same')(x)
x = KL.Activation('tanh')(x)
outputs = KL.Add()([x, inputs]) #与最外层的输入完成一次大残差
#防止特征值域过大，进行除2操作（取平均数残差）
outputs = KL.Lambda(lambda z: z / 2)(outputs)
#构建模型
model = KM.Model(inputs=inputs, outputs=outputs, name='Generator')
return model
def test_switchnorm_training_argument():
bn1 = SwitchNormalization(input_shape=(10, ))
x1 = Input(shape=(10, ))
y1 = bn1(x1, training=True)
assert bn1.updates
model1 = Model(x1, y1)
np.random.seed(123)
x = np.random.normal(loc=5.0, scale=10.0, size=(20, 10))
output_a = model1.predict(x)
model1.compile(loss='mse', optimizer='rmsprop')
model1.fit(x, x, epochs=1, verbose=0)
output_b = model1.predict(x)
assert np.abs(np.sum(output_a - output_b)) > 0.1
assert_allclose(output_b.mean(), 0.0, atol=1e-1)
assert_allclose(output_b.std(), 0.8,
atol=1e-1) # due to averaging over 3 norms
bn2 = SwitchNormalization(input_shape=(10, ))
x2 = Input(shape=(10, ))
bn2(x2, training=False)
assert not bn2.updates
def test_switchnorm_correctness_2d():
model = Sequential()
norm = SwitchNormalization(axis=1, input_shape=(10, 6), momentum=0.8)
model.add(norm)
model.compile(loss='mse', optimizer='rmsprop')
# centered on 5.0, variance 10.0
x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 10, 6))
model.fit(x, x, epochs=5, verbose=0)
out = model.predict(x)
out -= np.reshape(K.eval(norm.beta), (1, 10, 1))
out /= np.reshape(K.eval(norm.gamma), (1, 10, 1))
assert_allclose(out.mean(axis=(0, 2)), 0.0, atol=1e-1)
assert_allclose(out.std(axis=(0, 2)), 1.0, atol=1e-1)
def get_model(bn_mean, bn_std):
input = Input(shape=(1, ))
x = SwitchNormalization()(input)
model = Model(input, x)
model.set_weights([
np.array([1.]),
np.array([0.]),
np.array([-1e3, -1e3, 1.0]),
np.array([-1e3, -1e3, 1.0]),
np.array([bn_mean]),
np.array([bn_std**2])
])
return model
def test_that_trainable_disables_updates():
val_a = np.random.random((10, 4))
val_out = np.random.random((10, 4))
a = Input(shape=(4, ))
layer = SwitchNormalization(input_shape=(4, ))
b = layer(a)
model = Model(a, b)
model.trainable = False
assert not model.updates
model.compile('sgd', 'mse')
assert not model.updates
x1 = model.predict(val_a)
model.train_on_batch(val_a, val_out)
x2 = model.predict(val_a)
assert_allclose(x1, x2, atol=1e-7)
model.trainable = True
model.compile('sgd', 'mse')
assert model.updates
model.train_on_batch(val_a, val_out)
x2 = model.predict(val_a)
assert np.abs(np.sum(x1 - x2)) > 1e-5
layer.trainable = False
model.compile('sgd', 'mse')
assert not model.updates
x1 = model.predict(val_a)
model.train_on_batch(val_a, val_out)
x2 = model.predict(val_a)
assert_allclose(x1, x2, atol=1e-7)
def test_switchnorm_convnet():
model = Sequential()
norm = SwitchNormalization(axis=1, input_shape=(3, 4, 4), momentum=0.8)
model.add(norm)
model.compile(loss='mse', optimizer='sgd')
# centered on 5.0, variance 10.0
np.random.seed(123)
x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 3, 4, 4))
model.fit(x, x, epochs=4, verbose=0)
out = model.predict(x)
out -= np.reshape(K.eval(norm.beta), (1, 3, 1, 1))
out /= np.reshape(K.eval(norm.gamma), (1, 3, 1, 1))
assert_allclose(np.mean(out, axis=(0, 2, 3)), 0.0, atol=1e-1)
assert_allclose(np.std(out, axis=(0, 2, 3)), 1.0, atol=1e-1)
def rnn(self):
start_cr_a_fit_net = time.time()
self.split_dataset_rnn()
rnn_model = Sequential()
# RNN层设计
rnn_model.add(
SimpleRNN(15,
input_shape=(None, self.look_back),
return_sequences=True))
rnn_model.add(
SimpleRNN(10,
input_shape=(None, self.look_back),
return_sequences=True))
# SN层
if self.isdropout:
rnn_model.add(SwitchNormalization(axis=-1))
rnn_model.add(
SimpleRNN(15,
input_shape=(None, self.look_back),
return_sequences=True))
rnn_model.add(SimpleRNN(10, input_shape=(None, self.look_back)))
rnn_model.add(Dense(1))
# dropout层
if self.isdropout:
rnn_model.add(GaussianDropout(0.2))
rnn_model.summary()
rnn_model.compile(loss='mean_squared_error', optimizer='adam')
rnn_model.fit(self.x_train,
self.y_train,
epochs=self.epochs,
batch_size=self.batch_size,
verbose=1)
end_cr_a_fit_net = time.time() - start_cr_a_fit_net
print(
'Running time of creating and fitting the RNN network: %.2f Seconds'
% (end_cr_a_fit_net))
# LSTM prediction/LSTM进行预测
trainPredict = rnn_model.predict(
self.x_train) # Predict by training data set/训练集预测
testPredict = rnn_model.predict(
self.x_test) # Predict by test data set/测试集预测
return trainPredict, testPredict, self.y_train, self.y_test
def test_switchnorm_convnet_no_center_no_scale():
model = Sequential()
norm = SwitchNormalization(axis=-1,
center=False,
scale=False,
input_shape=(3, 4, 4),
momentum=0.8)
model.add(norm)
model.compile(loss='mse', optimizer='sgd')
# centered on 5.0, variance 10.0
x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 3, 4, 4))
model.fit(x, x, epochs=4, verbose=0)
out = model.predict(x)
assert_allclose(np.mean(out, axis=(0, 2, 3)), 0.0, atol=1e-1)
assert_allclose(np.std(out, axis=(0, 2, 3)), 1.0, atol=1e-1)
def Generator(normlization='instance'):
initializer = tf.random_normal_initializer(0., 0.02)
inputs = keras.layers.Input(name='blur_image', shape=input_shape)
x = keras.layers.Conv2D(filters=ngf,
kernel_size=(7, 7),
padding='same',
kernel_initializer=initializer,
use_bias=False)(inputs)
if normlization == 'instance':
x = tfa.layers.InstanceNormalization(
axis=3,
center=True,
scale=True,
beta_initializer="random_uniform",
gamma_initializer="random_uniform")(x)
else:
x = SwitchNormalization(axis=-1)(x)
x = keras.layers.Activation('relu')(x)
downsampling = 2
for i in range(downsampling): #两次下采样
mult = 2**i
x = keras.layers.Conv2D(filters=ngf * mult * 2,
kernel_size=(3, 3),
strides=2,
padding='same',
kernel_initializer=initializer,
use_bias=False)(x)
if normlization == 'instance':
x = tfa.layers.InstanceNormalization(
axis=3,
center=True,
scale=True,
beta_initializer="random_uniform",
gamma_initializer="random_uniform")(x)
else:
x = SwitchNormalization(axis=-1)(x)
x = keras.layers.Activation('relu')(x)
res_n = 2**downsampling
for i in range(n_blocks_gen): #定义多个残差层
x = res_block(x, ngf * res_n, normlization=normlization)
for i in range(downsampling): #两次上采样
n_f = 2**(downsampling - i)
x = keras.layers.Conv2DTranspose(filters=int(ngf * n_f),
kernel_size=(3, 3),
strides=2,
padding='same',
kernel_initializer=initializer,
use_bias=False)(x)
# x = keras.layers.UpSampling2D()(x)
x = keras.layers.Conv2D(filters=int(ngf * n_f / 2),
kernel_size=(3, 3),
padding='same',
kernel_initializer=initializer,
use_bias=False)(x)
if normlization == 'instance':
x = tfa.layers.InstanceNormalization(
axis=3,
center=True,
scale=True,
beta_initializer="random_uniform",
gamma_initializer="random_uniform")(x)
else:
x = SwitchNormalization(axis=-1)(x)
x = keras.layers.Activation('relu')(x)
#步长为1的卷积操作
x = keras.layers.Conv2D(filters=output_nc,
kernel_size=(7, 7),
padding='same',
kernel_initializer=initializer,
use_bias=False)(x)
x = keras.layers.Activation('tanh')(x)
outputs = keras.layers.Add()([x, inputs]) #与最外层的输入完成一次大残差
#防止特征值域过大，进行除2操作（取平均数残差）
outputs = keras.layers.Lambda(lambda z: z / 2.0)(outputs)
#构建模型
model = keras.Model(inputs=inputs, outputs=outputs, name='Generator')
return model
img_rows, img_cols = 32, 32
img_channels = 3
img_dim = (img_channels, img_rows,
img_cols) if K.image_dim_ordering() == "th" else (img_rows,
img_cols,
img_channels)
ip = Input(shape=img_dim)
# 0.98 is stable, whereas 0.99 is unstable for this model.
# Stability here comes at the cost of slight performance.
x = Conv2D(64, (3, 3), padding='same', kernel_initializer='he_normal')(ip)
x = SwitchNormalization(axis=-1, momentum=momentum)(x)
x = Activation('relu')(x)
x = Conv2D(64, (3, 3), padding='same', kernel_initializer='he_normal')(x)
x = SwitchNormalization(axis=-1, momentum=momentum)(x)
x = Activation('relu')(x)
x = Conv2D(128, (3, 3),
padding='same',
kernel_initializer='he_normal',
strides=(2, 2))(x)
x = SwitchNormalization(axis=-1, momentum=momentum)(x)
x = Activation('relu')(x)
x = Conv2D(128, (3, 3), padding='same', kernel_initializer='he_normal')(x)
x = SwitchNormalization(axis=-1, momentum=momentum)(x)
nb_classes = 10
img_rows, img_cols = 32, 32
img_channels = 3
img_dim = (img_channels, img_rows,
img_cols) if K.image_dim_ordering() == "th" else (img_rows,
img_cols,
img_channels)
bottleneck = False
width = 1
weight_decay = 1e-4
ip = Input(shape=img_dim)
x = Conv2D(64, (3, 3), padding='same', kernel_initializer='he_normal')(ip)
x = SwitchNormalization(axis=-1, momentum=0.98)(x)
x = Activation('relu')(x)
x = Conv2D(64, (3, 3), padding='same', kernel_initializer='he_normal')(x)
x = SwitchNormalization(axis=-1, momentum=0.98)(x)
x = Activation('relu')(x)
x = Conv2D(128, (3, 3),
padding='same',
kernel_initializer='he_normal',
strides=(2, 2))(x)
x = SwitchNormalization(axis=-1, momentum=0.98)(x)
x = Activation('relu')(x)
x = Conv2D(128, (3, 3), padding='same', kernel_initializer='he_normal')(x)
x = SwitchNormalization(axis=-1, momentum=0.98)(x)