def _build_attention(self, rank):
"""Builds multi-head dot-product attention computations.
This function builds attributes necessary for `_compute_attention` to
costomize attention computation to replace the default dot-product
attention.
Args:
rank: the rank of query, key, value tensors.
"""
if self._attention_axes is None:
self._attention_axes = tuple(range(1, rank - 2))
else:
self._attention_axes = tuple(self._attention_axes)
self._dot_product_equation, self._combine_equation, attn_scores_rank = (
_build_attention_equation(rank, attn_axes=self._attention_axes))
norm_axes = tuple(
range(attn_scores_rank - len(self._attention_axes),
attn_scores_rank))
self._softmax = advanced_activations.Softmax(axis=norm_axes)
self._dropout_layer = core.Dropout(rate=self._dropout)
def __init__(self, sizes,
cell = RNNCell.LSTM,
dropout = 0.2,
activation = 'linear',
loss = 'mse',
optimizer = 'rmsprop'):
self.model = Sequential()
self.model.add(cell(
input_dim = sizes[0],
output_dim = sizes[1],
return_sequences = True
))
for i in range(2, len(sizes) - 1):
self.model.add(cell(sizes[i], return_sequences = False))
self.model.add(core.Dropout(dropout))
self.model.add(core.Dense(output_dim = sizes[-1]))
self.model.add(core.Activation(activation))
self.model.compile(loss = loss, optimizer = optimizer)
def _fit(self, X, y):
logger.info(u"Building the network architecture")
self._model = models.Sequential()
previous_layer_size = self.input_dim
for layer_size in self._hidden_layers:
self._model.add(
core.Dense(input_dim=previous_layer_size,
output_dim=layer_size,
init=self._weight_init,
activation=self._activation))
self._model.add(
core.Dropout(self._dropout_ratio, input_shape=(layer_size, )))
previous_layer_size = layer_size
self._model.add(
core.Dense(input_dim=previous_layer_size,
output_dim=self.output_dim,
activation='softmax',
init=self._weight_init,
W_regularizer=regularizers.WeightRegularizer(
l1=self._l1_regularizer, l2=self._l2_regularizer),
activity_regularizer=regularizers.ActivityRegularizer(
l1=self._l1_regularizer, l2=self._l2_regularizer)))
logger.info(u"Compiling the network")
self._model.compile(optimizer=self._optimizer,
loss='categorical_crossentropy')
logger.info(u"Fitting the data to the network")
self._model.fit(X,
y,
batch_size=self._batch_size,
nb_epoch=self._fine_tune_epochs,
show_accuracy=True)
def new_encdecs(self, compile=True, use_dropout=None, use_noise=None):
self.enc_decs = []
if not use_dropout is None:
self.enc_use_drop = self.drop_rate > 0 and use_dropout
if not use_noise is None:
self.enc_use_noise = self.sigma_base > 0 and use_noise
if self.l1 != 0 or self.l2 != 0:
regularizer = WeightRegularizer(l1=self.l1, l2=self.l2)
else:
regularizer = None
for (i, (n_in, n_out)) in enumerate(
zip(self.layer_sizes[:-1], self.layer_sizes[1:])):
ae = Sequential()
enc_l = []
if self.enc_use_noise:
enc_l.append(
noise.GaussianNoise(self.sigma_base *
(self.sigma_fact**-i),
input_shape=(n_in, )))
enc_l.append(
core.Dense(input_dim=n_in,
output_dim=n_out,
activation='sigmoid',
W_regularizer=regularizer))
if self.enc_use_drop:
enc_l.append(core.Dropout(self.drop_rate))
enc = containers.Sequential(enc_l)
dec = containers.Sequential([
core.Dense(input_dim=n_out,
output_dim=n_in,
activation='sigmoid')
])
ae.add(
core.AutoEncoder(encoder=enc,
decoder=dec,
output_reconstruction=True))
if compile:
ae.compile(loss='mse', optimizer=self.enc_opt)
self.enc_decs.append(ae)
def NN_generator_dropout(n_layers : int, n_nodes : int, input_dim : int, output_dim : int) -> Sequential:
model = Sequential()
# Layer 1, input
layer1 = Dense(n_nodes, activation = 'softplus',
kernel_initializer=VarianceScaling(),
input_shape = (input_dim,))
model.add(layer1)
# Hidden layers
if n_layers > 1:
for i in range(n_layers - 1):
layer = Dense(n_nodes, activation = 'softplus',
kernel_initializer=VarianceScaling())
dropout = core.Dropout(0.25)
model.add(layer)
model.add(dropout)
# Output layer, price
price_layer = Dense(output_dim, kernel_initializer=VarianceScaling())
model.add(price_layer)
return model
def testOptimizerWithKerasModel(self):
a = input_layer.Input(shape=(3,), name='input_a')
b = input_layer.Input(shape=(3,), name='input_b')
dense = core.Dense(4, name='dense')
c = dense(a)
d = dense(b)
e = core.Dropout(0.5, name='dropout')(c)
model = training.Model([a, b], [d, e])
optimizer = gradient_descent.SGD(learning_rate=0.001)
loss = 'mse'
model.compile(optimizer, loss, metrics=['mae'])
input_a_np = np.random.random((10, 3))
input_b_np = np.random.random((10, 3))
output_d_np = np.random.random((10, 4))
output_e_np = np.random.random((10, 4))
model.fit([input_a_np, input_b_np], [output_d_np, output_e_np],
epochs=1,
batch_size=5)
time_x2 = Dense(12,
activation=keras.layers.LeakyReLU(alpha=0.2),
name="FullConnectionTime_2")(time_residual_output)
# AOD + 空间
station_x2 = Dense(12,
activation=keras.layers.LeakyReLU(alpha=0.2),
name="FullConnectionStation_2")(station_residual_output)
# 全部特征
allin_x2 = Dense(12,
activation=keras.layers.LeakyReLU(alpha=0.2),
name="FullConnectionAIA_2")(allin_residual_output)
# Dropout
# AOD + AODs
aods_y = core.Dropout(rate=0.01, name="Aods_Module")(aods_x2)
# AOD + 气象
meteorology_y = core.Dropout(rate=0.01,
name="Meteorology_Module")(meteorology_x2)
# AOD + 时滞
weather_y = core.Dropout(rate=0.01, name="Weather_Module")(weather_x2)
# AOD + NDVI
ndvi_y = core.Dropout(rate=0.01, name="NDVI_Module")(ndvi_x2)
# AOD + 时间
time_y = core.Dropout(rate=0.01, name="Time_Module")(time_x2)
# AOD + 空间
inputAODs = Input(shape=(len(data_aods_test.columns), ))
# 输入1
x1 = layers.Dense(24,
activation=keras.layers.advanced_activations.LeakyReLU(
alpha=0.5))(inputAOD)
x1_residual_connection1 = Dense(
24, activation=keras.layers.LeakyReLU(alpha=0.2))(x1)
x1_residual_connection2 = Dense(
24, activation=keras.layers.advanced_activations.ELU(
alpha=1.0))(x1_residual_connection1)
x1_residual_output = add([x1, x1_residual_connection2])
x1 = layers.Dense(12,
activation=keras.layers.advanced_activations.LeakyReLU(
alpha=0.5))(x1_residual_output)
x1 = core.Dropout(rate=0.01)(x1)
x1 = Dense(8, activation=keras.layers.LeakyReLU(alpha=0.2))(x1)
x1 = Dense(4, activation=keras.layers.LeakyReLU(alpha=0.2))(x1)
x1 = Model(inputs=inputAOD, outputs=x1)
# 输入2
x2 = layers.Dense(24,
activation=keras.layers.advanced_activations.LeakyReLU(
alpha=0.5))(inputSky)
x2_residual_connection1 = Dense(
24, activation=keras.layers.LeakyReLU(alpha=0.2))(x2)
x2_residual_connection2 = Dense(
24, activation=keras.layers.advanced_activations.ELU(
alpha=1.0))(x2_residual_connection1)
x2_residual_output = add([x2, x2_residual_connection2])
x2 = layers.Dense(12,
activation=keras.layers.advanced_activations.LeakyReLU(
class LayerCorrectnessTest(keras_parameterized.TestCase):
def setUp(self):
super(LayerCorrectnessTest, self).setUp()
# Set two virtual CPUs to test MirroredStrategy with multiple devices
cpus = tf.config.list_physical_devices('CPU')
tf.config.set_logical_device_configuration(cpus[0], [
tf.config.LogicalDeviceConfiguration(),
tf.config.LogicalDeviceConfiguration(),
])
def _create_model_from_layer(self, layer, input_shapes):
inputs = [layers.Input(batch_input_shape=s) for s in input_shapes]
if len(inputs) == 1:
inputs = inputs[0]
y = layer(inputs)
model = models.Model(inputs, y)
model.compile('sgd', 'mse')
return model
@parameterized.named_parameters(
('LeakyReLU', advanced_activations.LeakyReLU, (2, 2)),
('PReLU', advanced_activations.PReLU, (2, 2)),
('ELU', advanced_activations.ELU, (2, 2)),
('ThresholdedReLU', advanced_activations.ThresholdedReLU, (2, 2)),
('Softmax', advanced_activations.Softmax, (2, 2)),
('ReLU', advanced_activations.ReLU, (2, 2)),
('Conv1D', lambda: convolutional.Conv1D(2, 2), (2, 2, 1)),
('Conv2D', lambda: convolutional.Conv2D(2, 2), (2, 2, 2, 1)),
('Conv3D', lambda: convolutional.Conv3D(2, 2), (2, 2, 2, 2, 1)),
('Conv2DTranspose', lambda: convolutional.Conv2DTranspose(2, 2),
(2, 2, 2, 2)),
('SeparableConv2D', lambda: convolutional.SeparableConv2D(2, 2),
(2, 2, 2, 1)),
('DepthwiseConv2D', lambda: convolutional.DepthwiseConv2D(2, 2),
(2, 2, 2, 1)),
('UpSampling2D', convolutional.UpSampling2D, (2, 2, 2, 1)),
('ZeroPadding2D', convolutional.ZeroPadding2D, (2, 2, 2, 1)),
('Cropping2D', convolutional.Cropping2D, (2, 3, 3, 1)),
('ConvLSTM2D',
lambda: convolutional_recurrent.ConvLSTM2D(4, kernel_size=(2, 2)),
(4, 4, 4, 4, 4)),
('Dense', lambda: core.Dense(2), (2, 2)),
('Dropout', lambda: core.Dropout(0.5), (2, 2)),
('SpatialDropout2D', lambda: core.SpatialDropout2D(0.5), (2, 2, 2, 2)),
('Activation', lambda: core.Activation('sigmoid'), (2, 2)),
('Reshape', lambda: core.Reshape((1, 4, 1)), (2, 2, 2)),
('Permute', lambda: core.Permute((2, 1)), (2, 2, 2)),
('Attention', dense_attention.Attention, [(2, 2, 3), (2, 3, 3),
(2, 3, 3)]),
('AdditiveAttention', dense_attention.AdditiveAttention, [(2, 2, 3),
(2, 3, 3),
(2, 3, 3)]),
('Embedding', lambda: embeddings.Embedding(4, 4),
(2, 4), 2e-3, 2e-3, np.random.randint(4, size=(2, 4))),
('LocallyConnected1D', lambda: local.LocallyConnected1D(2, 2),
(2, 2, 1)),
('LocallyConnected2D', lambda: local.LocallyConnected2D(2, 2),
(2, 2, 2, 1)),
('Add', merge.Add, [(2, 2), (2, 2)]),
('Subtract', merge.Subtract, [(2, 2), (2, 2)]),
('Multiply', merge.Multiply, [(2, 2), (2, 2)]),
('Average', merge.Average, [(2, 2), (2, 2)]),
('Maximum', merge.Maximum, [(2, 2), (2, 2)]),
('Minimum', merge.Minimum, [(2, 2), (2, 2)]),
('Concatenate', merge.Concatenate, [(2, 2), (2, 2)]),
('Dot', lambda: merge.Dot(1), [(2, 2), (2, 2)]),
('GaussianNoise', lambda: noise.GaussianNoise(0.5), (2, 2)),
('GaussianDropout', lambda: noise.GaussianDropout(0.5), (2, 2)),
('AlphaDropout', lambda: noise.AlphaDropout(0.5), (2, 2)),
('BatchNormalization', normalization_v2.BatchNormalization,
(2, 2), 1e-2, 1e-2),
('LayerNormalization', normalization.LayerNormalization, (2, 2)),
('LayerNormalizationUnfused',
lambda: normalization.LayerNormalization(axis=1), (2, 2, 2)),
('MaxPooling2D', pooling.MaxPooling2D, (2, 2, 2, 1)),
('AveragePooling2D', pooling.AveragePooling2D, (2, 2, 2, 1)),
('GlobalMaxPooling2D', pooling.GlobalMaxPooling2D, (2, 2, 2, 1)),
('GlobalAveragePooling2D', pooling.GlobalAveragePooling2D,
(2, 2, 2, 1)),
('SimpleRNN', lambda: recurrent.SimpleRNN(units=4),
(4, 4, 4), 1e-2, 1e-2),
('GRU', lambda: recurrent.GRU(units=4), (4, 4, 4)),
('LSTM', lambda: recurrent.LSTM(units=4), (4, 4, 4)),
('GRUV2', lambda: recurrent_v2.GRU(units=4), (4, 4, 4)),
('LSTMV2', lambda: recurrent_v2.LSTM(units=4), (4, 4, 4)),
('TimeDistributed', lambda: wrappers.TimeDistributed(core.Dense(2)),
(2, 2, 2)),
('Bidirectional',
lambda: wrappers.Bidirectional(recurrent.SimpleRNN(units=4)),
(2, 2, 2)),
('AttentionLayerCausal',
lambda: dense_attention.Attention(causal=True), [(2, 2, 3), (2, 3, 3),
(2, 3, 3)]),
('AdditiveAttentionLayerCausal',
lambda: dense_attention.AdditiveAttention(causal=True), [(2, 3, 4),
(2, 3, 4),
(2, 3, 4)]),
)
def test_layer(self,
f32_layer_fn,
input_shape,
rtol=2e-3,
atol=2e-3,
input_data=None):
"""Tests a layer by comparing the float32 and mixed precision weights.
A float32 layer, a mixed precision layer, and a distributed mixed precision
layer are run. The three layers are identical other than their dtypes and
distribution strategies. The outputs after predict() and weights after fit()
are asserted to be close.
Args:
f32_layer_fn: A function returning a float32 layer. The other two layers
will automatically be created from this
input_shape: The shape of the input to the layer, including the batch
dimension. Or a list of shapes if the layer takes multiple inputs.
rtol: The relative tolerance to be asserted.
atol: The absolute tolerance to be asserted.
input_data: A Numpy array with the data of the input. If None, input data
will be randomly generated
"""
if f32_layer_fn == convolutional.ZeroPadding2D and \
tf.test.is_built_with_rocm():
return
if isinstance(input_shape[0], int):
input_shapes = [input_shape]
else:
input_shapes = input_shape
strategy = create_mirrored_strategy()
f32_layer = f32_layer_fn()
# Create the layers
assert f32_layer.dtype == f32_layer._compute_dtype == 'float32'
config = f32_layer.get_config()
config['dtype'] = policy.Policy('mixed_float16')
mp_layer = f32_layer.__class__.from_config(config)
distributed_mp_layer = f32_layer.__class__.from_config(config)
# Compute per_replica_input_shapes for the distributed model
global_batch_size = input_shapes[0][0]
assert global_batch_size % strategy.num_replicas_in_sync == 0, (
'The number of replicas, %d, does not divide the global batch size of '
'%d' % (strategy.num_replicas_in_sync, global_batch_size))
per_replica_batch_size = (global_batch_size //
strategy.num_replicas_in_sync)
per_replica_input_shapes = [(per_replica_batch_size, ) + s[1:]
for s in input_shapes]
# Create the models
f32_model = self._create_model_from_layer(f32_layer, input_shapes)
mp_model = self._create_model_from_layer(mp_layer, input_shapes)
with strategy.scope():
distributed_mp_model = self._create_model_from_layer(
distributed_mp_layer, per_replica_input_shapes)
# Set all model weights to the same values
f32_weights = f32_model.get_weights()
mp_model.set_weights(f32_weights)
distributed_mp_model.set_weights(f32_weights)
# Generate input data
if input_data is None:
# Cast inputs to float16 to avoid measuring error from having f16 layers
# cast to float16.
input_data = [
np.random.normal(size=s).astype('float16')
for s in input_shapes
]
if len(input_data) == 1:
input_data = input_data[0]
# Assert all models have close outputs.
f32_output = f32_model.predict(input_data)
mp_output = mp_model.predict(input_data)
self.assertAllClose(mp_output, f32_output, rtol=rtol, atol=atol)
self.assertAllClose(distributed_mp_model.predict(input_data),
f32_output,
rtol=rtol,
atol=atol)
# Run fit() on models
output = np.random.normal(
size=f32_model.outputs[0].shape).astype('float16')
for model in f32_model, mp_model, distributed_mp_model:
model.fit(input_data, output, batch_size=global_batch_size)
# Assert all models have close weights
f32_weights = f32_model.get_weights()
self.assertAllClose(mp_model.get_weights(),
f32_weights,
rtol=rtol,
atol=atol)
self.assertAllClose(distributed_mp_model.get_weights(),
f32_weights,
rtol=rtol,
atol=atol)
res_residual_connection2 = Dense(
16, name="FullConnectionLast_RC")(res_residual_connection1)
res_residual_output = add([res_x1, res_residual_connection2],
name="ResidualConnectionLast_Add")
# 全连接层 2
res_x2 = Dense(4,
activation=keras.layers.LeakyReLU(alpha=0.2),
name="FullConnectionLast_2")(res_residual_output)
res_x3 = Dense(8,
activation=keras.layers.LeakyReLU(alpha=0.2),
name="FullConnectionLast_2x")(res_x2)
# Dropout
res_y = core.Dropout(rate=0.01, name="Res_Module")(res_x3)
res_y2 = Dense(4,
activation=keras.layers.LeakyReLU(alpha=0.2),
name="FullConnectionLast_y")(res_y)
res_y2 = Dense(2,
activation=keras.layers.LeakyReLU(alpha=0.2),
name="FullConnectionLast_y2")(res_y2)
# 最终融合结果
res_outcome = Dense(1,
activation=keras.layers.LeakyReLU(alpha=0.2),
kernel_regularizer=keras.regularizers.l2(0.001),
name="sigmoid_FC")(res_y2)
# 正则化
padding='valid',
data_format=None)(conv_layer4_2)
conv_layer5_2 = convolutional.Conv2D(32, (3, 3),
strides=(1, 1),
padding='same',
activation='relu')(conv_layer3_2)
pool_layer4_2 = pooling.MaxPooling2D(pool_size=(2, 2),
strides=None,
padding='valid',
data_format=None)(conv_layer5_2)
flatten_layer_2 = core.Flatten()(pool_layer4_2)
hidden1_2 = core.Dense(64, activation='relu')(flatten_layer_2)
hidden_merge = concatenate([hidden1, hidden1_2], axis=-1)
dropout1 = core.Dropout(0.2)(hidden_merge)
hidden2 = core.Dense(64, activation='relu')(dropout1)
out = core.Dense(num_classes, activation='softmax')(hidden2)
model1 = Model([inp, inp_2], out)
model1.compile(loss="categorical_crossentropy",
optimizer='rmsprop',
metrics=["accuracy"])
seed = 7
def custom_iterator(Xp, Xs):
from itertools import izip
from keras.preprocessing.image import ImageDataGenerator
conv_dim_2 = 3 cnn.add(kconv.ZeroPadding2D((1, 1))) cnn.add(kconv.Convolution2D(num_filters_2, conv_dim_2, conv_dim_2, activation="relu")) cnn.add(kpool.MaxPooling2D(strides=(2, 2))) cnn.add(kconv.ZeroPadding2D((1, 1))) cnn.add(kconv.Convolution2D(num_filters_2, conv_dim_2, conv_dim_2, activation="relu")) cnn.add(kpool.MaxPooling2D(strides=(2, 2))) cnn.add(kconv.ZeroPadding2D((1, 1))) cnn.add(kconv.Convolution2D(num_filters_2, conv_dim_2, conv_dim_2, activation="relu")) cnn.add(kcore.Flatten()) cnn.add(kcore.Dropout(0.5)) cnn.add(kcore.Dense(128, activation="relu")) # 4096 cnn.add(kcore.Dense(num_class, activation="softmax")) cnn.summary() myadadelta = kopt.Adadelta(lr=0.65, rho=0.95, epsilon=1e-08, decay=0.001) cnn.compile(loss="categorical_crossentropy", optimizer=myadadelta, metrics=["accuracy"]) cnn.fit(train_X, train_Y, batch_size=128, nb_epoch=180, verbose=1) # test_X = test_data_raw.reshape(test_data_raw.shape[0], 1, 28, 28) test_X = test_X.astype(float) test_X /= 255.0 yPred = cnn.predict_classes(test_X)
meteorology_residual_connection = Dense(
24,
activation=keras.layers.advanced_activations.ELU(alpha=1.0),
kernel_regularizer=keras.regularizers.l2(0.1),
name="FullConnectionMA_RC")(meteorology_residual_connection)
meteorology_residual_connection = add(
[meteorology_x, meteorology_residual_connection],
name="ResidualConnectionMA_Add")
# 全连接层
meteorology_x = Dense(
3,
activation=keras.layers.advanced_activations.ELU(alpha=1.0),
kernel_regularizer=keras.regularizers.l2(0.1),
name="FullConnectionMA_2")(meteorology_residual_connection)
# Dropout
meteorology_y = core.Dropout(rate=0.5,
name="Meteorology_Module")(meteorology_x)
# 模型层
model_meteorology = Model(inputs=[Meteorology_input, AODs_input],
outputs=meteorology_y)
##########################################################################
# 第二个模块 时滞天气模块
Weather_input = Input(shape=(len(data_t1.columns), ),
name="Weather_input") # 只添加新的因素即可
# AODs_input = Input(shape=(17,), name="AODs_input")
# 因素融合层
weather_concat = concatenate([Weather_input, AODs_input])
# 全连接层
weather_x = Dense(24,
activation=keras.layers.advanced_activations.ELU(alpha=1.0),
def convnet_alexnet_lion_keras(image_dims):
# model = Sequential()
# model.add(Lambda(lambda x: (x / 255.0) - 0.5, input_shape=image_dims))
NR_CLASSES = 6
input = layers.Input(shape=image_dims, name="Input")
conv_1 = convolutional.Convolution2D(96,
11,
11,
border_mode='valid',
name="conv_1",
activation='relu',
init='glorot_uniform')(input)
pool_1 = convolutional.MaxPooling2D(pool_size=(3, 3),
name="pool_1")(conv_1)
zero_padding_1 = convolutional.ZeroPadding2D(padding=(1, 1),
name="zero_padding_1")(pool_1)
conv_2 = convolutional.Convolution2D(256,
3,
3,
border_mode='valid',
name="conv_2",
activation='relu',
init='glorot_uniform')(zero_padding_1)
pool_2 = convolutional.MaxPooling2D(pool_size=(3, 3),
name="pool_2")(conv_2)
zero_padding_2 = keras.layers.convolutional.ZeroPadding2D(
padding=(1, 1), name="zero_padding_2")(pool_2)
conv_3 = convolutional.Convolution2D(384,
3,
3,
border_mode='valid',
name="conv_3",
activation='relu',
init='glorot_uniform')(zero_padding_2)
conv_4 = convolutional.Convolution2D(384,
3,
3,
border_mode='valid',
name="conv_4",
activation='relu',
init='glorot_uniform')(conv_3)
conv_5 = convolutional.Convolution2D(256,
3,
3,
border_mode='valid',
name="conv_5",
activation='relu',
init='glorot_uniform')(conv_4)
pool_3 = convolutional.MaxPooling2D(pool_size=(3, 3),
name="pool_3")(conv_5)
flatten = core.Flatten(name="flatten")(pool_3)
fc_1 = core.Dense(4096,
name="fc_1",
activation='relu',
init='glorot_uniform')(flatten)
fc_1 = core.Dropout(0.5, name="fc_1_dropout")(fc_1)
output = core.Dense(4096,
name="Output",
activation='relu',
init='glorot_uniform')(fc_1)
output = core.Dropout(0.5, name="Output_dropout")(output)
fc_2 = core.Dense(NR_CLASSES,
name="fc_2",
activation='softmax',
init='glorot_uniform')(output)
return models.Model([input], [fc_2])
activation='relu')(pool_layer2)
pool_layer3 = pooling.MaxPooling2D(pool_size=(2, 2),
strides=None,
padding='valid',
data_format=None)(conv_layer4)
conv_layer5 = convolutional.Conv2D(32, (3, 3),
strides=(1, 1),
padding='same',
activation='relu')(conv_layer3)
pool_layer4 = pooling.MaxPooling2D(pool_size=(2, 2),
strides=None,
padding='valid',
data_format=None)(conv_layer5)
flatten_layer = core.Flatten()(pool_layer4)
hidden1 = core.Dense(64, activation='relu')(flatten_layer)
dropout1 = core.Dropout(0.2)(hidden1)
hidden2 = core.Dense(64, activation='relu')(dropout1)
out = core.Dense(num_classes, activation='softmax')(hidden2)
model = Model(inp, out)
model.compile(loss="categorical_crossentropy",
optimizer='rmsprop',
metrics=["accuracy"])
"""
Layer (type) Output Shape Param #
=================================================================
input_1 (InputLayer) (None, 256, 256, 3) 0
_________________________________________________________________
block1_conv1 (Conv2D) (None, 256, 256, 64) 1792
_________________________________________________________________
label_args = val_Label_args,
batch_size= 8,
input_size=input_dim)
#================================================MODEL_ARCHITECTURE============================================================
# RGB MODALITY BRANCH OF CNN
inputs_rgb = Input(shape=(input_dim[0],input_dim[1],3))
vgg_model_rgb = VGG16(weights='imagenet', include_top= False)
conv_model_rgb = vgg_model_rgb(inputs_rgb)
conv_model_rgb = Conv2D(1024, (3,3), strides=(1, 1), padding = 'same', activation='relu',data_format="channels_last") (conv_model_rgb)
conv_model_rgb = Conv2D(1024, (3,3), strides=(1, 1), padding = 'same', activation='relu',data_format="channels_last") (conv_model_rgb)
deconv_rgb_1 = Conv2DTranspose(num_class*C,(4,4), strides=(2, 2), padding='same', data_format="channels_last", activation='relu',kernel_initializer='glorot_normal')(conv_model_rgb)
#============================================================================================================
conv_rgb_1 = Conv2D(num_class*C, (3,3), strides=(1,1), padding = 'same', activation='relu', data_format='channels_last')(deconv_rgb_1)
dropout_rgb = core.Dropout(0.4)(conv_rgb_1)
#===============================================================================================================
deconv_rgb_2 = Conv2DTranspose(num_class*C,(4,4), strides=(2, 2), padding='same', data_format="channels_last", activation='relu',kernel_initializer='glorot_normal')(dropout_rgb)
conv_rgb_2 = Conv2D(num_class*C, (3,3), strides=(1,1), padding = 'same', activation='relu', data_format='channels_last')(deconv_rgb_2)
deconv_rgb_3 = Conv2DTranspose(num_class*C,(4,4), strides=(2, 2), padding='same', data_format="channels_last", activation='relu',kernel_initializer='glorot_normal')(conv_rgb_2)
conv_rgb_3 = Conv2D(num_class*C, (3,3), strides=(1,1), padding = 'same', activation='relu', data_format='channels_last')(deconv_rgb_3)
deconv_rgb_4 = Conv2DTranspose(num_class*C,(4,4), strides=(2, 2), padding='same', data_format="channels_last", activation='relu',kernel_initializer='glorot_normal')(conv_rgb_3)
conv_rgb_4 = Conv2D(num_class*C, (3,3), strides=(1,1), padding = 'same', activation='relu', data_format='channels_last')(deconv_rgb_4)
deconv_rgb_5 = Conv2DTranspose(num_class*C,(4,4), strides=(2, 2), padding='same', data_format="channels_last", activation='relu',kernel_initializer='glorot_normal')(conv_rgb_4)
# NIR MODALITY BRANCH OF CNN
inputs_nir = Input(shape=(input_dim[0],input_dim[1],3))
vgg_model_nir = VGG16(weights='imagenet', include_top= False)
conv_model_nir = vgg_model_rgb(inputs_nir)
conv_model_nir = Conv2D(1024, (3,3), strides=(1, 1), padding = 'same', activation='relu',data_format="channels_last") (conv_model_nir)
conv_model_nir = Conv2D(1024, (3,3), strides=(1, 1), padding = 'same', activation='relu',data_format="channels_last") (conv_model_nir)
trainX /= 255.0
trainY = kutils.to_categorical(trainData[:, 0])
nb_classes = trainY.shape[1]
cnn = models.Sequential()
cnn.add(
conv.Convolution2D(nb_filters,
nb_conv,
nb_conv,
border_mode="valid",
input_shape=(1, 28, 28),
activation="relu"))
cnn.add(conv.Convolution2D(nb_filters, nb_conv, nb_conv, activation="relu"))
cnn.add(conv.MaxPooling2D())
cnn.add(core.Dropout(0.25))
cnn.add(core.Flatten())
cnn.add(core.Dense(128, activation="relu"))
cnn.add(core.Dropout(0.15))
cnn.add(core.Dense(64, activation="relu"))
cnn.add(core.Dropout(0.15))
cnn.add(core.Dense(nb_classes, activation="softmax"))
sgd = optm.sgd(lr=0.01, momentum=0.9, decay=1e-6, nesterov=True)
cnn.compile(
loss="mean_squared_error",
optimizer="adadelta",
)
cnn.fit(trainX,
trainY,
noFeatures = trainX.shape[1] scaler = preproc.StandardScaler() trainX = scaler.fit_transform(trainX) """ Final Model """ epochs = 8 nn = models.Sequential() nn.add(core.Dense(noFeatures, input_shape=(noFeatures, ))) nn.add(advact.PReLU()) nn.add(norm.BatchNormalization()) nn.add(core.Dropout(0.2)) nn.add(core.Dense(2 * noFeatures, )) nn.add(advact.PReLU()) nn.add(norm.BatchNormalization()) nn.add(core.Dropout(0.25)) nn.add(core.Dense(noFeatures, )) nn.add(advact.PReLU()) nn.add(norm.BatchNormalization()) nn.add(core.Dropout(0.2)) nn.add(core.Dense(noOfClasses, activation="softmax")) print(nn.summary())
input_dim=input_shape))
model.add(
normalization.BatchNormalization(axis=-1,
momentum=0.99,
epsilon=0.001,
center=True,
scale=True,
beta_initializer='zeros',
gamma_initializer='ones',
moving_mean_initializer='zeros',
moving_variance_initializer='ones',
beta_regularizer=None,
gamma_regularizer=None,
beta_constraint=None,
gamma_constraint=None))
model.add(core.Dropout(0.25, noise_shape=None, seed=None))
for i in range(9):
model.add(
Dense(units=input_shape,
activation='relu',
use_bias=True,
kernel_initializer='random_uniform',
bias_initializer='he_normal'))
model.add(
normalization.BatchNormalization(axis=-1,
momentum=0.99,
epsilon=0.001,
center=True,
scale=True,
beta_initializer='zeros',
gamma_initializer='ones',
y_test = np_utils.to_categorical(y_test, 10)
model = Sequential()
#first conv layer with 32 filters output (32,14,14)
model.add(
Convolution2D(32, (2, 2),
padding='same',
data_format='channels_last',
input_shape=(28, 28, 1)))
model.add(normalization.BatchNormalization())
model.add(Activation('relu'))
model.add(
MaxPooling2D(pool_size=(2, 2), padding='same',
data_format='channels_last'))
model.add(core.Dropout(0.25))
#second conv layer with 64 filter output (64,7,7)
model.add(
Convolution2D(64, (2, 2), padding='same', data_format='channels_last'))
model.add(normalization.BatchNormalization())
model.add(Activation('relu'))
model.add(
MaxPooling2D(pool_size=(2, 2), padding='same',
data_format='channels_last'))
model.add(core.Dropout(0.25))
#fully connected layer with output 128
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.25))
cnn.add(conv.ZeroPadding2D((1, 1)))
cnn.add(conv.Convolution2D(64, 3, 3, activation="relu"))
cnn.add(conv.MaxPooling2D(strides=(2, 2)))
# cnn.add(conv.ZeroPadding2D((1, 1)))
# cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu"))
# cnn.add(conv.ZeroPadding2D((1, 1)))
# cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu"))
# cnn.add(conv.ZeroPadding2D((1, 1)))
# cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu"))
# cnn.add(conv.ZeroPadding2D((1, 1)))
# cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu"))
# cnn.add(conv.MaxPooling2D(strides=(2,2)))
cnn.add(core.Flatten())
cnn.add(core.Dropout(0.2))
cnn.add(core.Dense(128, activation="relu")) # 4096
cnn.add(core.Dense(nb_classes, activation="softmax"))
cnn.summary()
cnn.compile(loss="categorical_crossentropy",
optimizer="adadelta",
metrics=["accuracy"])
cnn.fit(trainX, trainY, batch_size=128, nb_epoch=1, verbose=1)
testX = test.reshape(test.shape[0], 1, 48, 48)
testX = testX.astype(float)
testX /= 255.0
yPred = cnn.predict_classes(testX)
def test_dropout():
layer = core.Dropout(0.5)
_runner(layer)
def test_dropout(self):
layer = core.Dropout(0.5)
self._runner(layer)
H = klc.Activation('relu')(H)
H = klconv.Convolution2D(n_channels/4, 3, 3, border_mode='same',
init='glorot_uniform')(H)
H = kln.BatchNormalization(mode=2)(H)
H = klc.Activation('relu')(H)
H = klconv.Convolution2D(1, 1, 1, border_mode='same', init='glorot_uniform')(H)
g_V = klc.Activation('sigmoid')(H)
generator = km.Model(g_input, g_V)
generator.compile(loss='binary_crossentropy', optimizer=g_opt)
# discriminator
d_input = kl.Input(shape=[1,img_rows,img_cols])
H = klconv.Convolution2D(256, 5, 5, subsample=(2,2), border_mode='same',
activation='relu')(d_input)
H = kla.LeakyReLU(0.2)(H)
H = klc.Dropout(dropout_rate)(H)
H = klconv.Convolution2D(512, 5, 5, subsample=(2,2), border_mode='same',
activation='relu')(H)
H = kla.LeakyReLU(0.2)(H)
H = klc.Dropout(dropout_rate)(H)
H = klc.Flatten()(H)
H = klc.Dense(256)(H)
H = kla.LeakyReLU(0.2)(H)
H = klc.Dropout(dropout_rate)(H)
d_V = klc.Dense(2, activation='softmax')(H)
discriminator = km.Model(d_input, d_V)
discriminator.compile(loss='categorical_crossentropy', optimizer=d_opt)
# stack generator and discriminator
gan_input = kl.Input(shape=[noise_size])
H = generator(gan_input)
# Read splitted data
df_train = pd.read_csv('train.csv')
df_valid = pd.read_csv('test.csv')
# CNN Model Architecture
model = models.Sequential()
model.add(convolutional.Convolution2D(16, 3, 3, input_shape=(32, 128, 3), activation='relu'))
model.add(pooling.MaxPooling2D(pool_size=(2, 2)))
model.add(convolutional.Convolution2D(32, 3, 3, activation='relu'))
model.add(pooling.MaxPooling2D(pool_size=(2, 2)))
model.add(convolutional.Convolution2D(64, 3, 3, activation='relu'))
model.add(pooling.MaxPooling2D(pool_size=(2, 2)))
model.add(core.Flatten())
model.add(core.Dense(500, activation='relu'))
model.add(core.Dropout(.5))
model.add(core.Dense(100, activation='relu'))
model.add(core.Dropout(.25))
model.add(core.Dense(20, activation='relu'))
model.add(core.Dense(1))
model.compile(optimizer=optimizers.Adam(lr=1e-04), loss='mean_squared_error')
# load the exist model
model.load_weights("model.h5")
history = model.fit_generator(# continue training model for 17 epochs
generate_samples(df_train, ''),
samples_per_epoch=df_train.shape[0],
nb_epoch=17,#0.016
validation_data=generate_samples(df_valid, '', augment=False),
nb_val_samples=df_valid.shape[0],
x7 = layers.Dense(24,
activation=keras.layers.advanced_activations.LeakyReLU(
alpha=0.5))(inputAODs)
x7 = layers.Dense(
3, activation=keras.layers.advanced_activations.LeakyReLU(alpha=0.5))(x7)
x7 = Model(inputs=inputAODs, outputs=x7)
combined = layers.concatenate([
x1.output, x2.output, x3.output, x4.output, x5.output, x6.output, x7.output
])
# 输出层
z = layers.Dense(
12,
activation=keras.layers.advanced_activations.LeakyReLU(alpha=0.5),
kernel_regularizer=keras.regularizers.l2(0.01))(combined)
z = core.Dropout(rate=0.01)(z)
z = layers.Dense(
4,
activation=keras.layers.advanced_activations.LeakyReLU(alpha=0.5),
kernel_regularizer=keras.regularizers.l2(0.01))(z)
z = layers.Dense(
1,
activation=keras.layers.advanced_activations.LeakyReLU(alpha=0.5),
kernel_regularizer=keras.regularizers.l2(0.01))(z)
# 建立模型
model = Model(inputs=[
x1.input, x2.input, x3.input, x4.input, x5.input, x6.input, x7.input
],
outputs=z)
# 模型编译
def architecture():
# put the normalization fucntion inside the model ensure preprocess using Lambda layer
# There were many issue depending on using the lambda layer and in this waa it works
def resize_normalize(image):
import cv2
from keras.backend import tf as ktf
"""
Applies preprocessing pipeline to an image: crops `top` and `bottom`
portions of image, resizes to 66*200 px and scales pixel values to [0, 1].
"""
# resize to width 200 and high 66 liek recommended
# in the nvidia paper for the used CNN
# image = cv2.resize(image, (66, 200)) #first try
resized = ktf.image.resize_images(image, (32, 128))
#normalize 0-1
resized = resized / 255.0 - 0.5
return resized
print('I am inside call of architecture')
#initialize model
model = Sequential()
#dropout = 0.5
nonlinear = 'tanh'
print('I am before call of cropping layer')
### Convolution layers and parameters were taken from the "nvidia paper" on end-to-end autonomous steering.
model.add(
Cropping2D(cropping=((60, 20), (1, 1)), input_shape=(160, 320, 3)))
print('I am before call of Lambda')
model.add(
Lambda(resize_normalize,
input_shape=(160, 320, 3),
output_shape=(32, 128, 3)))
# Model architecture
model.add(
Convolution2D(16, 3, 3, input_shape=(32, 128, 3), activation='relu'))
model.add(pooling.MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(32, 3, 3, activation='relu'))
model.add(pooling.MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(64, 3, 3, activation='relu'))
model.add(pooling.MaxPooling2D(pool_size=(2, 2)))
model.add(core.Flatten())
model.add(core.Dense(500, activation='relu'))
model.add(core.Dropout(.5))
model.add(core.Dense(100, activation='relu'))
model.add(core.Dropout(.25))
model.add(core.Dense(20, activation='relu'))
model.add(core.Dense(1))
model.compile(optimizer=optimizers.Adam(lr=1e-04),
loss='mean_squared_error')
#model.add(Lambda(lambda x: resize_normalize(x), input_shape=(80,318,3), output_shape=(66, 200, 3)))
# model.add(Convolution2D(24, 5, 5, name='conv1', subsample=(2, 2), activation=nonlinear))
# model.add(Convolution2D(36, 5, 5, name='conv2', subsample=(2, 2), activation=nonlinear))
# model.add(Convolution2D(48, 5, 5, name='conv3', subsample=(2, 2), activation=nonlinear))
# model.add(Convolution2D(64, 3, 3, name='conv4', activation=nonlinear))
# model.add(Convolution2D(64, 3, 3, name='conv5', activation=nonlinear))
# ### Regression
# model.add(Flatten())
# model.add(Dropout(dropout))
# model.add(Dense(1164, name='hidden1', activation=nonlinear))
# model.add(Dropout(dropout))
# model.add(Dense(100, name='hidden2', activation=nonlinear))
# model.add(Dropout(dropout))
# model.add(Dense(50, name='hidden3', activation=nonlinear))
# model.add(Dropout(dropout))
# model.add(Dense(10, name='hidden4', activation=nonlinear))
# model.add(Dropout(dropout))
# model.add(Dense(1, name='output', activation=nonlinear))
# #model.compile(optimizer=optimizers.Adam(lr=1e-04), loss='mean_squared_error')
# model.compile(optimizer='adam', loss='mse')
print('I am finished build the model')
print(model.summary())
return model
meteorology_residual_connection = Dense(
24,
activation=keras.layers.advanced_activations.ELU(alpha=1.0),
kernel_regularizer=keras.regularizers.l2(0.1),
name="FullConnectionMA_RC")(meteorology_residual_connection)
meteorology_residual_connection = add(
[meteorology_x, meteorology_residual_connection],
name="ResidualConnectionMA_Add")
# 全连接层
meteorology_x = Dense(
3,
activation=keras.layers.advanced_activations.ELU(alpha=1.0),
kernel_regularizer=keras.regularizers.l2(0.1),
name="FullConnectionMA_2")(meteorology_residual_connection)
# Dropout
meteorology_y = core.Dropout(rate=0.5,
name="Meteorology_Module")(meteorology_x)
# 模型层
model_meteorology = Model(inputs=[Meteorology_input, AODs_input],
outputs=meteorology_y)
######################################################################################################################
# 第二个模块 时滞天气模块
Weather_input = Input(shape=(12, ), name="Weather_input") # 只添加新的因素即可
# AODs_input = Input(shape=(17,), name="AODs_input")
# 因素融合层
weather_concat = concatenate([Weather_input, AODs_input])
# 全连接层
weather_x = Dense(24,
activation=keras.layers.advanced_activations.ELU(alpha=1.0),
kernel_regularizer=keras.regularizers.l2(0.1),
name="ResidualConnectionAIA")(allin_x1)
allin_residual_connection2 = Dense(
24,
activation=keras.layers.LeakyReLU(alpha=0.2),
name="FullConnectionAIA_RC")(allin_residual_connection1)
allin_residual_output = add([allin_x1, allin_residual_connection2],
name="ResidualConnectionAIA_Add")
# 全连接层 2
allin_x2 = Dense(12,
activation=keras.layers.LeakyReLU(alpha=0.2),
name="FullConnectionAIA_2")(allin_residual_output)
# Dropout
# 全部特征
allin_y = core.Dropout(rate=0.01, name="AllIn_Module")(allin_x2)
# 全连接层 3
# 全部特征
allin_y2 = Dense(8,
activation=keras.layers.LeakyReLU(alpha=0.2),
name="FullConnectionAIA_3x")(allin_y)
# 全连接层 3
# 全部特征
allin_y2 = Dense(4,
activation=keras.layers.LeakyReLU(alpha=0.2),
name="FullConnectionAIA_3")(allin_y2)
# 模型层
# 全部特征
152) # pm+id ==2 # 不要（，，1）
timesteps = 152 # 这里换成监测站
data_dim = len(input_list)
# 期望输入数据尺寸: (batch_size, timesteps, data_dim)
model = Sequential()
model.add(
LSTM(32, return_sequences=True,
input_shape=(timesteps, data_dim))) # 返回维度为 32 的向量序列
model.add(LSTM(32, return_sequences=True))
model.add(LSTM(32, return_sequences=True))
model.add(LSTM(32, return_sequences=True))
model.add(LSTM(32, return_sequences=True)) # 返回维度为 32 的向量序列
model.add(core.Dropout(rate=0.01))
model.add(LSTM(32, )) # 返回维度为 32 的单个向量
model.add(
Dense(152,
activation=keras.layers.LeakyReLU(alpha=0.2),
kernel_regularizer=keras.regularizers.l2(0.01)))
model.compile(loss=['mean_absolute_error'],
optimizer=keras.optimizers.Adam(lr=0.01,
beta_1=0.9,
beta_2=0.999),
metrics=['accuracy'])
# 计算耗时
starttime = datetime.datetime.now()
# 程序
