def _build_attention(self, rank): """Builds multi-head dot-product attention computations. This function builds attributes necessary for `_compute_attention` to costomize attention computation to replace the default dot-product attention. Args: rank: the rank of query, key, value tensors. """ if self._attention_axes is None: self._attention_axes = tuple(range(1, rank - 2)) else: self._attention_axes = tuple(self._attention_axes) self._dot_product_equation, self._combine_equation, attn_scores_rank = ( _build_attention_equation(rank, attn_axes=self._attention_axes)) norm_axes = tuple( range(attn_scores_rank - len(self._attention_axes), attn_scores_rank)) self._softmax = advanced_activations.Softmax(axis=norm_axes) self._dropout_layer = core.Dropout(rate=self._dropout)
def __init__(self, sizes, cell = RNNCell.LSTM, dropout = 0.2, activation = 'linear', loss = 'mse', optimizer = 'rmsprop'): self.model = Sequential() self.model.add(cell( input_dim = sizes[0], output_dim = sizes[1], return_sequences = True )) for i in range(2, len(sizes) - 1): self.model.add(cell(sizes[i], return_sequences = False)) self.model.add(core.Dropout(dropout)) self.model.add(core.Dense(output_dim = sizes[-1])) self.model.add(core.Activation(activation)) self.model.compile(loss = loss, optimizer = optimizer)
def _fit(self, X, y): logger.info(u"Building the network architecture") self._model = models.Sequential() previous_layer_size = self.input_dim for layer_size in self._hidden_layers: self._model.add( core.Dense(input_dim=previous_layer_size, output_dim=layer_size, init=self._weight_init, activation=self._activation)) self._model.add( core.Dropout(self._dropout_ratio, input_shape=(layer_size, ))) previous_layer_size = layer_size self._model.add( core.Dense(input_dim=previous_layer_size, output_dim=self.output_dim, activation='softmax', init=self._weight_init, W_regularizer=regularizers.WeightRegularizer( l1=self._l1_regularizer, l2=self._l2_regularizer), activity_regularizer=regularizers.ActivityRegularizer( l1=self._l1_regularizer, l2=self._l2_regularizer))) logger.info(u"Compiling the network") self._model.compile(optimizer=self._optimizer, loss='categorical_crossentropy') logger.info(u"Fitting the data to the network") self._model.fit(X, y, batch_size=self._batch_size, nb_epoch=self._fine_tune_epochs, show_accuracy=True)
def new_encdecs(self, compile=True, use_dropout=None, use_noise=None): self.enc_decs = [] if not use_dropout is None: self.enc_use_drop = self.drop_rate > 0 and use_dropout if not use_noise is None: self.enc_use_noise = self.sigma_base > 0 and use_noise if self.l1 != 0 or self.l2 != 0: regularizer = WeightRegularizer(l1=self.l1, l2=self.l2) else: regularizer = None for (i, (n_in, n_out)) in enumerate( zip(self.layer_sizes[:-1], self.layer_sizes[1:])): ae = Sequential() enc_l = [] if self.enc_use_noise: enc_l.append( noise.GaussianNoise(self.sigma_base * (self.sigma_fact**-i), input_shape=(n_in, ))) enc_l.append( core.Dense(input_dim=n_in, output_dim=n_out, activation='sigmoid', W_regularizer=regularizer)) if self.enc_use_drop: enc_l.append(core.Dropout(self.drop_rate)) enc = containers.Sequential(enc_l) dec = containers.Sequential([ core.Dense(input_dim=n_out, output_dim=n_in, activation='sigmoid') ]) ae.add( core.AutoEncoder(encoder=enc, decoder=dec, output_reconstruction=True)) if compile: ae.compile(loss='mse', optimizer=self.enc_opt) self.enc_decs.append(ae)
def NN_generator_dropout(n_layers : int, n_nodes : int, input_dim : int, output_dim : int) -> Sequential: model = Sequential() # Layer 1, input layer1 = Dense(n_nodes, activation = 'softplus', kernel_initializer=VarianceScaling(), input_shape = (input_dim,)) model.add(layer1) # Hidden layers if n_layers > 1: for i in range(n_layers - 1): layer = Dense(n_nodes, activation = 'softplus', kernel_initializer=VarianceScaling()) dropout = core.Dropout(0.25) model.add(layer) model.add(dropout) # Output layer, price price_layer = Dense(output_dim, kernel_initializer=VarianceScaling()) model.add(price_layer) return model
def testOptimizerWithKerasModel(self): a = input_layer.Input(shape=(3,), name='input_a') b = input_layer.Input(shape=(3,), name='input_b') dense = core.Dense(4, name='dense') c = dense(a) d = dense(b) e = core.Dropout(0.5, name='dropout')(c) model = training.Model([a, b], [d, e]) optimizer = gradient_descent.SGD(learning_rate=0.001) loss = 'mse' model.compile(optimizer, loss, metrics=['mae']) input_a_np = np.random.random((10, 3)) input_b_np = np.random.random((10, 3)) output_d_np = np.random.random((10, 4)) output_e_np = np.random.random((10, 4)) model.fit([input_a_np, input_b_np], [output_d_np, output_e_np], epochs=1, batch_size=5)
time_x2 = Dense(12, activation=keras.layers.LeakyReLU(alpha=0.2), name="FullConnectionTime_2")(time_residual_output) # AOD + 空间 station_x2 = Dense(12, activation=keras.layers.LeakyReLU(alpha=0.2), name="FullConnectionStation_2")(station_residual_output) # 全部特征 allin_x2 = Dense(12, activation=keras.layers.LeakyReLU(alpha=0.2), name="FullConnectionAIA_2")(allin_residual_output) # Dropout # AOD + AODs aods_y = core.Dropout(rate=0.01, name="Aods_Module")(aods_x2) # AOD + 气象 meteorology_y = core.Dropout(rate=0.01, name="Meteorology_Module")(meteorology_x2) # AOD + 时滞 weather_y = core.Dropout(rate=0.01, name="Weather_Module")(weather_x2) # AOD + NDVI ndvi_y = core.Dropout(rate=0.01, name="NDVI_Module")(ndvi_x2) # AOD + 时间 time_y = core.Dropout(rate=0.01, name="Time_Module")(time_x2) # AOD + 空间
inputAODs = Input(shape=(len(data_aods_test.columns), )) # 输入1 x1 = layers.Dense(24, activation=keras.layers.advanced_activations.LeakyReLU( alpha=0.5))(inputAOD) x1_residual_connection1 = Dense( 24, activation=keras.layers.LeakyReLU(alpha=0.2))(x1) x1_residual_connection2 = Dense( 24, activation=keras.layers.advanced_activations.ELU( alpha=1.0))(x1_residual_connection1) x1_residual_output = add([x1, x1_residual_connection2]) x1 = layers.Dense(12, activation=keras.layers.advanced_activations.LeakyReLU( alpha=0.5))(x1_residual_output) x1 = core.Dropout(rate=0.01)(x1) x1 = Dense(8, activation=keras.layers.LeakyReLU(alpha=0.2))(x1) x1 = Dense(4, activation=keras.layers.LeakyReLU(alpha=0.2))(x1) x1 = Model(inputs=inputAOD, outputs=x1) # 输入2 x2 = layers.Dense(24, activation=keras.layers.advanced_activations.LeakyReLU( alpha=0.5))(inputSky) x2_residual_connection1 = Dense( 24, activation=keras.layers.LeakyReLU(alpha=0.2))(x2) x2_residual_connection2 = Dense( 24, activation=keras.layers.advanced_activations.ELU( alpha=1.0))(x2_residual_connection1) x2_residual_output = add([x2, x2_residual_connection2]) x2 = layers.Dense(12, activation=keras.layers.advanced_activations.LeakyReLU(
class LayerCorrectnessTest(keras_parameterized.TestCase): def setUp(self): super(LayerCorrectnessTest, self).setUp() # Set two virtual CPUs to test MirroredStrategy with multiple devices cpus = tf.config.list_physical_devices('CPU') tf.config.set_logical_device_configuration(cpus[0], [ tf.config.LogicalDeviceConfiguration(), tf.config.LogicalDeviceConfiguration(), ]) def _create_model_from_layer(self, layer, input_shapes): inputs = [layers.Input(batch_input_shape=s) for s in input_shapes] if len(inputs) == 1: inputs = inputs[0] y = layer(inputs) model = models.Model(inputs, y) model.compile('sgd', 'mse') return model @parameterized.named_parameters( ('LeakyReLU', advanced_activations.LeakyReLU, (2, 2)), ('PReLU', advanced_activations.PReLU, (2, 2)), ('ELU', advanced_activations.ELU, (2, 2)), ('ThresholdedReLU', advanced_activations.ThresholdedReLU, (2, 2)), ('Softmax', advanced_activations.Softmax, (2, 2)), ('ReLU', advanced_activations.ReLU, (2, 2)), ('Conv1D', lambda: convolutional.Conv1D(2, 2), (2, 2, 1)), ('Conv2D', lambda: convolutional.Conv2D(2, 2), (2, 2, 2, 1)), ('Conv3D', lambda: convolutional.Conv3D(2, 2), (2, 2, 2, 2, 1)), ('Conv2DTranspose', lambda: convolutional.Conv2DTranspose(2, 2), (2, 2, 2, 2)), ('SeparableConv2D', lambda: convolutional.SeparableConv2D(2, 2), (2, 2, 2, 1)), ('DepthwiseConv2D', lambda: convolutional.DepthwiseConv2D(2, 2), (2, 2, 2, 1)), ('UpSampling2D', convolutional.UpSampling2D, (2, 2, 2, 1)), ('ZeroPadding2D', convolutional.ZeroPadding2D, (2, 2, 2, 1)), ('Cropping2D', convolutional.Cropping2D, (2, 3, 3, 1)), ('ConvLSTM2D', lambda: convolutional_recurrent.ConvLSTM2D(4, kernel_size=(2, 2)), (4, 4, 4, 4, 4)), ('Dense', lambda: core.Dense(2), (2, 2)), ('Dropout', lambda: core.Dropout(0.5), (2, 2)), ('SpatialDropout2D', lambda: core.SpatialDropout2D(0.5), (2, 2, 2, 2)), ('Activation', lambda: core.Activation('sigmoid'), (2, 2)), ('Reshape', lambda: core.Reshape((1, 4, 1)), (2, 2, 2)), ('Permute', lambda: core.Permute((2, 1)), (2, 2, 2)), ('Attention', dense_attention.Attention, [(2, 2, 3), (2, 3, 3), (2, 3, 3)]), ('AdditiveAttention', dense_attention.AdditiveAttention, [(2, 2, 3), (2, 3, 3), (2, 3, 3)]), ('Embedding', lambda: embeddings.Embedding(4, 4), (2, 4), 2e-3, 2e-3, np.random.randint(4, size=(2, 4))), ('LocallyConnected1D', lambda: local.LocallyConnected1D(2, 2), (2, 2, 1)), ('LocallyConnected2D', lambda: local.LocallyConnected2D(2, 2), (2, 2, 2, 1)), ('Add', merge.Add, [(2, 2), (2, 2)]), ('Subtract', merge.Subtract, [(2, 2), (2, 2)]), ('Multiply', merge.Multiply, [(2, 2), (2, 2)]), ('Average', merge.Average, [(2, 2), (2, 2)]), ('Maximum', merge.Maximum, [(2, 2), (2, 2)]), ('Minimum', merge.Minimum, [(2, 2), (2, 2)]), ('Concatenate', merge.Concatenate, [(2, 2), (2, 2)]), ('Dot', lambda: merge.Dot(1), [(2, 2), (2, 2)]), ('GaussianNoise', lambda: noise.GaussianNoise(0.5), (2, 2)), ('GaussianDropout', lambda: noise.GaussianDropout(0.5), (2, 2)), ('AlphaDropout', lambda: noise.AlphaDropout(0.5), (2, 2)), ('BatchNormalization', normalization_v2.BatchNormalization, (2, 2), 1e-2, 1e-2), ('LayerNormalization', normalization.LayerNormalization, (2, 2)), ('LayerNormalizationUnfused', lambda: normalization.LayerNormalization(axis=1), (2, 2, 2)), ('MaxPooling2D', pooling.MaxPooling2D, (2, 2, 2, 1)), ('AveragePooling2D', pooling.AveragePooling2D, (2, 2, 2, 1)), ('GlobalMaxPooling2D', pooling.GlobalMaxPooling2D, (2, 2, 2, 1)), ('GlobalAveragePooling2D', pooling.GlobalAveragePooling2D, (2, 2, 2, 1)), ('SimpleRNN', lambda: recurrent.SimpleRNN(units=4), (4, 4, 4), 1e-2, 1e-2), ('GRU', lambda: recurrent.GRU(units=4), (4, 4, 4)), ('LSTM', lambda: recurrent.LSTM(units=4), (4, 4, 4)), ('GRUV2', lambda: recurrent_v2.GRU(units=4), (4, 4, 4)), ('LSTMV2', lambda: recurrent_v2.LSTM(units=4), (4, 4, 4)), ('TimeDistributed', lambda: wrappers.TimeDistributed(core.Dense(2)), (2, 2, 2)), ('Bidirectional', lambda: wrappers.Bidirectional(recurrent.SimpleRNN(units=4)), (2, 2, 2)), ('AttentionLayerCausal', lambda: dense_attention.Attention(causal=True), [(2, 2, 3), (2, 3, 3), (2, 3, 3)]), ('AdditiveAttentionLayerCausal', lambda: dense_attention.AdditiveAttention(causal=True), [(2, 3, 4), (2, 3, 4), (2, 3, 4)]), ) def test_layer(self, f32_layer_fn, input_shape, rtol=2e-3, atol=2e-3, input_data=None): """Tests a layer by comparing the float32 and mixed precision weights. A float32 layer, a mixed precision layer, and a distributed mixed precision layer are run. The three layers are identical other than their dtypes and distribution strategies. The outputs after predict() and weights after fit() are asserted to be close. Args: f32_layer_fn: A function returning a float32 layer. The other two layers will automatically be created from this input_shape: The shape of the input to the layer, including the batch dimension. Or a list of shapes if the layer takes multiple inputs. rtol: The relative tolerance to be asserted. atol: The absolute tolerance to be asserted. input_data: A Numpy array with the data of the input. If None, input data will be randomly generated """ if f32_layer_fn == convolutional.ZeroPadding2D and \ tf.test.is_built_with_rocm(): return if isinstance(input_shape[0], int): input_shapes = [input_shape] else: input_shapes = input_shape strategy = create_mirrored_strategy() f32_layer = f32_layer_fn() # Create the layers assert f32_layer.dtype == f32_layer._compute_dtype == 'float32' config = f32_layer.get_config() config['dtype'] = policy.Policy('mixed_float16') mp_layer = f32_layer.__class__.from_config(config) distributed_mp_layer = f32_layer.__class__.from_config(config) # Compute per_replica_input_shapes for the distributed model global_batch_size = input_shapes[0][0] assert global_batch_size % strategy.num_replicas_in_sync == 0, ( 'The number of replicas, %d, does not divide the global batch size of ' '%d' % (strategy.num_replicas_in_sync, global_batch_size)) per_replica_batch_size = (global_batch_size // strategy.num_replicas_in_sync) per_replica_input_shapes = [(per_replica_batch_size, ) + s[1:] for s in input_shapes] # Create the models f32_model = self._create_model_from_layer(f32_layer, input_shapes) mp_model = self._create_model_from_layer(mp_layer, input_shapes) with strategy.scope(): distributed_mp_model = self._create_model_from_layer( distributed_mp_layer, per_replica_input_shapes) # Set all model weights to the same values f32_weights = f32_model.get_weights() mp_model.set_weights(f32_weights) distributed_mp_model.set_weights(f32_weights) # Generate input data if input_data is None: # Cast inputs to float16 to avoid measuring error from having f16 layers # cast to float16. input_data = [ np.random.normal(size=s).astype('float16') for s in input_shapes ] if len(input_data) == 1: input_data = input_data[0] # Assert all models have close outputs. f32_output = f32_model.predict(input_data) mp_output = mp_model.predict(input_data) self.assertAllClose(mp_output, f32_output, rtol=rtol, atol=atol) self.assertAllClose(distributed_mp_model.predict(input_data), f32_output, rtol=rtol, atol=atol) # Run fit() on models output = np.random.normal( size=f32_model.outputs[0].shape).astype('float16') for model in f32_model, mp_model, distributed_mp_model: model.fit(input_data, output, batch_size=global_batch_size) # Assert all models have close weights f32_weights = f32_model.get_weights() self.assertAllClose(mp_model.get_weights(), f32_weights, rtol=rtol, atol=atol) self.assertAllClose(distributed_mp_model.get_weights(), f32_weights, rtol=rtol, atol=atol)
res_residual_connection2 = Dense( 16, name="FullConnectionLast_RC")(res_residual_connection1) res_residual_output = add([res_x1, res_residual_connection2], name="ResidualConnectionLast_Add") # 全连接层 2 res_x2 = Dense(4, activation=keras.layers.LeakyReLU(alpha=0.2), name="FullConnectionLast_2")(res_residual_output) res_x3 = Dense(8, activation=keras.layers.LeakyReLU(alpha=0.2), name="FullConnectionLast_2x")(res_x2) # Dropout res_y = core.Dropout(rate=0.01, name="Res_Module")(res_x3) res_y2 = Dense(4, activation=keras.layers.LeakyReLU(alpha=0.2), name="FullConnectionLast_y")(res_y) res_y2 = Dense(2, activation=keras.layers.LeakyReLU(alpha=0.2), name="FullConnectionLast_y2")(res_y2) # 最终融合结果 res_outcome = Dense(1, activation=keras.layers.LeakyReLU(alpha=0.2), kernel_regularizer=keras.regularizers.l2(0.001), name="sigmoid_FC")(res_y2) # 正则化
padding='valid', data_format=None)(conv_layer4_2) conv_layer5_2 = convolutional.Conv2D(32, (3, 3), strides=(1, 1), padding='same', activation='relu')(conv_layer3_2) pool_layer4_2 = pooling.MaxPooling2D(pool_size=(2, 2), strides=None, padding='valid', data_format=None)(conv_layer5_2) flatten_layer_2 = core.Flatten()(pool_layer4_2) hidden1_2 = core.Dense(64, activation='relu')(flatten_layer_2) hidden_merge = concatenate([hidden1, hidden1_2], axis=-1) dropout1 = core.Dropout(0.2)(hidden_merge) hidden2 = core.Dense(64, activation='relu')(dropout1) out = core.Dense(num_classes, activation='softmax')(hidden2) model1 = Model([inp, inp_2], out) model1.compile(loss="categorical_crossentropy", optimizer='rmsprop', metrics=["accuracy"]) seed = 7 def custom_iterator(Xp, Xs): from itertools import izip from keras.preprocessing.image import ImageDataGenerator
conv_dim_2 = 3 cnn.add(kconv.ZeroPadding2D((1, 1))) cnn.add(kconv.Convolution2D(num_filters_2, conv_dim_2, conv_dim_2, activation="relu")) cnn.add(kpool.MaxPooling2D(strides=(2, 2))) cnn.add(kconv.ZeroPadding2D((1, 1))) cnn.add(kconv.Convolution2D(num_filters_2, conv_dim_2, conv_dim_2, activation="relu")) cnn.add(kpool.MaxPooling2D(strides=(2, 2))) cnn.add(kconv.ZeroPadding2D((1, 1))) cnn.add(kconv.Convolution2D(num_filters_2, conv_dim_2, conv_dim_2, activation="relu")) cnn.add(kcore.Flatten()) cnn.add(kcore.Dropout(0.5)) cnn.add(kcore.Dense(128, activation="relu")) # 4096 cnn.add(kcore.Dense(num_class, activation="softmax")) cnn.summary() myadadelta = kopt.Adadelta(lr=0.65, rho=0.95, epsilon=1e-08, decay=0.001) cnn.compile(loss="categorical_crossentropy", optimizer=myadadelta, metrics=["accuracy"]) cnn.fit(train_X, train_Y, batch_size=128, nb_epoch=180, verbose=1) # test_X = test_data_raw.reshape(test_data_raw.shape[0], 1, 28, 28) test_X = test_X.astype(float) test_X /= 255.0 yPred = cnn.predict_classes(test_X)
meteorology_residual_connection = Dense( 24, activation=keras.layers.advanced_activations.ELU(alpha=1.0), kernel_regularizer=keras.regularizers.l2(0.1), name="FullConnectionMA_RC")(meteorology_residual_connection) meteorology_residual_connection = add( [meteorology_x, meteorology_residual_connection], name="ResidualConnectionMA_Add") # 全连接层 meteorology_x = Dense( 3, activation=keras.layers.advanced_activations.ELU(alpha=1.0), kernel_regularizer=keras.regularizers.l2(0.1), name="FullConnectionMA_2")(meteorology_residual_connection) # Dropout meteorology_y = core.Dropout(rate=0.5, name="Meteorology_Module")(meteorology_x) # 模型层 model_meteorology = Model(inputs=[Meteorology_input, AODs_input], outputs=meteorology_y) ########################################################################## # 第二个模块 时滞天气模块 Weather_input = Input(shape=(len(data_t1.columns), ), name="Weather_input") # 只添加新的因素即可 # AODs_input = Input(shape=(17,), name="AODs_input") # 因素融合层 weather_concat = concatenate([Weather_input, AODs_input]) # 全连接层 weather_x = Dense(24, activation=keras.layers.advanced_activations.ELU(alpha=1.0),
def convnet_alexnet_lion_keras(image_dims): # model = Sequential() # model.add(Lambda(lambda x: (x / 255.0) - 0.5, input_shape=image_dims)) NR_CLASSES = 6 input = layers.Input(shape=image_dims, name="Input") conv_1 = convolutional.Convolution2D(96, 11, 11, border_mode='valid', name="conv_1", activation='relu', init='glorot_uniform')(input) pool_1 = convolutional.MaxPooling2D(pool_size=(3, 3), name="pool_1")(conv_1) zero_padding_1 = convolutional.ZeroPadding2D(padding=(1, 1), name="zero_padding_1")(pool_1) conv_2 = convolutional.Convolution2D(256, 3, 3, border_mode='valid', name="conv_2", activation='relu', init='glorot_uniform')(zero_padding_1) pool_2 = convolutional.MaxPooling2D(pool_size=(3, 3), name="pool_2")(conv_2) zero_padding_2 = keras.layers.convolutional.ZeroPadding2D( padding=(1, 1), name="zero_padding_2")(pool_2) conv_3 = convolutional.Convolution2D(384, 3, 3, border_mode='valid', name="conv_3", activation='relu', init='glorot_uniform')(zero_padding_2) conv_4 = convolutional.Convolution2D(384, 3, 3, border_mode='valid', name="conv_4", activation='relu', init='glorot_uniform')(conv_3) conv_5 = convolutional.Convolution2D(256, 3, 3, border_mode='valid', name="conv_5", activation='relu', init='glorot_uniform')(conv_4) pool_3 = convolutional.MaxPooling2D(pool_size=(3, 3), name="pool_3")(conv_5) flatten = core.Flatten(name="flatten")(pool_3) fc_1 = core.Dense(4096, name="fc_1", activation='relu', init='glorot_uniform')(flatten) fc_1 = core.Dropout(0.5, name="fc_1_dropout")(fc_1) output = core.Dense(4096, name="Output", activation='relu', init='glorot_uniform')(fc_1) output = core.Dropout(0.5, name="Output_dropout")(output) fc_2 = core.Dense(NR_CLASSES, name="fc_2", activation='softmax', init='glorot_uniform')(output) return models.Model([input], [fc_2])
activation='relu')(pool_layer2) pool_layer3 = pooling.MaxPooling2D(pool_size=(2, 2), strides=None, padding='valid', data_format=None)(conv_layer4) conv_layer5 = convolutional.Conv2D(32, (3, 3), strides=(1, 1), padding='same', activation='relu')(conv_layer3) pool_layer4 = pooling.MaxPooling2D(pool_size=(2, 2), strides=None, padding='valid', data_format=None)(conv_layer5) flatten_layer = core.Flatten()(pool_layer4) hidden1 = core.Dense(64, activation='relu')(flatten_layer) dropout1 = core.Dropout(0.2)(hidden1) hidden2 = core.Dense(64, activation='relu')(dropout1) out = core.Dense(num_classes, activation='softmax')(hidden2) model = Model(inp, out) model.compile(loss="categorical_crossentropy", optimizer='rmsprop', metrics=["accuracy"]) """ Layer (type) Output Shape Param # ================================================================= input_1 (InputLayer) (None, 256, 256, 3) 0 _________________________________________________________________ block1_conv1 (Conv2D) (None, 256, 256, 64) 1792 _________________________________________________________________
label_args = val_Label_args, batch_size= 8, input_size=input_dim) #================================================MODEL_ARCHITECTURE============================================================ # RGB MODALITY BRANCH OF CNN inputs_rgb = Input(shape=(input_dim[0],input_dim[1],3)) vgg_model_rgb = VGG16(weights='imagenet', include_top= False) conv_model_rgb = vgg_model_rgb(inputs_rgb) conv_model_rgb = Conv2D(1024, (3,3), strides=(1, 1), padding = 'same', activation='relu',data_format="channels_last") (conv_model_rgb) conv_model_rgb = Conv2D(1024, (3,3), strides=(1, 1), padding = 'same', activation='relu',data_format="channels_last") (conv_model_rgb) deconv_rgb_1 = Conv2DTranspose(num_class*C,(4,4), strides=(2, 2), padding='same', data_format="channels_last", activation='relu',kernel_initializer='glorot_normal')(conv_model_rgb) #============================================================================================================ conv_rgb_1 = Conv2D(num_class*C, (3,3), strides=(1,1), padding = 'same', activation='relu', data_format='channels_last')(deconv_rgb_1) dropout_rgb = core.Dropout(0.4)(conv_rgb_1) #=============================================================================================================== deconv_rgb_2 = Conv2DTranspose(num_class*C,(4,4), strides=(2, 2), padding='same', data_format="channels_last", activation='relu',kernel_initializer='glorot_normal')(dropout_rgb) conv_rgb_2 = Conv2D(num_class*C, (3,3), strides=(1,1), padding = 'same', activation='relu', data_format='channels_last')(deconv_rgb_2) deconv_rgb_3 = Conv2DTranspose(num_class*C,(4,4), strides=(2, 2), padding='same', data_format="channels_last", activation='relu',kernel_initializer='glorot_normal')(conv_rgb_2) conv_rgb_3 = Conv2D(num_class*C, (3,3), strides=(1,1), padding = 'same', activation='relu', data_format='channels_last')(deconv_rgb_3) deconv_rgb_4 = Conv2DTranspose(num_class*C,(4,4), strides=(2, 2), padding='same', data_format="channels_last", activation='relu',kernel_initializer='glorot_normal')(conv_rgb_3) conv_rgb_4 = Conv2D(num_class*C, (3,3), strides=(1,1), padding = 'same', activation='relu', data_format='channels_last')(deconv_rgb_4) deconv_rgb_5 = Conv2DTranspose(num_class*C,(4,4), strides=(2, 2), padding='same', data_format="channels_last", activation='relu',kernel_initializer='glorot_normal')(conv_rgb_4) # NIR MODALITY BRANCH OF CNN inputs_nir = Input(shape=(input_dim[0],input_dim[1],3)) vgg_model_nir = VGG16(weights='imagenet', include_top= False) conv_model_nir = vgg_model_rgb(inputs_nir) conv_model_nir = Conv2D(1024, (3,3), strides=(1, 1), padding = 'same', activation='relu',data_format="channels_last") (conv_model_nir) conv_model_nir = Conv2D(1024, (3,3), strides=(1, 1), padding = 'same', activation='relu',data_format="channels_last") (conv_model_nir)
trainX /= 255.0 trainY = kutils.to_categorical(trainData[:, 0]) nb_classes = trainY.shape[1] cnn = models.Sequential() cnn.add( conv.Convolution2D(nb_filters, nb_conv, nb_conv, border_mode="valid", input_shape=(1, 28, 28), activation="relu")) cnn.add(conv.Convolution2D(nb_filters, nb_conv, nb_conv, activation="relu")) cnn.add(conv.MaxPooling2D()) cnn.add(core.Dropout(0.25)) cnn.add(core.Flatten()) cnn.add(core.Dense(128, activation="relu")) cnn.add(core.Dropout(0.15)) cnn.add(core.Dense(64, activation="relu")) cnn.add(core.Dropout(0.15)) cnn.add(core.Dense(nb_classes, activation="softmax")) sgd = optm.sgd(lr=0.01, momentum=0.9, decay=1e-6, nesterov=True) cnn.compile( loss="mean_squared_error", optimizer="adadelta", ) cnn.fit(trainX, trainY,
noFeatures = trainX.shape[1] scaler = preproc.StandardScaler() trainX = scaler.fit_transform(trainX) """ Final Model """ epochs = 8 nn = models.Sequential() nn.add(core.Dense(noFeatures, input_shape=(noFeatures, ))) nn.add(advact.PReLU()) nn.add(norm.BatchNormalization()) nn.add(core.Dropout(0.2)) nn.add(core.Dense(2 * noFeatures, )) nn.add(advact.PReLU()) nn.add(norm.BatchNormalization()) nn.add(core.Dropout(0.25)) nn.add(core.Dense(noFeatures, )) nn.add(advact.PReLU()) nn.add(norm.BatchNormalization()) nn.add(core.Dropout(0.2)) nn.add(core.Dense(noOfClasses, activation="softmax")) print(nn.summary())
input_dim=input_shape)) model.add( normalization.BatchNormalization(axis=-1, momentum=0.99, epsilon=0.001, center=True, scale=True, beta_initializer='zeros', gamma_initializer='ones', moving_mean_initializer='zeros', moving_variance_initializer='ones', beta_regularizer=None, gamma_regularizer=None, beta_constraint=None, gamma_constraint=None)) model.add(core.Dropout(0.25, noise_shape=None, seed=None)) for i in range(9): model.add( Dense(units=input_shape, activation='relu', use_bias=True, kernel_initializer='random_uniform', bias_initializer='he_normal')) model.add( normalization.BatchNormalization(axis=-1, momentum=0.99, epsilon=0.001, center=True, scale=True, beta_initializer='zeros', gamma_initializer='ones',
y_test = np_utils.to_categorical(y_test, 10) model = Sequential() #first conv layer with 32 filters output (32,14,14) model.add( Convolution2D(32, (2, 2), padding='same', data_format='channels_last', input_shape=(28, 28, 1))) model.add(normalization.BatchNormalization()) model.add(Activation('relu')) model.add( MaxPooling2D(pool_size=(2, 2), padding='same', data_format='channels_last')) model.add(core.Dropout(0.25)) #second conv layer with 64 filter output (64,7,7) model.add( Convolution2D(64, (2, 2), padding='same', data_format='channels_last')) model.add(normalization.BatchNormalization()) model.add(Activation('relu')) model.add( MaxPooling2D(pool_size=(2, 2), padding='same', data_format='channels_last')) model.add(core.Dropout(0.25)) #fully connected layer with output 128 model.add(Flatten()) model.add(Dense(128, activation='relu')) model.add(Dropout(0.25))
cnn.add(conv.ZeroPadding2D((1, 1))) cnn.add(conv.Convolution2D(64, 3, 3, activation="relu")) cnn.add(conv.MaxPooling2D(strides=(2, 2))) # cnn.add(conv.ZeroPadding2D((1, 1))) # cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu")) # cnn.add(conv.ZeroPadding2D((1, 1))) # cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu")) # cnn.add(conv.ZeroPadding2D((1, 1))) # cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu")) # cnn.add(conv.ZeroPadding2D((1, 1))) # cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu")) # cnn.add(conv.MaxPooling2D(strides=(2,2))) cnn.add(core.Flatten()) cnn.add(core.Dropout(0.2)) cnn.add(core.Dense(128, activation="relu")) # 4096 cnn.add(core.Dense(nb_classes, activation="softmax")) cnn.summary() cnn.compile(loss="categorical_crossentropy", optimizer="adadelta", metrics=["accuracy"]) cnn.fit(trainX, trainY, batch_size=128, nb_epoch=1, verbose=1) testX = test.reshape(test.shape[0], 1, 48, 48) testX = testX.astype(float) testX /= 255.0 yPred = cnn.predict_classes(testX)
def test_dropout(): layer = core.Dropout(0.5) _runner(layer)
def test_dropout(self): layer = core.Dropout(0.5) self._runner(layer)
H = klc.Activation('relu')(H) H = klconv.Convolution2D(n_channels/4, 3, 3, border_mode='same', init='glorot_uniform')(H) H = kln.BatchNormalization(mode=2)(H) H = klc.Activation('relu')(H) H = klconv.Convolution2D(1, 1, 1, border_mode='same', init='glorot_uniform')(H) g_V = klc.Activation('sigmoid')(H) generator = km.Model(g_input, g_V) generator.compile(loss='binary_crossentropy', optimizer=g_opt) # discriminator d_input = kl.Input(shape=[1,img_rows,img_cols]) H = klconv.Convolution2D(256, 5, 5, subsample=(2,2), border_mode='same', activation='relu')(d_input) H = kla.LeakyReLU(0.2)(H) H = klc.Dropout(dropout_rate)(H) H = klconv.Convolution2D(512, 5, 5, subsample=(2,2), border_mode='same', activation='relu')(H) H = kla.LeakyReLU(0.2)(H) H = klc.Dropout(dropout_rate)(H) H = klc.Flatten()(H) H = klc.Dense(256)(H) H = kla.LeakyReLU(0.2)(H) H = klc.Dropout(dropout_rate)(H) d_V = klc.Dense(2, activation='softmax')(H) discriminator = km.Model(d_input, d_V) discriminator.compile(loss='categorical_crossentropy', optimizer=d_opt) # stack generator and discriminator gan_input = kl.Input(shape=[noise_size]) H = generator(gan_input)
# Read splitted data df_train = pd.read_csv('train.csv') df_valid = pd.read_csv('test.csv') # CNN Model Architecture model = models.Sequential() model.add(convolutional.Convolution2D(16, 3, 3, input_shape=(32, 128, 3), activation='relu')) model.add(pooling.MaxPooling2D(pool_size=(2, 2))) model.add(convolutional.Convolution2D(32, 3, 3, activation='relu')) model.add(pooling.MaxPooling2D(pool_size=(2, 2))) model.add(convolutional.Convolution2D(64, 3, 3, activation='relu')) model.add(pooling.MaxPooling2D(pool_size=(2, 2))) model.add(core.Flatten()) model.add(core.Dense(500, activation='relu')) model.add(core.Dropout(.5)) model.add(core.Dense(100, activation='relu')) model.add(core.Dropout(.25)) model.add(core.Dense(20, activation='relu')) model.add(core.Dense(1)) model.compile(optimizer=optimizers.Adam(lr=1e-04), loss='mean_squared_error') # load the exist model model.load_weights("model.h5") history = model.fit_generator(# continue training model for 17 epochs generate_samples(df_train, ''), samples_per_epoch=df_train.shape[0], nb_epoch=17,#0.016 validation_data=generate_samples(df_valid, '', augment=False), nb_val_samples=df_valid.shape[0],
x7 = layers.Dense(24, activation=keras.layers.advanced_activations.LeakyReLU( alpha=0.5))(inputAODs) x7 = layers.Dense( 3, activation=keras.layers.advanced_activations.LeakyReLU(alpha=0.5))(x7) x7 = Model(inputs=inputAODs, outputs=x7) combined = layers.concatenate([ x1.output, x2.output, x3.output, x4.output, x5.output, x6.output, x7.output ]) # 输出层 z = layers.Dense( 12, activation=keras.layers.advanced_activations.LeakyReLU(alpha=0.5), kernel_regularizer=keras.regularizers.l2(0.01))(combined) z = core.Dropout(rate=0.01)(z) z = layers.Dense( 4, activation=keras.layers.advanced_activations.LeakyReLU(alpha=0.5), kernel_regularizer=keras.regularizers.l2(0.01))(z) z = layers.Dense( 1, activation=keras.layers.advanced_activations.LeakyReLU(alpha=0.5), kernel_regularizer=keras.regularizers.l2(0.01))(z) # 建立模型 model = Model(inputs=[ x1.input, x2.input, x3.input, x4.input, x5.input, x6.input, x7.input ], outputs=z) # 模型编译
def architecture(): # put the normalization fucntion inside the model ensure preprocess using Lambda layer # There were many issue depending on using the lambda layer and in this waa it works def resize_normalize(image): import cv2 from keras.backend import tf as ktf """ Applies preprocessing pipeline to an image: crops `top` and `bottom` portions of image, resizes to 66*200 px and scales pixel values to [0, 1]. """ # resize to width 200 and high 66 liek recommended # in the nvidia paper for the used CNN # image = cv2.resize(image, (66, 200)) #first try resized = ktf.image.resize_images(image, (32, 128)) #normalize 0-1 resized = resized / 255.0 - 0.5 return resized print('I am inside call of architecture') #initialize model model = Sequential() #dropout = 0.5 nonlinear = 'tanh' print('I am before call of cropping layer') ### Convolution layers and parameters were taken from the "nvidia paper" on end-to-end autonomous steering. model.add( Cropping2D(cropping=((60, 20), (1, 1)), input_shape=(160, 320, 3))) print('I am before call of Lambda') model.add( Lambda(resize_normalize, input_shape=(160, 320, 3), output_shape=(32, 128, 3))) # Model architecture model.add( Convolution2D(16, 3, 3, input_shape=(32, 128, 3), activation='relu')) model.add(pooling.MaxPooling2D(pool_size=(2, 2))) model.add(Convolution2D(32, 3, 3, activation='relu')) model.add(pooling.MaxPooling2D(pool_size=(2, 2))) model.add(Convolution2D(64, 3, 3, activation='relu')) model.add(pooling.MaxPooling2D(pool_size=(2, 2))) model.add(core.Flatten()) model.add(core.Dense(500, activation='relu')) model.add(core.Dropout(.5)) model.add(core.Dense(100, activation='relu')) model.add(core.Dropout(.25)) model.add(core.Dense(20, activation='relu')) model.add(core.Dense(1)) model.compile(optimizer=optimizers.Adam(lr=1e-04), loss='mean_squared_error') #model.add(Lambda(lambda x: resize_normalize(x), input_shape=(80,318,3), output_shape=(66, 200, 3))) # model.add(Convolution2D(24, 5, 5, name='conv1', subsample=(2, 2), activation=nonlinear)) # model.add(Convolution2D(36, 5, 5, name='conv2', subsample=(2, 2), activation=nonlinear)) # model.add(Convolution2D(48, 5, 5, name='conv3', subsample=(2, 2), activation=nonlinear)) # model.add(Convolution2D(64, 3, 3, name='conv4', activation=nonlinear)) # model.add(Convolution2D(64, 3, 3, name='conv5', activation=nonlinear)) # ### Regression # model.add(Flatten()) # model.add(Dropout(dropout)) # model.add(Dense(1164, name='hidden1', activation=nonlinear)) # model.add(Dropout(dropout)) # model.add(Dense(100, name='hidden2', activation=nonlinear)) # model.add(Dropout(dropout)) # model.add(Dense(50, name='hidden3', activation=nonlinear)) # model.add(Dropout(dropout)) # model.add(Dense(10, name='hidden4', activation=nonlinear)) # model.add(Dropout(dropout)) # model.add(Dense(1, name='output', activation=nonlinear)) # #model.compile(optimizer=optimizers.Adam(lr=1e-04), loss='mean_squared_error') # model.compile(optimizer='adam', loss='mse') print('I am finished build the model') print(model.summary()) return model
meteorology_residual_connection = Dense( 24, activation=keras.layers.advanced_activations.ELU(alpha=1.0), kernel_regularizer=keras.regularizers.l2(0.1), name="FullConnectionMA_RC")(meteorology_residual_connection) meteorology_residual_connection = add( [meteorology_x, meteorology_residual_connection], name="ResidualConnectionMA_Add") # 全连接层 meteorology_x = Dense( 3, activation=keras.layers.advanced_activations.ELU(alpha=1.0), kernel_regularizer=keras.regularizers.l2(0.1), name="FullConnectionMA_2")(meteorology_residual_connection) # Dropout meteorology_y = core.Dropout(rate=0.5, name="Meteorology_Module")(meteorology_x) # 模型层 model_meteorology = Model(inputs=[Meteorology_input, AODs_input], outputs=meteorology_y) ###################################################################################################################### # 第二个模块 时滞天气模块 Weather_input = Input(shape=(12, ), name="Weather_input") # 只添加新的因素即可 # AODs_input = Input(shape=(17,), name="AODs_input") # 因素融合层 weather_concat = concatenate([Weather_input, AODs_input]) # 全连接层 weather_x = Dense(24, activation=keras.layers.advanced_activations.ELU(alpha=1.0), kernel_regularizer=keras.regularizers.l2(0.1),
name="ResidualConnectionAIA")(allin_x1) allin_residual_connection2 = Dense( 24, activation=keras.layers.LeakyReLU(alpha=0.2), name="FullConnectionAIA_RC")(allin_residual_connection1) allin_residual_output = add([allin_x1, allin_residual_connection2], name="ResidualConnectionAIA_Add") # 全连接层 2 allin_x2 = Dense(12, activation=keras.layers.LeakyReLU(alpha=0.2), name="FullConnectionAIA_2")(allin_residual_output) # Dropout # 全部特征 allin_y = core.Dropout(rate=0.01, name="AllIn_Module")(allin_x2) # 全连接层 3 # 全部特征 allin_y2 = Dense(8, activation=keras.layers.LeakyReLU(alpha=0.2), name="FullConnectionAIA_3x")(allin_y) # 全连接层 3 # 全部特征 allin_y2 = Dense(4, activation=keras.layers.LeakyReLU(alpha=0.2), name="FullConnectionAIA_3")(allin_y2) # 模型层 # 全部特征
152) # pm+id ==2 # 不要(,,1) timesteps = 152 # 这里换成监测站 data_dim = len(input_list) # 期望输入数据尺寸: (batch_size, timesteps, data_dim) model = Sequential() model.add( LSTM(32, return_sequences=True, input_shape=(timesteps, data_dim))) # 返回维度为 32 的向量序列 model.add(LSTM(32, return_sequences=True)) model.add(LSTM(32, return_sequences=True)) model.add(LSTM(32, return_sequences=True)) model.add(LSTM(32, return_sequences=True)) # 返回维度为 32 的向量序列 model.add(core.Dropout(rate=0.01)) model.add(LSTM(32, )) # 返回维度为 32 的单个向量 model.add( Dense(152, activation=keras.layers.LeakyReLU(alpha=0.2), kernel_regularizer=keras.regularizers.l2(0.01))) model.compile(loss=['mean_absolute_error'], optimizer=keras.optimizers.Adam(lr=0.01, beta_1=0.9, beta_2=0.999), metrics=['accuracy']) # 计算耗时 starttime = datetime.datetime.now() # 程序