def residual_network(x): """ ResNeXt by default. For ResNet set `cardinality` = 1 above. """ def add_common_layers(y): y = layers.BatchNormalization()(y) y = layers.LeakyReLU()(y) y = layers.Dropout(dropout_level)(y) return y def grouped_convolution(y, nb_channels, _strides): # when `cardinality` == 1 this is just a standard convolution if cardinality == 1: return layers.Conv1D( nb_channels, kernel_size=3, strides=_strides, padding='same', kernel_regularizer=regularizers.l2(L2_regularizer))(y) assert not nb_channels % cardinality _d = nb_channels // cardinality # in a grouped convolution layer, input and output channels are divided into `cardinality` groups, # and convolutions are separately performed within each group groups = [] for j in range(cardinality): group = layers.Lambda(lambda z: z[:, :, j * _d:j * _d + _d])(y) groups.append( layers.Conv1D( _d, kernel_size=3, strides=_strides, padding='same', kernel_regularizer=regularizers.l2(L2_regularizer))(group)) # the grouped convolutional layer concatenates them as the outputs of the layer y = layers.concatenate(groups) return y def residual_block(y, nb_channels_in, nb_channels_out, _strides=1, _project_shortcut=False): """ Our network consists of a stack of residual blocks. These blocks have the same topology, and are subject to two simple rules: - If producing spatial maps of the same size, the blocks share the same hyper-parameters (width and filter sizes). - Each time the spatial map is down-sampled by a factor of 2, the width of the blocks is multiplied by a factor of 2. """ shortcut = y # we modify the residual building block as a bottleneck design to make the network more economical y = layers.Conv1D( nb_channels_in, kernel_size=1, strides=1, padding='same', kernel_regularizer=regularizers.l2(L2_regularizer))(y) y = add_common_layers(y) # ResNeXt (identical to ResNet when `cardinality` == 1) y = grouped_convolution(y, nb_channels_in, _strides=_strides) y = add_common_layers(y) y = layers.Conv1D( nb_channels_out, kernel_size=1, strides=1, padding='same', kernel_regularizer=regularizers.l2(L2_regularizer))(y) # batch normalization is employed after aggregating the transformations and before adding to the shortcut y = layers.BatchNormalization()(y) # identity shortcuts used directly when the input and output are of the same dimensions if _project_shortcut or _strides != 1: # when the dimensions increase projection shortcut is used to match dimensions (done by 1×1 convolutions) # when the shortcuts go across feature maps of two sizes, they are performed with a stride of 2 shortcut = layers.Conv1D( nb_channels_out, kernel_size=1, strides=_strides, padding='same', kernel_regularizer=regularizers.l2(L2_regularizer))(shortcut) shortcut = layers.BatchNormalization()(shortcut) y = layers.add([shortcut, y]) # relu is performed right after each batch normalization, # expect for the output of the block where relu is performed after the adding to the shortcut y = layers.Activation('relu')(y) return y # conv1 x = layers.Conv1D(64, kernel_size=7, strides=2, padding='same', kernel_regularizer=regularizers.l2(L2_regularizer))(x) x = add_common_layers(x) # conv2 x = layers.MaxPooling1D(pool_size=3, strides=2, padding='same')(x) for i in range(2): project_shortcut = True if i == 0 else False x = residual_block(x, 64, 128, _project_shortcut=project_shortcut) # conv3 for i in range(2): # down-sampling is performed by conv3_1, conv4_1, and conv5_1 with a stride of 2 strides = 2 if i == 0 else 1 x = residual_block(x, 128, 256, _strides=strides) # conv4 for i in range(2): strides = 2 if i == 0 else 1 x = residual_block(x, 256, 512, _strides=strides) # conv5 for i in range(2): strides = 2 if i == 0 else 1 x = residual_block(x, 512, 1024, _strides=strides) x = layers.GlobalAveragePooling1D()(x) return x
def _build_layers_v2(self, input_dict, num_outputs, options): """Define the layers of a custom model. Arguments: input_dict (dict): Dictionary of input tensors, including "obs", "prev_action", "prev_reward", "is_training". num_outputs (int): Output tensor must be of size [BATCH_SIZE, num_outputs]. options (dict): Model options. Returns: (outputs, feature_layer): Tensors of size [BATCH_SIZE, num_outputs] and [BATCH_SIZE, desired_feature_size]. """ TRANSFORMER_MODEL_DIM = options["custom_options"][ "transformer_model_dim"] TRANSFORMER_NUM_HEADS = options["custom_options"][ "transformer_num_heads"] TRANSFORMER_DEPTH = options["custom_options"]["transformer_depth"] CONV_PADDING = options["custom_options"]["conv_padding"] # Agent architecture p.15 of Zambaldi et al # "The input module contained two convolutional layers with 12 and 24 kernels, 2 × 2 kernel sizes # and a stride of 1, followed by a rectified linear unit (ReLU) activation function. The output # was tagged with two extra channels indicating the spatial position (x and y) of each cell in # the feature map using evenly spaced values between −1 and 1. This was passed to the relational # module, consisting of relational blocks, with shared parameters. Queries, keys and values were # produced by 2 to 4 attention heads and had an embedding size (d) of 64. The output of this module # was aggregated using a feature-wise max pooling function and passed to 4 fully connected layers, # each followed by a ReLU. Policy logits (pi, size 4) and baseline function (B, size 1) were produced # by a linear projection. The policy logits were normalized and used as multinomial distribution from # which the action (a) was sampled." # NOTE: there is no dropout in Zambaldi et al inputs = input_dict["obs"] sess = tf.get_default_session() K.set_session(sess) # NOTE: the weights in the self-attention mechanism # and feed-forward layers are shared between all Transformer blocks (as in # Zambaldi et al, but unlike every other Transformer paper) # The published version of Zambaldi et al does not tell us what the MLP g_\theta is, but # # - in Santoro et al "A simple neural network module for relational reasoning" # the analogous g_\theta is is a four-layer MLP with 256 dimensional hidden layers # with ReLU non-linearities # - in Keras-Transformer the default is a two layer model with hidden dimension # equal to 4 * the embedding dimension, which in the case of 64 dimensional embeddings # gives 256 (this is also the convention in the Sparse Transformer paper) # - in the first version of Zambaldi et al they write "passed to a multilayer perceptron # (2-layer MLP with ReLU non-linearities) with the same layers sizes as ei" # # Hence, attempting to follow Zambaldi, we use layer size TRANSFORMER_MODEL_DIM # (in v6 we used 4 times this) attention_layer = MultiHeadSelfAttentionZambaldi( name='self_attention', num_heads=TRANSFORMER_NUM_HEADS, use_masking=False, dropout=0, compression_window_size=None) dense_layer1 = layers.Dense(TRANSFORMER_MODEL_DIM, activation='relu') dense_layer2 = layers.Dense(TRANSFORMER_MODEL_DIM) def transformer_block(input): #_, seq_len, d_model = K.int_shape(input) a = LayerNormalization()(input) a = attention_layer( a ) # a = attention(h) has shape -1, seq_len, TRANSFORMER_MODEL_DIM b = dense_layer1(a) b = dense_layer2(b) # b = ff(a) r = layers.Add()([input, b]) Hprime = LayerNormalization()(r) return Hprime # CONVOLUTIONS ------ # # Question: should we use max-pooling here? It seems not, as the downsampling in the # Santoro et al paper "A simple neural network module for relational reasoning" # occurs using 3x3 patches with stride 2, rather than max-pooling, and it is not # mentioned anywhere in the papers. # # It is worth comparing to e.g. the models for deep RL on 3D environments in the IMPALA # paper, see Figure 3, which also have no max-pooling layers and downsample instead using # strided convolutional layers. You'll see there also they prefer hidden layers of width # 256 for the FC layers after the initial convolutional layers, in processing visual scenes. # So the Zambaldi paper is consistent with their other work on deep RL, in terms of the model. x = layers.Lambda(lambda x: x / 255)(inputs) # rescale RGB to [0,1] x = layers.Conv2D(12, (2, 2), activation='relu', padding=CONV_PADDING)(x) x = layers.Conv2D(24, (2, 2), activation='relu', padding=CONV_PADDING)( x) # output shape -1, num_rows, num_cols, 62 x = layers.Dense( TRANSFORMER_MODEL_DIM - 2, activation=None, use_bias=False )(x) # output shape -1, num_rows, num_cols, TRANSFORMER_MODEL_DIM-2 # NOTE: we are using the default "valid" padding, so actually our width and height decrease # by one in each convolutional layer # POSITION EMBEDDING ----- # # Here we follow Zambaldi et al, rather than the standard Transformer # positional embeddings num_rows, num_cols, d_model = x.get_shape().as_list()[-3:] ps = np.zeros([num_rows, num_cols, 2], dtype=K.floatx()) # shape (12,13,2) for ty in range(num_rows): for tx in range(num_cols): ps[ty, tx, :] = [(2 / (num_rows - 1)) * ty - 1, (2 / (num_cols - 1)) * tx - 1] ps_expand = K.expand_dims(K.constant(ps), axis=0) # shape (1,num_rows,num_cols,2) ps_tiled = K.tile( ps_expand, [K.shape(x)[0], 1, 1, 1]) # shape (None,num_rows,num_cols,2) # (None,num_rows,num_cols,62) concatenate with (None,num_rows,num_cols,2) # to get (None,num_rows,num_cols,TRANSFORMER_MODEL_DIM) x = Concatenate(axis=3)([x, ps_tiled]) x = layers.Reshape((num_rows * num_cols, d_model + 2))(x) # TRANSFORMER ----- for i in range(TRANSFORMER_DEPTH): x = transformer_block(x) # MAX-POOLING ----- # from p.4 "The E~ matrix, with shape Nxf is reudced to an f-dimensional vector by max-pooling # over the entity dimension. This pooled vector is then passed to a small MLP..." num_entities, d_model = x.get_shape().as_list()[-2:] x = layers.MaxPooling1D(pool_size=num_entities)(x) x = layers.Flatten()(x) # FULLY-CONNECTED LAYERS ---- x = layers.Dense(256, activation='relu')(x) x = layers.Dense(256, activation='relu')(x) x = layers.Dense(256, activation='relu')(x) x = layers.Dense(256, activation='relu')(x) output_tensor = layers.Dense(4)(x) # final output is logits return output_tensor, x
validation_samples:training_samples + validation_samples + texting_samples] #卷积神经网络设计 from keras.models import Sequential from keras.layers import Flatten, Dense, Embedding from keras.layers import SimpleRNN from keras import layers from keras import regularizers model = Sequential() model.add(Embedding(10000, 32, input_shape=(2000, ))) #model.add(SimpleRNN(32)) #model.add(Dense(25,activation='softmax')) model.add(layers.Conv1D(256, 5, activation='relu')) #原来是256 model.add(layers.MaxPooling1D(2)) model.add(layers.Dropout(0.5)) model.add(layers.Conv1D(256, 5, activation='relu')) #原来是256 model.add(layers.MaxPooling1D(64)) model.add(layers.Dropout(0.5)) model.add(layers.GlobalAveragePooling1D()) model.add(Dense(17, activation='softmax')) model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) model.summary() model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) history = model.fit(x_train,
def build_conv_layers(frames, freq_bins, mod_options): ''' Build the convolutionnal base of the model based on the Dielemann model for music recommandation (useful for transfer learning). This model is trained on time-frequency representation of music. The input data fed to the model must be shaped like (batch_size, time, frequency). Once FC layers are added via the add_fc_layers function, the model can be trained and saved for re-use in transfer learning applications. Parameters ---------- frames: int Number of time frames in a single input freq_bin: int Number of frequency bands in a single input mod_option: dictionnary Specific options for the model. This dictionnary must contain: 'activation': string name of the activation function for keras layers 'batchNormConv': bool Choose if the model should apply babtch normalization after each convnet Returns ------- model: keras model The model is not compiled since it requires FC layers on top. Expects inputs of shape (batch size, frames, freq_bins) and outputs tensor of shape: (batch size, frames_pool, freq_bins) where frames_pool has been through 3 poolings of size 2 (divise by 2) and with 4 zeros added before each pooling (add 4). ''' inputs = layers.Input(shape=(frames, freq_bins)) #%%=========== First layer =============== #zero-padding the input padding_1 = layers.ZeroPadding1D(padding=2)(inputs) #Convnet 256 neurons with 4 sample window. Activation defined in mod_option dictionnary conv1 = layers.Conv1D(256, 4, padding='same', activation=mod_options['activation'])(padding_1) #Normalise batch if defined in mod_options if mod_options['batchNormConv']: conv1 = layers.BatchNormalization()(conv1) #Reduce data by max pooling between 2 values pool_1 = layers.MaxPooling1D(pool_size=2)(conv1) #%%============ Second layer ============== #Same layer as the previous one padding_2 = layers.ZeroPadding1D(padding=2)(pool_1) conv_2 = layers.Conv1D(256, 4, padding='same', activation=mod_options['activation'])(padding_2) if mod_options['batchNormConv']: conv_2 = layers.BatchNormalization()(conv_2) pool_2 = layers.MaxPooling1D(pool_size=2)(conv_2) ''' #%%=========== Third layer ???================ #zero-padding the input model.Add(layers.ZeroPadding1D(padding = 2)) #Convnet 512 neurons with 4 sample window. model.add(layers.Conv1D(512, 4, padding = 'same', activation = mod_options['activation'])) #Normalize batch if defined if mod_options['batchNormConv']: model.add(layers.BatchNormalization()) ''' #%%=========== Fourth layer ================= #zero-padding the input padding_3 = layers.ZeroPadding1D(padding=2)(pool_2) #Convnet 512 neurons with 4 sample window. conv_3 = layers.Conv1D(512, 4, padding='same', activation=mod_options['activation'])(padding_3) #Normalize batch if defined if mod_options['batchNormConv']: conv_3 = layers.BatchNormalization()(conv_3) model = Model(inputs=inputs, outputs=conv_3) return model
Xtrain = pad_sequences(encoded_train, maxlen=max_length, padding='post') Xtest = pad_sequences(encoded_test, maxlen=max_length, padding='post') vocab_size = len(tokenizer.word_index) + 1 # raw_embedding = load_embedding('glove.6B.100d.txt') raw_embedding = load_embedding('embedding_word2vec.txt') embedding_vectors = get_weight_matrix(raw_embedding, tokenizer.word_index) embedding_layer = layers.Embedding(vocab_size, 100, weights=[embedding_vectors], input_length=max_length, trainable=False) model = keras.Sequential() model.add(embedding_layer) model.add(layers.Conv1D(filters=128, kernel_size=5, activation='relu')) model.add(layers.MaxPooling1D(pool_size=2)) model.add(layers.Flatten()) model.add(layers.Dense(1, activation='sigmoid')) print(model.summary()) Xtrain = Xtrain.astype(np.float32) # y_train = y_train.astype(np.float32) # Xtest = pad_sequences(encoded_test, maxlen=max_length, padding='post') Xtest = Xtest.astype(np.float32) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) print( '-------------------------------------------------------------------------' ) print(f'Training for fold {fold_no} ...')
from keras import layers from keras import Input from keras.models import Model vocabulary_size = 50000 num_income_groups = 10 posts_input = Input(shape=(None, ), dtype='int32', name='posts') embedded_posts = layers.Embedding(256, vocabulary_size)(posts_input) x = layers.Conv1D(128, 5, activation='relu')(embedded_posts) x = layers.MaxPooling1D(5)(x) x = layers.Conv1D(256, 5, activation='relu')(x) x = layers.Conv1D(256, 5, activation='relu')(x) x = layers.MaxPooling1D(5)(x) x = layers.Conv1D(256, 5, activation='relu')(x) x = layers.Conv1D(256, 5, activation='relu')(x) x = layers.GlobalMaxPooling1D()(x) x = layers.Dense(128, activation='relu')(x) age_prediction = layers.Dense(1, name='age')(x) income_prediction = layers.Dense(num_income_groups, activation='softmax', name='income')(x) gender_prediction = layers.Dense(1, activation='sigmoid', name='gender')(x) model = Model(posts_input, [age_prediction, income_prediction, gender_prediction]) model.compile(optimizer='rmsprop', loss=['mse', 'categorical_crossentropy', 'binary_crossentropy']) model.compile(optimizer='rmsprop', loss={ 'age': 'mse', 'income': 'categorical_crossentropy', 'gender': 'binary_crossentropy'
layers.Bidirectional(layers.LSTM(32, dropout=0.2, recurrent_dropout=0.5), input_shape=[None, float_data.shape[-1]])) model.add(layers.Dense(1000, activation='relu')) model.add(layers.Dense(1)) model.compile(optimizer='adam', loss='mae') history = model.fit_generator(train_gen, steps_per_epoch=500, epochs=40, validation_data=val_gen, validation_steps=val_steps) ## conv1d ## model = models.Sequential() model.add(layers.Embedding(max_features, output_dim=128, input_length=maxlen)) model.add(layers.Conv1D(filters=32, kernel_size=7, activation='relu')) model.add(layers.MaxPooling1D(pool_size=5)) model.add(layers.Conv1D(32, 7, activation='relu')) model.add(layers.GlobalAveragePooling1D()) model.add(layers.Dense(1)) model.summary() model.compile(optimizer=optimizers.rmsprop(lr=1e-4), loss='binary_crossentropy', metrics=['acc']) history = model.fit(x_train, y_train, epochs=10, batch_size=128, validation_split=0.2) ## 1dconv --> RNN ## step = 3
def Conv1DRegressorIn1(flag): K.clear_session() current_neighbor = space['neighbor'] current_idx_idx = space['idx_idx'] current_dense_num = space['dense_num'] current_dilation1D_layers = space['dilation1D_layers'] current_dilation1D_filter_num = space['dilation1D_filter_num'] current_reduce1D_filter_num = space['reduce1D_filter_num'] summary = True verbose = 0 # # setHyperParams # ## hypers for data neighbor = {{choice([50, 60, 70, 80, 90, 100, 110, 120, 130, 140])}} idx_idx = {{choice([0, 1, 2, 3, 4, 5, 6, 7, 8])}} idx_lst = [ [x for x in range(158) if x not in [24, 26]], # 去除无用特征 [ x for x in range(158) if x not in [24, 26] + [x for x in range(1, 6)] + [x for x in range(16, 22)] + [40, 42] ], # 去除无用特征+冗余特征 [ x for x in range(158) if x not in [24, 26] + [x for x in range(0, 22)] ], # 去除无用特征+方位特征 [x for x in range(158) if x not in [24, 26] + [22, 23, 26, 37, 38]], # 去除无用特征+深度特征 [ x for x in range(158) if x not in [24, 26] + [x for x in range(27, 37)] + [x for x in range(40, 46)] ], # 去除无用特征+二级结构信息 # [x for x in range(158) if x not in [24, 26] + [x for x in range(27, 34)] + [x for x in range(40, 46)]],# 去除无用特征+二级结构信息1 # [x for x in range(158) if x not in [24, 26] + [x for x in range(34, 37)] + [x for x in range(40, 46)]],# 去除无用特征+二级结构信息2 [x for x in range(158) if x not in [24, 26] + [46, 47]], # 去除无用特征+实验条件 [ x for x in range(158) if x not in [24, 26] + [39] + [x for x in range(57, 61)] + [x for x in range(48, 57)] + [x for x in range(61, 81)] + [x for x in range(140, 155)] ], # 去除无用特征+所有原子编码 # [x for x in range(158) if x not in [24, 26] + [39] + [x for x in range(57, 61)] + [x for x in range(48, 57)] + [x for x in range(140, 145)]],# 去除无用特征+原子编码1 # [x for x in range(158) if x not in [24, 26] + [39] + [x for x in range(57, 61)] + [x for x in range(61, 77)] + [x for x in range(145, 153)]],# 去除无用特征+原子编码2 # [x for x in range(158) if x not in [24, 26] + [39] + [x for x in range(57, 61)] + [x for x in range(77, 81)] + [x for x in range(153, 155)]],# 去除无用特征+原子编码3 [ x for x in range(158) if x not in [24, 26] + [x for x in range(81, 98)] ], # 去除无用特征+rosetta_energy [ x for x in range(158) if x not in [24, 26] + [x for x in range(98, 140)] + [x for x in range(155, 158)] ] # 去除无用特征+msa ] idx = idx_lst[idx_idx] ## hypers for net lr = 1e-4 # 0.0001 batch_size = 32 epochs = 200 padding_style = 'same' activator_Conv1D = 'elu' activator_Dense = 'tanh' dense_num = {{choice([64, 96, 128])}} # 64 dilation1D_layers = {{choice([8, 16, 32])}} #8 dilation1D_filter_num = {{choice([16, 32])}} #16 dilation_lower = 1 dilation_upper = 16 dropout_rate_dilation = 0.25 dropout_rate_reduce = 0.25 dropout_rate_dense = 0.25 initializer_Conv1D = initializers.lecun_uniform(seed=527) initializer_Dense = initializers.he_normal(seed=527) kernel_size = 5 l2_rate = 0.001 loss_type = logcosh metrics = ('mae', pearson_r, rmse) pool_size = 2 reduce_layers = 5 # 110 -> 55 -> 28 -> 14 -> 7 -> 4, 50 - 25 - 13 - 7 - 4 - 2 reduce1D_filter_num = {{choice([16, 32, 64])}} #32 residual_stride = 2 def _data(fold_num, neighbor, idx): train_data_pth = '/dl/sry/mCNN/dataset/deepddg/npz/wild/cross_valid/cro_fold%s_train_center_CA_PCA_False_neighbor_140.npz' % fold_num val_data_pth = '/dl/sry/mCNN/dataset/deepddg/npz/wild/cross_valid/cro_fold%s_valid_center_CA_PCA_False_neighbor_140.npz' % fold_num ## train data train_data = np.load(train_data_pth) x_train = train_data['x'] y_train = train_data['y'] ddg_train = train_data['ddg'].reshape(-1) ## select kneighbor atoms x_train_kneighbor_lst = [] for sample in x_train: dist_arr = sample[:, 0] indices = sorted(dist_arr.argsort()[:neighbor]) x_train_kneighbor_lst.append(sample[indices, :]) x_train = np.array(x_train_kneighbor_lst) ## idx x_train = x_train[:, :, idx] ## val data val_data = np.load(val_data_pth) x_val = val_data['x'] y_val = val_data['y'] ddg_val = val_data['ddg'].reshape(-1) ## select kneighbor atoms x_val_kneighbor_lst = [] for sample in x_val: dist_arr = sample[:, 0] indices = sorted(dist_arr.argsort()[:neighbor]) x_val_kneighbor_lst.append(sample[indices, :]) x_val = np.array(x_val_kneighbor_lst) ## idx x_val = x_val[:, :, idx] # sort row default is chain, pass # reshape and one-hot y_train = to_categorical(y_train) y_val = to_categorical(y_val) # normalization train_shape = x_train.shape val_shape = x_val.shape col_train = train_shape[-1] col_val = val_shape[-1] x_train = x_train.reshape((-1, col_train)) x_val = x_val.reshape((-1, col_val)) mean = x_train.mean(axis=0) std = x_train.std(axis=0) std[np.argwhere(std == 0)] = 0.01 x_train -= mean x_train /= std x_val -= mean x_val /= std x_train = x_train.reshape(train_shape) x_val = x_val.reshape(val_shape) print('x_train: %s' '\ny_train: %s' '\nddg_train: %s' '\nx_val: %s' '\ny_val: %s' '\nddg_val: %s' % (x_train.shape, y_train.shape, ddg_train.shape, x_val.shape, y_val.shape, ddg_val.shape)) return x_train, y_train, ddg_train, x_val, y_val, ddg_val # # cross_valid # hyper_param_tag = '%s_%s_%s_%s_%s_%s' % ( current_neighbor, current_idx_idx, current_dense_num, current_dilation1D_layers, current_dilation1D_filter_num, current_reduce1D_filter_num) modeldir = '/dl/sry/projects/from_hp/mCNN/src/Network/deepddg/opt_all_resnet/model/%s-%s' % ( hyper_param_tag, time.strftime("%Y.%m.%d.%H.%M.%S", time.localtime())) os.makedirs(modeldir, exist_ok=True) opt_lst = [] for k_count in range(1, 11): print('\n** fold %s is processing **\n' % k_count) filepth = '%s/fold_%s_weights-best.h5' % (modeldir, k_count) my_callbacks = [ callbacks.ReduceLROnPlateau( monitor='val_loss', factor=0.33, patience=5, verbose=verbose, mode='min', min_lr=1e-8, ), callbacks.EarlyStopping(monitor='val_loss', patience=10, verbose=verbose), callbacks.ModelCheckpoint(filepath=filepth, monitor='val_mean_absolute_error', verbose=verbose, save_best_only=True, mode='min', save_weights_only=True) ] x_train, y_train, ddg_train, x_val, y_val, ddg_val = _data( k_count, neighbor, idx) row_num, col_num = x_train.shape[1:3] # # build net # input_layer = Input(shape=(row_num, col_num), name='input_layer') y = layers.Conv1D(filters=dilation1D_filter_num, kernel_size=1, padding=padding_style, activation=activator_Conv1D, kernel_initializer=initializer_Conv1D, kernel_regularizer=regularizers.l2(l2_rate), name='first_conv_layer', trainable=True)(input_layer) res = layers.BatchNormalization(axis=-1, name='first_BN_layer')(y) ## loop with Conv1D with dilation (padding='same') for _ in range(dilation1D_layers): y = layers.SeparableConv1D( filters=dilation1D_filter_num, kernel_size=kernel_size, padding=padding_style, dilation_rate=dilation_lower, activation=activator_Conv1D, depthwise_initializer=initializer_Conv1D, pointwise_initializer=initializer_Conv1D, depthwise_regularizer=regularizers.l2(l2_rate), pointwise_regularizer=regularizers.l2(l2_rate))(res) y = layers.BatchNormalization(axis=-1)(y) y = layers.Dropout(dropout_rate_dilation)(y) y = layers.SeparableConv1D( filters=dilation1D_filter_num, kernel_size=kernel_size, padding=padding_style, dilation_rate=dilation_lower, activation=activator_Conv1D, depthwise_initializer=initializer_Conv1D, pointwise_initializer=initializer_Conv1D, depthwise_regularizer=regularizers.l2(l2_rate), pointwise_regularizer=regularizers.l2(l2_rate))(y) y = layers.BatchNormalization(axis=-1)(y) res = layers.add([y, res]) dilation_lower *= 2 if dilation_lower > dilation_upper: dilation_lower = 1 ## Conv1D with dilation (padding='valaid') and residual block to reduce dimention. for _ in range(reduce_layers): y = layers.SeparableConv1D( filters=reduce1D_filter_num, kernel_size=kernel_size, padding=padding_style, activation=activator_Conv1D, depthwise_initializer=initializer_Conv1D, pointwise_initializer=initializer_Conv1D, depthwise_regularizer=regularizers.l2(l2_rate), pointwise_regularizer=regularizers.l2(l2_rate))(res) y = layers.BatchNormalization(axis=-1)(y) y = layers.Dropout(dropout_rate_reduce)(y) y = layers.MaxPooling1D(pool_size, padding=padding_style)(y) res = layers.SeparableConv1D( filters=reduce1D_filter_num, kernel_size=kernel_size, strides=residual_stride, padding=padding_style, activation=activator_Conv1D, depthwise_initializer=initializer_Conv1D, pointwise_initializer=initializer_Conv1D, depthwise_regularizer=regularizers.l2(l2_rate), pointwise_regularizer=regularizers.l2(l2_rate))(res) res = layers.add([y, res]) ## flat & dense y = layers.Flatten()(y) y = layers.Dense(dense_num, activation=activator_Dense, kernel_initializer=initializer_Dense)(y) y = layers.BatchNormalization(axis=-1)(y) y = layers.Dropout(dropout_rate_dense)(y) output_layer = layers.Dense(1, activation='linear')(y) model = models.Model(inputs=input_layer, outputs=output_layer) # print(model.get_layer(index=2).trainable) # print(model.get_layer(name='first_conv').trainable) if summary: trainable_count = int( np.sum( [K.count_params(p) for p in set(model.trainable_weights)])) non_trainable_count = int( np.sum([ K.count_params(p) for p in set(model.non_trainable_weights) ])) print('Total params: {:,}'.format(trainable_count + non_trainable_count)) print('Trainable params: {:,}'.format(trainable_count)) print('Non-trainable params: {:,}'.format(non_trainable_count)) # model.summary() adam = optimizers.Adam(lr=lr) model.compile( optimizer=adam, # 'rmsprop', # SGD,adam,rmsprop loss=loss_type, metrics=list(metrics)) # mae平均绝对误差(mean absolute error) accuracy result = model.fit( x=x_train, y=ddg_train, batch_size=batch_size, epochs=epochs, verbose=verbose, callbacks=my_callbacks, validation_data=(x_val, ddg_val), shuffle=True, ) # print('\n----------History:\n%s'%result.history) # # save # save_train_cv(model, modeldir, result.history, k_count) opt_lst.append(np.mean( result.history['val_mean_absolute_error'][-10:])) opt_loss = np.mean(opt_lst) # # print hyper combination group and current loss value # print('\n@current_hyper_tag: %s' '\n@current optmized_loss: %s' % (hyper_param_tag, opt_loss)) # return {'loss': validation_loss, 'status': STATUS_OK, 'model':model} return {'loss': opt_loss, 'status': STATUS_OK}
def Conv1DRegressorIn1(flag): K.clear_session() current_neighbor = space['neighbor'] current_idx_idx = space['idx_idx'] current_batch_size = space['batch_size'] current_conv1D_filter_num1 = space['conv1D_filter_num1'] current_conv1D_filter_num2 = space['conv1D_filter_num2'] current_conv1D_filter_num3 = space['conv1D_filter_num3'] current_dropout_rate_dense = space['dropout_rate_dense'] summary = True verbose = 0 # # setHyperParams # ## hypers for data neighbor = {{choice([50, 60, 70, 80, 90, 100, 110, 120, 130, 140])}} idx_idx = {{choice([0, 1, 2, 3, 4, 5, 6, 7, 8])}} idx_lst = [ [x for x in range(158) if x not in [24, 26]], # 去除无用特征 [ x for x in range(158) if x not in [24, 26] + [x for x in range(1, 6)] + [x for x in range(16, 22)] + [40, 42] ], # 去除无用特征+冗余特征 [ x for x in range(158) if x not in [24, 26] + [x for x in range(0, 22)] ], # 去除无用特征+方位特征 [x for x in range(158) if x not in [24, 26] + [22, 23, 26, 37, 38]], # 去除无用特征+深度特征 [ x for x in range(158) if x not in [24, 26] + [x for x in range(27, 37)] + [x for x in range(40, 46)] ], # 去除无用特征+二级结构信息 # [x for x in range(158) if x not in [24, 26] + [x for x in range(27, 34)] + [x for x in range(40, 46)]],# 去除无用特征+二级结构信息1 # [x for x in range(158) if x not in [24, 26] + [x for x in range(34, 37)] + [x for x in range(40, 46)]],# 去除无用特征+二级结构信息2 [x for x in range(158) if x not in [24, 26] + [46, 47]], # 去除无用特征+实验条件 [ x for x in range(158) if x not in [24, 26] + [39] + [x for x in range(57, 61)] + [x for x in range(48, 57)] + [x for x in range(61, 81)] + [x for x in range(140, 155)] ], # 去除无用特征+所有原子编码 # [x for x in range(158) if x not in [24, 26] + [39] + [x for x in range(57, 61)] + [x for x in range(48, 57)] + [x for x in range(140, 145)]],# 去除无用特征+原子编码1 # [x for x in range(158) if x not in [24, 26] + [39] + [x for x in range(57, 61)] + [x for x in range(61, 77)] + [x for x in range(145, 153)]],# 去除无用特征+原子编码2 # [x for x in range(158) if x not in [24, 26] + [39] + [x for x in range(57, 61)] + [x for x in range(77, 81)] + [x for x in range(153, 155)]],# 去除无用特征+原子编码3 [ x for x in range(158) if x not in [24, 26] + [x for x in range(81, 98)] ], # 去除无用特征+rosetta_energy [ x for x in range(158) if x not in [24, 26] + [x for x in range(98, 140)] + [x for x in range(155, 158)] ] # 去除无用特征+msa ] idx = idx_lst[idx_idx] ## hypers for net lr = 1e-4 # 0.0001 batch_size = {{choice([1, 32, 64, 128])}} epochs = 200 conv1D_filter_num1 = {{choice([16, 32])}} conv1D_filter_num2 = {{choice([16, 32, 64])}} conv1D_filter_num3 = {{choice([32, 64])}} dropout_rate_dense = {{choice([0.1, 0.2, 0.3, 0.4, 0.5])}} metrics = ('mae', pearson_r, rmse) def _data(fold_num, neighbor, idx): train_data_pth = '/dl/sry/mCNN/dataset/deepddg/npz/wild/cross_valid/cro_fold%s_train_center_CA_PCA_False_neighbor_140.npz' % fold_num val_data_pth = '/dl/sry/mCNN/dataset/deepddg/npz/wild/cross_valid/cro_fold%s_valid_center_CA_PCA_False_neighbor_140.npz' % fold_num ## train data train_data = np.load(train_data_pth) x_train = train_data['x'] y_train = train_data['y'] ddg_train = train_data['ddg'].reshape(-1) ## select kneighbor atoms x_train_kneighbor_lst = [] for sample in x_train: dist_arr = sample[:, 0] indices = sorted(dist_arr.argsort()[:neighbor]) x_train_kneighbor_lst.append(sample[indices, :]) x_train = np.array(x_train_kneighbor_lst) ## idx x_train = x_train[:, :, idx] ## val data val_data = np.load(val_data_pth) x_val = val_data['x'] y_val = val_data['y'] ddg_val = val_data['ddg'].reshape(-1) ## select kneighbor atoms x_val_kneighbor_lst = [] for sample in x_val: dist_arr = sample[:, 0] indices = sorted(dist_arr.argsort()[:neighbor]) x_val_kneighbor_lst.append(sample[indices, :]) x_val = np.array(x_val_kneighbor_lst) ## idx x_val = x_val[:, :, idx] # sort row default is chain, pass # reshape and one-hot y_train = to_categorical(y_train) y_val = to_categorical(y_val) # normalization train_shape = x_train.shape val_shape = x_val.shape col_train = train_shape[-1] col_val = val_shape[-1] x_train = x_train.reshape((-1, col_train)) x_val = x_val.reshape((-1, col_val)) mean = x_train.mean(axis=0) std = x_train.std(axis=0) std[np.argwhere(std == 0)] = 0.01 x_train -= mean x_train /= std x_val -= mean x_val /= std x_train = x_train.reshape(train_shape) x_val = x_val.reshape(val_shape) print('x_train: %s' '\ny_train: %s' '\nddg_train: %s' '\nx_val: %s' '\ny_val: %s' '\nddg_val: %s' % (x_train.shape, y_train.shape, ddg_train.shape, x_val.shape, y_val.shape, ddg_val.shape)) return x_train, y_train, ddg_train, x_val, y_val, ddg_val # # cross_valid # hyper_param_tag = '%s_%s_%s_%s_%s_%s_%s' % ( current_neighbor, current_idx_idx, current_batch_size, current_conv1D_filter_num1, current_conv1D_filter_num2, current_conv1D_filter_num3, current_dropout_rate_dense) modeldir = '/dl/sry/projects/from_hp/mCNN/src/Network/deepddg/opt_all_simpleNet_v4/model/%s-%s' % ( hyper_param_tag, time.strftime("%Y.%m.%d.%H.%M.%S", time.localtime())) os.makedirs(modeldir, exist_ok=True) opt_lst = [] for k_count in range(1, 11): print('\n** fold %s is processing **\n' % k_count) filepth = '%s/fold_%s_weights-best.h5' % (modeldir, k_count) my_callbacks = [ callbacks.ReduceLROnPlateau( monitor='val_loss', factor=0.33, patience=5, verbose=verbose, mode='min', min_lr=1e-8, ), callbacks.EarlyStopping(monitor='val_loss', patience=10, verbose=verbose), callbacks.ModelCheckpoint(filepath=filepth, monitor='val_mean_absolute_error', verbose=verbose, save_best_only=True, mode='min', save_weights_only=True) ] x_train, y_train, ddg_train, x_val, y_val, ddg_val = _data( k_count, neighbor, idx) row_num, col_num = x_train.shape[1:3] # # build net # network = models.Sequential() network.add( layers.SeparableConv1D(filters=conv1D_filter_num1, kernel_size=5, activation='relu', input_shape=(row_num, col_num))) network.add(layers.MaxPooling1D(pool_size=2)) network.add( layers.SeparableConv1D(filters=conv1D_filter_num2, kernel_size=5, activation='relu')) network.add(layers.MaxPooling1D(pool_size=2)) network.add( layers.SeparableConv1D(filters=conv1D_filter_num3, kernel_size=3, activation='relu')) network.add(layers.MaxPooling1D(pool_size=2)) network.add(layers.Flatten()) network.add(layers.Dense(128, activation='relu')) network.add(layers.Dropout(dropout_rate_dense)) network.add(layers.Dense(16, activation='relu')) network.add(layers.Dropout(0.3)) network.add(layers.Dense(1)) if summary: trainable_count = int( np.sum([ K.count_params(p) for p in set(network.trainable_weights) ])) non_trainable_count = int( np.sum([ K.count_params(p) for p in set(network.non_trainable_weights) ])) print('Total params: {:,}'.format(trainable_count + non_trainable_count)) print('Trainable params: {:,}'.format(trainable_count)) print('Non-trainable params: {:,}'.format(non_trainable_count)) # print(network.summary()) # rmsp = optimizers.RMSprop(lr=0.0001, decay=0.1) rmsp = optimizers.RMSprop(lr=lr) network.compile( optimizer=rmsp, # 'rmsprop', # SGD,adam,rmsprop loss='mae', metrics=list(metrics)) # mae平均绝对误差(mean absolute error) accuracy result = network.fit( x=x_train, y=ddg_train, batch_size=batch_size, epochs=epochs, verbose=verbose, callbacks=my_callbacks, validation_data=(x_val, ddg_val), shuffle=True, ) # print('\n----------History:\n%s'%result.history) # # save # save_train_cv(network, modeldir, result.history, k_count) opt_lst.append(np.mean( result.history['val_mean_absolute_error'][-10:])) opt_loss = np.mean(opt_lst) # # print hyper combination group and current loss value # print('\n@current_hyper_tag: %s' '\n@current optmized_loss: %s' % (hyper_param_tag, opt_loss)) # return {'loss': validation_loss, 'status': STATUS_OK, 'model':model} return {'loss': opt_loss, 'status': STATUS_OK}
def au_Exp(): now = datetime.datetime.now() now_s = now.strftime("%Y-%m-%d-%H-%M-%S") config = tf.compat.v1.ConfigProto(gpu_options=tf.compat.v1.GPUOptions(allow_growth=True)) sess = tf.compat.v1.Session(config=config) ## 准备几个参数,用于后续的自动化 epochs_au = 50 batch_size_au = 1 jihuo = 'tanh' callback_list_test =[ keras.callbacks.ModelCheckpoint( filepath= now_s+'.h5', ##文件路径 存在当前路径下吧 还好找 monitor= 'val_loss', ## 监控指标 save_best_only= True ## 保持最佳模型 ) ] moisture,d5,dp5,dp6 = sF.getOdata() test_data,test_lable,train_data,train_lable = sF.getTestData(moisture,d5,dp5,dp6 ) model = models.Sequential() model.add(layers.Conv1D(64,7,activation=jihuo,input_shape=(700,1))) model.add(layers.MaxPooling1D(2)) model.add(layers.Conv1D(32,7,activation=jihuo)) model.add(layers.MaxPooling1D(2)) model.add(layers.Conv1D(32,7,activation=jihuo)) model.add(layers.MaxPooling1D(2)) model.add(layers.Conv1D(16,7,activation=jihuo)) # model.add(layers.GlobalMaxPooling1D()) ## 实际效果极差1 model.add(layers.Flatten()) model.add(layers.Dense(16)) model.add(layers.Dense(8)) # model.add(layers.Dense(4)) # model.add(layers.Dense(2)) model.add(layers.Dense(1)) model.summary() model.compile(optimizer=RMSprop(),loss='mse') history = model.fit(train_data,train_lable, epochs=epochs_au, batch_size=batch_size_au, validation_data=(test_data,test_lable), callbacks= callback_list_test ) sF.drawLoss(history) ## 绘制当前的验证曲线 model = load_model(now_s+'.h5') result_trian = model.predict(train_data) result_predict = model.predict(test_data) rmsec = sF.calculate_RMSE(result_trian,train_lable) ## 训练集上的RMSE rmsep = sF.calculate_RMSE(result_predict,test_lable) ## 测试集上的RMSE r_2_t = sF.calculate_R21(result_trian,train_lable)## 训练集上的R_2 r_2_p = sF.calculate_R21(result_predict,test_lable)## 测试集上得R_2 # print("Root Mean Square Error of Calibrationh is : %g"%(rmsec)) # print("训练集上得决定系数:%f"%(r_2_t)) # print("Root Mean Square Error of Prediction is : %g"%(rmsep)) # print("测试集上得决定系数:%f"%(r_2_p)) ###### 下面的代码用于自动记录实验数据 write_data=[(now_s,epochs_au,batch_size_au,rmsec,r_2_t,rmsep,r_2_p)]#需要新写入的数据 sF.write_To_Csv(write_data) return rmsep
def build_model(): model_weights = np.load('sound8.npy', encoding='latin1').item() print(type(model_weights)) model = models.Sequential() model.add(layers.InputLayer(batch_input_shape=(1, None, 1))) filter_parameters = [ { 'name': 'conv1', 'num_filters': 16, 'padding': 32, 'kernel_size': 64, 'conv_strides': 2, 'pool_size': 8, 'pool_strides': 8 }, { 'name': 'conv2', 'num_filters': 32, 'padding': 16, 'kernel_size': 32, 'conv_strides': 2, 'pool_size': 8, 'pool_strides': 8 }, { 'name': 'conv3', 'num_filters': 64, 'padding': 8, 'kernel_size': 16, 'conv_strides': 2 }, { 'name': 'conv4', 'num_filters': 128, 'padding': 4, 'kernel_size': 8, 'conv_strides': 2 }, { 'name': 'conv5', 'num_filters': 256, 'padding': 2, 'kernel_size': 4, 'conv_strides': 2, 'pool_size': 4, 'pool_strides': 4 }, { 'name': 'conv6', 'num_filters': 512, 'padding': 2, 'kernel_size': 4, 'conv_strides': 2 }, { 'name': 'conv7', 'num_filters': 1024, 'padding': 2, 'kernel_size': 4, 'conv_strides': 2 }, { 'name': 'conv8_2', 'num_filters': 401, 'padding': 1, 'kernel_size': 8, 'conv_strides': 2 }, ] for x in filter_parameters: model.add(layers.ZeroPadding1D(padding=x['padding'])) model.add( layers.Conv1D(x['num_filters'], kernel_size=x['kernel_size'], strides=x['conv_strides'], padding='valid')) weights = model_weights[x['name']]['weights'].reshape( model.layers[-1].get_weights()[0].shape) biases = model_weights[x['name']]['biases'] model.layers[-1].set_weights([weights, biases]) if 'conv8' not in x['name']: gamma = model_weights[x['name']]['gamma'] beta = model_weights[x['name']]['beta'] mean = model_weights[x['name']]['mean'] var = model_weights[x['name']]['var'] model.add(layers.BatchNormalization()) model.layers[-1].set_weights([gamma, beta, mean, var]) model.add(layers.Activation('relu')) if 'pool_size' in x: model.add( layers.MaxPooling1D(pool_size=x['pool_size'], strides=x['pool_strides'], padding='valid')) return model
def build_birnn_feature_coattention_cnn_model(voca_dim, time_steps, num_features, feature_dim, output_dim, model_dim, mlp_dim, num_filters, filter_sizes, item_embedding=None, rnn_depth=1, mlp_depth=1, drop_out=0.5, rnn_drop_out=0., rnn_state_drop_out=0., cnn_drop_out=0.5, pooling='max', trainable_embedding=False, gpu=False, return_customized_layers=False): """ Create A Bidirectional Attention Model. :param voca_dim: vocabulary dimension size. :param time_steps: the length of input :param output_dim: the output dimension size :param model_dim: rrn dimension size :param mlp_dim: the dimension size of fully connected layer :param item_embedding: integer, numpy 2D array, or None (default=None) If item_embedding is a integer, connect a randomly initialized embedding matrix to the input tensor. If item_embedding is a matrix, this matrix will be used as the embedding matrix. If item_embedding is None, then connect input tensor to RNN layer directly. :param rnn_depth: rnn depth :param mlp_depth: the depth of fully connected layers :param num_att_channel: the number of attention channels, this can be used to mimic multi-head attention mechanism :param drop_out: dropout rate of fully connected layers :param rnn_drop_out: dropout rate of rnn layers :param rnn_state_drop_out: dropout rate of rnn state tensor :param trainable_embedding: boolean :param gpu: boolean, default=False If True, CuDNNLSTM is used instead of LSTM for RNN layer. :param return_customized_layers: boolean, default=False If True, return model and customized object dictionary, otherwise return model only :return: keras model """ if model_dim % 2 == 1: model_dim += 1 if item_embedding is not None: inputs = models.Input(shape=(time_steps, ), dtype='int32', name='input0') x1 = inputs # item embedding if isinstance(item_embedding, np.ndarray): assert voca_dim == item_embedding.shape[0] x1 = layers.Embedding(voca_dim, item_embedding.shape[1], input_length=time_steps, weights=[ item_embedding, ], trainable=trainable_embedding, mask_zero=False, name='embedding_layer0')(x1) elif utils.is_integer(item_embedding): x1 = layers.Embedding(voca_dim, item_embedding, input_length=time_steps, trainable=trainable_embedding, mask_zero=False, name='embedding_layer0')(x1) else: raise ValueError( "item_embedding must be either integer or numpy matrix") else: inputs = models.Input(shape=(time_steps, voca_dim), dtype='float32', name='input0') x1 = inputs inputs1 = models.Input(shape=(num_features, feature_dim), dtype='float32', name='input1') x2 = layers.Dense(feature_dim, name="feature_map_layer", activation="relu")(inputs1) if gpu: # rnn encoding for i in range(rnn_depth): x1 = layers.Bidirectional(layers.CuDNNLSTM(int(model_dim / 2), return_sequences=True), name='bi_lstm_layer' + str(i))(x1) x1 = layers.BatchNormalization(name='rnn_batch_norm_layer' + str(i))(x1) x1 = layers.Dropout(rnn_drop_out, name="rnn_dropout_layer" + str(i))(x1) else: # rnn encoding for i in range(rnn_depth): x1 = layers.Bidirectional(layers.LSTM( int(model_dim / 2), return_sequences=True, dropout=rnn_drop_out, recurrent_dropout=rnn_state_drop_out), name='bi_lstm_layer' + str(i))(x1) x1 = layers.BatchNormalization(name='rnn_batch_norm_layer' + str(i))(x1) # attention attens = clayers.CoAttentionWeight(name="coattention_weights_layer")( [x1, x2]) attens1 = clayers.FeatureNormalization( name="normalized_coattention_weights_layer1", axis=1)(attens) attens2 = clayers.FeatureNormalization( name="normalized_coattention_weights_layer2", axis=2)(attens) # compare focus1 = layers.Dot((1, 1), name="focus_layer1")([attens1, x1]) focus2 = layers.Dot((2, 1), name="focus_layer2")([attens2, x2]) pair1 = layers.Concatenate(axis=-1, name="pair_layer1")([x1, focus2]) pair2 = layers.Concatenate(axis=-1, name="pair_layer2")([x2, focus1]) x1 = layers.TimeDistributed(layers.Dense(model_dim, activation="relu"), name="compare_layer1")(pair1) x2 = layers.TimeDistributed(layers.Dense(model_dim, activation="relu"), name="compare_layer2")(pair2) # Multi-Channel CNN for x1 pooled_outputs = [] for i in range(len(filter_sizes)): conv = layers.Conv1D(num_filters, kernel_size=filter_sizes[i], padding='valid', activation='relu')(x1) if pooling == 'max': conv = layers.MaxPooling1D(pool_size=time_steps - filter_sizes[i] + 1, strides=1, padding='valid')(conv) else: conv = layers.AveragePooling1D(pool_size=time_steps - filter_sizes[i] + 1, strides=1, padding='valid')(conv) pooled_outputs.append(conv) x1 = layers.Concatenate(name='concated_layer')(pooled_outputs) x1 = layers.Flatten()(x1) x1 = layers.Dropout(cnn_drop_out, name='conv_dropout_layer')(x1) x1 = layers.BatchNormalization(name="batch_norm_layer")(x1) # Average Pool for x2 x2 = layers.GlobalAveragePooling1D(name="average_pool_layer")(x2) x = layers.Concatenate(axis=1, name="concat_deep_feature_layer")([x1, x2]) # MLP Layers for i in range(mlp_depth - 1): x = layers.Dense(mlp_dim, activation='selu', kernel_initializer='lecun_normal', name='selu_layer' + str(i))(x) x = layers.AlphaDropout(drop_out, name='alpha_layer' + str(i))(x) outputs = layers.Dense(output_dim, activation="softmax", name="softmax_layer0")(x) model = models.Model(inputs, outputs) if return_customized_layers: return model, { 'CoAttentionWeight': clayers.CoAttentionWeight, "FeatureNormalization": clayers.FeatureNormalization } return model
def build_birnn_cnn_model(voca_dim, time_steps, output_dim, rnn_dim, mlp_dim, num_filters, filter_sizes, item_embedding=None, rnn_depth=1, mlp_depth=1, drop_out=0.5, rnn_drop_out=0.5, rnn_state_drop_out=0.5, cnn_drop_out=0.5, pooling='max', trainable_embedding=False, gpu=False, return_customized_layers=False): """ Create A Bidirectional CNN Model. :param voca_dim: vocabulary dimension size. :param time_steps: the length of input :param output_dim: the output dimension size :param rnn_dim: rrn dimension size :param num_filters: the number of filters :param filter_sizes: list of integers The kernel size. :param mlp_dim: the dimension size of fully connected layer :param item_embedding: integer, numpy 2D array, or None (default=None) If item_embedding is a integer, connect a randomly initialized embedding matrix to the input tensor. If item_embedding is a matrix, this matrix will be used as the embedding matrix. If item_embedding is None, then connect input tensor to RNN layer directly. :param rnn_depth: rnn depth :param mlp_depth: the depth of fully connected layers :param num_att_channel: the number of attention channels, this can be used to mimic multi-head attention mechanism :param drop_out: dropout rate of fully connected layers :param rnn_drop_out: dropout rate of rnn layers :param rnn_state_drop_out: dropout rate of rnn state tensor :param cnn_drop_out: dropout rate of between cnn layer and fully connected layers :param pooling: str, either 'max' or 'average' Pooling method. :param trainable_embedding: boolean :param gpu: boolean, default=False If True, CuDNNLSTM is used instead of LSTM for RNN layer. :param return_customized_layers: boolean, default=False If True, return model and customized object dictionary, otherwise return model only :return: keras model """ if item_embedding is not None: inputs = models.Input(shape=(time_steps, ), dtype='int32', name='input0') x = inputs # item embedding if isinstance(item_embedding, np.ndarray): assert voca_dim == item_embedding.shape[0] x = layers.Embedding(voca_dim, item_embedding.shape[1], input_length=time_steps, weights=[ item_embedding, ], trainable=trainable_embedding, mask_zero=False, name='embedding_layer0')(x) elif utils.is_integer(item_embedding): x = layers.Embedding(voca_dim, item_embedding, input_length=time_steps, trainable=trainable_embedding, mask_zero=False, name='embedding_layer0')(x) else: raise ValueError( "item_embedding must be either integer or numpy matrix") else: inputs = models.Input(shape=(time_steps, voca_dim), dtype='float32', name='input0') x = inputs if gpu: # rnn encoding for i in range(rnn_depth): x = layers.Bidirectional(layers.CuDNNLSTM(rnn_dim, return_sequences=True), name='bi_lstm_layer' + str(i))(x) x = layers.BatchNormalization(name='rnn_batch_norm_layer' + str(i))(x) x = layers.Dropout(rnn_drop_out, name="rnn_dropout_layer" + str(i))(x) else: # rnn encoding for i in range(rnn_depth): x = layers.Bidirectional(layers.LSTM( rnn_dim, return_sequences=True, dropout=rnn_drop_out, recurrent_dropout=rnn_state_drop_out), name='bi_lstm_layer' + str(i))(x) x = layers.BatchNormalization(name='rnn_batch_norm_layer' + str(i))(x) pooled_outputs = [] for i in range(len(filter_sizes)): conv = layers.Conv1D(num_filters, kernel_size=filter_sizes[i], padding='valid', activation='relu')(x) if pooling == 'max': conv = layers.MaxPooling1D(pool_size=time_steps - filter_sizes[i] + 1, strides=1, padding='valid')(conv) else: conv = layers.AveragePooling1D(pool_size=time_steps - filter_sizes[i] + 1, strides=1, padding='valid')(conv) pooled_outputs.append(conv) x = layers.Concatenate(name='concated_layer')(pooled_outputs) x = layers.Flatten()(x) x = layers.Dropout(cnn_drop_out, name='conv_dropout_layer')(x) x = layers.BatchNormalization(name="batch_norm_layer")(x) # MLP Layers for i in range(mlp_depth - 1): x = layers.Dense(mlp_dim, activation='selu', kernel_initializer='lecun_normal', name='selu_layer' + str(i))(x) x = layers.AlphaDropout(drop_out, name='alpha_layer' + str(i))(x) outputs = layers.Dense(output_dim, activation="softmax", name="softmax_layer0")(x) model = models.Model(inputs, outputs) if return_customized_layers: return model, dict() return model
K.set_session(sess) ################################################### from keras.models import Model, load_model from keras import layers as kl from keras import optimizers def average_pred(y_true, y_pred): return K.mean(y_pred) li = kl.Input(shape=(TIMESTEP_SIZE, 1)) l = li for n_units in [16, 32, 64, 128]: l = kl.Conv1D(n_units, 3, activation='elu', padding='same')(l) l = kl.MaxPooling1D()(l) l = kl.Flatten()(l) l = kl.Dense(64, activation='elu')(l) l = kl.Dense(1, activation='sigmoid')(l) model = Model(li, l) model.compile(optimizers.Adamax(lr=0.002), "binary_crossentropy", ['acc', average_pred]) if not args.test: from keras.callbacks import TensorBoard import os iteration = 0 res = [] print_msg = "iteration: {} : loss: {:.6f}, acc: {:.4%}, avg_pred: {:.4f}, avg_y: {:.4f}, left_iter_to_test: {}"
def main(): ##Variables ftrVct0 = [] #feature vector 0 ftrVct1 = [] #feature vector 1 rawData1 = [] #raw Data rawData2 = [] #raw Data trnData = [] #data used to train tstData = [] #data to test lbls = [] accVct = [] val_accVct = [] lossVct = [] val_lossVct = [] epochsVct = [] ##Read firts data rawData0 = readFile(filename0) rawData1 = readFile(filename1) rawData2 = readFile(filename2) rawData = np.concatenate((rawData0[:,0:850],rawData1[:,0:850],rawData2[:,0:850]),axis = 0) normData = normalizeData(*rawData) ## normData1 = normalizeData(*rawData1[:,0:850]) ## normData2 = normalizeData(*rawData2[:,0:850]) print('Shape normData: ', np.asarray(normData).shape) ## print('Shape normData1: ', np.asarray(normData1).shape) ## print('Shape normData2: ', np.asarray(normData2).shape) lbls = np.concatenate((np.zeros((len(rawData0[:,0:850]),1)),np.ones((len(rawData1[:,0:850]),1)), 2*np.ones((len(rawData2[:,0:850]),1))),axis=0) print('Shape lbls: ', lbls.shape) ## print('lbls: ', lbls) ## trnData = np.concatenate((normData0,normData1,normData2),axis = 0) ## print('Shape trnData: ', trnData.shape) x_train, x_test, y_train, y_test = train_test_split(normData, lbls, test_size=0.2, random_state=0) x_train = np.expand_dims(x_train,2) x_val = np.expand_dims(x_test,2) print('Shape X_train: ', np.asarray(x_train).shape) print('Shape Y_train: ', np.asarray(y_train).shape) print('Shape X_val: ', np.asarray(x_val).shape) print('Shape Y_val: ', np.asarray(y_test).shape) ## print('Shape train_gen: ', train_gen) steps = range(350, 350+np.asarray(normData).shape[-1]) print('Time steps: ', np.asarray(steps).shape) print('Time steps: ', steps[0]) #plt.plot(steps,np.asarray(x_train[0]), color="g") #plt.show() #ANN building model = Sequential() model.add(layers.Conv1D(20, 170, activation='relu',input_shape=(850,1)))#16 #T 16,85 model.add(layers.MaxPooling1D(2)) #model.add(layers.Conv1D(8,15, activation = 'relu'))#16 #T16,10 ## model.add(layers.MaxPooling1D(2)) ## model.add(layers.Conv1D(32,16, activation = 'relu')) ## model.add(layers.MaxPooling1D(2)) ## model.add(layers.Conv1D(32,16, activation = 'relu')) ## model.add(layers.MaxPooling1D(2)) ## model.add(layers.Conv1D(32,16, activation = 'relu')) #### model.add(layers.GlobalMaxPooling1D()) ## model.add(layers.Bidirectional(layers.GRU(32, dropout=0.1, recurrent_dropout=0.3, return_sequences=True))) ## model.add(layers.Bidirectional(layers.GRU(32, dropout=0.1, recurrent_dropout=0.3, return_sequences=True))) ## model.add(layers.Bidirectional(layers.GRU(64, dropout=0.1, recurrent_dropout=0.3, return_sequences=True))) model.add(layers.Bidirectional(layers.GRU(32, dropout=0.1, recurrent_dropout=0.3)))#16 ## model.add(layers.GRU(8, activation='relu', dropout=0.1, recurrent_dropout=0.3)) ## model.add(layers.Dense(32, activation = 'relu')) #model.add(layers.Flatten()) ## model.add(layers.Dense(32, activation = 'relu')) ## model.add(layers.Dropout(0.1)) model.add(layers.Dense(3, activation = 'sigmoid')) model.summary() model.compile(optimizer=Adagrad(lr=0.003, epsilon=None, decay=0.0), loss='sparse_categorical_crossentropy', metrics=['acc']) ## model.compile(optimizer=RMSprop(), loss='mae') #history = model.fit(x_train,y_train, epochs=2000, validation_data = (x_val, y_test)) history = model.fit(x_train,y_train, epochs=2000, batch_size=10000000, validation_data = (x_val, y_test)) ## history = model.fit_generator(train_gen, steps_per_epoch = 10,epochs=100,validation_data=val_gen, validation_steps = 100) acc = history.history['acc'] val_acc = history.history['val_acc'] loss = history.history['loss'] val_loss = history.history['val_loss'] epochs = range(1, len(loss) + 1) ## accVct.append(acc) ## val_accVct.append(val_acc) ## lossVct.append(loss) ## val_lossVct.append(val_loss) ## epochsVct = [] df = pandas.DataFrame(data={"epochs": epochs, "loss": loss, "val_loss": val_loss,"acc": acc,"val_acc": val_acc}) df.to_csv("./Results_2.csv", sep=',',index=False) #model.save_weights('my_model_weights.h5') #model.save('my_model.h5') plt.plot(epochs, acc, 'r', label='Training acc') plt.plot(epochs, val_acc, 'b', label='Validation acc') plt.title('Training and validation accuracy') plt.legend() plt.figure() plt.plot(epochs, loss, 'r', label='Training loss') plt.plot(epochs, val_loss, 'b', label='Validation loss') plt.title('Training and validation loss') plt.legend() plt.show()
def cnn_2x_lstm_siamese(voc_size, max_len, dropout=0.2): """Two siamese branches, each embedding a statement. Binary classifier on top. Args: voc_size: size of the vocabulary for the input statements. max_len: maximum length for the input statements. dropout: Fraction of units to drop. Returns: A Keras model instance. """ pivot_input = layers.Input(shape=(max_len, ), dtype='int32') statement_input = layers.Input(shape=(max_len, ), dtype='int32') x = layers.Embedding(output_dim=256, input_dim=voc_size, input_length=max_len)(pivot_input) x = layers.Conv1D(32, 7, activation='relu', kernel_regularizer=regularizers.l2(0.01))(x) x = layers.ZeroPadding1D(padding=1)(x) x = layers.Conv1D(32, 7, activation='relu', kernel_regularizer=regularizers.l2(0.01))(x) x = layers.MaxPooling1D(pool_size=2, strides=2)(x) x = layers.Conv1D(64, 5, activation='relu', kernel_regularizer=regularizers.l2(0.01))(x) x = layers.ZeroPadding1D(padding=1)(x) x = layers.Conv1D(64, 5, activation='relu', kernel_regularizer=regularizers.l2(0.01))(x) x = layers.MaxPooling1D(pool_size=2, strides=2)(x) x = layers.Conv1D(128, 3, activation='relu', kernel_regularizer=regularizers.l2(0.01))(x) x = layers.ZeroPadding1D(padding=1)(x) x = layers.Conv1D(128, 3, activation='relu', kernel_regularizer=regularizers.l2(0.01))(x) x = layers.MaxPooling1D(pool_size=2, strides=2)(x) x = layers.Conv1D(256, 3, activation='relu', kernel_regularizer=regularizers.l2(0.01))(x) x = layers.ZeroPadding1D(padding=1)(x) x = layers.Conv1D(256, 3, activation='relu', kernel_regularizer=regularizers.l2(0.01))(x) x = layers.MaxPooling1D(pool_size=2, strides=2)(x) embedded_pivot = layers.LSTM(256)(x) encoder_model = Model(pivot_input, embedded_pivot) embedded_statement = encoder_model(statement_input) concat = layers.merge([embedded_pivot, embedded_statement], mode='concat') x = layers.Dense(256, activation='relu')(concat) x = layers.Dropout(dropout)(x) prediction = layers.Dense(1, activation='relu')(x) model = Model([pivot_input, statement_input], prediction) return model
input_layer = layers.Input(shape=(max_num_amino, len(dataindex)), name='input') net = layers.Conv1D(conv, 5, padding='same')(input_layer) net = layers.Activation(activation)(net) #act1 = layers.MaxPooling1D(pool_size=poolSize,strides=1,padding='same')(act1) net = layers.Dropout(drop)(net) net = layers.Conv1D(conv, 5, padding='same')(net) net = layers.Activation(activation)(net) net = layers.MaxPooling1D(pool_size=poolSize, strides=1, padding='same')(net) net = layers.Dropout(drop)(net) net = layers.Conv1D(conv, 5, padding='same')(net) net = layers.Activation(activation)(net) net = layers.MaxPooling1D(pool_size=poolSize, strides=1, padding='same')(net) #pool3 = layers.Conv1D(128, 5, activation='relu', padding='same')(pool3) net = layers.Dropout(drop)(net) net = layers.Dense(conv)(net) net = layers.Activation(activation)(net)
from keras.models import Sequential from keras.layers import Embedding, SimpleRNN, LSTM, Dense model = Sequential() model.add(Embedding(10000, 32)) model.add(SimpleRNN(32)) model.summary() modelLSTM = Sequential() modelLSTM.add(Embedding(10000, 32)) modelLSTM.add(LSTM(32)) modelLSTM.add(Dense(1, activation='sigmoid')) from keras import layers modelCNN1D = Sequential() modelCNN1D.add(layers.Conv1D(32, 5, input_shape=(None, 100), activation='relu')) modelCNN1D.add(layers.MaxPooling1D(5)) modelCNN1D.add(layers.Conv1D(32, 5, activation='relu')) modelCNN1D.add(layers.GRU(64, dropout=0.2, recurrent_dropout=0.5)) modelCNN1D.add(Dense(1)) modelCNN1D.summary()
if ('--learning_rate' in sys.argv): try: lr_arg = sys.argv[sys.argv.index('--learning_rate') + 1] lr = float(lr_arg) except: lr = 0.001 else: lr = 0.001 # --------------------------------------------------------------------------- # ENCODER x_input = layers.Input(shape=input_shape) x = layers.Conv1D(8, 15, padding='same', activation='relu')(x_input) shape_mp1 = K.int_shape(x) x = layers.MaxPooling1D(5, padding='same')(x) x = layers.Conv1D(16, 15, padding='same', activation='relu')(x) shape_mp2 = K.int_shape(x) x = layers.MaxPooling1D(5, padding='same')(x) x = layers.Dropout(rate=0.1)(x) x = layers.Conv1D(16, 15, padding='same', activation='relu')(x) shape = K.int_shape(x) x = layers.Flatten()(x) x = layers.Dropout(rate=0.1)(x) x = layers.Dense(16, activation='relu')(x) z_mean = layers.Dense(latent_dim)(x) z_log_var = layers.Dense(latent_dim)(x) encoder = models.Model(x_input, [z_mean, z_log_var]) encoder.summary()
dtype='float32') # In[16]: # apply the 1-D Convolutional Neural Network n_timesteps = x_train.shape[1] n_features = x_train.shape[2] n_outputs = 35 model = models.Sequential() model.add( layers.Conv1D(filters=100, kernel_size=10, activation='relu', input_shape=(n_timesteps, n_features))) model.add(layers.MaxPooling1D()) model.add(layers.BatchNormalization()) model.add(layers.Conv1D(100, 10, activation='relu')) model.add(layers.MaxPooling1D()) model.add(layers.BatchNormalization()) model.add(layers.Conv1D(100, 10, activation='relu')) model.add(layers.MaxPooling1D()) model.add(layers.BatchNormalization()) model.add(layers.Flatten()) model.add(layers.Dropout(0.5)) model.add(layers.Dropout(0.25)) model.add(layers.Dense(n_outputs, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) print(model.summary())
def __init__(self, df=None, text=None, label=None, test_split=0.10, lr=0.01, num_words=10000, embedding_dim=50, dense_nodes=None, maxlen=None, tb_dir=None, loss_func='categorical_crossentropy', pre_trained_EM=None, test_df=None, cnn=False, act='softmax', lstm=True, opt='adam', drop=False, slim_bert=False): ''' num_words: is the max limit of words test_df: expect to be a list. Ex. [df, text_label, code_label] if tb_dir: is not none, will create a dir pre_trained_EM: pass in a dict of word embeddings embedding_dim: if pass in a pre trained EM, embedding_dim might needs to be reset ''' if isinstance(df, pd.DataFrame) and text != None and label != None: #print('var that outside the class: {}'.format(a)) self.version = '6.0' self.df = df self.text = text #column name for text self.label = label #column name for label self.test_split = test_split self.classes = np.unique(df[label].values) pre_train = False self.opt = opt self.lr = lr self.maxlen = maxlen self.num_words = num_words self.labels_not_match = False self.loss_func = loss_func if (len(self.classes) == 2) and (self.loss_func == 'categorical_crossentropy'): self.loss_func = 'binary_crossentropy' #visualization on TensorBoard self.tb_dir = tb_dir sentences = df[text].values encoder = LabelBinarizer() y = encoder.fit_transform(df[label].values) if test_df == None: # train test split & tokenization self.sentences_train, self.sentences_test, self.y_train, self.y_test = train_test_split( sentences, y, test_size=self.test_split, random_state=42) self.tokenizer = Tokenizer(num_words=self.num_words) self.tokenizer.fit_on_texts(self.sentences_train) self.X_train = self.tokenizer.texts_to_sequences( self.sentences_train) self.X_test = self.tokenizer.texts_to_sequences( self.sentences_test) #vocab_size = len(self.tokenizer.word_index) + 1 else: ''' if user pass in a separate validation data set format: test_df = [df, text, label] ''' # over write test_split -> 0 self.test_split = 0 self.valid_df = test_df[0] self.valid_text = test_df[1] self.valid_label = test_df[2] self.test_label = np.unique( self.valid_df[self.valid_label].values) if not np.array_equal(self.classes, self.test_label): print('labels not match:') self.labels_not_match = True print('training set: {}'.format(self.classes)) print('testing set: {}'.format( np.unique(self.valid_df[self.valid_label].values))) #raise Exception('label not match') self.sentences_train = sentences self.y_train = y self.sentences_test = self.valid_df[self.valid_text].values self.y_test = encoder.transform( self.valid_df[self.valid_label].values) self.tokenizer = Tokenizer(num_words=self.num_words) self.tokenizer.fit_on_texts(self.sentences_train) self.X_train = self.tokenizer.texts_to_sequences( self.sentences_train) self.X_test = self.tokenizer.texts_to_sequences( self.sentences_test) self.vocab_size = len(self.tokenizer.word_index) + 1 #maxlen_op = ['maxlen','mid','mean'] if (type(maxlen) == int) or (type(maxlen) == float): self.maxlen = int(maxlen) else: #back to default #self.maxlen = 'maxlen' length_list = [] for val in self.X_train: length_list.append(len(val)) self.maxlen = max(length_list) ''' if type(self.maxlen) == str: length_list = [] for val in self.X_train: length_list.append(len(val)) if self.maxlen == 'maxlen': self.maxlen = max(length_list) ''' self.X_train = pad_sequences(self.X_train, padding='post', maxlen=self.maxlen) self.X_test = pad_sequences(self.X_test, padding='post', maxlen=self.maxlen) # set pre trained embeddings if isinstance(pre_trained_EM, dict): # over write vocab_size # set pre_train = True NB_WORDS = len(pre_trained_EM) self.vocab_size = NB_WORDS pre_train = True missing_w = 0 emb_matrix = np.zeros((NB_WORDS, embedding_dim)) for w, i in self.tokenizer.word_index.items(): if i < NB_WORDS: vect = pre_trained_EM.get(w) if vect is not None: emb_matrix[i] = vect else: missing_w += 1 else: break ''' Hyperparameter 1. embedding_dim 2. Dense nodes Different layers 1. dropout layers ''' self.embedding_dim = embedding_dim self.output_num = self.y_train.shape[1] if dense_nodes == None: self.dense_nodes = self.output_num * 10 else: self.dense_nodes = dense_nodes #self.loss_func = ['mean_squared_error', 'logcosh', 'mean_absolute_error', 'categorical_crossentropy'] ''' 1. em-de-de-output 2. em-lstm-output 3. em-cnn-de-output 4. em-cnn-lstm-output 5. note: an extra dropout layer can be added ''' model = Sequential() #model.add(layers.Embedding(vocab_size, self.embedding_dim, input_length=self.maxlen)) if pre_train: model.add( layers.Embedding(self.vocab_size, self.embedding_dim, input_length=self.maxlen, trainable=False)) model.layers[0].set_weights([emb_matrix]) else: model.add( layers.Embedding(self.vocab_size, self.embedding_dim, input_length=self.maxlen)) ''' adding layers to Sequential model ''' if slim_bert: model.add( layers.Bidirectional( layers.LSTM(128, return_sequences=True, dropout=0.2, recurrent_dropout=0.2))) model.add(SeqSelfAttention(attention_activation='sigmoid')) model.add(layers.Flatten()) else: if cnn: model.add( layers.Conv1D(128, 5, activation='relu', padding='same')) #model.add(layers.MaxPooling1D()) if not lstm: model.add(layers.MaxPooling1D()) model.add(layers.Dropout(0.3)) if lstm: model.add( layers.LSTM(self.dense_nodes, dropout=0.2, recurrent_dropout=0.2, return_sequences=True)) model.add(layers.Flatten()) if drop: model.add(layers.Dropout(0.5)) if (cnn and not lstm) or (not cnn and not lstm): model.add(layers.Dense(self.dense_nodes, activation='relu')) ''' add a output Dense layer to model ''' model.add(layers.Dense(self.output_num, activation=act)) ''' sgd = SGD(lr=self.lr, momentum=0.9, decay=0.0, nesterov=False) adam = Adam(lr=self.lr, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False) if self.opt == 'sgd': model.compile(loss=self.loss_func, optimizer=sgd, metrics=['accuracy']) else: self.opt = 'adam' model.compile(loss=self.loss_func, optimizer=adam, metrics=['accuracy']) ''' self.model = model #self.model.summary() else: print("please load a model")
) # build network model: CNN + RNN from keras.models import Sequential from keras import layers from keras.optimizers import RMSprop model = Sequential() # CNN layers: 1D ConV + max pooling model.add( layers.Conv1D( 32, 5, activation='relu', input_shape=(None, float_data.shape[-1] ))) # 5: ConV window. P190 the feature vectors count model.add(layers.MaxPooling1D(3)) # 1/3 model.add(layers.Conv1D(32, 5, activation='relu')) # RNN layer model.add(layers.GRU(32, dropout=0.1, recurrent_dropout=0.5)) # Dense layer classifier model.add(layers.Dense(1)) print(model.summary()) ''' _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= conv1d_1 (Conv1D) (None, None, 32) 2272 _________________________________________________________________ max_pooling1d_1 (MaxPooling1 (None, None, 32) 0 _________________________________________________________________
val_steps = (valnum - lookback) // batch_size # 查看训练集需要抽取的次数 train_steps = trainnum // batch_size from keras.callbacks import EarlyStopping early_stopping = EarlyStopping(monitor="val_loss", patience=30) #创建模型 model1 = models.Sequential() model1.add( layers.Dense(512, activation="relu", input_shape=(lookback // step, data.shape[-1]))) model1.add(layers.Conv1D(filters=1024, kernel_size=5, activation="relu")) model1.add(layers.MaxPooling1D(5)) model1.add(layers.Conv1D(filters=1024, kernel_size=3, activation="relu")) model1.add(layers.GlobalMaxPool1D()) model1.add(layers.Dropout(0.5)) model1.add(layers.Dense(8, activation="softmax")) model1.summary() model1.compile(optimizer=optimizers.RMSprop(), loss="categorical_crossentropy", metrics=["acc"]) history1 = model1.fit_generator(train_gen, steps_per_epoch=train_steps, epochs=200, validation_data=val_gen, validation_steps=val_steps, callbacks=[early_stopping])
def build_model(self, inception=True, res=True, strided=False, maxpool=True, avgpool=False, batchnorm=True): self.i = 0 pad = 'same' padp = 'same' c_act = self.config['c_act'] r_act = self.config['r_act'] rk_act = self.config['rk_act'] r = kr.regularizers.l2(self.config['reg']) c = self.input stride_size = self.config['strides'] if strided else 1 if inception: c0 = layers.Conv1D(self.config['filters'], kernel_size=4, strides=stride_size, padding=pad, activation=c_act)(c) c1 = layers.Conv1D(self.config['filters'], kernel_size=8, strides=stride_size, padding=pad, activation=c_act)(c) c2 = layers.Conv1D(self.config['filters'], kernel_size=32, strides=stride_size, padding=pad, activation=c_act)(c) c = layers.concatenate([c0, c1, c2]) if maxpool: c = layers.MaxPooling1D(2, padding=padp)(c) elif avgpool: c = layers.AveragePooling1D(2, padding=padp)(c) if batchnorm: c = layers.BatchNormalization()(c) c = layers.SpatialDropout1D(self.config['cnn_drop'])(c) c0 = layers.Conv1D(self.config['filters'], kernel_size=4, strides=stride_size, padding=pad, activation=c_act)(c) c1 = layers.Conv1D(self.config['filters'], kernel_size=8, strides=stride_size, padding=pad, activation=c_act)(c) c2 = layers.Conv1D(self.config['filters'], kernel_size=32, strides=stride_size, padding=pad, activation=c_act)(c) c = layers.concatenate([c0, c1, c2]) if maxpool: c = layers.MaxPooling1D(2, padding=padp)(c) elif avgpool: c = layers.AveragePooling1D(2, padding=padp)(c) if batchnorm: c = layers.BatchNormalization()(c) c = layers.SpatialDropout1D(self.config['cnn_drop'])(c) c0 = layers.Conv1D(self.config['filters'], kernel_size=4, strides=stride_size, padding=pad, activation=c_act)(c) c1 = layers.Conv1D(self.config['filters'], kernel_size=8, strides=stride_size, padding=pad, activation=c_act)(c) c2 = layers.Conv1D(self.config['filters'], kernel_size=32, strides=stride_size, padding=pad, activation=c_act)(c) c = layers.concatenate([c0, c1, c2]) if maxpool: c = layers.MaxPooling1D(2, padding=padp)(c) elif avgpool: c = layers.AveragePooling1D(2, padding=padp)(c) if batchnorm: c = layers.BatchNormalization()(c) c = layers.SpatialDropout1D(self.config['cnn_drop'])(c) else: # No inception Modules c = layers.Conv1D(self.config['filters'], kernel_size=4, strides=stride_size, padding=pad, activation=c_act)(self.input) if maxpool: c = layers.MaxPooling1D(2, padding=padp)(c) elif avgpool: c = layers.AveragePooling1D(2, padding=padp)(c) if batchnorm: c = layers.BatchNormalization()(c) c = layers.SpatialDropout1D(self.config['cnn_drop'])(c) c = layers.Conv1D(self.config['filters'], kernel_size=4, strides=stride_size, padding=pad, activation=c_act)(c) if maxpool: c = layers.MaxPooling1D(2, padding=padp)(c) elif avgpool: c = layers.AveragePooling1D(2, padding=padp)(c) if batchnorm: c = layers.BatchNormalization()(c) c = layers.SpatialDropout1D(self.config['cnn_drop'])(c) c = layers.Conv1D(self.config['filters'], kernel_size=4, strides=stride_size, padding=pad, activation=c_act)(c) if maxpool: c = layers.MaxPooling1D(2, padding=padp)(c) elif avgpool: c = layers.AveragePooling1D(2, padding=padp)(c) if batchnorm: c = layers.BatchNormalization()(c) c = layers.SpatialDropout1D(self.config['cnn_drop'])(c) if res: # Residual RNN g1 = layers.GRU(self.config['state_size'], return_sequences=True, activation=rk_act, recurrent_activation=r_act, dropout=self.config['rec_drop'], recurrent_dropout=self.config['rec_drop'], recurrent_regularizer=r, kernel_regularizer=r)(c) g2 = layers.GRU(self.config['state_size'], return_sequences=True, activation=rk_act, recurrent_activation=r_act, dropout=self.config['rec_drop'], recurrent_dropout=self.config['rec_drop'], recurrent_regularizer=r, kernel_regularizer=r)(g1) g_concat1 = layers.concatenate([g1, g2]) g3 = layers.GRU(self.config['state_size'], return_sequences=True, activation=rk_act, recurrent_activation=r_act, dropout=self.config['rec_drop'], recurrent_dropout=self.config['rec_drop'], recurrent_regularizer=r, kernel_regularizer=r)(g_concat1) g_concat2 = layers.concatenate([g1, g2, g3]) g = layers.GRU(self.config['state_size'], return_sequences=False, activation=rk_act, recurrent_activation=r_act, dropout=self.config['rec_drop'], recurrent_dropout=self.config['rec_drop'], recurrent_regularizer=r, kernel_regularizer=r)(g_concat2) else: # No Residual RNN g = layers.GRU(self.config['state_size'], return_sequences=True, activation=rk_act, recurrent_activation=r_act, dropout=self.config['rec_drop'], recurrent_dropout=self.config['rec_drop'], recurrent_regularizer=r, kernel_regularizer=r)(c) g = layers.GRU(self.config['state_size'], return_sequences=True, activation=rk_act, recurrent_activation=r_act, dropout=self.config['rec_drop'], recurrent_dropout=self.config['rec_drop'], recurrent_regularizer=r, kernel_regularizer=r)(g) g = layers.GRU(self.config['state_size'], return_sequences=True, activation=rk_act, recurrent_activation=r_act, dropout=self.config['rec_drop'], recurrent_dropout=self.config['rec_drop'], recurrent_regularizer=r, kernel_regularizer=r)(g) g = layers.GRU(self.config['state_size'], return_sequences=False, activation=rk_act, recurrent_activation=r_act, dropout=self.config['rec_drop'], recurrent_dropout=self.config['rec_drop'], recurrent_regularizer=r, kernel_regularizer=r)(g) d = layers.Dense(self.config['output_size'])(g) out = layers.Softmax()(d) self.model = Model(self.input, out) print("{} initialized.".format(self.model.name))
# model.add(kn.Dense(500, activation='relu')) # model.add(kn.Dropout(0.2)) # model.add(kn.Dense(90, activation='sigmoid')) # model.compile(optimizer='adam', loss='binary_crossentropy', # metrics=["accuracy", keras.metrics.FalseNegatives(), keras.metrics.FalsePositives()]) # model.summary() # 80% f1 score inp = kn.Input(shape=(mdn, ), dtype='int32') x = kn.Embedding(vocab_size, 100, weights=[embedding_matrix], input_length=mdn, trainable=False)(inp) x = kn.Conv1D(100, 5, activation='relu')(x) x = kn.MaxPooling1D(5)(x) x = kn.Dropout(0.2)(x) x = kn.Conv1D(100, 5, activation='relu')(x) x = kn.MaxPooling1D(22)(x) x = kn.Dropout(0.2)(x) x = kn.Flatten()(x) x = kn.Dense(128, activation='relu')(x) x = kn.Dropout(0.05)(x) preds = kn.Dense(90, activation='sigmoid')(x) model = keras.Model(inp, preds) model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=[ 'accuracy', keras.metrics.FalseNegatives(),
def build_full_model(frames, freq_bins, mod_options): ''' Build a keras model based on the Dielemann model for music recommandation. This model is trained on time-frequency representation of music. The input data fed to the model must be shaped like (batch_size, time, frequency). Parameters ---------- frames: int Number of time frames in a single input freq_bin: int Number of frequency bands in a single input mod_option: dictionnary Specific options for the model. This dictionnary must contain: 'activation': string name of the activation function for keras layers 'batchNormConv': bool Choose if the model should apply babtch normalization after each convnet 'FC number': int Number of cells for each FC layer 'batchNormDense': bool Choose if the model should apply babtch normalization after each FC layer 'Alphabet size': int Size of the training alphabet (last layer, size of the output) Returns ------- model: keras model The model built. Expects inputs of shape (batch_size, frames, freq_bins) and outputs tensor of shape (batch_size, alphabet_size). ''' inputs = layers.Input(shape=(frames, freq_bins)) #%%=========== First layer =============== #zero-padding the input padding_1 = layers.ZeroPadding1D(padding=2)(inputs) #Convnet 256 neurons with 4 sample window. Activation defined in mod_option dictionnary conv1 = layers.Conv1D(256, 4, padding='same', activation=mod_options['activation'])(padding_1) #Normalise batch if defined in mod_options if mod_options['batchNormConv']: conv1 = layers.BatchNormalization()(conv1) #Reduce data by max pooling between 2 values pool_1 = layers.MaxPooling1D(pool_size=2)(conv1) #%%============ Second layer ============== #Same layer as the previous one padding_2 = layers.ZeroPadding1D(padding=2)(pool_1) conv_2 = layers.Conv1D(256, 4, padding='same', activation=mod_options['activation'])(padding_2) if mod_options['batchNormConv']: conv_2 = layers.BatchNormalization()(conv_2) pool_2 = layers.MaxPooling1D(pool_size=2)(conv_2) ''' #%%=========== Third layer ???================ #zero-padding the input model.Add(layers.ZeroPadding1D(padding = 2)) #Convnet 512 neurons with 4 sample window. model.add(layers.Conv1D(512, 4, padding = 'same', activation = mod_options['activation'])) #Normalize batch if defined if mod_options['batchNormConv']: model.add(layers.BatchNormalization()) ''' #%%=========== Fourth layer ================= #zero-padding the input padding_3 = layers.ZeroPadding1D(padding=2)(pool_2) #Convnet 512 neurons with 4 sample window. conv_3 = layers.Conv1D(512, 4, padding='same', activation=mod_options['activation'])(padding_3) #Normalize batch if defined if mod_options['batchNormConv']: conv_3 = layers.BatchNormalization()(conv_3) #%%========== Global temporal pooling layer ========= pool_max = layers.GlobalMaxPooling1D()(conv_3) pool_average = layers.GlobalAveragePooling1D()(conv_3) pool_LP = layers.Lambda(lambda x: GlobalLPPooling1D(x))(conv_3) pool_time = layers.Concatenate()([pool_max, pool_average, pool_LP]) #%%========== FC Layers ========================= FC_1 = layers.Dense(mod_options['FC number'], activation=mod_options['activation'])(pool_time) if mod_options['batchNormDense']: FC_1 = layers.BatchNormalization()(FC_1) FC_2 = layers.Dense(mod_options['FC number'], activation=mod_options['activation'])(FC_1) if mod_options['batchNormDense']: FC_2 = layers.BatchNormalization()(FC_2) FC_3 = layers.Dense(mod_options['Alphabet size'], activation='softmax')(FC_2) model = Model(inputs=inputs, outputs=FC_3) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) return model
def InceptionResNetV2(include_top=True, weights=None, input_tensor=None, input_shape=None, pooling=None, classes=100, **kwargs): """Instantiates the Inception-ResNet v2 architecture. Note that the data format convention used by the model is the one specified in your Keras config at `~/.keras/keras.json`. # Arguments include_top: whether to include the fully-connected layer at the top of the network. weights: one of `None` (random initialization) or the path to the weights file to be loaded. input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as input for the model. input_shape: optional shape tuple if input_tensor is not specified. and width should be no smaller than 75. pooling: Optional pooling mode for feature extraction when `include_top` is `False`. - `None` means that the output of the model will be the 3D tensor output of the last convolutional block. - `'avg'` means that global average pooling will be applied to the output of the last convolutional block, and thus the output of the model will be a 2D tensor. - `'max'` means that global max pooling will be applied. classes: optional number of classes to classify inputs into, only to be specified if `include_top` is `True`, and if no `weights` argument is specified. # Returns A Keras `Model` instance. # Raises ValueError: in case of invalid argument for `weights`, or invalid input shape. """ if weights is not None and not os.path.exists(weights): raise ValueError( 'The `weights` argument should be either `None` (random initialization) or the path to the weights file to be loaded.' ) if input_tensor is None: inputs = layers.Input(shape=input_shape) else: if not backend.is_keras_tensor(input_tensor): inputs = layers.Input(tensor=input_tensor, shape=input_shape) else: inputs = input_tensor # Stem block: 35 x 192 x = conv1d_bn(inputs, 32, 3, strides=2, padding='valid') x = conv1d_bn(x, 32, 3, padding='valid') x = conv1d_bn(x, 64, 3) x = layers.MaxPooling1D(3, strides=2)(x) x = conv1d_bn(x, 80, 1, padding='valid') x = conv1d_bn(x, 192, 3, padding='valid') x = layers.MaxPooling1D(3, strides=2)(x) # Mixed 5b (Inception-A block): 35 x 320 branch_0 = conv1d_bn(x, 96, 1) branch_1 = conv1d_bn(x, 48, 1) branch_1 = conv1d_bn(branch_1, 64, 5) branch_2 = conv1d_bn(x, 64, 1) branch_2 = conv1d_bn(branch_2, 96, 3) branch_2 = conv1d_bn(branch_2, 96, 3) branch_pool = layers.AveragePooling1D(3, strides=1, padding='same')(x) branch_pool = conv1d_bn(branch_pool, 64, 1) branches = [branch_0, branch_1, branch_2, branch_pool] channel_axis = 1 if backend.image_data_format() == 'channels_first' else 2 x = layers.Concatenate(axis=channel_axis, name='mixed_5b')(branches) # 10x block35 (Inception-ResNet-A block): 35 x 320 for block_idx in range(1, 11): x = inception_resnet_block(x, scale=0.17, block_type='block35', block_idx=block_idx) # Mixed 6a (Reduction-A block): 17 x 1088 branch_0 = conv1d_bn(x, 384, 3, strides=2, padding='valid') branch_1 = conv1d_bn(x, 256, 1) branch_1 = conv1d_bn(branch_1, 256, 3) branch_1 = conv1d_bn(branch_1, 384, 3, strides=2, padding='valid') branch_pool = layers.MaxPooling1D(3, strides=2, padding='valid')(x) branches = [branch_0, branch_1, branch_pool] x = layers.Concatenate(axis=channel_axis, name='mixed_6a')(branches) # 20x block17 (Inception-ResNet-B block): 17 x 1088 for block_idx in range(1, 21): x = inception_resnet_block(x, scale=0.1, block_type='block17', block_idx=block_idx) # Mixed 7a (Reduction-B block): 8 x 2080 branch_0 = conv1d_bn(x, 256, 1) branch_0 = conv1d_bn(branch_0, 384, 3, strides=2, padding='valid') branch_1 = conv1d_bn(x, 256, 1) branch_1 = conv1d_bn(branch_1, 288, 3, strides=2, padding='valid') branch_2 = conv1d_bn(x, 256, 1) branch_2 = conv1d_bn(branch_2, 288, 3) branch_2 = conv1d_bn(branch_2, 320, 3, strides=2, padding='valid') branch_pool = layers.MaxPooling1D(3, strides=2, padding='valid')(x) branches = [branch_0, branch_1, branch_2, branch_pool] x = layers.Concatenate(axis=channel_axis, name='mixed_7a')(branches) # 10x block8 (Inception-ResNet-C block): 8 x 2080 for block_idx in range(1, 10): x = inception_resnet_block(x, scale=0.2, block_type='block8', block_idx=block_idx) x = inception_resnet_block(x, scale=1., activation=None, block_type='block8', block_idx=10) # Final convolution block: 8 x 1536 x = conv1d_bn(x, 1536, 1, name='conv_7b') if include_top: # Classification block x = layers.GlobalAveragePooling1D(name='avg_pool')(x) x = layers.Dense(classes, activation='softmax', name='predictions')(x) else: if pooling == 'avg': x = layers.GlobalAveragePooling1D()(x) elif pooling == 'max': x = layers.GlobalMaxPooling1D()(x) # Ensure that the model takes into account # any potential predecessors of `input_tensor`. if input_tensor is not None: inputs = utils.get_source_inputs(input_tensor) else: inputs = inputs # Create model. model = models.Model(inputs, x, name='1d_inception_resnet_v2') # Load weights. if weights is not None: model.load_weights(weights) return model
print('Pad sequences (sample x time)') x_train = sequence.pad_sequences(x_train, maxlen=max_len) x_test = sequence.pad_sequences(x_test, maxlen=max_len) print('x_train shape:', x_train.shape) print('x_test shape:', x_test.shape) # 在IMDB数据上训练并评估一个简单的一维卷积神经网络 from keras.models import Sequential from keras import layers from keras.optimizers import RMSprop model = Sequential() model.add(layers.Embedding(max_features, 128, input_length=max_len)) model.add(layers.Conv1D(32, 7, activation='relu')) model.add(layers.MaxPooling1D(5)) model.add(layers.Conv1D(32, 7, activation='relu')) model.add(layers.GlobalMaxPooling1D()) model.add(layers.Dense(1)) model.summary() model.compile(optimizer=RMSprop(lr=1e-4), loss='binary_crossentropy', metrics=['acc']) history = model.fit(x_train, y_train, epochs=10, batch_size=128, validation_split=0.2)
def CTC(signals, out_len, label_len, kernel_size, conv_window_len, lr, dropoutRate, training=True): inner = signals n_label = 5 # CNN inner = layers.Conv1D(kernel_size[0], conv_window_len, padding='same', activation="relu")(inner) inner = layers.Conv1D(kernel_size[0], conv_window_len, padding='same', activation="relu")(inner) inner = layers.MaxPooling1D(2)(inner) inner = layers.Conv1D(kernel_size[1], conv_window_len, padding='same', activation="relu")(inner) inner = layers.Conv1D(kernel_size[1], conv_window_len, padding='same', activation="relu")(inner) inner = layers.MaxPooling1D(2)(inner) # CNN to RNN #print(inner.get_shape()) inter = layers.Dense(64, activation='relu', kernel_initializer='he_normal')(inner) # 2"RNN layer gru_1 = layers.GRU(128, return_sequences=True, kernel_initializer='he_normal')(inner) gru_1b = layers.GRU(128, return_sequences=True, go_backwards=True, kernel_initializer='he_normal')(inner) gru1_merged = layers.Add()([gru_1, gru_1b]) gru_2 = layers.GRU(128, return_sequences=True, kernel_initializer='he_normal')(gru1_merged) gru_2b = layers.GRU(128, return_sequences=True, go_backwards=True, kernel_initializer='he_normal')(gru1_merged) gru2_merged = layers.Concatenate()([gru_2, gru_2b]) inner = gru2_merged inner = layers.Dense(n_label)(inner) y_pred = layers.Activation('softmax')(inner) # loss compute module labels = Input(name="labels", shape=[label_len], dtype='int64') input_length = Input(name="input_length", shape=[1], dtype='int64') label_length = Input(name="label_length", shape=[1], dtype='int64') loss_out = layers.Lambda(ctc_lambda_func, output_shape=(1, ), name='ctc')( [y_pred, labels, input_length, label_length]) if training: return models.Model([signals, labels, input_length, label_length], outputs=loss_out) else: return models.Model(inputs=[signals], outputs=y_pred)