def residual_network(x):
"""
ResNeXt by default. For ResNet set `cardinality` = 1 above.
"""
def add_common_layers(y):
y = layers.BatchNormalization()(y)
y = layers.LeakyReLU()(y)
y = layers.Dropout(dropout_level)(y)
return y
def grouped_convolution(y, nb_channels, _strides):
# when `cardinality` == 1 this is just a standard convolution
if cardinality == 1:
return layers.Conv1D(
nb_channels,
kernel_size=3,
strides=_strides,
padding='same',
kernel_regularizer=regularizers.l2(L2_regularizer))(y)
assert not nb_channels % cardinality
_d = nb_channels // cardinality
# in a grouped convolution layer, input and output channels are divided into `cardinality` groups,
# and convolutions are separately performed within each group
groups = []
for j in range(cardinality):
group = layers.Lambda(lambda z: z[:, :, j * _d:j * _d + _d])(y)
groups.append(
layers.Conv1D(
_d,
kernel_size=3,
strides=_strides,
padding='same',
kernel_regularizer=regularizers.l2(L2_regularizer))(group))
# the grouped convolutional layer concatenates them as the outputs of the layer
y = layers.concatenate(groups)
return y
def residual_block(y,
nb_channels_in,
nb_channels_out,
_strides=1,
_project_shortcut=False):
"""
Our network consists of a stack of residual blocks. These blocks have the same topology,
and are subject to two simple rules:
- If producing spatial maps of the same size, the blocks share the same hyper-parameters (width and filter sizes).
- Each time the spatial map is down-sampled by a factor of 2, the width of the blocks is multiplied by a factor of 2.
"""
shortcut = y
# we modify the residual building block as a bottleneck design to make the network more economical
y = layers.Conv1D(
nb_channels_in,
kernel_size=1,
strides=1,
padding='same',
kernel_regularizer=regularizers.l2(L2_regularizer))(y)
y = add_common_layers(y)
# ResNeXt (identical to ResNet when `cardinality` == 1)
y = grouped_convolution(y, nb_channels_in, _strides=_strides)
y = add_common_layers(y)
y = layers.Conv1D(
nb_channels_out,
kernel_size=1,
strides=1,
padding='same',
kernel_regularizer=regularizers.l2(L2_regularizer))(y)
# batch normalization is employed after aggregating the transformations and before adding to the shortcut
y = layers.BatchNormalization()(y)
# identity shortcuts used directly when the input and output are of the same dimensions
if _project_shortcut or _strides != 1:
# when the dimensions increase projection shortcut is used to match dimensions (done by 1×1 convolutions)
# when the shortcuts go across feature maps of two sizes, they are performed with a stride of 2
shortcut = layers.Conv1D(
nb_channels_out,
kernel_size=1,
strides=_strides,
padding='same',
kernel_regularizer=regularizers.l2(L2_regularizer))(shortcut)
shortcut = layers.BatchNormalization()(shortcut)
y = layers.add([shortcut, y])
# relu is performed right after each batch normalization,
# expect for the output of the block where relu is performed after the adding to the shortcut
y = layers.Activation('relu')(y)
return y
# conv1
x = layers.Conv1D(64,
kernel_size=7,
strides=2,
padding='same',
kernel_regularizer=regularizers.l2(L2_regularizer))(x)
x = add_common_layers(x)
# conv2
x = layers.MaxPooling1D(pool_size=3, strides=2, padding='same')(x)
for i in range(2):
project_shortcut = True if i == 0 else False
x = residual_block(x, 64, 128, _project_shortcut=project_shortcut)
# conv3
for i in range(2):
# down-sampling is performed by conv3_1, conv4_1, and conv5_1 with a stride of 2
strides = 2 if i == 0 else 1
x = residual_block(x, 128, 256, _strides=strides)
# conv4
for i in range(2):
strides = 2 if i == 0 else 1
x = residual_block(x, 256, 512, _strides=strides)
# conv5
for i in range(2):
strides = 2 if i == 0 else 1
x = residual_block(x, 512, 1024, _strides=strides)
x = layers.GlobalAveragePooling1D()(x)
return x
def _build_layers_v2(self, input_dict, num_outputs, options):
"""Define the layers of a custom model.
Arguments:
input_dict (dict): Dictionary of input tensors, including "obs",
"prev_action", "prev_reward", "is_training".
num_outputs (int): Output tensor must be of size
[BATCH_SIZE, num_outputs].
options (dict): Model options.
Returns:
(outputs, feature_layer): Tensors of size [BATCH_SIZE, num_outputs]
and [BATCH_SIZE, desired_feature_size].
"""
TRANSFORMER_MODEL_DIM = options["custom_options"][
"transformer_model_dim"]
TRANSFORMER_NUM_HEADS = options["custom_options"][
"transformer_num_heads"]
TRANSFORMER_DEPTH = options["custom_options"]["transformer_depth"]
CONV_PADDING = options["custom_options"]["conv_padding"]
# Agent architecture p.15 of Zambaldi et al
# "The input module contained two convolutional layers with 12 and 24 kernels, 2 × 2 kernel sizes
# and a stride of 1, followed by a rectified linear unit (ReLU) activation function. The output
# was tagged with two extra channels indicating the spatial position (x and y) of each cell in
# the feature map using evenly spaced values between −1 and 1. This was passed to the relational
# module, consisting of relational blocks, with shared parameters. Queries, keys and values were
# produced by 2 to 4 attention heads and had an embedding size (d) of 64. The output of this module
# was aggregated using a feature-wise max pooling function and passed to 4 fully connected layers,
# each followed by a ReLU. Policy logits (pi, size 4) and baseline function (B, size 1) were produced
# by a linear projection. The policy logits were normalized and used as multinomial distribution from
# which the action (a) was sampled."
# NOTE: there is no dropout in Zambaldi et al
inputs = input_dict["obs"]
sess = tf.get_default_session()
K.set_session(sess)
# NOTE: the weights in the self-attention mechanism
# and feed-forward layers are shared between all Transformer blocks (as in
# Zambaldi et al, but unlike every other Transformer paper)
# The published version of Zambaldi et al does not tell us what the MLP g_\theta is, but
#
# - in Santoro et al "A simple neural network module for relational reasoning"
# the analogous g_\theta is is a four-layer MLP with 256 dimensional hidden layers
# with ReLU non-linearities
# - in Keras-Transformer the default is a two layer model with hidden dimension
# equal to 4 * the embedding dimension, which in the case of 64 dimensional embeddings
# gives 256 (this is also the convention in the Sparse Transformer paper)
# - in the first version of Zambaldi et al they write "passed to a multilayer perceptron
# (2-layer MLP with ReLU non-linearities) with the same layers sizes as ei"
#
# Hence, attempting to follow Zambaldi, we use layer size TRANSFORMER_MODEL_DIM
# (in v6 we used 4 times this)
attention_layer = MultiHeadSelfAttentionZambaldi(
name='self_attention',
num_heads=TRANSFORMER_NUM_HEADS,
use_masking=False,
dropout=0,
compression_window_size=None)
dense_layer1 = layers.Dense(TRANSFORMER_MODEL_DIM, activation='relu')
dense_layer2 = layers.Dense(TRANSFORMER_MODEL_DIM)
def transformer_block(input):
#_, seq_len, d_model = K.int_shape(input)
a = LayerNormalization()(input)
a = attention_layer(
a
) # a = attention(h) has shape -1, seq_len, TRANSFORMER_MODEL_DIM
b = dense_layer1(a)
b = dense_layer2(b) # b = ff(a)
r = layers.Add()([input, b])
Hprime = LayerNormalization()(r)
return Hprime
# CONVOLUTIONS ------
#
# Question: should we use max-pooling here? It seems not, as the downsampling in the
# Santoro et al paper "A simple neural network module for relational reasoning"
# occurs using 3x3 patches with stride 2, rather than max-pooling, and it is not
# mentioned anywhere in the papers.
#
# It is worth comparing to e.g. the models for deep RL on 3D environments in the IMPALA
# paper, see Figure 3, which also have no max-pooling layers and downsample instead using
# strided convolutional layers. You'll see there also they prefer hidden layers of width
# 256 for the FC layers after the initial convolutional layers, in processing visual scenes.
# So the Zambaldi paper is consistent with their other work on deep RL, in terms of the model.
x = layers.Lambda(lambda x: x / 255)(inputs) # rescale RGB to [0,1]
x = layers.Conv2D(12, (2, 2), activation='relu',
padding=CONV_PADDING)(x)
x = layers.Conv2D(24, (2, 2), activation='relu', padding=CONV_PADDING)(
x) # output shape -1, num_rows, num_cols, 62
x = layers.Dense(
TRANSFORMER_MODEL_DIM - 2, activation=None, use_bias=False
)(x) # output shape -1, num_rows, num_cols, TRANSFORMER_MODEL_DIM-2
# NOTE: we are using the default "valid" padding, so actually our width and height decrease
# by one in each convolutional layer
# POSITION EMBEDDING -----
#
# Here we follow Zambaldi et al, rather than the standard Transformer
# positional embeddings
num_rows, num_cols, d_model = x.get_shape().as_list()[-3:]
ps = np.zeros([num_rows, num_cols, 2],
dtype=K.floatx()) # shape (12,13,2)
for ty in range(num_rows):
for tx in range(num_cols):
ps[ty, tx, :] = [(2 / (num_rows - 1)) * ty - 1,
(2 / (num_cols - 1)) * tx - 1]
ps_expand = K.expand_dims(K.constant(ps),
axis=0) # shape (1,num_rows,num_cols,2)
ps_tiled = K.tile(
ps_expand,
[K.shape(x)[0], 1, 1, 1]) # shape (None,num_rows,num_cols,2)
# (None,num_rows,num_cols,62) concatenate with (None,num_rows,num_cols,2)
# to get (None,num_rows,num_cols,TRANSFORMER_MODEL_DIM)
x = Concatenate(axis=3)([x, ps_tiled])
x = layers.Reshape((num_rows * num_cols, d_model + 2))(x)
# TRANSFORMER -----
for i in range(TRANSFORMER_DEPTH):
x = transformer_block(x)
# MAX-POOLING -----
# from p.4 "The E~ matrix, with shape Nxf is reudced to an f-dimensional vector by max-pooling
# over the entity dimension. This pooled vector is then passed to a small MLP..."
num_entities, d_model = x.get_shape().as_list()[-2:]
x = layers.MaxPooling1D(pool_size=num_entities)(x)
x = layers.Flatten()(x)
# FULLY-CONNECTED LAYERS ----
x = layers.Dense(256, activation='relu')(x)
x = layers.Dense(256, activation='relu')(x)
x = layers.Dense(256, activation='relu')(x)
x = layers.Dense(256, activation='relu')(x)
output_tensor = layers.Dense(4)(x) # final output is logits
return output_tensor, x
validation_samples:training_samples +
validation_samples + texting_samples]
#卷积神经网络设计
from keras.models import Sequential
from keras.layers import Flatten, Dense, Embedding
from keras.layers import SimpleRNN
from keras import layers
from keras import regularizers
model = Sequential()
model.add(Embedding(10000, 32, input_shape=(2000, )))
#model.add(SimpleRNN(32))
#model.add(Dense(25,activation='softmax'))
model.add(layers.Conv1D(256, 5, activation='relu')) #原来是256
model.add(layers.MaxPooling1D(2))
model.add(layers.Dropout(0.5))
model.add(layers.Conv1D(256, 5, activation='relu')) #原来是256
model.add(layers.MaxPooling1D(64))
model.add(layers.Dropout(0.5))
model.add(layers.GlobalAveragePooling1D())
model.add(Dense(17, activation='softmax'))
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.summary()
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
history = model.fit(x_train,
def build_conv_layers(frames, freq_bins, mod_options):
'''
Build the convolutionnal base of the model based on the Dielemann model
for music recommandation (useful for transfer learning).
This model is trained on time-frequency representation of music. The input
data fed to the model must be shaped like (batch_size, time, frequency).
Once FC layers are added via the add_fc_layers function, the model can be
trained and saved for re-use in transfer learning applications.
Parameters
----------
frames: int
Number of time frames in a single input
freq_bin: int
Number of frequency bands in a single input
mod_option: dictionnary
Specific options for the model. This dictionnary must contain:
'activation': string
name of the activation function for keras layers
'batchNormConv': bool
Choose if the model should apply babtch normalization after each
convnet
Returns
-------
model: keras model
The model is not compiled since it requires FC layers on top. Expects
inputs of shape (batch size, frames, freq_bins) and outputs tensor of
shape: (batch size, frames_pool, freq_bins) where frames_pool has been
through 3 poolings of size 2 (divise by 2) and with 4 zeros added
before each pooling (add 4).
'''
inputs = layers.Input(shape=(frames, freq_bins))
#%%=========== First layer ===============
#zero-padding the input
padding_1 = layers.ZeroPadding1D(padding=2)(inputs)
#Convnet 256 neurons with 4 sample window. Activation defined in mod_option dictionnary
conv1 = layers.Conv1D(256,
4,
padding='same',
activation=mod_options['activation'])(padding_1)
#Normalise batch if defined in mod_options
if mod_options['batchNormConv']:
conv1 = layers.BatchNormalization()(conv1)
#Reduce data by max pooling between 2 values
pool_1 = layers.MaxPooling1D(pool_size=2)(conv1)
#%%============ Second layer ==============
#Same layer as the previous one
padding_2 = layers.ZeroPadding1D(padding=2)(pool_1)
conv_2 = layers.Conv1D(256,
4,
padding='same',
activation=mod_options['activation'])(padding_2)
if mod_options['batchNormConv']:
conv_2 = layers.BatchNormalization()(conv_2)
pool_2 = layers.MaxPooling1D(pool_size=2)(conv_2)
'''
#%%=========== Third layer ???================
#zero-padding the input
model.Add(layers.ZeroPadding1D(padding = 2))
#Convnet 512 neurons with 4 sample window.
model.add(layers.Conv1D(512, 4, padding = 'same', activation = mod_options['activation']))
#Normalize batch if defined
if mod_options['batchNormConv']:
model.add(layers.BatchNormalization())
'''
#%%=========== Fourth layer =================
#zero-padding the input
padding_3 = layers.ZeroPadding1D(padding=2)(pool_2)
#Convnet 512 neurons with 4 sample window.
conv_3 = layers.Conv1D(512,
4,
padding='same',
activation=mod_options['activation'])(padding_3)
#Normalize batch if defined
if mod_options['batchNormConv']:
conv_3 = layers.BatchNormalization()(conv_3)
model = Model(inputs=inputs, outputs=conv_3)
return model
Xtrain = pad_sequences(encoded_train, maxlen=max_length, padding='post')
Xtest = pad_sequences(encoded_test, maxlen=max_length, padding='post')
vocab_size = len(tokenizer.word_index) + 1
# raw_embedding = load_embedding('glove.6B.100d.txt')
raw_embedding = load_embedding('embedding_word2vec.txt')
embedding_vectors = get_weight_matrix(raw_embedding, tokenizer.word_index)
embedding_layer = layers.Embedding(vocab_size,
100,
weights=[embedding_vectors],
input_length=max_length,
trainable=False)
model = keras.Sequential()
model.add(embedding_layer)
model.add(layers.Conv1D(filters=128, kernel_size=5, activation='relu'))
model.add(layers.MaxPooling1D(pool_size=2))
model.add(layers.Flatten())
model.add(layers.Dense(1, activation='sigmoid'))
print(model.summary())
Xtrain = Xtrain.astype(np.float32)
# y_train = y_train.astype(np.float32)
# Xtest = pad_sequences(encoded_test, maxlen=max_length, padding='post')
Xtest = Xtest.astype(np.float32)
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print(
'-------------------------------------------------------------------------'
)
print(f'Training for fold {fold_no} ...')
from keras import layers
from keras import Input
from keras.models import Model
vocabulary_size = 50000
num_income_groups = 10
posts_input = Input(shape=(None, ), dtype='int32', name='posts')
embedded_posts = layers.Embedding(256, vocabulary_size)(posts_input)
x = layers.Conv1D(128, 5, activation='relu')(embedded_posts)
x = layers.MaxPooling1D(5)(x)
x = layers.Conv1D(256, 5, activation='relu')(x)
x = layers.Conv1D(256, 5, activation='relu')(x)
x = layers.MaxPooling1D(5)(x)
x = layers.Conv1D(256, 5, activation='relu')(x)
x = layers.Conv1D(256, 5, activation='relu')(x)
x = layers.GlobalMaxPooling1D()(x)
x = layers.Dense(128, activation='relu')(x)
age_prediction = layers.Dense(1, name='age')(x)
income_prediction = layers.Dense(num_income_groups,
activation='softmax',
name='income')(x)
gender_prediction = layers.Dense(1, activation='sigmoid', name='gender')(x)
model = Model(posts_input,
[age_prediction, income_prediction, gender_prediction])
model.compile(optimizer='rmsprop',
loss=['mse', 'categorical_crossentropy', 'binary_crossentropy'])
model.compile(optimizer='rmsprop',
loss={
'age': 'mse',
'income': 'categorical_crossentropy',
'gender': 'binary_crossentropy'
layers.Bidirectional(layers.LSTM(32, dropout=0.2, recurrent_dropout=0.5),
input_shape=[None, float_data.shape[-1]]))
model.add(layers.Dense(1000, activation='relu'))
model.add(layers.Dense(1))
model.compile(optimizer='adam', loss='mae')
history = model.fit_generator(train_gen,
steps_per_epoch=500,
epochs=40,
validation_data=val_gen,
validation_steps=val_steps)
## conv1d ##
model = models.Sequential()
model.add(layers.Embedding(max_features, output_dim=128, input_length=maxlen))
model.add(layers.Conv1D(filters=32, kernel_size=7, activation='relu'))
model.add(layers.MaxPooling1D(pool_size=5))
model.add(layers.Conv1D(32, 7, activation='relu'))
model.add(layers.GlobalAveragePooling1D())
model.add(layers.Dense(1))
model.summary()
model.compile(optimizer=optimizers.rmsprop(lr=1e-4),
loss='binary_crossentropy',
metrics=['acc'])
history = model.fit(x_train,
y_train,
epochs=10,
batch_size=128,
validation_split=0.2)
## 1dconv --> RNN ##
step = 3
def Conv1DRegressorIn1(flag):
K.clear_session()
current_neighbor = space['neighbor']
current_idx_idx = space['idx_idx']
current_dense_num = space['dense_num']
current_dilation1D_layers = space['dilation1D_layers']
current_dilation1D_filter_num = space['dilation1D_filter_num']
current_reduce1D_filter_num = space['reduce1D_filter_num']
summary = True
verbose = 0
#
# setHyperParams
#
## hypers for data
neighbor = {{choice([50, 60, 70, 80, 90, 100, 110, 120, 130, 140])}}
idx_idx = {{choice([0, 1, 2, 3, 4, 5, 6, 7, 8])}}
idx_lst = [
[x for x in range(158) if x not in [24, 26]], # 去除无用特征
[
x for x in range(158) if x not in [24, 26] +
[x for x in range(1, 6)] + [x for x in range(16, 22)] + [40, 42]
], # 去除无用特征+冗余特征
[
x for x in range(158)
if x not in [24, 26] + [x for x in range(0, 22)]
], # 去除无用特征+方位特征
[x for x in range(158)
if x not in [24, 26] + [22, 23, 26, 37, 38]], # 去除无用特征+深度特征
[
x for x in range(158) if x not in [24, 26] +
[x for x in range(27, 37)] + [x for x in range(40, 46)]
], # 去除无用特征+二级结构信息
# [x for x in range(158) if x not in [24, 26] + [x for x in range(27, 34)] + [x for x in range(40, 46)]],# 去除无用特征+二级结构信息1
# [x for x in range(158) if x not in [24, 26] + [x for x in range(34, 37)] + [x for x in range(40, 46)]],# 去除无用特征+二级结构信息2
[x for x in range(158) if x not in [24, 26] + [46, 47]], # 去除无用特征+实验条件
[
x for x in range(158) if x not in [24, 26] + [39] +
[x for x in range(57, 61)] + [x for x in range(48, 57)] +
[x for x in range(61, 81)] + [x for x in range(140, 155)]
], # 去除无用特征+所有原子编码
# [x for x in range(158) if x not in [24, 26] + [39] + [x for x in range(57, 61)] + [x for x in range(48, 57)] + [x for x in range(140, 145)]],# 去除无用特征+原子编码1
# [x for x in range(158) if x not in [24, 26] + [39] + [x for x in range(57, 61)] + [x for x in range(61, 77)] + [x for x in range(145, 153)]],# 去除无用特征+原子编码2
# [x for x in range(158) if x not in [24, 26] + [39] + [x for x in range(57, 61)] + [x for x in range(77, 81)] + [x for x in range(153, 155)]],# 去除无用特征+原子编码3
[
x for x in range(158)
if x not in [24, 26] + [x for x in range(81, 98)]
], # 去除无用特征+rosetta_energy
[
x for x in range(158) if x not in [24, 26] +
[x for x in range(98, 140)] + [x for x in range(155, 158)]
] # 去除无用特征+msa
]
idx = idx_lst[idx_idx]
## hypers for net
lr = 1e-4 # 0.0001
batch_size = 32
epochs = 200
padding_style = 'same'
activator_Conv1D = 'elu'
activator_Dense = 'tanh'
dense_num = {{choice([64, 96, 128])}} # 64
dilation1D_layers = {{choice([8, 16, 32])}} #8
dilation1D_filter_num = {{choice([16, 32])}} #16
dilation_lower = 1
dilation_upper = 16
dropout_rate_dilation = 0.25
dropout_rate_reduce = 0.25
dropout_rate_dense = 0.25
initializer_Conv1D = initializers.lecun_uniform(seed=527)
initializer_Dense = initializers.he_normal(seed=527)
kernel_size = 5
l2_rate = 0.001
loss_type = logcosh
metrics = ('mae', pearson_r, rmse)
pool_size = 2
reduce_layers = 5 # 110 -> 55 -> 28 -> 14 -> 7 -> 4, 50 - 25 - 13 - 7 - 4 - 2
reduce1D_filter_num = {{choice([16, 32, 64])}} #32
residual_stride = 2
def _data(fold_num, neighbor, idx):
train_data_pth = '/dl/sry/mCNN/dataset/deepddg/npz/wild/cross_valid/cro_fold%s_train_center_CA_PCA_False_neighbor_140.npz' % fold_num
val_data_pth = '/dl/sry/mCNN/dataset/deepddg/npz/wild/cross_valid/cro_fold%s_valid_center_CA_PCA_False_neighbor_140.npz' % fold_num
## train data
train_data = np.load(train_data_pth)
x_train = train_data['x']
y_train = train_data['y']
ddg_train = train_data['ddg'].reshape(-1)
## select kneighbor atoms
x_train_kneighbor_lst = []
for sample in x_train:
dist_arr = sample[:, 0]
indices = sorted(dist_arr.argsort()[:neighbor])
x_train_kneighbor_lst.append(sample[indices, :])
x_train = np.array(x_train_kneighbor_lst)
## idx
x_train = x_train[:, :, idx]
## val data
val_data = np.load(val_data_pth)
x_val = val_data['x']
y_val = val_data['y']
ddg_val = val_data['ddg'].reshape(-1)
## select kneighbor atoms
x_val_kneighbor_lst = []
for sample in x_val:
dist_arr = sample[:, 0]
indices = sorted(dist_arr.argsort()[:neighbor])
x_val_kneighbor_lst.append(sample[indices, :])
x_val = np.array(x_val_kneighbor_lst)
## idx
x_val = x_val[:, :, idx]
# sort row default is chain, pass
# reshape and one-hot
y_train = to_categorical(y_train)
y_val = to_categorical(y_val)
# normalization
train_shape = x_train.shape
val_shape = x_val.shape
col_train = train_shape[-1]
col_val = val_shape[-1]
x_train = x_train.reshape((-1, col_train))
x_val = x_val.reshape((-1, col_val))
mean = x_train.mean(axis=0)
std = x_train.std(axis=0)
std[np.argwhere(std == 0)] = 0.01
x_train -= mean
x_train /= std
x_val -= mean
x_val /= std
x_train = x_train.reshape(train_shape)
x_val = x_val.reshape(val_shape)
print('x_train: %s'
'\ny_train: %s'
'\nddg_train: %s'
'\nx_val: %s'
'\ny_val: %s'
'\nddg_val: %s' % (x_train.shape, y_train.shape, ddg_train.shape,
x_val.shape, y_val.shape, ddg_val.shape))
return x_train, y_train, ddg_train, x_val, y_val, ddg_val
#
# cross_valid
#
hyper_param_tag = '%s_%s_%s_%s_%s_%s' % (
current_neighbor, current_idx_idx, current_dense_num,
current_dilation1D_layers, current_dilation1D_filter_num,
current_reduce1D_filter_num)
modeldir = '/dl/sry/projects/from_hp/mCNN/src/Network/deepddg/opt_all_resnet/model/%s-%s' % (
hyper_param_tag, time.strftime("%Y.%m.%d.%H.%M.%S", time.localtime()))
os.makedirs(modeldir, exist_ok=True)
opt_lst = []
for k_count in range(1, 11):
print('\n** fold %s is processing **\n' % k_count)
filepth = '%s/fold_%s_weights-best.h5' % (modeldir, k_count)
my_callbacks = [
callbacks.ReduceLROnPlateau(
monitor='val_loss',
factor=0.33,
patience=5,
verbose=verbose,
mode='min',
min_lr=1e-8,
),
callbacks.EarlyStopping(monitor='val_loss',
patience=10,
verbose=verbose),
callbacks.ModelCheckpoint(filepath=filepth,
monitor='val_mean_absolute_error',
verbose=verbose,
save_best_only=True,
mode='min',
save_weights_only=True)
]
x_train, y_train, ddg_train, x_val, y_val, ddg_val = _data(
k_count, neighbor, idx)
row_num, col_num = x_train.shape[1:3]
#
# build net
#
input_layer = Input(shape=(row_num, col_num), name='input_layer')
y = layers.Conv1D(filters=dilation1D_filter_num,
kernel_size=1,
padding=padding_style,
activation=activator_Conv1D,
kernel_initializer=initializer_Conv1D,
kernel_regularizer=regularizers.l2(l2_rate),
name='first_conv_layer',
trainable=True)(input_layer)
res = layers.BatchNormalization(axis=-1, name='first_BN_layer')(y)
## loop with Conv1D with dilation (padding='same')
for _ in range(dilation1D_layers):
y = layers.SeparableConv1D(
filters=dilation1D_filter_num,
kernel_size=kernel_size,
padding=padding_style,
dilation_rate=dilation_lower,
activation=activator_Conv1D,
depthwise_initializer=initializer_Conv1D,
pointwise_initializer=initializer_Conv1D,
depthwise_regularizer=regularizers.l2(l2_rate),
pointwise_regularizer=regularizers.l2(l2_rate))(res)
y = layers.BatchNormalization(axis=-1)(y)
y = layers.Dropout(dropout_rate_dilation)(y)
y = layers.SeparableConv1D(
filters=dilation1D_filter_num,
kernel_size=kernel_size,
padding=padding_style,
dilation_rate=dilation_lower,
activation=activator_Conv1D,
depthwise_initializer=initializer_Conv1D,
pointwise_initializer=initializer_Conv1D,
depthwise_regularizer=regularizers.l2(l2_rate),
pointwise_regularizer=regularizers.l2(l2_rate))(y)
y = layers.BatchNormalization(axis=-1)(y)
res = layers.add([y, res])
dilation_lower *= 2
if dilation_lower > dilation_upper:
dilation_lower = 1
## Conv1D with dilation (padding='valaid') and residual block to reduce dimention.
for _ in range(reduce_layers):
y = layers.SeparableConv1D(
filters=reduce1D_filter_num,
kernel_size=kernel_size,
padding=padding_style,
activation=activator_Conv1D,
depthwise_initializer=initializer_Conv1D,
pointwise_initializer=initializer_Conv1D,
depthwise_regularizer=regularizers.l2(l2_rate),
pointwise_regularizer=regularizers.l2(l2_rate))(res)
y = layers.BatchNormalization(axis=-1)(y)
y = layers.Dropout(dropout_rate_reduce)(y)
y = layers.MaxPooling1D(pool_size, padding=padding_style)(y)
res = layers.SeparableConv1D(
filters=reduce1D_filter_num,
kernel_size=kernel_size,
strides=residual_stride,
padding=padding_style,
activation=activator_Conv1D,
depthwise_initializer=initializer_Conv1D,
pointwise_initializer=initializer_Conv1D,
depthwise_regularizer=regularizers.l2(l2_rate),
pointwise_regularizer=regularizers.l2(l2_rate))(res)
res = layers.add([y, res])
## flat & dense
y = layers.Flatten()(y)
y = layers.Dense(dense_num,
activation=activator_Dense,
kernel_initializer=initializer_Dense)(y)
y = layers.BatchNormalization(axis=-1)(y)
y = layers.Dropout(dropout_rate_dense)(y)
output_layer = layers.Dense(1, activation='linear')(y)
model = models.Model(inputs=input_layer, outputs=output_layer)
# print(model.get_layer(index=2).trainable)
# print(model.get_layer(name='first_conv').trainable)
if summary:
trainable_count = int(
np.sum(
[K.count_params(p) for p in set(model.trainable_weights)]))
non_trainable_count = int(
np.sum([
K.count_params(p) for p in set(model.non_trainable_weights)
]))
print('Total params: {:,}'.format(trainable_count +
non_trainable_count))
print('Trainable params: {:,}'.format(trainable_count))
print('Non-trainable params: {:,}'.format(non_trainable_count))
# model.summary()
adam = optimizers.Adam(lr=lr)
model.compile(
optimizer=adam, # 'rmsprop', # SGD,adam,rmsprop
loss=loss_type,
metrics=list(metrics)) # mae平均绝对误差（mean absolute error） accuracy
result = model.fit(
x=x_train,
y=ddg_train,
batch_size=batch_size,
epochs=epochs,
verbose=verbose,
callbacks=my_callbacks,
validation_data=(x_val, ddg_val),
shuffle=True,
)
# print('\n----------History:\n%s'%result.history)
#
# save
#
save_train_cv(model, modeldir, result.history, k_count)
opt_lst.append(np.mean(
result.history['val_mean_absolute_error'][-10:]))
opt_loss = np.mean(opt_lst)
#
# print hyper combination group and current loss value
#
print('\n@current_hyper_tag: %s'
'\n@current optmized_loss: %s' % (hyper_param_tag, opt_loss))
# return {'loss': validation_loss, 'status': STATUS_OK, 'model':model}
return {'loss': opt_loss, 'status': STATUS_OK}
def Conv1DRegressorIn1(flag):
K.clear_session()
current_neighbor = space['neighbor']
current_idx_idx = space['idx_idx']
current_batch_size = space['batch_size']
current_conv1D_filter_num1 = space['conv1D_filter_num1']
current_conv1D_filter_num2 = space['conv1D_filter_num2']
current_conv1D_filter_num3 = space['conv1D_filter_num3']
current_dropout_rate_dense = space['dropout_rate_dense']
summary = True
verbose = 0
#
# setHyperParams
#
## hypers for data
neighbor = {{choice([50, 60, 70, 80, 90, 100, 110, 120, 130, 140])}}
idx_idx = {{choice([0, 1, 2, 3, 4, 5, 6, 7, 8])}}
idx_lst = [
[x for x in range(158) if x not in [24, 26]], # 去除无用特征
[
x for x in range(158) if x not in [24, 26] +
[x for x in range(1, 6)] + [x for x in range(16, 22)] + [40, 42]
], # 去除无用特征+冗余特征
[
x for x in range(158)
if x not in [24, 26] + [x for x in range(0, 22)]
], # 去除无用特征+方位特征
[x for x in range(158)
if x not in [24, 26] + [22, 23, 26, 37, 38]], # 去除无用特征+深度特征
[
x for x in range(158) if x not in [24, 26] +
[x for x in range(27, 37)] + [x for x in range(40, 46)]
], # 去除无用特征+二级结构信息
# [x for x in range(158) if x not in [24, 26] + [x for x in range(27, 34)] + [x for x in range(40, 46)]],# 去除无用特征+二级结构信息1
# [x for x in range(158) if x not in [24, 26] + [x for x in range(34, 37)] + [x for x in range(40, 46)]],# 去除无用特征+二级结构信息2
[x for x in range(158) if x not in [24, 26] + [46, 47]], # 去除无用特征+实验条件
[
x for x in range(158) if x not in [24, 26] + [39] +
[x for x in range(57, 61)] + [x for x in range(48, 57)] +
[x for x in range(61, 81)] + [x for x in range(140, 155)]
], # 去除无用特征+所有原子编码
# [x for x in range(158) if x not in [24, 26] + [39] + [x for x in range(57, 61)] + [x for x in range(48, 57)] + [x for x in range(140, 145)]],# 去除无用特征+原子编码1
# [x for x in range(158) if x not in [24, 26] + [39] + [x for x in range(57, 61)] + [x for x in range(61, 77)] + [x for x in range(145, 153)]],# 去除无用特征+原子编码2
# [x for x in range(158) if x not in [24, 26] + [39] + [x for x in range(57, 61)] + [x for x in range(77, 81)] + [x for x in range(153, 155)]],# 去除无用特征+原子编码3
[
x for x in range(158)
if x not in [24, 26] + [x for x in range(81, 98)]
], # 去除无用特征+rosetta_energy
[
x for x in range(158) if x not in [24, 26] +
[x for x in range(98, 140)] + [x for x in range(155, 158)]
] # 去除无用特征+msa
]
idx = idx_lst[idx_idx]
## hypers for net
lr = 1e-4 # 0.0001
batch_size = {{choice([1, 32, 64, 128])}}
epochs = 200
conv1D_filter_num1 = {{choice([16, 32])}}
conv1D_filter_num2 = {{choice([16, 32, 64])}}
conv1D_filter_num3 = {{choice([32, 64])}}
dropout_rate_dense = {{choice([0.1, 0.2, 0.3, 0.4, 0.5])}}
metrics = ('mae', pearson_r, rmse)
def _data(fold_num, neighbor, idx):
train_data_pth = '/dl/sry/mCNN/dataset/deepddg/npz/wild/cross_valid/cro_fold%s_train_center_CA_PCA_False_neighbor_140.npz' % fold_num
val_data_pth = '/dl/sry/mCNN/dataset/deepddg/npz/wild/cross_valid/cro_fold%s_valid_center_CA_PCA_False_neighbor_140.npz' % fold_num
## train data
train_data = np.load(train_data_pth)
x_train = train_data['x']
y_train = train_data['y']
ddg_train = train_data['ddg'].reshape(-1)
## select kneighbor atoms
x_train_kneighbor_lst = []
for sample in x_train:
dist_arr = sample[:, 0]
indices = sorted(dist_arr.argsort()[:neighbor])
x_train_kneighbor_lst.append(sample[indices, :])
x_train = np.array(x_train_kneighbor_lst)
## idx
x_train = x_train[:, :, idx]
## val data
val_data = np.load(val_data_pth)
x_val = val_data['x']
y_val = val_data['y']
ddg_val = val_data['ddg'].reshape(-1)
## select kneighbor atoms
x_val_kneighbor_lst = []
for sample in x_val:
dist_arr = sample[:, 0]
indices = sorted(dist_arr.argsort()[:neighbor])
x_val_kneighbor_lst.append(sample[indices, :])
x_val = np.array(x_val_kneighbor_lst)
## idx
x_val = x_val[:, :, idx]
# sort row default is chain, pass
# reshape and one-hot
y_train = to_categorical(y_train)
y_val = to_categorical(y_val)
# normalization
train_shape = x_train.shape
val_shape = x_val.shape
col_train = train_shape[-1]
col_val = val_shape[-1]
x_train = x_train.reshape((-1, col_train))
x_val = x_val.reshape((-1, col_val))
mean = x_train.mean(axis=0)
std = x_train.std(axis=0)
std[np.argwhere(std == 0)] = 0.01
x_train -= mean
x_train /= std
x_val -= mean
x_val /= std
x_train = x_train.reshape(train_shape)
x_val = x_val.reshape(val_shape)
print('x_train: %s'
'\ny_train: %s'
'\nddg_train: %s'
'\nx_val: %s'
'\ny_val: %s'
'\nddg_val: %s' % (x_train.shape, y_train.shape, ddg_train.shape,
x_val.shape, y_val.shape, ddg_val.shape))
return x_train, y_train, ddg_train, x_val, y_val, ddg_val
#
# cross_valid
#
hyper_param_tag = '%s_%s_%s_%s_%s_%s_%s' % (
current_neighbor, current_idx_idx, current_batch_size,
current_conv1D_filter_num1, current_conv1D_filter_num2,
current_conv1D_filter_num3, current_dropout_rate_dense)
modeldir = '/dl/sry/projects/from_hp/mCNN/src/Network/deepddg/opt_all_simpleNet_v4/model/%s-%s' % (
hyper_param_tag, time.strftime("%Y.%m.%d.%H.%M.%S", time.localtime()))
os.makedirs(modeldir, exist_ok=True)
opt_lst = []
for k_count in range(1, 11):
print('\n** fold %s is processing **\n' % k_count)
filepth = '%s/fold_%s_weights-best.h5' % (modeldir, k_count)
my_callbacks = [
callbacks.ReduceLROnPlateau(
monitor='val_loss',
factor=0.33,
patience=5,
verbose=verbose,
mode='min',
min_lr=1e-8,
),
callbacks.EarlyStopping(monitor='val_loss',
patience=10,
verbose=verbose),
callbacks.ModelCheckpoint(filepath=filepth,
monitor='val_mean_absolute_error',
verbose=verbose,
save_best_only=True,
mode='min',
save_weights_only=True)
]
x_train, y_train, ddg_train, x_val, y_val, ddg_val = _data(
k_count, neighbor, idx)
row_num, col_num = x_train.shape[1:3]
#
# build net
#
network = models.Sequential()
network.add(
layers.SeparableConv1D(filters=conv1D_filter_num1,
kernel_size=5,
activation='relu',
input_shape=(row_num, col_num)))
network.add(layers.MaxPooling1D(pool_size=2))
network.add(
layers.SeparableConv1D(filters=conv1D_filter_num2,
kernel_size=5,
activation='relu'))
network.add(layers.MaxPooling1D(pool_size=2))
network.add(
layers.SeparableConv1D(filters=conv1D_filter_num3,
kernel_size=3,
activation='relu'))
network.add(layers.MaxPooling1D(pool_size=2))
network.add(layers.Flatten())
network.add(layers.Dense(128, activation='relu'))
network.add(layers.Dropout(dropout_rate_dense))
network.add(layers.Dense(16, activation='relu'))
network.add(layers.Dropout(0.3))
network.add(layers.Dense(1))
if summary:
trainable_count = int(
np.sum([
K.count_params(p) for p in set(network.trainable_weights)
]))
non_trainable_count = int(
np.sum([
K.count_params(p)
for p in set(network.non_trainable_weights)
]))
print('Total params: {:,}'.format(trainable_count +
non_trainable_count))
print('Trainable params: {:,}'.format(trainable_count))
print('Non-trainable params: {:,}'.format(non_trainable_count))
# print(network.summary())
# rmsp = optimizers.RMSprop(lr=0.0001, decay=0.1)
rmsp = optimizers.RMSprop(lr=lr)
network.compile(
optimizer=rmsp, # 'rmsprop', # SGD,adam,rmsprop
loss='mae',
metrics=list(metrics)) # mae平均绝对误差（mean absolute error） accuracy
result = network.fit(
x=x_train,
y=ddg_train,
batch_size=batch_size,
epochs=epochs,
verbose=verbose,
callbacks=my_callbacks,
validation_data=(x_val, ddg_val),
shuffle=True,
)
# print('\n----------History:\n%s'%result.history)
#
# save
#
save_train_cv(network, modeldir, result.history, k_count)
opt_lst.append(np.mean(
result.history['val_mean_absolute_error'][-10:]))
opt_loss = np.mean(opt_lst)
#
# print hyper combination group and current loss value
#
print('\n@current_hyper_tag: %s'
'\n@current optmized_loss: %s' % (hyper_param_tag, opt_loss))
# return {'loss': validation_loss, 'status': STATUS_OK, 'model':model}
return {'loss': opt_loss, 'status': STATUS_OK}
def au_Exp():
now = datetime.datetime.now()
now_s = now.strftime("%Y-%m-%d-%H-%M-%S")
config = tf.compat.v1.ConfigProto(gpu_options=tf.compat.v1.GPUOptions(allow_growth=True))
sess = tf.compat.v1.Session(config=config)
## 准备几个参数，用于后续的自动化
epochs_au = 50
batch_size_au = 1
jihuo = 'tanh'
callback_list_test =[
keras.callbacks.ModelCheckpoint(
filepath= now_s+'.h5', ##文件路径 存在当前路径下吧 还好找
monitor= 'val_loss', ## 监控指标
save_best_only= True ## 保持最佳模型
)
]
moisture,d5,dp5,dp6 = sF.getOdata()
test_data,test_lable,train_data,train_lable = sF.getTestData(moisture,d5,dp5,dp6 )
model = models.Sequential()
model.add(layers.Conv1D(64,7,activation=jihuo,input_shape=(700,1)))
model.add(layers.MaxPooling1D(2))
model.add(layers.Conv1D(32,7,activation=jihuo))
model.add(layers.MaxPooling1D(2))
model.add(layers.Conv1D(32,7,activation=jihuo))
model.add(layers.MaxPooling1D(2))
model.add(layers.Conv1D(16,7,activation=jihuo))
# model.add(layers.GlobalMaxPooling1D()) ## 实际效果极差1
model.add(layers.Flatten())
model.add(layers.Dense(16))
model.add(layers.Dense(8))
# model.add(layers.Dense(4))
# model.add(layers.Dense(2))
model.add(layers.Dense(1))
model.summary()
model.compile(optimizer=RMSprop(),loss='mse')
history = model.fit(train_data,train_lable,
epochs=epochs_au,
batch_size=batch_size_au,
validation_data=(test_data,test_lable),
callbacks= callback_list_test
)
sF.drawLoss(history) ## 绘制当前的验证曲线
model = load_model(now_s+'.h5')
result_trian = model.predict(train_data)
result_predict = model.predict(test_data)
rmsec = sF.calculate_RMSE(result_trian,train_lable) ## 训练集上的RMSE
rmsep = sF.calculate_RMSE(result_predict,test_lable) ## 测试集上的RMSE
r_2_t = sF.calculate_R21(result_trian,train_lable)## 训练集上的R_2
r_2_p = sF.calculate_R21(result_predict,test_lable)## 测试集上得R_2
# print("Root Mean Square Error of Calibrationh is : %g"%(rmsec))
# print("训练集上得决定系数：%f"%(r_2_t))
# print("Root Mean Square Error of Prediction is : %g"%(rmsep))
# print("测试集上得决定系数：%f"%(r_2_p))
###### 下面的代码用于自动记录实验数据
write_data=[(now_s,epochs_au,batch_size_au,rmsec,r_2_t,rmsep,r_2_p)]#需要新写入的数据
sF.write_To_Csv(write_data)
return rmsep
def build_model():
model_weights = np.load('sound8.npy', encoding='latin1').item()
print(type(model_weights))
model = models.Sequential()
model.add(layers.InputLayer(batch_input_shape=(1, None, 1)))
filter_parameters = [
{
'name': 'conv1',
'num_filters': 16,
'padding': 32,
'kernel_size': 64,
'conv_strides': 2,
'pool_size': 8,
'pool_strides': 8
},
{
'name': 'conv2',
'num_filters': 32,
'padding': 16,
'kernel_size': 32,
'conv_strides': 2,
'pool_size': 8,
'pool_strides': 8
},
{
'name': 'conv3',
'num_filters': 64,
'padding': 8,
'kernel_size': 16,
'conv_strides': 2
},
{
'name': 'conv4',
'num_filters': 128,
'padding': 4,
'kernel_size': 8,
'conv_strides': 2
},
{
'name': 'conv5',
'num_filters': 256,
'padding': 2,
'kernel_size': 4,
'conv_strides': 2,
'pool_size': 4,
'pool_strides': 4
},
{
'name': 'conv6',
'num_filters': 512,
'padding': 2,
'kernel_size': 4,
'conv_strides': 2
},
{
'name': 'conv7',
'num_filters': 1024,
'padding': 2,
'kernel_size': 4,
'conv_strides': 2
},
{
'name': 'conv8_2',
'num_filters': 401,
'padding': 1,
'kernel_size': 8,
'conv_strides': 2
},
]
for x in filter_parameters:
model.add(layers.ZeroPadding1D(padding=x['padding']))
model.add(
layers.Conv1D(x['num_filters'],
kernel_size=x['kernel_size'],
strides=x['conv_strides'],
padding='valid'))
weights = model_weights[x['name']]['weights'].reshape(
model.layers[-1].get_weights()[0].shape)
biases = model_weights[x['name']]['biases']
model.layers[-1].set_weights([weights, biases])
if 'conv8' not in x['name']:
gamma = model_weights[x['name']]['gamma']
beta = model_weights[x['name']]['beta']
mean = model_weights[x['name']]['mean']
var = model_weights[x['name']]['var']
model.add(layers.BatchNormalization())
model.layers[-1].set_weights([gamma, beta, mean, var])
model.add(layers.Activation('relu'))
if 'pool_size' in x:
model.add(
layers.MaxPooling1D(pool_size=x['pool_size'],
strides=x['pool_strides'],
padding='valid'))
return model
def build_birnn_feature_coattention_cnn_model(voca_dim,
time_steps,
num_features,
feature_dim,
output_dim,
model_dim,
mlp_dim,
num_filters,
filter_sizes,
item_embedding=None,
rnn_depth=1,
mlp_depth=1,
drop_out=0.5,
rnn_drop_out=0.,
rnn_state_drop_out=0.,
cnn_drop_out=0.5,
pooling='max',
trainable_embedding=False,
gpu=False,
return_customized_layers=False):
"""
Create A Bidirectional Attention Model.
:param voca_dim: vocabulary dimension size.
:param time_steps: the length of input
:param output_dim: the output dimension size
:param model_dim: rrn dimension size
:param mlp_dim: the dimension size of fully connected layer
:param item_embedding: integer, numpy 2D array, or None (default=None)
If item_embedding is a integer, connect a randomly initialized embedding matrix to the input tensor.
If item_embedding is a matrix, this matrix will be used as the embedding matrix.
If item_embedding is None, then connect input tensor to RNN layer directly.
:param rnn_depth: rnn depth
:param mlp_depth: the depth of fully connected layers
:param num_att_channel: the number of attention channels, this can be used to mimic multi-head attention mechanism
:param drop_out: dropout rate of fully connected layers
:param rnn_drop_out: dropout rate of rnn layers
:param rnn_state_drop_out: dropout rate of rnn state tensor
:param trainable_embedding: boolean
:param gpu: boolean, default=False
If True, CuDNNLSTM is used instead of LSTM for RNN layer.
:param return_customized_layers: boolean, default=False
If True, return model and customized object dictionary, otherwise return model only
:return: keras model
"""
if model_dim % 2 == 1:
model_dim += 1
if item_embedding is not None:
inputs = models.Input(shape=(time_steps, ),
dtype='int32',
name='input0')
x1 = inputs
# item embedding
if isinstance(item_embedding, np.ndarray):
assert voca_dim == item_embedding.shape[0]
x1 = layers.Embedding(voca_dim,
item_embedding.shape[1],
input_length=time_steps,
weights=[
item_embedding,
],
trainable=trainable_embedding,
mask_zero=False,
name='embedding_layer0')(x1)
elif utils.is_integer(item_embedding):
x1 = layers.Embedding(voca_dim,
item_embedding,
input_length=time_steps,
trainable=trainable_embedding,
mask_zero=False,
name='embedding_layer0')(x1)
else:
raise ValueError(
"item_embedding must be either integer or numpy matrix")
else:
inputs = models.Input(shape=(time_steps, voca_dim),
dtype='float32',
name='input0')
x1 = inputs
inputs1 = models.Input(shape=(num_features, feature_dim),
dtype='float32',
name='input1')
x2 = layers.Dense(feature_dim, name="feature_map_layer",
activation="relu")(inputs1)
if gpu:
# rnn encoding
for i in range(rnn_depth):
x1 = layers.Bidirectional(layers.CuDNNLSTM(int(model_dim / 2),
return_sequences=True),
name='bi_lstm_layer' + str(i))(x1)
x1 = layers.BatchNormalization(name='rnn_batch_norm_layer' +
str(i))(x1)
x1 = layers.Dropout(rnn_drop_out,
name="rnn_dropout_layer" + str(i))(x1)
else:
# rnn encoding
for i in range(rnn_depth):
x1 = layers.Bidirectional(layers.LSTM(
int(model_dim / 2),
return_sequences=True,
dropout=rnn_drop_out,
recurrent_dropout=rnn_state_drop_out),
name='bi_lstm_layer' + str(i))(x1)
x1 = layers.BatchNormalization(name='rnn_batch_norm_layer' +
str(i))(x1)
# attention
attens = clayers.CoAttentionWeight(name="coattention_weights_layer")(
[x1, x2])
attens1 = clayers.FeatureNormalization(
name="normalized_coattention_weights_layer1", axis=1)(attens)
attens2 = clayers.FeatureNormalization(
name="normalized_coattention_weights_layer2", axis=2)(attens)
# compare
focus1 = layers.Dot((1, 1), name="focus_layer1")([attens1, x1])
focus2 = layers.Dot((2, 1), name="focus_layer2")([attens2, x2])
pair1 = layers.Concatenate(axis=-1, name="pair_layer1")([x1, focus2])
pair2 = layers.Concatenate(axis=-1, name="pair_layer2")([x2, focus1])
x1 = layers.TimeDistributed(layers.Dense(model_dim, activation="relu"),
name="compare_layer1")(pair1)
x2 = layers.TimeDistributed(layers.Dense(model_dim, activation="relu"),
name="compare_layer2")(pair2)
# Multi-Channel CNN for x1
pooled_outputs = []
for i in range(len(filter_sizes)):
conv = layers.Conv1D(num_filters,
kernel_size=filter_sizes[i],
padding='valid',
activation='relu')(x1)
if pooling == 'max':
conv = layers.MaxPooling1D(pool_size=time_steps - filter_sizes[i] +
1,
strides=1,
padding='valid')(conv)
else:
conv = layers.AveragePooling1D(pool_size=time_steps -
filter_sizes[i] + 1,
strides=1,
padding='valid')(conv)
pooled_outputs.append(conv)
x1 = layers.Concatenate(name='concated_layer')(pooled_outputs)
x1 = layers.Flatten()(x1)
x1 = layers.Dropout(cnn_drop_out, name='conv_dropout_layer')(x1)
x1 = layers.BatchNormalization(name="batch_norm_layer")(x1)
# Average Pool for x2
x2 = layers.GlobalAveragePooling1D(name="average_pool_layer")(x2)
x = layers.Concatenate(axis=1, name="concat_deep_feature_layer")([x1, x2])
# MLP Layers
for i in range(mlp_depth - 1):
x = layers.Dense(mlp_dim,
activation='selu',
kernel_initializer='lecun_normal',
name='selu_layer' + str(i))(x)
x = layers.AlphaDropout(drop_out, name='alpha_layer' + str(i))(x)
outputs = layers.Dense(output_dim,
activation="softmax",
name="softmax_layer0")(x)
model = models.Model(inputs, outputs)
if return_customized_layers:
return model, {
'CoAttentionWeight': clayers.CoAttentionWeight,
"FeatureNormalization": clayers.FeatureNormalization
}
return model
def build_birnn_cnn_model(voca_dim,
time_steps,
output_dim,
rnn_dim,
mlp_dim,
num_filters,
filter_sizes,
item_embedding=None,
rnn_depth=1,
mlp_depth=1,
drop_out=0.5,
rnn_drop_out=0.5,
rnn_state_drop_out=0.5,
cnn_drop_out=0.5,
pooling='max',
trainable_embedding=False,
gpu=False,
return_customized_layers=False):
"""
Create A Bidirectional CNN Model.
:param voca_dim: vocabulary dimension size.
:param time_steps: the length of input
:param output_dim: the output dimension size
:param rnn_dim: rrn dimension size
:param num_filters: the number of filters
:param filter_sizes: list of integers
The kernel size.
:param mlp_dim: the dimension size of fully connected layer
:param item_embedding: integer, numpy 2D array, or None (default=None)
If item_embedding is a integer, connect a randomly initialized embedding matrix to the input tensor.
If item_embedding is a matrix, this matrix will be used as the embedding matrix.
If item_embedding is None, then connect input tensor to RNN layer directly.
:param rnn_depth: rnn depth
:param mlp_depth: the depth of fully connected layers
:param num_att_channel: the number of attention channels, this can be used to mimic multi-head attention mechanism
:param drop_out: dropout rate of fully connected layers
:param rnn_drop_out: dropout rate of rnn layers
:param rnn_state_drop_out: dropout rate of rnn state tensor
:param cnn_drop_out: dropout rate of between cnn layer and fully connected layers
:param pooling: str, either 'max' or 'average'
Pooling method.
:param trainable_embedding: boolean
:param gpu: boolean, default=False
If True, CuDNNLSTM is used instead of LSTM for RNN layer.
:param return_customized_layers: boolean, default=False
If True, return model and customized object dictionary, otherwise return model only
:return: keras model
"""
if item_embedding is not None:
inputs = models.Input(shape=(time_steps, ),
dtype='int32',
name='input0')
x = inputs
# item embedding
if isinstance(item_embedding, np.ndarray):
assert voca_dim == item_embedding.shape[0]
x = layers.Embedding(voca_dim,
item_embedding.shape[1],
input_length=time_steps,
weights=[
item_embedding,
],
trainable=trainable_embedding,
mask_zero=False,
name='embedding_layer0')(x)
elif utils.is_integer(item_embedding):
x = layers.Embedding(voca_dim,
item_embedding,
input_length=time_steps,
trainable=trainable_embedding,
mask_zero=False,
name='embedding_layer0')(x)
else:
raise ValueError(
"item_embedding must be either integer or numpy matrix")
else:
inputs = models.Input(shape=(time_steps, voca_dim),
dtype='float32',
name='input0')
x = inputs
if gpu:
# rnn encoding
for i in range(rnn_depth):
x = layers.Bidirectional(layers.CuDNNLSTM(rnn_dim,
return_sequences=True),
name='bi_lstm_layer' + str(i))(x)
x = layers.BatchNormalization(name='rnn_batch_norm_layer' +
str(i))(x)
x = layers.Dropout(rnn_drop_out,
name="rnn_dropout_layer" + str(i))(x)
else:
# rnn encoding
for i in range(rnn_depth):
x = layers.Bidirectional(layers.LSTM(
rnn_dim,
return_sequences=True,
dropout=rnn_drop_out,
recurrent_dropout=rnn_state_drop_out),
name='bi_lstm_layer' + str(i))(x)
x = layers.BatchNormalization(name='rnn_batch_norm_layer' +
str(i))(x)
pooled_outputs = []
for i in range(len(filter_sizes)):
conv = layers.Conv1D(num_filters,
kernel_size=filter_sizes[i],
padding='valid',
activation='relu')(x)
if pooling == 'max':
conv = layers.MaxPooling1D(pool_size=time_steps - filter_sizes[i] +
1,
strides=1,
padding='valid')(conv)
else:
conv = layers.AveragePooling1D(pool_size=time_steps -
filter_sizes[i] + 1,
strides=1,
padding='valid')(conv)
pooled_outputs.append(conv)
x = layers.Concatenate(name='concated_layer')(pooled_outputs)
x = layers.Flatten()(x)
x = layers.Dropout(cnn_drop_out, name='conv_dropout_layer')(x)
x = layers.BatchNormalization(name="batch_norm_layer")(x)
# MLP Layers
for i in range(mlp_depth - 1):
x = layers.Dense(mlp_dim,
activation='selu',
kernel_initializer='lecun_normal',
name='selu_layer' + str(i))(x)
x = layers.AlphaDropout(drop_out, name='alpha_layer' + str(i))(x)
outputs = layers.Dense(output_dim,
activation="softmax",
name="softmax_layer0")(x)
model = models.Model(inputs, outputs)
if return_customized_layers:
return model, dict()
return model
K.set_session(sess)
###################################################
from keras.models import Model, load_model
from keras import layers as kl
from keras import optimizers
def average_pred(y_true, y_pred):
return K.mean(y_pred)
li = kl.Input(shape=(TIMESTEP_SIZE, 1))
l = li
for n_units in [16, 32, 64, 128]:
l = kl.Conv1D(n_units, 3, activation='elu', padding='same')(l)
l = kl.MaxPooling1D()(l)
l = kl.Flatten()(l)
l = kl.Dense(64, activation='elu')(l)
l = kl.Dense(1, activation='sigmoid')(l)
model = Model(li, l)
model.compile(optimizers.Adamax(lr=0.002), "binary_crossentropy",
['acc', average_pred])
if not args.test:
from keras.callbacks import TensorBoard
import os
iteration = 0
res = []
print_msg = "iteration: {} : loss: {:.6f}, acc: {:.4%}, avg_pred: {:.4f}, avg_y: {:.4f}, left_iter_to_test: {}"
def main():
##Variables
ftrVct0 = [] #feature vector 0
ftrVct1 = [] #feature vector 1
rawData1 = [] #raw Data
rawData2 = [] #raw Data
trnData = [] #data used to train
tstData = [] #data to test
lbls = []
accVct = []
val_accVct = []
lossVct = []
val_lossVct = []
epochsVct = []
##Read firts data
rawData0 = readFile(filename0)
rawData1 = readFile(filename1)
rawData2 = readFile(filename2)
rawData = np.concatenate((rawData0[:,0:850],rawData1[:,0:850],rawData2[:,0:850]),axis = 0)
normData = normalizeData(*rawData)
## normData1 = normalizeData(*rawData1[:,0:850])
## normData2 = normalizeData(*rawData2[:,0:850])
print('Shape normData: ', np.asarray(normData).shape)
## print('Shape normData1: ', np.asarray(normData1).shape)
## print('Shape normData2: ', np.asarray(normData2).shape)
lbls = np.concatenate((np.zeros((len(rawData0[:,0:850]),1)),np.ones((len(rawData1[:,0:850]),1)), 2*np.ones((len(rawData2[:,0:850]),1))),axis=0)
print('Shape lbls: ', lbls.shape)
## print('lbls: ', lbls)
## trnData = np.concatenate((normData0,normData1,normData2),axis = 0)
## print('Shape trnData: ', trnData.shape)
x_train, x_test, y_train, y_test = train_test_split(normData, lbls, test_size=0.2, random_state=0)
x_train = np.expand_dims(x_train,2)
x_val = np.expand_dims(x_test,2)
print('Shape X_train: ', np.asarray(x_train).shape)
print('Shape Y_train: ', np.asarray(y_train).shape)
print('Shape X_val: ', np.asarray(x_val).shape)
print('Shape Y_val: ', np.asarray(y_test).shape)
## print('Shape train_gen: ', train_gen)
steps = range(350, 350+np.asarray(normData).shape[-1])
print('Time steps: ', np.asarray(steps).shape)
print('Time steps: ', steps[0])
#plt.plot(steps,np.asarray(x_train[0]), color="g")
#plt.show()
#ANN building
model = Sequential()
model.add(layers.Conv1D(20, 170, activation='relu',input_shape=(850,1)))#16 #T 16,85
model.add(layers.MaxPooling1D(2))
#model.add(layers.Conv1D(8,15, activation = 'relu'))#16 #T16,10
## model.add(layers.MaxPooling1D(2))
## model.add(layers.Conv1D(32,16, activation = 'relu'))
## model.add(layers.MaxPooling1D(2))
## model.add(layers.Conv1D(32,16, activation = 'relu'))
## model.add(layers.MaxPooling1D(2))
## model.add(layers.Conv1D(32,16, activation = 'relu'))
#### model.add(layers.GlobalMaxPooling1D())
## model.add(layers.Bidirectional(layers.GRU(32, dropout=0.1, recurrent_dropout=0.3, return_sequences=True)))
## model.add(layers.Bidirectional(layers.GRU(32, dropout=0.1, recurrent_dropout=0.3, return_sequences=True)))
## model.add(layers.Bidirectional(layers.GRU(64, dropout=0.1, recurrent_dropout=0.3, return_sequences=True)))
model.add(layers.Bidirectional(layers.GRU(32, dropout=0.1, recurrent_dropout=0.3)))#16
## model.add(layers.GRU(8, activation='relu', dropout=0.1, recurrent_dropout=0.3))
## model.add(layers.Dense(32, activation = 'relu'))
#model.add(layers.Flatten())
## model.add(layers.Dense(32, activation = 'relu'))
## model.add(layers.Dropout(0.1))
model.add(layers.Dense(3, activation = 'sigmoid'))
model.summary()
model.compile(optimizer=Adagrad(lr=0.003, epsilon=None, decay=0.0), loss='sparse_categorical_crossentropy', metrics=['acc'])
## model.compile(optimizer=RMSprop(), loss='mae')
#history = model.fit(x_train,y_train, epochs=2000, validation_data = (x_val, y_test))
history = model.fit(x_train,y_train, epochs=2000, batch_size=10000000, validation_data = (x_val, y_test))
## history = model.fit_generator(train_gen, steps_per_epoch = 10,epochs=100,validation_data=val_gen, validation_steps = 100)
acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss) + 1)
## accVct.append(acc)
## val_accVct.append(val_acc)
## lossVct.append(loss)
## val_lossVct.append(val_loss)
## epochsVct = []
df = pandas.DataFrame(data={"epochs": epochs, "loss": loss, "val_loss": val_loss,"acc": acc,"val_acc": val_acc})
df.to_csv("./Results_2.csv", sep=',',index=False)
#model.save_weights('my_model_weights.h5')
#model.save('my_model.h5')
plt.plot(epochs, acc, 'r', label='Training acc')
plt.plot(epochs, val_acc, 'b', label='Validation acc')
plt.title('Training and validation accuracy')
plt.legend()
plt.figure()
plt.plot(epochs, loss, 'r', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.legend()
plt.show()
def cnn_2x_lstm_siamese(voc_size, max_len, dropout=0.2):
"""Two siamese branches, each embedding a statement.
Binary classifier on top.
Args:
voc_size: size of the vocabulary for the input statements.
max_len: maximum length for the input statements.
dropout: Fraction of units to drop.
Returns:
A Keras model instance.
"""
pivot_input = layers.Input(shape=(max_len, ), dtype='int32')
statement_input = layers.Input(shape=(max_len, ), dtype='int32')
x = layers.Embedding(output_dim=256,
input_dim=voc_size,
input_length=max_len)(pivot_input)
x = layers.Conv1D(32,
7,
activation='relu',
kernel_regularizer=regularizers.l2(0.01))(x)
x = layers.ZeroPadding1D(padding=1)(x)
x = layers.Conv1D(32,
7,
activation='relu',
kernel_regularizer=regularizers.l2(0.01))(x)
x = layers.MaxPooling1D(pool_size=2, strides=2)(x)
x = layers.Conv1D(64,
5,
activation='relu',
kernel_regularizer=regularizers.l2(0.01))(x)
x = layers.ZeroPadding1D(padding=1)(x)
x = layers.Conv1D(64,
5,
activation='relu',
kernel_regularizer=regularizers.l2(0.01))(x)
x = layers.MaxPooling1D(pool_size=2, strides=2)(x)
x = layers.Conv1D(128,
3,
activation='relu',
kernel_regularizer=regularizers.l2(0.01))(x)
x = layers.ZeroPadding1D(padding=1)(x)
x = layers.Conv1D(128,
3,
activation='relu',
kernel_regularizer=regularizers.l2(0.01))(x)
x = layers.MaxPooling1D(pool_size=2, strides=2)(x)
x = layers.Conv1D(256,
3,
activation='relu',
kernel_regularizer=regularizers.l2(0.01))(x)
x = layers.ZeroPadding1D(padding=1)(x)
x = layers.Conv1D(256,
3,
activation='relu',
kernel_regularizer=regularizers.l2(0.01))(x)
x = layers.MaxPooling1D(pool_size=2, strides=2)(x)
embedded_pivot = layers.LSTM(256)(x)
encoder_model = Model(pivot_input, embedded_pivot)
embedded_statement = encoder_model(statement_input)
concat = layers.merge([embedded_pivot, embedded_statement], mode='concat')
x = layers.Dense(256, activation='relu')(concat)
x = layers.Dropout(dropout)(x)
prediction = layers.Dense(1, activation='relu')(x)
model = Model([pivot_input, statement_input], prediction)
return model
input_layer = layers.Input(shape=(max_num_amino, len(dataindex)), name='input') net = layers.Conv1D(conv, 5, padding='same')(input_layer) net = layers.Activation(activation)(net) #act1 = layers.MaxPooling1D(pool_size=poolSize,strides=1,padding='same')(act1) net = layers.Dropout(drop)(net) net = layers.Conv1D(conv, 5, padding='same')(net) net = layers.Activation(activation)(net) net = layers.MaxPooling1D(pool_size=poolSize, strides=1, padding='same')(net) net = layers.Dropout(drop)(net) net = layers.Conv1D(conv, 5, padding='same')(net) net = layers.Activation(activation)(net) net = layers.MaxPooling1D(pool_size=poolSize, strides=1, padding='same')(net) #pool3 = layers.Conv1D(128, 5, activation='relu', padding='same')(pool3) net = layers.Dropout(drop)(net) net = layers.Dense(conv)(net) net = layers.Activation(activation)(net)
from keras.models import Sequential
from keras.layers import Embedding, SimpleRNN, LSTM, Dense
model = Sequential()
model.add(Embedding(10000, 32))
model.add(SimpleRNN(32))
model.summary()
modelLSTM = Sequential()
modelLSTM.add(Embedding(10000, 32))
modelLSTM.add(LSTM(32))
modelLSTM.add(Dense(1, activation='sigmoid'))
from keras import layers
modelCNN1D = Sequential()
modelCNN1D.add(layers.Conv1D(32, 5, input_shape=(None, 100),
activation='relu'))
modelCNN1D.add(layers.MaxPooling1D(5))
modelCNN1D.add(layers.Conv1D(32, 5, activation='relu'))
modelCNN1D.add(layers.GRU(64, dropout=0.2, recurrent_dropout=0.5))
modelCNN1D.add(Dense(1))
modelCNN1D.summary()
if ('--learning_rate' in sys.argv):
try:
lr_arg = sys.argv[sys.argv.index('--learning_rate') + 1]
lr = float(lr_arg)
except:
lr = 0.001
else:
lr = 0.001
# ---------------------------------------------------------------------------
# ENCODER
x_input = layers.Input(shape=input_shape)
x = layers.Conv1D(8, 15, padding='same', activation='relu')(x_input)
shape_mp1 = K.int_shape(x)
x = layers.MaxPooling1D(5, padding='same')(x)
x = layers.Conv1D(16, 15, padding='same', activation='relu')(x)
shape_mp2 = K.int_shape(x)
x = layers.MaxPooling1D(5, padding='same')(x)
x = layers.Dropout(rate=0.1)(x)
x = layers.Conv1D(16, 15, padding='same', activation='relu')(x)
shape = K.int_shape(x)
x = layers.Flatten()(x)
x = layers.Dropout(rate=0.1)(x)
x = layers.Dense(16, activation='relu')(x)
z_mean = layers.Dense(latent_dim)(x)
z_log_var = layers.Dense(latent_dim)(x)
encoder = models.Model(x_input, [z_mean, z_log_var])
encoder.summary()
dtype='float32')
# In[16]:
# apply the 1-D Convolutional Neural Network
n_timesteps = x_train.shape[1]
n_features = x_train.shape[2]
n_outputs = 35
model = models.Sequential()
model.add(
layers.Conv1D(filters=100,
kernel_size=10,
activation='relu',
input_shape=(n_timesteps, n_features)))
model.add(layers.MaxPooling1D())
model.add(layers.BatchNormalization())
model.add(layers.Conv1D(100, 10, activation='relu'))
model.add(layers.MaxPooling1D())
model.add(layers.BatchNormalization())
model.add(layers.Conv1D(100, 10, activation='relu'))
model.add(layers.MaxPooling1D())
model.add(layers.BatchNormalization())
model.add(layers.Flatten())
model.add(layers.Dropout(0.5))
model.add(layers.Dropout(0.25))
model.add(layers.Dense(n_outputs, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print(model.summary())
def __init__(self,
df=None,
text=None,
label=None,
test_split=0.10,
lr=0.01,
num_words=10000,
embedding_dim=50,
dense_nodes=None,
maxlen=None,
tb_dir=None,
loss_func='categorical_crossentropy',
pre_trained_EM=None,
test_df=None,
cnn=False,
act='softmax',
lstm=True,
opt='adam',
drop=False,
slim_bert=False):
'''
num_words: is the max limit of words
test_df: expect to be a list. Ex. [df, text_label, code_label]
if tb_dir: is not none, will create a dir
pre_trained_EM: pass in a dict of word embeddings
embedding_dim: if pass in a pre trained EM, embedding_dim might needs to be reset
'''
if isinstance(df, pd.DataFrame) and text != None and label != None:
#print('var that outside the class: {}'.format(a))
self.version = '6.0'
self.df = df
self.text = text #column name for text
self.label = label #column name for label
self.test_split = test_split
self.classes = np.unique(df[label].values)
pre_train = False
self.opt = opt
self.lr = lr
self.maxlen = maxlen
self.num_words = num_words
self.labels_not_match = False
self.loss_func = loss_func
if (len(self.classes) == 2) and (self.loss_func
== 'categorical_crossentropy'):
self.loss_func = 'binary_crossentropy'
#visualization on TensorBoard
self.tb_dir = tb_dir
sentences = df[text].values
encoder = LabelBinarizer()
y = encoder.fit_transform(df[label].values)
if test_df == None:
# train test split & tokenization
self.sentences_train, self.sentences_test, self.y_train, self.y_test = train_test_split(
sentences, y, test_size=self.test_split, random_state=42)
self.tokenizer = Tokenizer(num_words=self.num_words)
self.tokenizer.fit_on_texts(self.sentences_train)
self.X_train = self.tokenizer.texts_to_sequences(
self.sentences_train)
self.X_test = self.tokenizer.texts_to_sequences(
self.sentences_test)
#vocab_size = len(self.tokenizer.word_index) + 1
else:
'''
if user pass in a separate validation data set
format: test_df = [df, text, label]
'''
# over write test_split -> 0
self.test_split = 0
self.valid_df = test_df[0]
self.valid_text = test_df[1]
self.valid_label = test_df[2]
self.test_label = np.unique(
self.valid_df[self.valid_label].values)
if not np.array_equal(self.classes, self.test_label):
print('labels not match:')
self.labels_not_match = True
print('training set: {}'.format(self.classes))
print('testing set: {}'.format(
np.unique(self.valid_df[self.valid_label].values)))
#raise Exception('label not match')
self.sentences_train = sentences
self.y_train = y
self.sentences_test = self.valid_df[self.valid_text].values
self.y_test = encoder.transform(
self.valid_df[self.valid_label].values)
self.tokenizer = Tokenizer(num_words=self.num_words)
self.tokenizer.fit_on_texts(self.sentences_train)
self.X_train = self.tokenizer.texts_to_sequences(
self.sentences_train)
self.X_test = self.tokenizer.texts_to_sequences(
self.sentences_test)
self.vocab_size = len(self.tokenizer.word_index) + 1
#maxlen_op = ['maxlen','mid','mean']
if (type(maxlen) == int) or (type(maxlen) == float):
self.maxlen = int(maxlen)
else:
#back to default
#self.maxlen = 'maxlen'
length_list = []
for val in self.X_train:
length_list.append(len(val))
self.maxlen = max(length_list)
'''
if type(self.maxlen) == str:
length_list = []
for val in self.X_train:
length_list.append(len(val))
if self.maxlen == 'maxlen':
self.maxlen = max(length_list)
'''
self.X_train = pad_sequences(self.X_train,
padding='post',
maxlen=self.maxlen)
self.X_test = pad_sequences(self.X_test,
padding='post',
maxlen=self.maxlen)
# set pre trained embeddings
if isinstance(pre_trained_EM, dict):
# over write vocab_size
# set pre_train = True
NB_WORDS = len(pre_trained_EM)
self.vocab_size = NB_WORDS
pre_train = True
missing_w = 0
emb_matrix = np.zeros((NB_WORDS, embedding_dim))
for w, i in self.tokenizer.word_index.items():
if i < NB_WORDS:
vect = pre_trained_EM.get(w)
if vect is not None:
emb_matrix[i] = vect
else:
missing_w += 1
else:
break
'''
Hyperparameter
1. embedding_dim
2. Dense nodes
Different layers
1. dropout layers
'''
self.embedding_dim = embedding_dim
self.output_num = self.y_train.shape[1]
if dense_nodes == None:
self.dense_nodes = self.output_num * 10
else:
self.dense_nodes = dense_nodes
#self.loss_func = ['mean_squared_error', 'logcosh', 'mean_absolute_error', 'categorical_crossentropy']
'''
1. em-de-de-output
2. em-lstm-output
3. em-cnn-de-output
4. em-cnn-lstm-output
5. note: an extra dropout layer can be added
'''
model = Sequential()
#model.add(layers.Embedding(vocab_size, self.embedding_dim, input_length=self.maxlen))
if pre_train:
model.add(
layers.Embedding(self.vocab_size,
self.embedding_dim,
input_length=self.maxlen,
trainable=False))
model.layers[0].set_weights([emb_matrix])
else:
model.add(
layers.Embedding(self.vocab_size,
self.embedding_dim,
input_length=self.maxlen))
'''
adding layers to Sequential model
'''
if slim_bert:
model.add(
layers.Bidirectional(
layers.LSTM(128,
return_sequences=True,
dropout=0.2,
recurrent_dropout=0.2)))
model.add(SeqSelfAttention(attention_activation='sigmoid'))
model.add(layers.Flatten())
else:
if cnn:
model.add(
layers.Conv1D(128,
5,
activation='relu',
padding='same'))
#model.add(layers.MaxPooling1D())
if not lstm:
model.add(layers.MaxPooling1D())
model.add(layers.Dropout(0.3))
if lstm:
model.add(
layers.LSTM(self.dense_nodes,
dropout=0.2,
recurrent_dropout=0.2,
return_sequences=True))
model.add(layers.Flatten())
if drop:
model.add(layers.Dropout(0.5))
if (cnn and not lstm) or (not cnn and not lstm):
model.add(layers.Dense(self.dense_nodes,
activation='relu'))
'''
add a output Dense layer to model
'''
model.add(layers.Dense(self.output_num, activation=act))
'''
sgd = SGD(lr=self.lr, momentum=0.9, decay=0.0, nesterov=False)
adam = Adam(lr=self.lr, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
if self.opt == 'sgd':
model.compile(loss=self.loss_func, optimizer=sgd, metrics=['accuracy'])
else:
self.opt = 'adam'
model.compile(loss=self.loss_func, optimizer=adam, metrics=['accuracy'])
'''
self.model = model
#self.model.summary()
else:
print("please load a model")
)
# build network model: CNN + RNN
from keras.models import Sequential
from keras import layers
from keras.optimizers import RMSprop
model = Sequential()
# CNN layers: 1D ConV + max pooling
model.add(
layers.Conv1D(
32, 5, activation='relu',
input_shape=(None, float_data.shape[-1]
))) # 5: ConV window. P190 the feature vectors count
model.add(layers.MaxPooling1D(3)) # 1/3
model.add(layers.Conv1D(32, 5, activation='relu'))
# RNN layer
model.add(layers.GRU(32, dropout=0.1, recurrent_dropout=0.5))
# Dense layer classifier
model.add(layers.Dense(1))
print(model.summary())
'''
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d_1 (Conv1D) (None, None, 32) 2272
_________________________________________________________________
max_pooling1d_1 (MaxPooling1 (None, None, 32) 0
_________________________________________________________________
val_steps = (valnum - lookback) // batch_size
# 查看训练集需要抽取的次数
train_steps = trainnum // batch_size
from keras.callbacks import EarlyStopping
early_stopping = EarlyStopping(monitor="val_loss", patience=30)
#创建模型
model1 = models.Sequential()
model1.add(
layers.Dense(512,
activation="relu",
input_shape=(lookback // step, data.shape[-1])))
model1.add(layers.Conv1D(filters=1024, kernel_size=5, activation="relu"))
model1.add(layers.MaxPooling1D(5))
model1.add(layers.Conv1D(filters=1024, kernel_size=3, activation="relu"))
model1.add(layers.GlobalMaxPool1D())
model1.add(layers.Dropout(0.5))
model1.add(layers.Dense(8, activation="softmax"))
model1.summary()
model1.compile(optimizer=optimizers.RMSprop(),
loss="categorical_crossentropy",
metrics=["acc"])
history1 = model1.fit_generator(train_gen,
steps_per_epoch=train_steps,
epochs=200,
validation_data=val_gen,
validation_steps=val_steps,
callbacks=[early_stopping])
def build_model(self,
inception=True,
res=True,
strided=False,
maxpool=True,
avgpool=False,
batchnorm=True):
self.i = 0
pad = 'same'
padp = 'same'
c_act = self.config['c_act']
r_act = self.config['r_act']
rk_act = self.config['rk_act']
r = kr.regularizers.l2(self.config['reg'])
c = self.input
stride_size = self.config['strides'] if strided else 1
if inception:
c0 = layers.Conv1D(self.config['filters'],
kernel_size=4,
strides=stride_size,
padding=pad,
activation=c_act)(c)
c1 = layers.Conv1D(self.config['filters'],
kernel_size=8,
strides=stride_size,
padding=pad,
activation=c_act)(c)
c2 = layers.Conv1D(self.config['filters'],
kernel_size=32,
strides=stride_size,
padding=pad,
activation=c_act)(c)
c = layers.concatenate([c0, c1, c2])
if maxpool:
c = layers.MaxPooling1D(2, padding=padp)(c)
elif avgpool:
c = layers.AveragePooling1D(2, padding=padp)(c)
if batchnorm:
c = layers.BatchNormalization()(c)
c = layers.SpatialDropout1D(self.config['cnn_drop'])(c)
c0 = layers.Conv1D(self.config['filters'],
kernel_size=4,
strides=stride_size,
padding=pad,
activation=c_act)(c)
c1 = layers.Conv1D(self.config['filters'],
kernel_size=8,
strides=stride_size,
padding=pad,
activation=c_act)(c)
c2 = layers.Conv1D(self.config['filters'],
kernel_size=32,
strides=stride_size,
padding=pad,
activation=c_act)(c)
c = layers.concatenate([c0, c1, c2])
if maxpool:
c = layers.MaxPooling1D(2, padding=padp)(c)
elif avgpool:
c = layers.AveragePooling1D(2, padding=padp)(c)
if batchnorm:
c = layers.BatchNormalization()(c)
c = layers.SpatialDropout1D(self.config['cnn_drop'])(c)
c0 = layers.Conv1D(self.config['filters'],
kernel_size=4,
strides=stride_size,
padding=pad,
activation=c_act)(c)
c1 = layers.Conv1D(self.config['filters'],
kernel_size=8,
strides=stride_size,
padding=pad,
activation=c_act)(c)
c2 = layers.Conv1D(self.config['filters'],
kernel_size=32,
strides=stride_size,
padding=pad,
activation=c_act)(c)
c = layers.concatenate([c0, c1, c2])
if maxpool:
c = layers.MaxPooling1D(2, padding=padp)(c)
elif avgpool:
c = layers.AveragePooling1D(2, padding=padp)(c)
if batchnorm:
c = layers.BatchNormalization()(c)
c = layers.SpatialDropout1D(self.config['cnn_drop'])(c)
else: # No inception Modules
c = layers.Conv1D(self.config['filters'],
kernel_size=4,
strides=stride_size,
padding=pad,
activation=c_act)(self.input)
if maxpool:
c = layers.MaxPooling1D(2, padding=padp)(c)
elif avgpool:
c = layers.AveragePooling1D(2, padding=padp)(c)
if batchnorm:
c = layers.BatchNormalization()(c)
c = layers.SpatialDropout1D(self.config['cnn_drop'])(c)
c = layers.Conv1D(self.config['filters'],
kernel_size=4,
strides=stride_size,
padding=pad,
activation=c_act)(c)
if maxpool:
c = layers.MaxPooling1D(2, padding=padp)(c)
elif avgpool:
c = layers.AveragePooling1D(2, padding=padp)(c)
if batchnorm:
c = layers.BatchNormalization()(c)
c = layers.SpatialDropout1D(self.config['cnn_drop'])(c)
c = layers.Conv1D(self.config['filters'],
kernel_size=4,
strides=stride_size,
padding=pad,
activation=c_act)(c)
if maxpool:
c = layers.MaxPooling1D(2, padding=padp)(c)
elif avgpool:
c = layers.AveragePooling1D(2, padding=padp)(c)
if batchnorm:
c = layers.BatchNormalization()(c)
c = layers.SpatialDropout1D(self.config['cnn_drop'])(c)
if res: # Residual RNN
g1 = layers.GRU(self.config['state_size'],
return_sequences=True,
activation=rk_act,
recurrent_activation=r_act,
dropout=self.config['rec_drop'],
recurrent_dropout=self.config['rec_drop'],
recurrent_regularizer=r,
kernel_regularizer=r)(c)
g2 = layers.GRU(self.config['state_size'],
return_sequences=True,
activation=rk_act,
recurrent_activation=r_act,
dropout=self.config['rec_drop'],
recurrent_dropout=self.config['rec_drop'],
recurrent_regularizer=r,
kernel_regularizer=r)(g1)
g_concat1 = layers.concatenate([g1, g2])
g3 = layers.GRU(self.config['state_size'],
return_sequences=True,
activation=rk_act,
recurrent_activation=r_act,
dropout=self.config['rec_drop'],
recurrent_dropout=self.config['rec_drop'],
recurrent_regularizer=r,
kernel_regularizer=r)(g_concat1)
g_concat2 = layers.concatenate([g1, g2, g3])
g = layers.GRU(self.config['state_size'],
return_sequences=False,
activation=rk_act,
recurrent_activation=r_act,
dropout=self.config['rec_drop'],
recurrent_dropout=self.config['rec_drop'],
recurrent_regularizer=r,
kernel_regularizer=r)(g_concat2)
else: # No Residual RNN
g = layers.GRU(self.config['state_size'],
return_sequences=True,
activation=rk_act,
recurrent_activation=r_act,
dropout=self.config['rec_drop'],
recurrent_dropout=self.config['rec_drop'],
recurrent_regularizer=r,
kernel_regularizer=r)(c)
g = layers.GRU(self.config['state_size'],
return_sequences=True,
activation=rk_act,
recurrent_activation=r_act,
dropout=self.config['rec_drop'],
recurrent_dropout=self.config['rec_drop'],
recurrent_regularizer=r,
kernel_regularizer=r)(g)
g = layers.GRU(self.config['state_size'],
return_sequences=True,
activation=rk_act,
recurrent_activation=r_act,
dropout=self.config['rec_drop'],
recurrent_dropout=self.config['rec_drop'],
recurrent_regularizer=r,
kernel_regularizer=r)(g)
g = layers.GRU(self.config['state_size'],
return_sequences=False,
activation=rk_act,
recurrent_activation=r_act,
dropout=self.config['rec_drop'],
recurrent_dropout=self.config['rec_drop'],
recurrent_regularizer=r,
kernel_regularizer=r)(g)
d = layers.Dense(self.config['output_size'])(g)
out = layers.Softmax()(d)
self.model = Model(self.input, out)
print("{} initialized.".format(self.model.name))
# model.add(kn.Dense(500, activation='relu'))
# model.add(kn.Dropout(0.2))
# model.add(kn.Dense(90, activation='sigmoid'))
# model.compile(optimizer='adam', loss='binary_crossentropy',
# metrics=["accuracy", keras.metrics.FalseNegatives(), keras.metrics.FalsePositives()])
# model.summary()
# 80% f1 score
inp = kn.Input(shape=(mdn, ), dtype='int32')
x = kn.Embedding(vocab_size,
100,
weights=[embedding_matrix],
input_length=mdn,
trainable=False)(inp)
x = kn.Conv1D(100, 5, activation='relu')(x)
x = kn.MaxPooling1D(5)(x)
x = kn.Dropout(0.2)(x)
x = kn.Conv1D(100, 5, activation='relu')(x)
x = kn.MaxPooling1D(22)(x)
x = kn.Dropout(0.2)(x)
x = kn.Flatten()(x)
x = kn.Dense(128, activation='relu')(x)
x = kn.Dropout(0.05)(x)
preds = kn.Dense(90, activation='sigmoid')(x)
model = keras.Model(inp, preds)
model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=[
'accuracy',
keras.metrics.FalseNegatives(),
def build_full_model(frames, freq_bins, mod_options):
'''
Build a keras model based on the Dielemann model for music recommandation.
This model is trained on time-frequency representation of music. The input
data fed to the model must be shaped like (batch_size, time, frequency).
Parameters
----------
frames: int
Number of time frames in a single input
freq_bin: int
Number of frequency bands in a single input
mod_option: dictionnary
Specific options for the model. This dictionnary must contain:
'activation': string
name of the activation function for keras layers
'batchNormConv': bool
Choose if the model should apply babtch normalization after each
convnet
'FC number': int
Number of cells for each FC layer
'batchNormDense': bool
Choose if the model should apply babtch normalization after each
FC layer
'Alphabet size': int
Size of the training alphabet (last layer, size of the output)
Returns
-------
model: keras model
The model built. Expects inputs of shape (batch_size, frames,
freq_bins) and outputs tensor of shape (batch_size, alphabet_size).
'''
inputs = layers.Input(shape=(frames, freq_bins))
#%%=========== First layer ===============
#zero-padding the input
padding_1 = layers.ZeroPadding1D(padding=2)(inputs)
#Convnet 256 neurons with 4 sample window. Activation defined in mod_option dictionnary
conv1 = layers.Conv1D(256,
4,
padding='same',
activation=mod_options['activation'])(padding_1)
#Normalise batch if defined in mod_options
if mod_options['batchNormConv']:
conv1 = layers.BatchNormalization()(conv1)
#Reduce data by max pooling between 2 values
pool_1 = layers.MaxPooling1D(pool_size=2)(conv1)
#%%============ Second layer ==============
#Same layer as the previous one
padding_2 = layers.ZeroPadding1D(padding=2)(pool_1)
conv_2 = layers.Conv1D(256,
4,
padding='same',
activation=mod_options['activation'])(padding_2)
if mod_options['batchNormConv']:
conv_2 = layers.BatchNormalization()(conv_2)
pool_2 = layers.MaxPooling1D(pool_size=2)(conv_2)
'''
#%%=========== Third layer ???================
#zero-padding the input
model.Add(layers.ZeroPadding1D(padding = 2))
#Convnet 512 neurons with 4 sample window.
model.add(layers.Conv1D(512, 4, padding = 'same', activation = mod_options['activation']))
#Normalize batch if defined
if mod_options['batchNormConv']:
model.add(layers.BatchNormalization())
'''
#%%=========== Fourth layer =================
#zero-padding the input
padding_3 = layers.ZeroPadding1D(padding=2)(pool_2)
#Convnet 512 neurons with 4 sample window.
conv_3 = layers.Conv1D(512,
4,
padding='same',
activation=mod_options['activation'])(padding_3)
#Normalize batch if defined
if mod_options['batchNormConv']:
conv_3 = layers.BatchNormalization()(conv_3)
#%%========== Global temporal pooling layer =========
pool_max = layers.GlobalMaxPooling1D()(conv_3)
pool_average = layers.GlobalAveragePooling1D()(conv_3)
pool_LP = layers.Lambda(lambda x: GlobalLPPooling1D(x))(conv_3)
pool_time = layers.Concatenate()([pool_max, pool_average, pool_LP])
#%%========== FC Layers =========================
FC_1 = layers.Dense(mod_options['FC number'],
activation=mod_options['activation'])(pool_time)
if mod_options['batchNormDense']:
FC_1 = layers.BatchNormalization()(FC_1)
FC_2 = layers.Dense(mod_options['FC number'],
activation=mod_options['activation'])(FC_1)
if mod_options['batchNormDense']:
FC_2 = layers.BatchNormalization()(FC_2)
FC_3 = layers.Dense(mod_options['Alphabet size'],
activation='softmax')(FC_2)
model = Model(inputs=inputs, outputs=FC_3)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def InceptionResNetV2(include_top=True,
weights=None,
input_tensor=None,
input_shape=None,
pooling=None,
classes=100,
**kwargs):
"""Instantiates the Inception-ResNet v2 architecture.
Note that the data format convention used by the model is
the one specified in your Keras config at `~/.keras/keras.json`.
# Arguments
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization)
or the path to the weights file to be loaded.
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as input for the model.
input_shape: optional shape tuple if input_tensor is not specified.
and width should be no smaller than 75.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 3D tensor output of the last convolutional block.
- `'avg'` means that global average pooling
will be applied to the output of the
last convolutional block, and thus
the output of the model will be a 2D tensor.
- `'max'` means that global max pooling will be applied.
classes: optional number of classes to classify inputs
into, only to be specified if `include_top` is `True`, and
if no `weights` argument is specified.
# Returns
A Keras `Model` instance.
# Raises
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
if weights is not None and not os.path.exists(weights):
raise ValueError(
'The `weights` argument should be either `None` (random initialization) or the path to the weights file to be loaded.'
)
if input_tensor is None:
inputs = layers.Input(shape=input_shape)
else:
if not backend.is_keras_tensor(input_tensor):
inputs = layers.Input(tensor=input_tensor, shape=input_shape)
else:
inputs = input_tensor
# Stem block: 35 x 192
x = conv1d_bn(inputs, 32, 3, strides=2, padding='valid')
x = conv1d_bn(x, 32, 3, padding='valid')
x = conv1d_bn(x, 64, 3)
x = layers.MaxPooling1D(3, strides=2)(x)
x = conv1d_bn(x, 80, 1, padding='valid')
x = conv1d_bn(x, 192, 3, padding='valid')
x = layers.MaxPooling1D(3, strides=2)(x)
# Mixed 5b (Inception-A block): 35 x 320
branch_0 = conv1d_bn(x, 96, 1)
branch_1 = conv1d_bn(x, 48, 1)
branch_1 = conv1d_bn(branch_1, 64, 5)
branch_2 = conv1d_bn(x, 64, 1)
branch_2 = conv1d_bn(branch_2, 96, 3)
branch_2 = conv1d_bn(branch_2, 96, 3)
branch_pool = layers.AveragePooling1D(3, strides=1, padding='same')(x)
branch_pool = conv1d_bn(branch_pool, 64, 1)
branches = [branch_0, branch_1, branch_2, branch_pool]
channel_axis = 1 if backend.image_data_format() == 'channels_first' else 2
x = layers.Concatenate(axis=channel_axis, name='mixed_5b')(branches)
# 10x block35 (Inception-ResNet-A block): 35 x 320
for block_idx in range(1, 11):
x = inception_resnet_block(x,
scale=0.17,
block_type='block35',
block_idx=block_idx)
# Mixed 6a (Reduction-A block): 17 x 1088
branch_0 = conv1d_bn(x, 384, 3, strides=2, padding='valid')
branch_1 = conv1d_bn(x, 256, 1)
branch_1 = conv1d_bn(branch_1, 256, 3)
branch_1 = conv1d_bn(branch_1, 384, 3, strides=2, padding='valid')
branch_pool = layers.MaxPooling1D(3, strides=2, padding='valid')(x)
branches = [branch_0, branch_1, branch_pool]
x = layers.Concatenate(axis=channel_axis, name='mixed_6a')(branches)
# 20x block17 (Inception-ResNet-B block): 17 x 1088
for block_idx in range(1, 21):
x = inception_resnet_block(x,
scale=0.1,
block_type='block17',
block_idx=block_idx)
# Mixed 7a (Reduction-B block): 8 x 2080
branch_0 = conv1d_bn(x, 256, 1)
branch_0 = conv1d_bn(branch_0, 384, 3, strides=2, padding='valid')
branch_1 = conv1d_bn(x, 256, 1)
branch_1 = conv1d_bn(branch_1, 288, 3, strides=2, padding='valid')
branch_2 = conv1d_bn(x, 256, 1)
branch_2 = conv1d_bn(branch_2, 288, 3)
branch_2 = conv1d_bn(branch_2, 320, 3, strides=2, padding='valid')
branch_pool = layers.MaxPooling1D(3, strides=2, padding='valid')(x)
branches = [branch_0, branch_1, branch_2, branch_pool]
x = layers.Concatenate(axis=channel_axis, name='mixed_7a')(branches)
# 10x block8 (Inception-ResNet-C block): 8 x 2080
for block_idx in range(1, 10):
x = inception_resnet_block(x,
scale=0.2,
block_type='block8',
block_idx=block_idx)
x = inception_resnet_block(x,
scale=1.,
activation=None,
block_type='block8',
block_idx=10)
# Final convolution block: 8 x 1536
x = conv1d_bn(x, 1536, 1, name='conv_7b')
if include_top:
# Classification block
x = layers.GlobalAveragePooling1D(name='avg_pool')(x)
x = layers.Dense(classes, activation='softmax', name='predictions')(x)
else:
if pooling == 'avg':
x = layers.GlobalAveragePooling1D()(x)
elif pooling == 'max':
x = layers.GlobalMaxPooling1D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = utils.get_source_inputs(input_tensor)
else:
inputs = inputs
# Create model.
model = models.Model(inputs, x, name='1d_inception_resnet_v2')
# Load weights.
if weights is not None:
model.load_weights(weights)
return model
print('Pad sequences (sample x time)')
x_train = sequence.pad_sequences(x_train, maxlen=max_len)
x_test = sequence.pad_sequences(x_test, maxlen=max_len)
print('x_train shape:', x_train.shape)
print('x_test shape:', x_test.shape)
# 在IMDB数据上训练并评估一个简单的一维卷积神经网络
from keras.models import Sequential
from keras import layers
from keras.optimizers import RMSprop
model = Sequential()
model.add(layers.Embedding(max_features, 128, input_length=max_len))
model.add(layers.Conv1D(32, 7, activation='relu'))
model.add(layers.MaxPooling1D(5))
model.add(layers.Conv1D(32, 7, activation='relu'))
model.add(layers.GlobalMaxPooling1D())
model.add(layers.Dense(1))
model.summary()
model.compile(optimizer=RMSprop(lr=1e-4),
loss='binary_crossentropy',
metrics=['acc'])
history = model.fit(x_train,
y_train,
epochs=10,
batch_size=128,
validation_split=0.2)
def CTC(signals,
out_len,
label_len,
kernel_size,
conv_window_len,
lr,
dropoutRate,
training=True):
inner = signals
n_label = 5
# CNN
inner = layers.Conv1D(kernel_size[0],
conv_window_len,
padding='same',
activation="relu")(inner)
inner = layers.Conv1D(kernel_size[0],
conv_window_len,
padding='same',
activation="relu")(inner)
inner = layers.MaxPooling1D(2)(inner)
inner = layers.Conv1D(kernel_size[1],
conv_window_len,
padding='same',
activation="relu")(inner)
inner = layers.Conv1D(kernel_size[1],
conv_window_len,
padding='same',
activation="relu")(inner)
inner = layers.MaxPooling1D(2)(inner)
# CNN to RNN
#print(inner.get_shape())
inter = layers.Dense(64, activation='relu',
kernel_initializer='he_normal')(inner)
# 2"RNN layer
gru_1 = layers.GRU(128,
return_sequences=True,
kernel_initializer='he_normal')(inner)
gru_1b = layers.GRU(128,
return_sequences=True,
go_backwards=True,
kernel_initializer='he_normal')(inner)
gru1_merged = layers.Add()([gru_1, gru_1b])
gru_2 = layers.GRU(128,
return_sequences=True,
kernel_initializer='he_normal')(gru1_merged)
gru_2b = layers.GRU(128,
return_sequences=True,
go_backwards=True,
kernel_initializer='he_normal')(gru1_merged)
gru2_merged = layers.Concatenate()([gru_2, gru_2b])
inner = gru2_merged
inner = layers.Dense(n_label)(inner)
y_pred = layers.Activation('softmax')(inner)
# loss compute module
labels = Input(name="labels", shape=[label_len], dtype='int64')
input_length = Input(name="input_length", shape=[1], dtype='int64')
label_length = Input(name="label_length", shape=[1], dtype='int64')
loss_out = layers.Lambda(ctc_lambda_func, output_shape=(1, ), name='ctc')(
[y_pred, labels, input_length, label_length])
if training:
return models.Model([signals, labels, input_length, label_length],
outputs=loss_out)
else:
return models.Model(inputs=[signals], outputs=y_pred)
