def create_submodel_with_skipconv1d(embedding_layer, submod_layer_descriptor,
target_cnn_ks, skip):
submodels = []
ks_and_masks = generate_ks_and_masks(target_cnn_ks, skip)
for mask in ks_and_masks[1]:
model = Sequential()
model.add(embedding_layer)
for layer_descriptor in submod_layer_descriptor.split(","):
if layer_descriptor.endswith("_"):
continue
ld = layer_descriptor.split("=")
layer_name = ld[0]
params = None
if len(ld) > 1:
params = ld[1].split("-")
if layer_name == "dropout":
model.add(Dropout(float(params[0])))
elif layer_name == "lstm":
if params[1] == "True":
return_seq = True
else:
return_seq = False
model.add(
LSTM(units=int(params[0]), return_sequences=return_seq))
elif layer_name == "gru":
if params[1] == "True":
return_seq = True
else:
return_seq = False
model.add(
GRU(units=int(params[0]), return_sequences=return_seq))
elif layer_name == "bilstm":
if params[1] == "True":
return_seq = True
else:
return_seq = False
model.add(
Bidirectional(
LSTM(units=int(params[0]),
return_sequences=return_seq)))
elif layer_name == "conv1d":
model.add(
SkipConv1D(filters=int(params[0]),
kernel_size=int(ks_and_masks[0]),
validGrams=mask,
padding='same',
activation='relu'))
elif layer_name == "maxpooling1d":
size = params[0]
if size == "v":
size = int(ks_and_masks[0])
else:
size = int(params[0])
model.add(MaxPooling1D(pool_size=size))
elif layer_name == "gmaxpooling1d":
model.add(GlobalMaxPooling1D())
elif layer_name == "dense":
model.add(Dense(int(params[0]), activation=params[1]))
submodels.append(model)
return submodels
# load pre-trained word embeddings into an Embedding layer
# note that we set trainable = False so as to keep the embeddings fixed
embedding_layer = Embedding(num_words,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=False)
print('Training model.')
# train a 1D convnet with global maxpooling
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH, ), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
x = Conv1D(128, 5, activation='relu')(embedded_sequences)
x = MaxPooling1D(5)(x)
x = Conv1D(128, 5, activation='relu')(x)
x = MaxPooling1D(5)(x)
x = Conv1D(128, 5, activation='relu')(x)
x = GlobalMaxPooling1D()(x)
x = Dense(128, activation='relu')(x)
preds = Dense(len(labels_index), activation='softmax')(x)
model = Model(sequence_input, preds)
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['acc'])
y_train = to_categorical(y_train)
x_train = to_categorical(x_train)
x_val = to_categorical(x_val)
dev_label_data_path = os.path.join('Data', 'cnews.dev.label')
x_train, y_train, x_test, y_test, x_dev, y_dev = load_train_data(train_emb_data_path, train_tag_data_path,
train_label_data_path,
test_emb_data_path, test_tag_data_path,
test_label_data_path,
dev_emb_data_path, dev_tag_data_path,
dev_label_data_path)
# read config
sentence_words_num = int(getConfig('cnn', 'WORDS_DIM'))
word_vector_dim = int(getConfig('cnn', 'VEC_DIM'))
# end config
model = Sequential()
model.add(Conv1D(500, 3, activation='relu', input_shape=(300, 900)))
model.add(MaxPooling1D(3))
model.add(Dropout(1.0))
model.add(Flatten())
model.add(Dense(11, activation='softmax'))
# sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
adam = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])
# test
early_stopping = EarlyStopping(monitor='acc', patience=3, mode='max')
model.fit(x_train, y_train, batch_size=128, epochs=50, callbacks=[early_stopping])
score_test = model.evaluate(x_test, y_test, batch_size=64)
print 'loss=', score_test[0], ' acc=', score_test[1]
embedding_layer = Embedding(len(word_index) + 1,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=True)
convs = []
filter_sizes = [3, 4, 5]
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH), dtype='int32')
embedding_sequences = embedding_layer(sequence_input)
for fsz in filter_sizes:
l_conv = Conv1D(filters=128, kernel_size=fsz,
activation='relu')(embedding_sequences)
l_pool = MaxPooling1D(5)(l_conv)
convs.append(l_conv)
l_merge = Merge(mode='concat', concat_axis=1)(convs)
l_conv1 = Conv1D(128, 5, activation='relu')(l_merge)
l_pool1 = MaxPooling1D(5)(l_conv1)
l_conv2 = Conv1D(128, 5, activation='relu')(l_pool1)
l_pool2 = MaxPooling1D(30)(l_conv2)
l_flat = Flatten()(l_pool2)
l_dense = Dense(128, activation='relu')(l_flat)
preds = Dense(2, activation='softmax')(l_dense)
model = Model(sequence_input, preds)
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
y_test = np_utils.to_categorical(lb.fit_transform(y_test))
y_train
X_train.shape
x_traincnn = np.expand_dims(X_train, axis=2)
x_testcnn = np.expand_dims(X_test, axis=2)
model = Sequential()
model.add(Conv1D(256, 5, padding='same', input_shape=(216, 1)))
model.add(Activation('relu'))
model.add(Conv1D(128, 5, padding='same'))
model.add(Activation('relu'))
model.add(Dropout(0.1))
model.add(MaxPooling1D(pool_size=(8)))
model.add(Conv1D(
128,
5,
padding='same',
))
model.add(Activation('relu'))
#model.add(Conv1D(128, 5,padding='same',))
#model.add(Activation('relu'))
#model.add(Conv1D(128, 5,padding='same',))
#model.add(Activation('relu'))
#model.add(Dropout(0.2))
model.add(Conv1D(
128,
5,
padding='same',
def CreateModel(self):
'''
定义CNN/LSTM/CTC模型，使用函数式模型
输入层：39维的特征值序列，一条语音数据的最大长度设为1500（大约15s）
隐藏层一：1024个神经元的卷积层
隐藏层二：池化层，池化窗口大小为2
隐藏层三：Dropout层，需要断开的神经元的比例为0.2，防止过拟合
隐藏层四：循环层、LSTM层
隐藏层五：Dropout层，需要断开的神经元的比例为0.2，防止过拟合
隐藏层六：全连接层，神经元数量为self.MS_OUTPUT_SIZE，使用softmax作为激活函数，
输出层：自定义层，即CTC层，使用CTC的loss作为损失函数
当前未完成，网络模型可能还需要修改
'''
# 每一帧使用13维mfcc特征及其13维一阶差分和13维二阶差分表示，最大信号序列长度为1500
input_data = Input(name='the_input',
shape=(self.AUDIO_LENGTH,
self.AUDIO_FEATURE_LENGTH))
layer_h1_c = Conv1D(filters=256,
kernel_size=5,
strides=1,
use_bias=True,
padding="valid")(input_data) # 卷积层
#layer_h1_a = Activation('relu', name='relu0')(layer_h1_c)
layer_h1_a = LeakyReLU(alpha=0.3)(layer_h1_c) # 高级激活层
layer_h1 = MaxPooling1D(pool_size=2, strides=None,
padding="valid")(layer_h1_a) # 池化层
layer_h2 = BatchNormalization()(layer_h1)
layer_h3_c = Conv1D(filters=256,
kernel_size=5,
strides=1,
use_bias=True,
padding="valid")(layer_h2) # 卷积层
layer_h3_a = LeakyReLU(alpha=0.3)(layer_h3_c) # 高级激活层
#layer_h3_a = Activation('relu', name='relu1')(layer_h3_c)
layer_h3 = MaxPooling1D(pool_size=2, strides=None,
padding="valid")(layer_h3_a) # 池化层
layer_h4 = Dropout(0.1)(layer_h3) # 随机中断部分神经网络连接，防止过拟合
layer_h5 = Dense(256, use_bias=True,
activation="softmax")(layer_h4) # 全连接层
layer_h6 = Dense(256, use_bias=True,
activation="softmax")(layer_h5) # 全连接层
#layer_h4 = Activation('softmax', name='softmax0')(layer_h4_d1)
layer_h7 = LSTM(256,
activation='softmax',
use_bias=True,
return_sequences=True)(layer_h6) # LSTM层
layer_h8 = LSTM(256,
activation='softmax',
use_bias=True,
return_sequences=True)(layer_h7) # LSTM层
layer_h9 = LSTM(256,
activation='softmax',
use_bias=True,
return_sequences=True)(layer_h8) # LSTM层
layer_h10 = LSTM(256,
activation='softmax',
use_bias=True,
return_sequences=True)(layer_h9) # LSTM层
#layer_h10 = Activation('softmax', name='softmax1')(layer_h9)
layer_h10_dropout = Dropout(0.1)(layer_h10) # 随机中断部分神经网络连接，防止过拟合
layer_h11 = Dense(512, use_bias=True,
activation="softmax")(layer_h10_dropout) # 全连接层
layer_h12 = Dense(self.MS_OUTPUT_SIZE,
use_bias=True,
activation="softmax")(layer_h11) # 全连接层
#layer_h6 = Dense(1283, activation="softmax")(layer_h5) # 全连接层
y_pred = Activation('softmax', name='softmax2')(layer_h12)
model_data = Model(inputs=input_data, outputs=y_pred)
#model_data.summary()
#labels = Input(name='the_labels', shape=[60], dtype='float32')
labels = Input(name='the_labels',
shape=[self.label_max_string_length],
dtype='float32')
input_length = Input(name='input_length', shape=[1], dtype='int64')
label_length = Input(name='label_length', shape=[1], dtype='int64')
# Keras doesn't currently support loss funcs with extra parameters
# so CTC loss is implemented in a lambda layer
#layer_out = Lambda(ctc_lambda_func,output_shape=(self.MS_OUTPUT_SIZE, ), name='ctc')([y_pred, labels, input_length, label_length])#(layer_h6) # CTC
loss_out = Lambda(self.ctc_lambda_func, output_shape=(1, ),
name='ctc')(
[y_pred, labels, input_length, label_length])
#top_k_decoded, _ = K.ctc_decode(y_pred, input_length)
#self.decoder = K.function([input_data, input_length], [top_k_decoded[0]])
#y_out = Activation('softmax', name='softmax3')(loss_out)
model = Model(inputs=[input_data, labels, input_length, label_length],
outputs=loss_out)
model.summary()
# clipnorm seems to speeds up convergence
#sgd = SGD(lr=0.0001, decay=1e-8, momentum=0.9, nesterov=True, clipnorm=5)
ada_d = Adadelta(lr=0.001, rho=0.95, epsilon=1e-06)
#model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer = sgd, metrics=['accuracy'])
#model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer = ada_d, metrics=['accuracy']) ctc_cost
model.compile(loss={
'ctc': lambda y_true, y_pred: y_pred
},
optimizer=ada_d,
metrics=['accuracy', self.ctc_cost])
# captures output of softmax so we can decode the output during visualization
self.test_func = K.function([input_data], [y_pred])
self.test_func_input_length = K.function([input_length],
[input_length])
print('[*提示] 创建模型成功，模型编译成功')
return model
def build(input_shape, classes):
model = Sequential()
#Block1
filter_num = ['None', 32, 64, 128, 256]
kernel_size = ['None', 8, 8, 8, 8]
conv_stride_size = ['None', 1, 1, 1, 1]
pool_stride_size = ['None', 4, 4, 4, 4]
pool_size = ['None', 8, 8, 8, 8]
model.add(
Conv1D(filters=filter_num[1],
kernel_size=kernel_size[1],
input_shape=input_shape,
strides=conv_stride_size[1],
padding='same',
name='block1_conv1'))
model.add(BatchNormalization(axis=-1))
model.add(ELU(alpha=1.0, name='block1_adv_act1'))
model.add(
Conv1D(filters=filter_num[1],
kernel_size=kernel_size[1],
strides=conv_stride_size[1],
padding='same',
name='block1_conv2'))
model.add(BatchNormalization(axis=-1))
model.add(ELU(alpha=1.0, name='block1_adv_act2'))
model.add(
MaxPooling1D(pool_size=pool_size[1],
strides=pool_stride_size[1],
padding='same',
name='block1_pool'))
model.add(Dropout(0.2, name='block1_dropout'))
model.add(
Conv1D(filters=filter_num[2],
kernel_size=kernel_size[2],
strides=conv_stride_size[2],
padding='same',
name='block2_conv1'))
model.add(BatchNormalization())
model.add(Activation('relu', name='block2_act1'))
model.add(
Conv1D(filters=filter_num[2],
kernel_size=kernel_size[2],
strides=conv_stride_size[2],
padding='same',
name='block2_conv2'))
model.add(BatchNormalization())
model.add(Activation('relu', name='block2_act2'))
model.add(
MaxPooling1D(pool_size=pool_size[2],
strides=pool_stride_size[3],
padding='same',
name='block2_pool'))
model.add(Dropout(0.2, name='block2_dropout'))
model.add(
Conv1D(filters=filter_num[3],
kernel_size=kernel_size[3],
strides=conv_stride_size[3],
padding='same',
name='block3_conv1'))
model.add(BatchNormalization())
model.add(Activation('relu', name='block3_act1'))
model.add(
Conv1D(filters=filter_num[3],
kernel_size=kernel_size[3],
strides=conv_stride_size[3],
padding='same',
name='block3_conv2'))
model.add(BatchNormalization())
model.add(Activation('relu', name='block3_act2'))
model.add(
MaxPooling1D(pool_size=pool_size[3],
strides=pool_stride_size[3],
padding='same',
name='block3_pool'))
model.add(Dropout(0.2, name='block3_dropout'))
model.add(
Conv1D(filters=filter_num[4],
kernel_size=kernel_size[4],
strides=conv_stride_size[4],
padding='same',
name='block4_conv1'))
model.add(BatchNormalization())
model.add(Activation('relu', name='block4_act1'))
model.add(
Conv1D(filters=filter_num[4],
kernel_size=kernel_size[4],
strides=conv_stride_size[4],
padding='same',
name='block4_conv2'))
model.add(BatchNormalization())
model.add(Activation('relu', name='block4_act2'))
model.add(
MaxPooling1D(pool_size=pool_size[4],
strides=pool_stride_size[4],
padding='same',
name='block4_pool'))
model.add(Dropout(0.2, name='block4_dropout'))
model.add(Flatten(name='flatten'))
model.add(
Dense(512, kernel_initializer=glorot_uniform(seed=0), name='fc1'))
model.add(BatchNormalization())
model.add(Activation('relu', name='fc1_act'))
model.add(Dropout(0.7, name='fc1_dropout'))
model.add(
Dense(512, kernel_initializer=glorot_uniform(seed=0), name='fc2'))
model.add(BatchNormalization())
model.add(Activation('relu', name='fc2_act'))
model.add(Dropout(0.5, name='fc2_dropout'))
model.add(
Dense(classes,
kernel_initializer=glorot_uniform(seed=0),
name='fc3'))
model.add(Activation('softmax', name="softmax"))
return model
stratify=Y,
test_size=0.2,
random_state=777,
shuffle=True)
from keras.layers import Dense, Dropout, Flatten, Conv1D, Input, MaxPooling1D
from keras.models import Model
from keras.callbacks import EarlyStopping, ModelCheckpoint
from keras import backend as K
K.clear_session()
inputs = Input(shape=(8000, 1))
# First conv1D layer
conv = Conv1D(8, 13, padding='valid', activation='relu', strides=1)(inputs)
conv = MaxPooling1D(3)(conv)
conv = Dropout(0.3)(conv)
# Second conv1D layer
conv = Conv1D(16, 11, padding='valid', activation='relu', strides=1)(conv)
conv = MaxPooling1D(3)(conv)
conv = Dropout(0.3)(conv)
# Third conv1D layer
conv = Conv1D(32, 9, padding='valid', activation='relu', strides=1)(conv)
conv = MaxPooling1D(3)(conv)
conv = Dropout(0.3)(conv)
# Fourth conv1D layer
conv = Conv1D(64, 7, padding='valid', activation='relu', strides=1)(conv)
conv = MaxPooling1D(3)(conv)
# 3.Fully Connected Model: The interpretation of extracted features in terms of a predictive output.<br/>
# In[14]:
model = Sequential()
model.add(
Embedding(vocab_size,
embedding_dim,
input_length=max_length,
weights=[embedding_matrix],
trainable=False))
model.add(
Conv1D(128, 3, activation='relu')
) #https://arxiv.org/abs/1408.5882 (Convolutional Neural Networks for Sentence Classification by Yoon Kim)
model.add(
MaxPooling1D(3)
) #https://arxiv.org/abs/1510.03820(A Sensitivity Analysis of (and Practitioners' Guide to) Convolutional Neural Networks for Sentence Classification)
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dense(31, activation='softmax'))
model.summary()
# ### Training the model
# In[17]:
model.compile(
optimizer='adam',
loss=
'binary_crossentropy', # https://stackoverflow.com/questions/42081257/keras-binary-crossentropy-vs-categorical-crossentropy-performance
metrics=['accuracy'])
def supervised_model(pad_len = args.pad_len, max_vocab = args.max_vocab, emb_dim = args.emb_dim,\
lstm_units = args.lstm_units, dropout = args.dropout, recurrent_dropout = args.recurrent_dropout,\
dense_units = args.dense_units,\
left_emb = src_embs, right_emb = tgt_embs):
"""
Creates a Keras model to predict whether sentence pair is a translation of one another
:param left_emb: multi-lingual embedding of left sentences
:param right_emb: multi-lingual embedding of right sentences
:param pad_len: maximum sentence length
:param max_vocab: words included in embedding matrices
:param emb_dim: dimensionality of embedding matrices
:param lstm_units: hidden units in LSTM, defaults to 32
:param dropout: LSTM dropout, default of 0.3
:param recurrent_dropout: LSTM recurrent dropout, default of 0.5
:param dense_units: hidden units in dense layers of classification block, defaults to 1,024
:return: Keras model
"""
## Encode left sentence
left_sent = Input(shape=(pad_len, ))
x = Embedding(max_vocab,
emb_dim,
input_length=pad_len,
weights=[left_emb],
trainable=False)(left_sent)
x = Bidirectional(
LSTM(lstm_units,
dropout=dropout,
recurrent_dropout=recurrent_dropout,
return_sequences=True))(x)
x = MaxPooling1D(pad_len)(x)
x = Flatten()(x)
left_enc = x
## Encode right sentence
right_sent = Input(shape=(pad_len, ))
x = Embedding(max_vocab,
emb_dim,
input_length=pad_len,
weights=[right_emb],
trainable=False)(right_sent)
x = Bidirectional(
LSTM(lstm_units,
dropout=dropout,
recurrent_dropout=recurrent_dropout,
return_sequences=True))(x)
x = MaxPooling1D(pad_len)(x)
x = Flatten()(x)
right_enc = x
## Classify
x = Concatenate()([left_enc, right_enc])
x = Dense(dense_units)(x)
x = BatchNormalization()(x)
x = LeakyReLU()(x)
x = Dense(dense_units)(x)
x = BatchNormalization()(x)
x = LeakyReLU()(x)
prediction = Dense(1, activation='sigmoid')(x)
model = Model([left_sent, right_sent], prediction)
model.summary()
return model
# In[ ]: print((x_train.shape, y_train.shape, x_test.shape, y_test.shape)) # ## Creating Model # In[ ]: model = Sequential() model.add(Conv1D(128,3,activation = 'relu', input_shape = (x_train.shape[1],1))) model.add(MaxPooling1D((1))) model.add(Conv1D(256,3,activation = 'relu')) model.add(MaxPooling1D((1))) model.add(Conv1D(512, 3, activation='relu')) model.add(MaxPooling1D((1))) model.add(Conv1D(1024, 3, activation='relu')) model.add(MaxPooling1D((1))) model.add(Flatten()) model.add(Dense(512,activation = 'relu')) model.add(Dropout(0.3))
train_e = keras.utils.to_categorical(train_e,num_classes)
test_e = keras.utils.to_categorical(test_e, num_classes)
train_f = keras.utils.to_categorical(train_f,num_classes)
test_f = keras.utils.to_categorical(test_f, num_classes)
'''
#build model (do PCA through Keras)
model = Sequential()
model.add(
MaxPooling1D(pool_size=(2),
strides=None,
padding='valid',
input_shape=(100, 351)))
model.add(Dropout(0.20, noise_shape=None, seed=None))
model.add(Flatten())
model.add(Dense(55, activation='relu'))
model.add(Dropout(0.4, noise_shape=None, seed=None))
model.add(Dense(2, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adadelta(),
metrics=['accuracy'])
# Y.append(np.array(temp2).reshape(1,1)) train_X,test_X,train_label,test_label = train_test_split(X, Y, test_size=0.1,shuffle=False) len_t = len(train_X) # train_X,valid_X,train_label,valid_label = train_test_split(train_X, train_label, test_size=0.2,shuffle=True) train_X = np.array(train_X) test_X = np.array(test_X) train_label = np.array(train_label) test_label = np.array(test_label) # valid_label = np.array(valid_label) # valid_X = np.array(valid_X) train_X = train_X.reshape(train_X.shape[0],7,50,1) test_X = test_X.reshape(test_X.shape[0],7,50,1) model = Sequential() #add model layers model.add(TimeDistributed(Conv1D(128, kernel_size=1, activation='relu', input_shape=(None,50,1)))) model.add(TimeDistributed(MaxPooling1D(2))) model.add(TimeDistributed(Conv1D(256, kernel_size=1, activation='relu'))) model.add(TimeDistributed(MaxPooling1D(2))) model.add(TimeDistributed(Conv1D(512, kernel_size=1, activation='relu'))) model.add(TimeDistributed(MaxPooling1D(2))) model.add(TimeDistributed(Flatten())) model.add(Bidirectional(LSTM(200,return_sequences=True))) model.add(Dropout(0.25)) model.add(Bidirectional(LSTM(200,return_sequences=False))) model.add(Dropout(0.5)) model.add(Dense(1, activation='linear')) model.compile(optimizer='adam', loss='mse') model.fit(train_X, train_label, validation_data=(test_X,test_label), epochs=200) print(model.summary()) print(model.evaluate(test_X,test_label)) # model.summary()
from keras.models import Sequential
from keras.layers.embeddings import Embedding
from keras.preprocessing import sequence
import numpy as np
from keras.datasets import imdb
(X_train, y_train), (X_test, y_test) = imdb.load_data()
max_word = 400
X_train = sequence.pad_sequences(X_train, maxlen=max_word)
X_test = sequence.pad_sequences(X_test, maxlen=max_word)
vocab_size = np.max([np.max(X_train[i]) for i in range(X_train.shape[0])]) + 1
model = Sequential()
model.add(Embedding(vocab_size, 64, input_length=max_word))
model.add(Conv1D(filters=64, kernel_size=3, padding='same', activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(Dropout(0.25))
model.add(Conv1D(filters=128, kernel_size=3, padding='same',
activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(64, activation='relu'))
model.add(Dense(32, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
print(model.summary())
def GAPConv1DOptLearnerStaticArchitecture(param_trainable, init_wrapper, smpl_params, input_info, faces, emb_size=1000, input_type="3D_POINTS"):
""" Optimised learner network architecture """
# An embedding layer is required to optimise the parameters
optlearner_input = Input(shape=(1,), name="embedding_index")
# Initialise the embedding layers
emb_layers = init_emb_layers(optlearner_input, emb_size, param_trainable, init_wrapper)
optlearner_params = Concatenate(name="parameter_embedding")(emb_layers)
optlearner_params = Reshape(target_shape=(85,), name="learned_params")(optlearner_params)
print("optlearner parameters shape: " +str(optlearner_params.shape))
#exit(1)
# Ground truth parameters and point cloud are inputs to the model as well
gt_params = Input(shape=(85,), name="gt_params")
gt_pc = Input(shape=(6890, 3), name="gt_pc")
print("gt parameters shape: " + str(gt_params.shape))
print("gt point cloud shape: " + str(gt_pc.shape))
# Compute the true offset (i.e. difference) between the ground truth and learned parameters
pi = K.constant(np.pi)
delta_d = Lambda(lambda x: x[0] - x[1], name="delta_d")([gt_params, optlearner_params])
#delta_d = Lambda(lambda x: x[0] - x[1], name="delta_d_no_mod")([gt_params, optlearner_params])
#delta_d = Lambda(lambda x: K.tf.math.floormod(x - pi, 2*pi) - pi, name="delta_d")(delta_d) # custom modulo 2pi of delta_d
#delta_d = custom_mod(delta_d, pi, name="delta_d") # custom modulo 2pi of delta_d
print("delta_d shape: " + str(delta_d.shape))
#exit(1)
# Calculate the (batched) MSE between the learned parameters and the ground truth parameters
false_loss_delta_d = Lambda(lambda x: K.mean(K.square(x), axis=1))(delta_d)
print("delta_d loss shape: " + str(false_loss_delta_d.shape))
#exit(1)
false_loss_delta_d = Reshape(target_shape=(1,), name="delta_d_mse")(false_loss_delta_d)
print("delta_d loss shape: " + str(false_loss_delta_d.shape))
# Load SMPL model and get necessary parameters
optlearner_pc = get_pc(optlearner_params, smpl_params, input_info, faces) # UNCOMMENT
print("optlearner_pc shape: " + str(optlearner_pc.shape))
#exit(1)
#optlearner_pc = Dense(6890*3)(delta_d)
#optlearner_pc = Reshape((6890, 3))(optlearner_pc)
# Get the (batched) Euclidean loss between the learned and ground truth point clouds
pc_euclidean_diff = Lambda(lambda x: x[0] - x[1])([gt_pc, optlearner_pc])
pc_euclidean_dist = Lambda(lambda x: K.sum(K.square(x),axis=-1))(pc_euclidean_diff)
print('pc euclidean dist '+str(pc_euclidean_dist.shape))
#exit(1)
false_loss_pc = Lambda(lambda x: K.mean(x, axis=1))(pc_euclidean_dist)
false_loss_pc = Reshape(target_shape=(1,), name="pc_mean_euc_dist")(false_loss_pc)
print("point cloud loss shape: " + str(false_loss_pc.shape))
#exit(1)
# Gather sets of points and compute their cross product to get mesh normals
# In order of: right hand, right wrist, right forearm, right bicep end, right bicep, right shoulder, top of cranium, left shoulder, left bicep, left bicep end, left forearm, left wrist, left hand,
# chest, belly/belly button, back of neck, upper back, central back, lower back/tailbone,
# left foot, left over-ankle, left shin, left over-knee, left quadricep, left hip, right, hip, right, quadricep, right over-knee, right shin, right, over-ankle, right foot
vertex_list = [5674, 5705, 5039, 5151, 4977, 4198, 411, 606, 1506, 1682, 1571, 2244, 2212,
3074, 3500, 460, 2878, 3014, 3021,
3365, 4606, 4588, 4671, 6877, 1799, 5262, 3479, 1187, 1102, 1120, 6740]
#face_array = np.array([11396, 8620, 7866, 5431, 6460, 1732, 4507])
pc_euclidean_diff_NOGRAD = Lambda(lambda x: K.stop_gradient(x))(pc_euclidean_diff) # This is added to avoid influencing embedding layer parameters by a "bad" gradient network
vertex_diff_NOGRAD = Lambda(lambda x: K.tf.gather(x, np.array(vertex_list).astype(np.int32), axis=-2))(pc_euclidean_diff_NOGRAD)
print("vertex_diff_NOGRAD shape: " + str(vertex_diff_NOGRAD.shape))
vertex_diff_NOGRAD = Flatten()(vertex_diff_NOGRAD)
#exit(1)
face_array = np.array([[face for face in faces if vertex in face][0] for vertex in vertex_list]) # only take a single face for each vertex
print("face_array shape: " + str(face_array.shape))
gt_normals = get_mesh_normals(gt_pc, face_array, layer_name="gt_cross_product")
print("gt_normals shape: " + str(gt_normals.shape))
opt_normals = get_mesh_normals(optlearner_pc, face_array, layer_name="opt_cross_product")
print("opt_normals shape: " + str(opt_normals.shape))
#exit(1)
# Learn the offset in parameters from the difference between the ground truth and learned mesh normals
diff_normals = Lambda(lambda x: K.tf.cross(x[0], x[1]), name="diff_cross_product")([gt_normals, opt_normals])
diff_normals_NOGRAD = Lambda(lambda x: K.stop_gradient(x))(diff_normals) # This is added to avoid influencing embedding layer parameters by a "bad" gradient network
diff_angles = Lambda(lambda x: K.tf.subtract(x[0], x[1]), name="diff_angle")([gt_normals, opt_normals])
diff_angles_NOGRAD = Lambda(lambda x: K.stop_gradient(x))(diff_angles)
diff_angles_norm_NOGRAD = Lambda(lambda x: K.tf.norm(x, axis=-1), name="diff_angle_norm")(diff_angles_NOGRAD)
dist_angles = Lambda(lambda x: K.mean(K.square(x), axis=-1), name="diff_angle_mse")(diff_angles)
dist_angles_NOGRAD = Lambda(lambda x: K.stop_gradient(x))(dist_angles)
print("diff_angles shape: " + str(diff_angles.shape))
print("dist_angles shape: " + str(dist_angles.shape))
#pc_euclidean_diff_NOGRAD = Lambda(lambda x: K.stop_gradient(x))(pc_euclidean_diff) # This is added to avoid influencing embedding layer parameters by a "bad" gradient network
#print("diff_normals_NOGRAD shape: " + str(diff_normals_NOGRAD.shape))
diff_normals_NOGRAD = Flatten()(diff_normals_NOGRAD)
diff_angles_NOGRAD = Flatten()(diff_angles_NOGRAD)
mesh_diff_NOGRAD = Concatenate()([diff_normals_NOGRAD, dist_angles_NOGRAD])
if input_type == "3D_POINTS":
optlearner_architecture = Dense(2**9, activation="relu")(vertex_diff_NOGRAD)
if input_type == "MESH_NORMALS":
#optlearner_architecture = Dense(2**11, activation="relu")(diff_angles_norm_NOGRAD)
#optlearner_architecture = Dense(2**11, activation="relu")(diff_angles_NOGRAD)
optlearner_architecture = Dense(2**9, activation="relu")(mesh_diff_NOGRAD)
optlearner_architecture = BatchNormalization()(optlearner_architecture)
optlearner_architecture = Dropout(0.5)(optlearner_architecture)
print('optlearner_architecture shape: '+str(optlearner_architecture.shape))
optlearner_architecture = Reshape((optlearner_architecture.shape[1].value, 1))(optlearner_architecture)
print('optlearner_architecture shape: '+str(optlearner_architecture.shape))
#optlearner_architecture = Conv1D(64, 5, activation="relu")(optlearner_architecture)
optlearner_architecture = Conv1D(64, 5, activation="relu")(optlearner_architecture)
optlearner_architecture = Conv1D(64, 5, activation="relu")(optlearner_architecture)
optlearner_architecture = BatchNormalization()(optlearner_architecture)
optlearner_architecture = Dropout(0.5)(optlearner_architecture)
optlearner_architecture = MaxPooling1D(3)(optlearner_architecture)
optlearner_architecture = Conv1D(128, 5, activation="relu")(optlearner_architecture)
optlearner_architecture = Conv1D(128, 5, activation="relu")(optlearner_architecture)
optlearner_architecture = BatchNormalization()(optlearner_architecture)
optlearner_architecture = Dropout(0.5)(optlearner_architecture)
optlearner_architecture = MaxPooling1D(3)(optlearner_architecture)
print('optlearner_architecture shape: '+str(optlearner_architecture.shape))
#optlearner_architecture = Conv1D(128, 3, activation="relu")(optlearner_architecture)
optlearner_architecture = Conv1D(256, 3, activation="relu")(optlearner_architecture)
optlearner_architecture = Conv1D(256, 3, activation="relu")(optlearner_architecture)
optlearner_architecture = BatchNormalization()(optlearner_architecture)
optlearner_architecture = Dropout(0.5)(optlearner_architecture)
optlearner_architecture = MaxPooling1D(2)(optlearner_architecture)
optlearner_architecture = Conv1D(512, 3, activation="relu")(optlearner_architecture)
optlearner_architecture = Conv1D(512, 3, activation="relu")(optlearner_architecture)
optlearner_architecture = BatchNormalization()(optlearner_architecture)
#optlearner_architecture = Dropout(0.5)(optlearner_architecture)
#optlearner_architecture = MaxPooling1D(2)(optlearner_architecture)
#optlearner_architecture = MaxPooling1D(3)(optlearner_architecture)
print('optlearner_architecture shape: '+str(optlearner_architecture.shape))
#optlearner_architecture = Conv1D(1024, 3, activation="relu")(optlearner_architecture)
#optlearner_architecture = Conv1D(1024, 3, activation="relu")(optlearner_architecture)
#optlearner_architecture = BatchNormalization()(optlearner_architecture)
#optlearner_architecture = MaxPooling1D(3)(optlearner_architecture)
#print('optlearner_architecture shape: '+str(optlearner_architecture.shape))
optlearner_architecture = GlobalAveragePooling1D()(optlearner_architecture)
#print('optlearner_architecture shape: '+str(optlearner_architecture.shape))
#optlearner_architecture = Flatten()(optlearner_architecture)
#conv_shape = int(np.prod(optlearner_architecture.shape[1:]))
#optlearner_architecture = Reshape((conv_shape,))(optlearner_architecture)
print('optlearner_architecture shape: '+str(optlearner_architecture.shape))
optlearner_architecture = Dropout(0.5)(optlearner_architecture)
#optlearner_architecture = Dense(2**7, activation="relu")(optlearner_architecture)
print('optlearner_architecture shape: '+str(optlearner_architecture.shape))
#delta_d_hat = Dense(85, activation=pos_scaled_tanh, name="delta_d_hat")(optlearner_architecture)
delta_d_hat = Dense(85, activation="linear", name="delta_d_hat")(optlearner_architecture)
#delta_d_hat = Dense(85, activation=centred_linear, name="delta_d_hat")(optlearner_architecture)
print('delta_d_hat shape: '+str(delta_d_hat.shape))
#exit(1)
# Calculate the (batched) MSE between the learned and ground truth offset in the parameters
delta_d_NOGRAD = Lambda(lambda x: K.stop_gradient(x))(delta_d)
false_loss_delta_d_hat = Lambda(lambda x: K.mean(K.square(x[0] - x[1]), axis=1))([delta_d_NOGRAD, delta_d_hat])
#false_loss_delta_d_hat = Lambda(lambda x: K.sum(K.square(x[0] - x[1]), axis=1))([delta_d_NOGRAD, delta_d_hat])
#false_loss_delta_d_hat = Lambda(lambda x: mape(x[0], x[1]))([delta_d_NOGRAD, delta_d_hat])
false_loss_delta_d_hat = Reshape(target_shape=(1,), name="delta_d_hat_mse")(false_loss_delta_d_hat)
print("delta_d_hat loss shape: " + str(false_loss_delta_d_hat.shape))
#false_sin_loss_delta_d_hat = get_sin_metric(delta_d_NOGRAD, delta_d_hat)
false_sin_loss_delta_d_hat = get_sin_metric(delta_d_NOGRAD, delta_d_hat, average=False)
false_sin_loss_delta_d_hat = Lambda(lambda x: x, name="delta_d_hat_sin_output")(false_sin_loss_delta_d_hat)
print("delta_d_hat sin loss shape: " + str(false_sin_loss_delta_d_hat.shape))
# Prevent model from using the delta_d_hat gradient in final loss
delta_d_hat_NOGRAD = Lambda(lambda x: K.stop_gradient(x), name='optlearner_output_NOGRAD')(delta_d_hat)
# False loss designed to pass the learned offset as a gradient to the embedding layer
false_loss_smpl = Multiply(name="smpl_diff")([optlearner_params, delta_d_hat_NOGRAD])
print("smpl loss shape: " + str(false_loss_smpl.shape))
#return [optlearner_input, gt_params, gt_pc], [optlearner_params, false_loss_delta_d, optlearner_pc, false_loss_pc, false_loss_delta_d_hat, false_sin_loss_delta_d_hat, false_loss_smpl, delta_d, delta_d_hat, delta_d_hat_NOGRAD]
return [optlearner_input, gt_params, gt_pc], [optlearner_params, false_loss_delta_d, optlearner_pc, false_loss_pc, false_loss_delta_d_hat, false_sin_loss_delta_d_hat, false_loss_smpl, delta_d, delta_d_hat, dist_angles]
# MODEL SETUP #
#########################
# Set training hyper-parameters.
epochs = 22
batch_size = 64
learn_rate = 0.001
drop_prob = 0.75
optimiser = Adam(lr=learn_rate)
branch1 = Sequential()
branch1.add(BatchNormalization(input_shape=(num_seq_steps, num_seq_features)))
branch1.add(Conv1D(50, 5, kernel_initializer='he_uniform', activation='relu',
input_shape=(num_seq_steps, num_seq_features)))
branch1.add(MaxPooling1D(2))
branch1.add(Dropout(drop_prob))
branch1.add(BatchNormalization())
branch1.add(Conv1D(50, 11, kernel_initializer='he_uniform', activation='relu'))
branch1.add(MaxPooling1D(2))
branch1.add(Dropout(drop_prob))
branch1.add(BatchNormalization())
branch1.add(Bidirectional(GRU(150, kernel_initializer='he_uniform', activation='relu')))
branch1.add(Dropout(drop_prob))
branch2 = Sequential()
branch2.add(Activation('linear', input_shape=(num_features,)))
model = Sequential()
def heartnet(load_path,
activation_function='relu',
bn_momentum=0.99,
bias=False,
dropout_rate=0.5,
dropout_rate_dense=0.0,
eps=1.1e-5,
kernel_size=5,
l2_reg=0.0,
l2_reg_dense=0.0,
lr=0.0012843784,
lr_decay=0.0001132885,
maxnorm=10000.,
padding='valid',
random_seed=1,
subsam=2,
num_filt=(8, 4),
num_dense=20,
FIR_train=False,
trainable=True,
type=1,
num_class=2,
num_class_domain=1,
hp_lambda=0,
batch_size=1024,
optim='SGD',
segments='0101'):
#num_dense = 20 default
input = Input(shape=(2500, 1))
xx = branch(input, num_filt, kernel_size, random_seed, padding, bias,
maxnorm, l2_reg, eps, bn_momentum, activation_function,
dropout_rate, subsam, trainable)
xx = branch(input, num_filt, kernel_size, random_seed, padding, bias,
maxnorm, l2_reg, eps, bn_momentum, activation_function,
dropout_rate, subsam, trainable)
xx = res_block(xx, 32, kernel_size, 2, 'same', random_seed, bias, maxnorm,
l2_reg, eps, bn_momentum, activation_function, dropout_rate,
subsam, trainable)
xx = res_block(xx, 64, kernel_size, 2, 'same', random_seed, bias, maxnorm,
l2_reg, eps, bn_momentum, activation_function, dropout_rate,
subsam, trainable)
xx = MaxPooling1D(pool_size=2)(xx)
# xx = res_block(xx,128,kernel_size,2,'same',random_seed,bias,maxnorm,l2_reg,
# eps,bn_momentum,activation_function,dropout_rate,subsam,trainable)
# xx = res_block(xx,128,kernel_size,1,'same',random_seed,bias,maxnorm,l2_reg,
# eps,bn_momentum,activation_function,dropout_rate,subsam,trainable)
# xx = res_block(xx,128,kernel_size,2,'same',random_seed,bias,maxnorm,l2_reg,
# eps,bn_momentum,activation_function,dropout_rate,subsam,trainable,cat=False)
# xx = res_block(xx,128,kernel_size,2,'same',random_seed,bias,maxnorm,l2_reg,
# eps,bn_momentum,activation_function,dropout_rate,subsam,trainable,cat=False)
xx = Conv1D(128,
kernel_size=kernel_size,
kernel_initializer=initializers.he_normal(seed=random_seed),
padding=padding,
strides=2,
use_bias=bias,
kernel_constraint=max_norm(maxnorm),
trainable=trainable,
kernel_regularizer=l2(l2_reg))(xx)
xx = MaxPooling1D(pool_size=2)(xx)
merged = Flatten()(xx)
merged = Dropout(rate=dropout_rate, seed=random_seed)(merged)
dann_in = GradientReversal(hp_lambda=hp_lambda, name='grl')(merged)
dsc = Dense(80,
activation=activation_function,
kernel_initializer=initializers.he_normal(seed=random_seed),
use_bias=bias,
kernel_constraint=max_norm(maxnorm),
kernel_regularizer=l2(l2_reg_dense),
name='domain_dense')(dann_in)
dsc = Dense(num_class_domain, activation='softmax', name="domain")(dsc)
merged = Dense(50,
activation=activation_function,
kernel_initializer=initializers.he_normal(seed=random_seed),
use_bias=bias,
kernel_constraint=max_norm(maxnorm),
kernel_regularizer=l2(l2_reg_dense),
name='class_dense')(merged)
merged = Dense(num_class, activation='softmax', name="class")(merged)
model = Model(inputs=input, outputs=[merged, dsc])
if load_path:
model.load_weights(filepath=load_path, by_name=False)
#if load_path: # If path for loading model was specified
#model.load_weights(filepath='../../models_dbt_dann/fold_a_gt 2019-09-09 16:53:52.063276/weights.0041-0.6907.hdf5', by_name=True)
# models/fold_a_gt 2019-09-04 17:36:52.860817/weights.0200-0.7135.hdf5
if optim == 'Adam':
opt = Adam(lr=lr, decay=lr_decay)
else:
opt = SGD(lr=lr, decay=lr_decay)
if (num_class_domain > 1):
domain_loss_function = 'categorical_crossentropy'
else:
domain_loss_function = 'binary_crossentropy'
model.compile(optimizer=opt,
loss={
'class': 'categorical_crossentropy',
'domain': domain_loss_function
},
loss_weights=[1, 1],
metrics=['accuracy'])
#model.compile(optimizer=opt, loss={'class':'categorical_crossentropy','domain':'categorical_crossentropy'}, metrics=['accuracy'])
return model
embedding_matrix[i] = embedding_vector
# load pre-trained word embeddings into an Embedding layer
# note that we set trainable = False so as to keep the embeddings fixed
embedding_layer = Embedding(num_words + 1,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=False)
print('Training model.')
# train a 1D convnet with global maxpooling
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH, ), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
x = Conv1D(128, 2, activation='relu')(embedded_sequences)
x = MaxPooling1D(2)(x)
x = Conv1D(128, 2, activation='relu')(x)
x = MaxPooling1D(2)(x)
x = Conv1D(128, 2, activation='relu')(x)
x = MaxPooling1D(2)(x)
x = Flatten()(x)
x = Dense(128, activation='relu')(x)
preds = Dense(2, activation='softmax')(x)
model = Model(sequence_input, preds)
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['acc'])
model.fit(x_train,
y_train,
# Static model do not have embedding layer
if model_type == "CNN-static":
z = Dropout(dropout_prob[0])(model_input)
else:
z = Embedding(len(vocabulary_inv), embedding_dim, input_length=sequence_length, name="embedding")(model_input)
z = Dropout(dropout_prob[0])(z)
# Convolutional block
conv_blocks = []
for sz in filter_sizes:
conv = Convolution1D(filters=num_filters,
kernel_size=sz,
padding="valid",
activation="relu",
strides=1)(z)
conv = MaxPooling1D(pool_size=2)(conv)
conv = Flatten()(conv)
conv_blocks.append(conv)
z = Concatenate()(conv_blocks) if len(conv_blocks) > 1 else conv_blocks[0]
z = Dropout(dropout_prob[1])(z)
z = Dense(hidden_dims, activation="relu")(z)
model_output = Dense(1, activation="sigmoid")(z)
model = Model(model_input, model_output)
model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])
# Initialize weights with word2vec
if model_type == "CNN-non-static":
embedding_layer = model.get_layer("embedding")
embedding_layer.set_weights(embedding_weights)
incorrect_doc3.add(
Embedding(
n_symbols + 1,
embedding_size,
mask_zero=True,
weights=[
embedding_weights
])) # note you have to put embedding weights in a list by convention
incorrect_doc3.add(Dropout(0.25))
incorrect_doc3.add(
Convolution1D(nb_filter=nb_filter,
filter_length=filter_length,
border_mode='valid',
activation='relu',
subsample_length=1))
incorrect_doc3.add(MaxPooling1D(pool_length=pool_length))
incorrect_doc3.add(LSTM(lstm_output_size))
incorrect_doc3.add(Dense(dense_units))
## Cosine Merged Layers, later each layer is reshaped and then a lamda layer is applied which gives the cosine similarity.
# Query - Correct Document
merged_q_cd = Sequential()
merged_q_cd.add(
Merge([query, correct_doc], mode='cos', name='q_cd', dot_axes=1))
merged_q_cd.add(Reshape((1, )))
merged_q_cd.add(Lambda(lambda x: 1 - x))
# Query - Incorrect Document 1
merged_q_incd1 = Sequential()
merged_q_incd1.add(
Merge([query, incorrect_doc1], mode='cos', name='q_incd1', dot_axes=1))
# load pre-trained word embeddings into an Embedding layer
# note that we set trainable = False so as to keep the embeddings fixed
embedding_layer = Embedding(num_words,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=False)
print('training model')
# train a 1D convnet with global maxpooling
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH, ),
dtype='float32') # input layer
embedded_sequences = embedding_layer(sequence_input) # embedding layer
x = Conv1D(50, 5, activation='relu')(embedded_sequences) # convolution layer
x = MaxPooling1D(5)(x) # pooling layer
x = Conv1D(50, 5, activation='relu')(x) # convolution layer
x = MaxPooling1D(5)(x) # pooling layer
# (un)comment to switch betweeen LSTM and pure CN
x = LSTM(128)(x) # lstm layer
# (un)comment to switch betweeen LSTM and pure CNN
#x = Conv1D(128, 5, activation='relu')(x) # convolution layer
#x = GlobalMaxPooling1D()(x) # pooling layer
x = Dense(128, activation='relu')(x) # dense deep layer
preds = Dense(len(labels_simp), activation='softmax')(x) # output layer
model = Model(sequence_input, preds)
# train
model.compile(loss='binary_crossentropy',
optimizer='sgd',
metrics=['accuracy'])
def DF(input_shape=None, emb_size=None):
# -----------------Entry flow -----------------
input_data = Input(shape=input_shape)
filter_num = ['None', 32, 64, 128, 256]
kernel_size = ['None', 8, 8, 8, 8]
conv_stride_size = ['None', 1, 1, 1, 1]
pool_stride_size = ['None', 4, 4, 4, 4]
pool_size = ['None', 8, 8, 8, 8]
model = Conv1D(filters=filter_num[1],
kernel_size=kernel_size[1],
strides=conv_stride_size[1],
padding='same',
name='block1_conv1')(input_data)
model = ELU(alpha=1.0, name='block1_adv_act1')(model)
model = Conv1D(filters=filter_num[1],
kernel_size=kernel_size[1],
strides=conv_stride_size[1],
padding='same',
name='block1_conv2')(model)
model = ELU(alpha=1.0, name='block1_adv_act2')(model)
model = MaxPooling1D(pool_size=pool_size[1],
strides=pool_stride_size[1],
padding='same',
name='block1_pool')(model)
model = Dropout(0.1, name='block1_dropout')(model)
model = Conv1D(filters=filter_num[2],
kernel_size=kernel_size[2],
strides=conv_stride_size[2],
padding='same',
name='block2_conv1')(model)
model = Activation('relu', name='block2_act1')(model)
model = Conv1D(filters=filter_num[2],
kernel_size=kernel_size[2],
strides=conv_stride_size[2],
padding='same',
name='block2_conv2')(model)
model = Activation('relu', name='block2_act2')(model)
model = MaxPooling1D(pool_size=pool_size[2],
strides=pool_stride_size[3],
padding='same',
name='block2_pool')(model)
model = Dropout(0.1, name='block2_dropout')(model)
model = Conv1D(filters=filter_num[3],
kernel_size=kernel_size[3],
strides=conv_stride_size[3],
padding='same',
name='block3_conv1')(model)
model = Activation('relu', name='block3_act1')(model)
model = Conv1D(filters=filter_num[3],
kernel_size=kernel_size[3],
strides=conv_stride_size[3],
padding='same',
name='block3_conv2')(model)
model = Activation('relu', name='block3_act2')(model)
model = MaxPooling1D(pool_size=pool_size[3],
strides=pool_stride_size[3],
padding='same',
name='block3_pool')(model)
model = Dropout(0.1, name='block3_dropout')(model)
model = Conv1D(filters=filter_num[4],
kernel_size=kernel_size[4],
strides=conv_stride_size[4],
padding='same',
name='block4_conv1')(model)
model = Activation('relu', name='block4_act1')(model)
model = Conv1D(filters=filter_num[4],
kernel_size=kernel_size[4],
strides=conv_stride_size[4],
padding='same',
name='block4_conv2')(model)
model = Activation('relu', name='block4_act2')(model)
model = MaxPooling1D(pool_size=pool_size[4],
strides=pool_stride_size[4],
padding='same',
name='block4_pool')(model)
output = Flatten()(model)
dense_layer = Dense(emb_size, name='FeaturesVec')(output)
shared_conv2 = Model(inputs=input_data, outputs=dense_layer)
return shared_conv2
n_signals = 1 #So far each instance is one signal. We will diversify them in next step
n_outputs = 1 #Binary Classification
#Build the model
verbose, epochs, batch_size = True, 15, 16
n_steps, n_length = 40, 10
X_train = X_train.reshape((X_train.shape[0], n_steps, n_length, n_signals))
# define model
model = Sequential()
model.add(
TimeDistributed(Conv1D(filters=64, kernel_size=3, activation='relu'),
input_shape=(None, n_length, n_signals)))
model.add(TimeDistributed(Conv1D(filters=64, kernel_size=3,
activation='relu')))
model.add(TimeDistributed(Dropout(0.5)))
model.add(TimeDistributed(MaxPooling1D(pool_size=2)))
model.add(TimeDistributed(Flatten()))
model.add(LSTM(100))
model.add(Dropout(0.5))
model.add(Dense(100, activation='relu'))
model.add(Dense(n_outputs, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=[keras_auc])
model.fit(X_train,
y_train,
epochs=epochs,
batch_size=batch_size,
verbose=verbose)
model.save_weights('model1.hdf5')
#%%time
def build_network(**params):
from keras.models import Model
from keras.layers import Input, Conv1D, BatchNormalization, Activation, Add, MaxPooling1D, Dropout
from keras.layers.core import Lambda, Dense
#from keras.layers.core import Activation
from keras.layers.wrappers import TimeDistributed
from keras.optimizers import Adam
def zeropad(x):
y = K.zeros_like(x)
return K.concatenate([x, K.zeros_like(x)], axis=2)
def zeropad_output_shape(input_shape):
shape = list(input_shape)
assert len(shape) == 3
shape[2] *= 2
return tuple(shape)
inputs = Input(shape=[None, 1], dtype='float32', name='inputs')
# 1st Conv layer (number of filters = 32)
layer = Conv1D(filters=32,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(inputs)
layer = BatchNormalization()(layer)
layer = Activation(params["conv_activation"])(layer)
# 0th resnet layer (number of filters = 32, subsampling = 1)
shortcut_0 = MaxPooling1D(pool_size=1)(layer)
layer = Conv1D(filters=32,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Dropout(0.2)(layer)
layer = Conv1D(filters=32,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = Add()([shortcut_0, layer])
# 1st resnet layer (number of filters = 32, subsampling = 2)
shortcut_1 = MaxPooling1D(pool_size=2)(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Conv1D(filters=32,
kernel_size=16,
strides=2,
padding='same',
kernel_initializer='he_normal')(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Dropout(0.2)(layer)
layer = Conv1D(filters=32,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = Add()([shortcut_1, layer])
# 2nd resnet layer (number of filters = 32, subsampling = 1)
shortcut_2 = MaxPooling1D(pool_size=1)(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Conv1D(filters=32,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Dropout(0.2)(layer)
layer = Conv1D(filters=32,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = Add()([shortcut_2, layer])
# 3rd resnet layer (number of filters = 32, subsampling = 2)
shortcut_3 = MaxPooling1D(pool_size=2)(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Conv1D(filters=32,
kernel_size=16,
strides=2,
padding='same',
kernel_initializer='he_normal')(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Dropout(0.2)(layer)
layer = Conv1D(filters=32,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = Add()([shortcut_3, layer])
# 4th resnet layer (number of filters = 64, subsampling = 1, zero-pad)
shortcut_4 = MaxPooling1D(pool_size=1)(layer)
shortcut_4 = Lambda(zeropad, output_shape=zeropad_output_shape)(shortcut_4)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Conv1D(filters=64,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Dropout(0.2)(layer)
layer = Conv1D(filters=64,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = Add()([shortcut_4, layer])
# 5th resnet layer (number of filters = 64, subsampling = 2)
shortcut_5 = MaxPooling1D(pool_size=2)(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Conv1D(filters=64,
kernel_size=16,
strides=2,
padding='same',
kernel_initializer='he_normal')(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Dropout(0.2)(layer)
layer = Conv1D(filters=64,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = Add()([shortcut_5, layer])
# 6th resnet layer (number of filters = 64, subsampling = 1)
shortcut_6 = MaxPooling1D(pool_size=1)(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Conv1D(filters=64,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Dropout(0.2)(layer)
layer = Conv1D(filters=64,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = Add()([shortcut_6, layer])
# 7th resnet layer (number of filters = 64, subsampling = 2)
shortcut_7 = MaxPooling1D(pool_size=2)(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Conv1D(filters=64,
kernel_size=16,
strides=2,
padding='same',
kernel_initializer='he_normal')(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Dropout(0.2)(layer)
layer = Conv1D(filters=64,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = Add()([shortcut_7, layer])
# 8th resnet layer (number of filters = 128, subsampling = 1, zero-pad)
shortcut_8 = MaxPooling1D(pool_size=1)(layer)
shortcut_8 = Lambda(zeropad, output_shape=zeropad_output_shape)(shortcut_8)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Conv1D(filters=128,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Dropout(0.2)(layer)
layer = Conv1D(filters=128,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = Add()([shortcut_8, layer])
# 9th resnet layer (number of filters = 128, subsampling = 2)
shortcut_9 = MaxPooling1D(pool_size=2)(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Conv1D(filters=128,
kernel_size=16,
strides=2,
padding='same',
kernel_initializer='he_normal')(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Dropout(0.2)(layer)
layer = Conv1D(filters=128,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = Add()([shortcut_9, layer])
# 10th resnet layer (number of filters = 128, subsampling = 1)
shortcut_10 = MaxPooling1D(pool_size=1)(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Conv1D(filters=128,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Dropout(0.2)(layer)
layer = Conv1D(filters=128,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = Add()([shortcut_10, layer])
# 11th resnet layer (number of filters = 128, subsampling = 2)
shortcut_11 = MaxPooling1D(pool_size=2)(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Conv1D(filters=128,
kernel_size=16,
strides=2,
padding='same',
kernel_initializer='he_normal')(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Dropout(0.2)(layer)
layer = Conv1D(filters=128,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = Add()([shortcut_11, layer])
# 12th resnet layer (number of filters = 256, subsampling = 1, zero-pad)
shortcut_12 = MaxPooling1D(pool_size=1)(layer)
shortcut_12 = Lambda(zeropad,
output_shape=zeropad_output_shape)(shortcut_12)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Conv1D(filters=256,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Dropout(0.2)(layer)
layer = Conv1D(filters=256,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = Add()([shortcut_12, layer])
# 13th resnet layer (number of filters = 256, subsampling = 2)
shortcut_13 = MaxPooling1D(pool_size=2)(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Conv1D(filters=256,
kernel_size=16,
strides=2,
padding='same',
kernel_initializer='he_normal')(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Dropout(0.2)(layer)
layer = Conv1D(filters=256,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = Add()([shortcut_13, layer])
# 14th resnet layer (number of filters = 256, subsampling = 1)
shortcut_14 = MaxPooling1D(pool_size=1)(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Conv1D(filters=256,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Dropout(0.2)(layer)
layer = Conv1D(filters=256,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = Add()([shortcut_14, layer])
# 15th resnet layer (number of filters = 256, subsampling = 2)
shortcut_15 = MaxPooling1D(pool_size=2)(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Conv1D(filters=256,
kernel_size=16,
strides=2,
padding='same',
kernel_initializer='he_normal')(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Dropout(0.2)(layer)
layer = Conv1D(filters=256,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = Add()([shortcut_15, layer])
# 16th resnet layer (number of filters = 256, subsampling = 1, zero-pad)
shortcut_16 = MaxPooling1D(pool_size=1)(layer)
shortcut_16 = Lambda(zeropad,
output_shape=zeropad_output_shape)(shortcut_16)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Conv1D(filters=256,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Dropout(0.2)(layer)
layer = Conv1D(filters=256,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = Add()([shortcut_16, layer])
# 17th resnet layer (number of filters = 256, subsampling = 2)
shortcut_17 = MaxPooling1D(pool_size=2)(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Conv1D(filters=256,
kernel_size=16,
strides=2,
padding='same',
kernel_initializer='he_normal')(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Dropout(0.2)(layer)
layer = Conv1D(filters=256,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = Add()([shortcut_17, layer])
# 18th resnet layer (number of filters = 256, subsampling = 1)
shortcut_18 = MaxPooling1D(pool_size=1)(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Conv1D(filters=256,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Dropout(0.2)(layer)
layer = Conv1D(filters=256,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = Add()([shortcut_18, layer])
# 19th resnet layer (number of filters = 256, subsampling = 2)
shortcut_19 = MaxPooling1D(pool_size=2)(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Conv1D(filters=256,
kernel_size=16,
strides=2,
padding='same',
kernel_initializer='he_normal')(layer)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = Dropout(0.2)(layer)
layer = Conv1D(filters=256,
kernel_size=16,
strides=1,
padding='same',
kernel_initializer='he_normal')(layer)
layer = Add()([shortcut_19, layer])
#Output layer (number of output = class number)
layer = BatchNormalization()(layer)
layer = Activation('relu')(layer)
layer = TimeDistributed(Dense(params["num_categories"]))(layer)
output = Activation('softmax')(layer)
model = Model(inputs=[inputs], outputs=[output])
optimizer = Adam(lr=params["learning_rate"])
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
return model
return (tf.math.sign(x) + 1) / 2
model = Sequential()
model.add(Conv1D(16, 128, input_shape=(428, 1)))
model.add(BatchNormalization(center=True, scale=True))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Conv1D(8, 64))
model.add(BatchNormalization(center=True, scale=True))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(MaxPooling1D(2))
model.add(Conv1D(8, 32))
model.add(BatchNormalization(center=True, scale=True))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Conv1D(4, 16))
model.add(BatchNormalization(center=True, scale=True))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(MaxPooling1D(2))
# model.add(Conv1D(32,2))
# model.add(BatchNormalization(center=True, scale=True))
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
print('Build model...')
model = Sequential()
model.add(Embedding(max_features, embedding_size, input_length=maxlen))
model.add(Dropout(0.25))
model.add(
Convolution1D(nb_filter=nb_filter,
filter_length=filter_length,
border_mode='valid',
activation='relu',
subsample_length=1))
model.add(MaxPooling1D(pool_length=pool_length))
model.add(LSTM(lstm_output_size))
model.add(Dense(1))
model.add(Activation('sigmoid'))
import keras.optimizers
opt = keras.optimizers.adam(0.01)
model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])
print('Train...')
model.fit(X_train,
y_train,
batch_size=batch_size,
y_test = out_test
y_valid = out_val
padded_docs = X_train
vpadded_docs = X_val
tpadded_docs = X_test
# create the model
embedding_vecor_length = 100
model = Sequential()
model.add(
Embedding(top_words,
embedding_vecor_length,
input_length=max_review_length))
model.add(
Conv1D(64, kernel_size, padding='valid', activation='relu', strides=1))
model.add(MaxPooling1D(pool_size=pool_size))
model.add(LSTM(lstm_output_size, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(nclass, activation='softmax'))
# compile the model
model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['accuracy'])
# summarize the model
print(model.summary())
# fit the model
model.fit(padded_docs,
y_train,
epochs=1,
verbose=0,
validation_data=(vpadded_docs, y_valid))
def main(args):
args = parse_train_args(args)
X_train = np.load(args.X_train)
X_validate = np.load(args.X_validate)
y_train = np.load(args.y_train)
y_validate = np.load(args.y_validate)
model_save_path = args.model_save_path
tensorboard_output_dir = args.tensorboard_output_dir
def lr_schedule(epoch, lr):
if epoch > 50:
if epoch % 10 == 0:
return lr * 0.95
return lr
lr_callback = LearningRateScheduler(lr_schedule)
callbacks = [
lr_callback,
TensorBoard(log_dir=tensorboard_output_dir,
write_images=True,
write_grads=True,
histogram_freq=5,
batch_size=10000)
]
input_shape = X_train.shape[1:]
num_output_classes = y_train.shape[1]
input_layer = Input(shape=input_shape)
conv_1 = Conv1D(filters=16,
kernel_size=4,
padding='same',
activation='selu',
kernel_regularizer=l2(1e-3))(input_layer)
pool_1 = MaxPooling1D(pool_size=(5), strides=1)(conv_1)
conv_2 = Conv1D(filters=32,
kernel_size=4,
padding='same',
activation='selu',
kernel_regularizer=l2(1e-3))(pool_1)
pool_2 = MaxPooling1D(pool_size=(4), strides=1)(conv_2)
conv_3 = Conv1D(filters=48,
kernel_size=4,
padding='same',
activation='selu',
kernel_regularizer=l2(1e-3))(pool_2)
pool_3 = MaxPooling1D(pool_size=(3), strides=1)(conv_3)
flatten = Flatten()(pool_3)
dn_1 = Dense(336, activation='selu')(flatten)
drop = Dropout(0.5)(dn_1)
predictions = Dense(num_output_classes, activation='softmax')(drop)
model = Model(input_layer, predictions)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print(model.summary())
batch_size = 10000
epochs = 50
model.fit(X_train,
y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(X_validate, y_validate),
callbacks=callbacks)
model.save(model_save_path)
None,
52,
), name='char_input')
embed_char_out = TimeDistributed(Embedding(
len(char2Idx),
30,
embeddings_initializer=RandomUniform(minval=-0.5, maxval=0.5)),
name='char_embedding')(character_input)
dropout = Dropout(0.5)(embed_char_out)
conv1d_out = TimeDistributed(
Conv1D(kernel_size=3,
filters=30,
padding='same',
activation='tanh',
strides=1))(dropout)
maxpool_out = TimeDistributed(MaxPooling1D(52))(conv1d_out)
char = TimeDistributed(Flatten())(maxpool_out)
char = Dropout(0.5)(char)
output = concatenate([words, casing, char])
output = Bidirectional(
LSTM(200, return_sequences=True, dropout=0.50,
recurrent_dropout=0.25))(output)
output = TimeDistributed(Dense(len(label2Idx), activation='softmax'))(output)
model = Model(inputs=[words_input, casing_input, character_input],
outputs=[output])
model.compile(loss='sparse_categorical_crossentropy', optimizer='nadam')
model.summary()
plot_model(model, to_file='model.png')
for epoch in range(epochs):
print("Epoch %d/%d" % (epoch, epochs))
def create_submodel(embedding_layers,
submod_layer_descriptor,
cnn_ks,
cnn_dilation=None):
model = Sequential()
if len(embedding_layers) == 1:
model.add(embedding_layers[0])
else:
concat_embedding_layers(embedding_layers, model)
for layer_descriptor in submod_layer_descriptor.split(","):
if "=" not in layer_descriptor:
continue
ld = layer_descriptor.split("=")
layer_name = ld[0]
params = None
if len(ld) > 1:
params = ld[1].split("-")
if layer_name == "dropout":
model.add(Dropout(float(params[0])))
elif layer_name == "lstm":
if params[1] == "True":
return_seq = True
else:
return_seq = False
model.add(LSTM(units=int(params[0]), return_sequences=return_seq))
elif layer_name == "gru":
if params[1] == "True":
return_seq = True
else:
return_seq = False
model.add(GRU(units=int(params[0]), return_sequences=return_seq))
elif layer_name == "bilstm":
if params[1] == "True":
return_seq = True
else:
return_seq = False
model.add(
Bidirectional(
LSTM(units=int(params[0]), return_sequences=return_seq)))
elif layer_name == "conv1d":
if cnn_dilation is None:
model.add(
Conv1D(filters=int(params[0]),
kernel_size=int(cnn_ks),
padding='same',
activation='relu'))
else:
model.add(
Conv1D(filters=int(params[0]),
kernel_size=int(cnn_ks),
dilation_rate=int(cnn_dilation),
padding='same',
activation='relu'))
elif layer_name == "maxpooling1d":
size = params[0]
if size == "v":
size = int(cnn_ks)
else:
size = int(params[0])
model.add(MaxPooling1D(pool_size=size))
elif layer_name == "gmaxpooling1d":
model.add(GlobalMaxPooling1D())
elif layer_name == "dense":
model.add(Dense(int(params[0]), activation=params[1]))
return model
