def train(self, words_filepath, tmp_dir, nb_samples=10000000):
'''
Тренируем модель на словах в указанном файле words_filepath.
:param words_filepath: путь к plain text utf8 файлу со списком слов (одно слово на строку)
:param tmp_dir: путь к каталогу, куда будем сохранять всякие сводки по процессу обучения
для визуализации и прочего контроля
'''
# составляем список слов для тренировки и валидации
known_words = self.load_words(words_filepath)
logging.info('There are {} known words'.format(len(known_words)))
max_word_len = max(map(len, known_words))
seq_len = max_word_len + 2 # 2 символа добавляются к каждому слову для маркировки начала и конца последовательности
logging.info('max_word_len={}'.format(max_word_len))
# ограничиваем число слов для обучения и валидации
if len(known_words) > nb_samples:
known_words = set(list(known_words)[:nb_samples])
val_share = 0.3
random.seed(self.seed)
train_words = set(
filter(lambda z: random.random() > val_share, known_words))
val_words = set(filter(lambda z: z not in train_words, known_words))
# В тренировочный набор добавляем особое "пустое" слово, которое нужно
# в качестве заполнителя при выравнивании цепочек слов разной длины.
train_words.add(u'')
train_words = raw_wordset(train_words, max_word_len)
val_words = raw_wordset(val_words, max_word_len)
logging.info('train set contains {} words'.format(len(train_words)))
logging.info('val set contains {} words'.format(len(val_words)))
all_chars = {FILLER_CHAR, BEG_CHAR, END_CHAR}
for word in known_words:
all_chars.update(word)
char2index = {FILLER_CHAR: 0}
for i, c in enumerate(all_chars):
if c != FILLER_CHAR:
char2index[c] = len(char2index)
index2char = dict([(i, c) for c, i in six.iteritems(char2index)])
nb_chars = len(all_chars)
logging.info('nb_chars={}'.format(nb_chars))
mask_zero = self.arch_type == 'rnn'
if self.char_dims > 0:
# Символы будут представляться векторами заданной длины,
# и по мере обучения вектора будут корректироваться для
# уменьшения общего лосса.
embedding = Embedding(output_dim=self.char_dims,
input_dim=nb_chars,
input_length=seq_len,
mask_zero=mask_zero,
trainable=True)
else:
# 1-hot encoding of characters.
# длина векторов пользователем не указана, поэтому задаем ее так, что
# поместилось 1-hot представление.
self.char_dims = nb_chars
char_matrix = np.zeros((nb_chars, self.char_dims))
for i in range(nb_chars):
char_matrix[i, i] = 1.0
embedding = Embedding(output_dim=self.char_dims,
input_dim=nb_chars,
input_length=seq_len,
weights=[char_matrix],
mask_zero=mask_zero,
trainable=self.tunable_char_embeddings)
input_chars = Input(shape=(seq_len, ), dtype='int32', name='input')
encoder = embedding(input_chars)
logging.info('Building "{}" neural network'.format(self.arch_type))
if self.arch_type == 'cnn':
conv_list = []
merged_size = 0
nb_filters = 32
for kernel_size in range(1, 4):
conv_layer = Conv1D(filters=nb_filters,
kernel_size=kernel_size,
padding='valid',
activation='relu',
strides=1)(encoder)
#conv_layer = GlobalMaxPooling1D()(conv_layer)
conv_layer = GlobalAveragePooling1D()(conv_layer)
conv_list.append(conv_layer)
merged_size += nb_filters
encoder = keras.layers.concatenate(inputs=conv_list)
encoder = Dense(units=self.vec_size, activation='sigmoid')(encoder)
elif self.arch_type == 'rnn':
encoder = recurrent.LSTM(units=self.vec_size,
return_sequences=False)(encoder)
elif self.arch_type == 'bidir_lstm':
encoder = Bidirectional(
recurrent.LSTM(units=self.vec_size // 2,
return_sequences=False))(encoder)
elif self.arch_type == 'lstm(lstm)':
encoder = Bidirectional(
recurrent.LSTM(units=self.vec_size // 2,
return_sequences=True))(encoder)
encoder = Bidirectional(
recurrent.LSTM(units=self.vec_size // 2,
return_sequences=False))(encoder)
elif self.arch_type == 'lstm+cnn':
conv_list = []
merged_size = 0
rnn_size = self.vec_size
conv_list.append(
recurrent.LSTM(units=rnn_size,
return_sequences=False)(encoder))
merged_size += rnn_size
nb_filters = 32
for kernel_size in range(1, 4):
conv_layer = Conv1D(filters=nb_filters,
kernel_size=kernel_size,
padding='valid',
activation='relu',
strides=1)(encoder)
#conv_layer = GlobalMaxPooling1D()(conv_layer)
conv_layer = GlobalAveragePooling1D()(conv_layer)
conv_list.append(conv_layer)
merged_size += nb_filters
encoder = keras.layers.concatenate(inputs=conv_list)
encoder = Dense(units=self.vec_size, activation='sigmoid')(encoder)
elif self.arch_type == 'lstm(cnn)':
conv_list = []
merged_size = 0
nb_filters = 32
rnn_size = nb_filters
for kernel_size in range(1, 4):
conv_layer = Conv1D(
filters=nb_filters,
kernel_size=kernel_size,
padding='valid',
activation='relu',
strides=1,
name='shared_conv_{}'.format(kernel_size))(encoder)
#conv_layer = keras.layers.MaxPooling1D(pool_size=kernel_size, strides=None, padding='valid')(conv_layer)
conv_layer = keras.layers.AveragePooling1D(
pool_size=kernel_size, strides=None,
padding='valid')(conv_layer)
conv_layer = recurrent.LSTM(rnn_size,
return_sequences=False)(conv_layer)
conv_list.append(conv_layer)
merged_size += rnn_size
encoder = keras.layers.concatenate(inputs=conv_list)
encoder = Dense(units=self.vec_size, activation='sigmoid')(encoder)
elif self.arch_type == 'gru(cnn)':
conv_list = []
merged_size = 0
for kernel_size, nb_filters in [(1, 16), (2, 32), (3, 64),
(4, 128)]:
conv_layer = Conv1D(
filters=nb_filters,
kernel_size=kernel_size,
padding='valid',
activation='relu',
strides=1,
name='shared_conv_{}'.format(kernel_size))(encoder)
conv_layer = keras.layers.AveragePooling1D(
pool_size=kernel_size, strides=None,
padding='valid')(conv_layer)
conv_layer = recurrent.GRU(nb_filters,
return_sequences=False)(conv_layer)
conv_list.append(conv_layer)
merged_size += nb_filters
encoder = keras.layers.concatenate(inputs=conv_list)
encoder = Dense(units=self.vec_size, activation='sigmoid')(encoder)
else:
raise RuntimeError('Unknown architecture of neural net: {}'.format(
self.arch_type))
decoder = RepeatVector(seq_len)(encoder)
decoder = recurrent.LSTM(self.vec_size, return_sequences=True)(decoder)
decoder = TimeDistributed(Dense(nb_chars, activation='softmax'),
name='output')(decoder)
model = Model(inputs=input_chars, outputs=decoder)
model.compile(loss='categorical_crossentropy', optimizer='nadam')
keras.utils.plot_model(model,
to_file=os.path.join(
self.model_dir, 'wordchar2vector.arch.png'),
show_shapes=False,
show_layer_names=True)
weigths_path = os.path.join(self.model_dir, 'wordchar2vector.model')
arch_filepath = os.path.join(self.model_dir, 'wordchar2vector.arch')
model_config = {
'max_word_len': max_word_len,
'seq_len': seq_len,
'char2index': char2index,
'FILLER_CHAR': FILLER_CHAR,
'BEG_CHAR': BEG_CHAR,
'END_CHAR': END_CHAR,
'arch_filepath': arch_filepath,
'weights_path': weigths_path,
'vec_size': self.vec_size,
'arch_type': self.arch_type
}
with open(os.path.join(self.model_dir, self.config_filename),
'w') as f:
json.dump(model_config, f)
# model_checkpoint = ModelCheckpoint( weigths_path,
# monitor='val_loss',
# verbose=1,
# save_best_only=True,
# mode='auto')
#
# early_stopping = EarlyStopping(monitor='val_loss',
# patience=20,
# verbose=1,
# mode='auto')
X_viz, y_viz = build_test(
list(val_words)[0:1000], max_word_len, char2index)
learning_curve_filename = os.path.join(
tmp_dir,
'learning_curve__{}_vecsize={}_tunable_char_embeddings={}_chardims={}_batchsize={}_seed={}.csv'
.format(self.arch_type, self.vec_size,
self.tunable_char_embeddings, self.char_dims,
self.batch_size, self.seed))
visualizer = VisualizeCallback(X_viz, y_viz, model, index2char,
weigths_path, learning_curve_filename)
# csv_logger = CSVLogger(learning_curve_filename, append=True, separator='\t')
remaining_epochs = 1000
workout_count = 0
batch_size = self.batch_size
while batch_size >= self.min_batch_size and remaining_epochs > 0:
logging.info(
'Workout #{}: start training with batch_size={}, remaining epochs={}'
.format(workout_count, batch_size, remaining_epochs))
if workout_count > 0:
model.load_weights(weigths_path)
workout_count += 1
visualizer.new_epochs()
hist = model.fit_generator(
generator=generate_rows(train_words, self.batch_size,
char2index, seq_len, 1),
steps_per_epoch=len(train_words) // self.batch_size,
epochs=remaining_epochs,
verbose=1,
callbacks=[visualizer
], # csv_logger, model_checkpoint, early_stopping],
validation_data=generate_rows(val_words, self.batch_size,
char2index, seq_len, 1),
validation_steps=len(val_words) // self.batch_size,
)
remaining_epochs -= len(hist.history)
batch_size = batch_size // 2
logging.info('Training complete, best_accuracy={}'.format(
visualizer.get_best_accuracy()))
# Загружаем наилучшее состояние модели
model.load_weights(weigths_path)
# Создадим модель с урезанным до кодирующей части графом.
model = Model(inputs=input_chars, outputs=encoder)
# Сохраним эту модель
with open(arch_filepath, 'w') as f:
f.write(model.to_json())
# Пересохраним веса усеченной модели
model.save_weights(weigths_path)
#............................THE TRACING NEWORK................................
sequence_input1 = Input(shape=(input_sentence_length1, ))
x = Embedding(num_words,
EMBEDDING_DIM,
embeddings_initializer=Constant(e),
trainable=True)(sequence_input1)
x = SpatialDropout1D(0.2)(x)
x = Bidirectional(
LSTM(32, return_sequences=True, dropout=0.5, recurrent_dropout=0.5))(x)
x = Conv1D(16,
kernel_size=3,
padding="valid",
kernel_initializer="glorot_uniform")(x)
avg_pool1 = GlobalAveragePooling1D()(x)
max_pool1 = GlobalMaxPooling1D()(x)
x = concatenate([avg_pool1, max_pool1])
#........................................................................
sequence_input2 = Input(shape=(input_sentence_length2, ))
y = Embedding(num_words,
EMBEDDING_DIM,
embeddings_initializer=Constant(e),
trainable=True)(sequence_input2)
y = SpatialDropout1D(0.2)(y)
y = Bidirectional(
LSTM(32, return_sequences=True, dropout=0.5, recurrent_dropout=0.5))(y)
y = Conv1D(16,
kernel_size=3,
padding="valid",
def build_model(self, x):
x = ConsumeMask()(x)
x = GlobalAveragePooling1D()(x)
return x
def define_model(at_layer_name='at_output', loc_layer_name='loc_output'):
time_pooling_factor = 1
input_shape = (64, 431, 1)
melInput = Input(input_shape)
# ---- mel convolution part ----
mBlock1 = Conv2D(filters=64, kernel_size=(3, 3), padding="same")(melInput)
mBlock1 = BatchNormalization()(mBlock1)
mBlock1 = Activation(activation="relu")(mBlock1)
mBlock1 = MaxPooling2D(pool_size=(4, 1))(mBlock1)
# mBlock1 = Dropout(0.1)(mBlock1)
mBlock1 = SpatialDropout2D(0.3, data_format=K.image_data_format())(mBlock1)
mBlock2 = Conv2D(filters=64, kernel_size=(3, 3), padding="same")(mBlock1)
mBlock2 = BatchNormalization()(mBlock2)
mBlock2 = Activation(activation="relu")(mBlock2)
mBlock2 = MaxPooling2D(pool_size=(4, time_pooling_factor))(mBlock2)
mBlock2 = SpatialDropout2D(0.3, data_format=K.image_data_format())(mBlock2)
# mBlock2 = Dropout(0.1)(mBlock2)
mBlock3 = Conv2D(filters=64, kernel_size=(3, 3), padding="same")(mBlock2)
mBlock3 = BatchNormalization()(mBlock3)
mBlock3 = Activation(activation="relu")(mBlock3)
mBlock3 = MaxPooling2D(pool_size=(4, time_pooling_factor))(mBlock3)
mBlock3 = SpatialDropout2D(0.3, data_format=K.image_data_format())(mBlock3)
# mBlock3 = Dropout(0.1)(mBlock3)
targetShape = int(mBlock3.shape[1] * mBlock3.shape[2])
mReshape = Reshape(target_shape=(targetShape, 64))(mBlock3)
gru = Bidirectional(
GRU(kernel_initializer='glorot_uniform', activation='tanh', recurrent_dropout=0.1, \
dropout=0.1, units=64, return_sequences=True)
)(mReshape)
gru = Dropout(0.1)(gru)
output = TimeDistributed(Dense(64, activation="relu"), )(gru)
output = Dropout(0.1)(output)
loc_output = TimeDistributed(
Dense(10, activation="sigmoid"),
name=loc_layer_name,
)(output)
# output = TimeDistributed(
# Lambda(lambda x: (x - K.min(x, axis=1, keepdims=True))/(K.max(x, axis=1, keepdims=True)- K.min(x, axis=1, keepdims=True)) ),
# )(output)
### output = GlobalAveragePooling1D()(output)
gap = GlobalAveragePooling1D()(loc_output)
gmp = GlobalMaxPooling1D()(loc_output)
# flat_gap = Flatten()(gap)
# flat_gmp = Flatten()(gmp)
concat = Concatenate()([gap, gmp])
d = Dense(100, activation="relu")(concat)
d = Dropout(rate=0.5)(d)
at_output = Dense(10, activation="sigmoid", name=at_layer_name)(d)
model = Model(inputs=[melInput], outputs=[loc_output, at_output])
return model
def train(s_data, s_out, nuc, bestwight, samplesize, epoch_num):
batch_size = 1024
num_classes_org = 1024
num_classes = 63
shape1 = (None, DATA_LENGTH, 2)
model = cnn_network_keras.build_network(shape=shape1,
num_classes=num_classes_org)
model.load_weights(bestwight)
model.layers.pop() # remove last layer
model.layers.pop() # remove last layer
model.layers.pop() # remove last layer
for layer in model.layers:
if layer.name == "conv1d_20":
break
else:
layer.trainable = False
flat = GlobalAveragePooling1D()(model.layers[-1].output)
model_t = Model(inputs=model.input, outputs=flat)
model_r = Network(inputs=model_t.input,
outputs=model_t.output,
name="shared_layer")
prediction = Dense(num_classes, activation='softmax')(model_t.output)
model_r = Model(inputs=model_r.input, outputs=prediction)
optimizer = SGD(lr=5e-5, decay=0.00005)
model_r.compile(optimizer=optimizer, loss="categorical_crossentropy")
model_t.compile(optimizer=optimizer, loss=original_loss)
x_ref, test_x_r, y_ref, test_y_r, num_classes_r = prepDataNear(
s_data, samplesize, nuc)
x_target, test_x, train_y, test_y, num_classes = prepData(
s_data, samplesize, nuc)
ref_samples = np.arange(x_ref.shape[0])
loss, loss_c = [], []
epochs = []
print("training...")
for epochnumber in range(epoch_num):
x_r, y_r, lc, ld = [], [], [], []
np.random.shuffle(x_target)
np.random.shuffle(ref_samples)
for i in range(len(x_target)):
x_r.append(x_ref[ref_samples[i]])
y_r.append(y_ref[ref_samples[i]])
x_r = np.array(x_r)
y_r = np.array(y_r)
for i in range(int(len(x_target) / batchsize)):
batch_target = x_target[i * batchsize:i * batchsize + batchsize]
batch_ref = x_r[i * batchsize:i * batchsize + batchsize]
batch_y = y_r[i * batchsize:i * batchsize + batchsize]
# target data
lc.append(
model_t.train_on_batch(batch_target,
np.zeros((batchsize, feature_out))))
# reference data
ld.append(model_r.train_on_batch(batch_ref, batch_y))
loss.append(np.mean(ld))
loss_c.append(np.mean(lc))
epochs.append(epochnumber)
print("epoch : {} ,Descriptive loss : {}, Compact loss : {}".format(
epochnumber + 1, loss[-1], loss_c[-1]))
if epochnumber % 1 == 0:
model_t.save_weights(s_out + '/' + nuc +
'/model_t_ep_{}.h5'.format(epochnumber))
model_r.save_weights(s_out + '/' + nuc +
'/model_r_ep_{}.h5'.format(epochnumber))
def conv1d_class(train_A, words_of_tweets, extra_features, feature_selection,
encoding, print_file):
reading = Twitter_Depression_Detection.Reader(
) # Import the ClassRead.py file, to get the encoding
# fix random seed for reproducibility
numpy.random.seed(7)
x = np.array(words_of_tweets)
y = train_A['label']
# -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
# Initialize the roc-auc score running average list
# Initialize a count to print the number of folds
# Initialize metrics to print their average
av_roc = 0.
count = 0
precision = 0
accuracy = 0
recall = 0
f1score = 0
# -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
# Initialize your 10 - cross vailidation
# Set shuffle equals True to randomize your splits on your training data
kf = KFold(n_splits=10, random_state=41, shuffle=True)
# Set up for loop to run for the number of cross vals you defined in your parameter
for train_index, test_index in kf.split(x):
count += 1
print('Fold #: ', count)
with open(print_file,
"a") as myfile: # Write above print into output file
myfile.write('Fold #: ' + str(count) + '\n')
# This indexs your train and test data for your cross validation and sorts them in random order, since we used shuffle equals True
x_train, x_test = reading.get_enc(x[train_index], 1, y[train_index],
train_index, extra_features,
feature_selection, encoding,
print_file), reading.get_enc(
x[test_index], 0, y[test_index],
test_index, extra_features,
feature_selection, encoding,
print_file)
y_train, y_test = y[train_index], y[test_index]
# -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
# Initializing Neural Network
classifier = Sequential()
# -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
# 'minority': resample only the minority class;
oversample = SMOTE(sampling_strategy='minority',
k_neighbors=10,
random_state=0)
x_train, y_train = oversample.fit_resample(x_train, y_train)
# -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
print(x_train.shape[0], ' ', x_train.shape[1])
print(x_test.shape[0], ' ', x_test.shape[1])
x_train = x_train.reshape(x_train.shape[0], 1, x_train.shape[1])
x_test = x_test.reshape(x_test.shape[0], 1, x_test.shape[1])
classifier.add(
Dense(20,
kernel_initializer='glorot_uniform',
activation='softsign',
kernel_constraint=maxnorm(2),
input_shape=(1, x_train.shape[2])))
classifier.add(
Conv1D(filters=32,
kernel_size=3,
padding='same',
activation='relu'))
classifier.add(Dropout(0.2))
classifier.add(
Conv1D(filters=32,
kernel_size=3,
padding='same',
activation='relu'))
classifier.add(
Conv1D(filters=32,
kernel_size=3,
padding='same',
activation='relu'))
classifier.add(
Conv1D(filters=32,
kernel_size=3,
padding='same',
activation='relu'))
classifier.add(
Conv1D(filters=32,
kernel_size=3,
padding='same',
activation='relu'))
classifier.add(Dropout(0.2))
classifier.add(GlobalAveragePooling1D())
classifier.add(
Dense(500,
kernel_initializer='glorot_uniform',
activation='softsign',
kernel_constraint=maxnorm(2)))
# Adding the output layer with 1 output
classifier.add(
Dense(1, kernel_initializer='glorot_uniform',
activation='sigmoid'))
optimizer = RMSprop(lr=0.001)
# Compiling Neural Network
classifier.compile(optimizer=optimizer,
loss='binary_crossentropy',
metrics=['accuracy'])
# -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=2,
verbose=0,
mode='auto')
# Fitting our model
classifier.fit(x_train, y_train, batch_size=20, epochs=50)
# -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
# Your model is fit. Time to predict our output and test our training data
print("Evaluating model...")
with open(print_file,
"a") as myfile: # Write above print into output file
myfile.write("Evaluating model..." + '\n')
test_preds = classifier.predict_proba(x_test, verbose=0)
roc = roc_auc_score(y_test, test_preds)
scores = classifier.evaluate(x_test, y_test)
print(scores)
# Print your model summary
print(classifier.summary())
# Print your ROC-AUC score for your kfold, and the running score average
print('ROC: ', roc)
av_roc += roc
print('Continued Avg: ', av_roc / count)
with open(print_file,
"a") as myfile: # Write above print into output file
myfile.write('Scores: ' + str(scores) + '\n' +
'Classifier summary: ' + str(classifier.summary()) +
'\n' + 'ROC: ' + str(roc) + '\n' + 'Continued Avg: ' +
str(av_roc / count) + '\n')
# -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
# Predicting the Test set results
y_pred = classifier.predict(x_test)
y_pred = (y_pred > 0.5)
# Creating the Confusion Matrix
cm = confusion_matrix(y_test, y_pred)
print(cm)
with open(print_file,
"a") as myfile: # Write above print into output file
myfile.write(str(cm) + '\n')
temp_accuracy = accuracy_score(y_test, y_pred)
temp_precision, temp_recall, temp_f1_score, _ = precision_recall_fscore_support(
y_test, y_pred, average='binary')
accuracy += temp_accuracy
precision += temp_precision
recall += temp_recall
f1score += temp_f1_score
print("Accuracy: ", temp_accuracy)
print("Precision: ", temp_precision)
print("Recall: ", temp_recall)
print("F1 score: ", temp_f1_score)
# Print average of metrics
print("Average Precision: ", precision / 10)
print("Average Accuracy: ", accuracy / 10)
print("Average Recall: ", recall / 10)
print("Average F1-score: ", f1score / 10)
# Print your final average ROC-AUC score and organize your models predictions in a dataframe
print('Average ROC:', av_roc / 10)
with open(print_file, "a") as myfile: # Write above print into output file
myfile.write("Average Precision: " + str(precision / 10) + '\n' +
"Average Accuracy: " + str(accuracy / 10) + '\n' +
"Average Recall: " + str(recall / 10) + '\n' +
"Average F1-score: " + str(f1score / 10) + '\n' +
'Average ROC:' + str(av_roc / 10) + '\n')
batch_conds, batch_labels = [], []
c_in = Input(shape=(1,))
c = Embedding(len(variants), 128)(c_in)
c = Reshape((128,))(c)
model = build_transformer_model(
config_path,
checkpoint_path,
model='roformer',
layer_norm_cond=c,
additional_input_layers=c_in
)
output = GlobalAveragePooling1D()(model.output)
output = Dense(2, activation='softmax')(output)
model = Model(model.inputs, output)
model.summary()
AdamEMA = extend_with_exponential_moving_average(Adam, name='AdamEMA')
optimizer = AdamEMA(learing_rate, ema_momentum=0.9999)
model.compile(
loss='sparse_categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy',F1]
)
# 转换数据集
train_generator = data_generator(train_data, batch_size)
input_length=conf.data_prep.pad_len_word,
trainable=False)(text_input)
m = Dropout(0.5)(m)
m = Conv1D(int(conf.modelling.num_filters),
int(conf.modelling.filter_size),
activation='relu',
padding='same')(m)
m = AveragePooling1D(2)(m)
m = Conv1D(conf.modelling.num_filters - 4,
conf.modelling.filter_size,
activation='relu',
padding='same')(m)
m = GlobalAveragePooling1D()(m)
category_emb = CatEmbLayer(train_cont['cat_s']['cat'],
conf.modelling.category_emb_dim, category_input)
parent_cat_emb = CatEmbLayer(train_cont['cat_s']["parent_cat"],
conf.modelling.category_emb_dim, parent_cat_input)
region_cat_emb = CatEmbLayer(train_cont['cat_s']["region_cat"],
conf.modelling.category_emb_dim, region_cat_input)
city_cat_emb = CatEmbLayer(train_cont['cat_s']["city_cat"],
conf.modelling.category_emb_dim, city_cat_input)
image_cat_emb = CatEmbLayer(train_cont['cat_s']["image_cat"],
conf.modelling.category_emb_dim, image_cat_input)
user_cat_emb = CatEmbLayer(train_cont['cat_s']["user_cat"],
conf.modelling.category_emb_dim, user_cat_input)
day_cat_emb = CatEmbLayer(train_cont['cat_s']["day_cat"],
conf.modelling.category_emb_dim, day_cat_input)
region_score_map = BatchNormalization()(region_score_map)
region_score_map = Activation('sigmoid', name='region_attention')(region_score_map)
#region_fea = merge([id_fea_map,region_score_map], mode='dot', dot_axes=(1,1))
region_fea = keras.layers.Dot(axes=(1,1))([id_fea_map,region_score_map])
region_fea = Lambda(lambda x: x*(1.0/L))(region_fea)
region_fea = BatchNormalization()(region_fea)
# attribute feature fusion
# attr_scores = merge(attr_score_list, mode='concat')
attr_scores = keras.layers.Concatenate()(attr_score_list)
attr_scores = BatchNormalization()(attr_scores)
attr_scores = Activation('sigmoid')(attr_scores)
# attr_fea = merge(attr_fea_list, mode='concat')
attr_fea = keras.layers.Concatenate()(attr_fea_list)
attr_fea = Reshape((emb_dim, len(nb_attributes)))(attr_fea)
equal_attr_fea = GlobalAveragePooling1D()(attr_fea)
# attr_fea = merge([attr_fea,attr_scores], mode='dot', dot_axes=(2,1))
attr_fea = keras.layers.Dot(axes=(2,1))([attr_fea,attr_scores])
attr_fea = Lambda(lambda x: x*(1.0/len(nb_attributes)))(attr_fea)
attr_fea = BatchNormalization()(attr_fea)
# loss-3: final classification
if(attr_equal):
attr_fea = equal_attr_fea
if(region_equal):
region_fea = id_pool
# final_fea = merge([attr_fea,region_fea], mode='concat')
final_fea = keras.layers.Concatenate()([attr_fea,region_fea])
final_fea = Activation('relu', name='final_fea')(final_fea)
final_fea = Dropout(dropout)(final_fea)
final_prob = Dense(nb_classes)(final_fea)
def complex_cnn_based_rnn(config, embedding_matrix=None):
'''Complex CNN based RNN
:param config:
:param embedding_matrix:
:return:
'''
print("Build Complex CNN based Attentive RNN...")
if embedding_matrix == None:
# # embedding_matrix = np.zeros((config.max_features, config.embedding_dims))
# numpy_rng = np.random.RandomState(4321)
# embedding_matrix = numpy_rng.uniform(low=-0.05, high=0.05, size=(config.max_features, config.embedding_dims))
embedding_layer = Embedding(config.max_features,
config.embedding_dims,
input_length=config.maxlen)
else:
embedding_layer = Embedding(config.max_features,
config.embedding_dims,
weights=[embedding_matrix],
input_length=config.maxlen,
trainable=False)
rnn_layer = Bidirectional(
GRU(config.lstm_output_size,
dropout=config.dropout,
recurrent_dropout=config.dropout))
# cnn_layer = Conv1D(filters=config.nb_filter,
# kernel_size=config.filter_length, padding = "valid", activation="relu", strides=1)
#
# conv1 = Conv1D(filters=config.nb_filter,
# kernel_size=1, padding="valid", strides=1, activation='relu')
conv2 = Conv1D(filters=config.nb_filter,
kernel_size=2,
padding="valid",
strides=1,
activation='relu')
conv3 = Conv1D(filters=config.nb_filter,
kernel_size=3,
padding="valid",
strides=1,
activation='relu')
conv4 = Conv1D(filters=config.nb_filter,
kernel_size=4,
padding="valid",
strides=1,
activation='relu')
# conv5 = Conv1D(filters=config.nb_filter,
# kernel_size=5, padding='same', activation='relu')
#
# conv6 = Conv1D(filters=config.nb_filter,
# kernel_size=6, padding='same', activation='relu')
#
# pooling_layer = GlobalMaxPooling1D()
cnn_dense = Dense(config.hidden_dims, activation='relu')
# cnn_dropout1 = Dropout(0.2)
# cnn_dropout2 = Dropout(0.2)
# cnn_batchnormalization = BatchNormalization()
# cnn_repeatvector = RepeatVector(config.embedding_dims)
# cnn_dense1 = Dense(300, activation='relu')
sequence_input = Input(shape=(config.maxlen, ), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
# conv1a = conv1(embedded_sequences)
# glob1a = GlobalAveragePooling1D()(conv1a)
# glob1a = Dropout(config.dropout)(glob1a)
# glob1a = BatchNormalization()(glob1a)
conv2a = conv2(embedded_sequences)
glob2a = GlobalAveragePooling1D()(conv2a)
glob2a = Dropout(config.dropout)(glob2a)
glob2a = BatchNormalization()(glob2a)
conv3a = conv3(embedded_sequences)
glob3a = GlobalAveragePooling1D()(conv3a)
glob3a = Dropout(config.dropout)(glob3a)
glob3a = BatchNormalization()(glob3a)
conv4a = conv4(embedded_sequences)
glob4a = GlobalAveragePooling1D()(conv4a)
glob4a = Dropout(config.dropout)(glob4a)
glob4a = BatchNormalization()(glob4a)
# conv5a = conv5(embedded_sequences)
# glob5a = GlobalAveragePooling1D()(conv5a)
# glob5a = Dropout(config.dropout)(glob5a)
# glob5a = BatchNormalization()(glob5a)
#
# conv6a = conv6(embedded_sequences)
# glob6a = GlobalAveragePooling1D()(conv6a)
# glob6a = Dropout(config.dropout)(glob6a)
# glob6a = BatchNormalization()(glob6a)
cnn = concatenate([glob2a, glob3a, glob4a])
# print(np.shape(cnn))
# print(cnn.shape)
cnn_t = cnn_dense(cnn)
a = multiply([cnn_t, embedded_sequences])
a = Permute([2, 1])(a)
a = Lambda(lambda x: K.sum(x, axis=1))(a)
a = Activation('sigmoid')(a)
embedded_sequences = Permute([2, 1])(embedded_sequences)
x = multiply([a, embedded_sequences])
x = Permute([2, 1])(x)
x = rnn_layer(x)
x = Dense(config.hidden_dims, activation='relu')(x)
x = Dropout(config.dropout)(x)
x = BatchNormalization()(x)
preds = Dense(1, activation='sigmoid')(x)
########################################
## train the model
########################################
model = Model(inputs=sequence_input, outputs=preds)
model.compile(loss='binary_crossentropy',
optimizer='nadam',
metrics=['acc'])
model.summary()
return model
kernel_initializer='he_uniform')(x1)
x2 = BatchNormalization()(x2)
x2 = Activation('relu')(x2)
x3 = Conv1D(64, 3, padding='same',
kernel_initializer='he_uniform')(x2)
x3 = BatchNormalization()(x3)
#do not need to expand channels for the sum
sx2 = BatchNormalization()(outputB2)
outputB3 = add([sx2, x3])
outputB3 = Activation('relu')(outputB2)
#### End
fea = GlobalAveragePooling1D()(outputB3)
out = Dense(3,
kernel_regularizer=regularizers.l2(0.001),
activation='softmax')(fea)
model = Model(ip, out)
#model.summary()
######---------------------
epochs_s = 100
batch_s = 16
learning_rate = 1e-3
weight_fn = "./weights/weights_fea_Resnet_t" + str(user) + ".h5"
model_checkpoint = ModelCheckpoint(weight_fn,
verbose=1,
mode='max',
def build_model(self):
# Create joint embedding layer (decay strings)
decstr_embedding = Embedding(
self.num_pdg_codes,
8,
input_length=self.shape_dict['decay_input'][1],
)
# Network to process decay string
decay_input = Input(shape=self.shape_dict['decay_input'][1:],
name='decay_input')
decay_embed = decstr_embedding(decay_input)
# Build wide CNN for decay string processing
decay_l = self._conv1D_node(
decay_embed,
filters=64,
kernel_size=3,
)
decay_l = AveragePooling1D(pool_size=2)(decay_l)
decay_l = LSTM(
units=64,
activation='tanh',
)(decay_l)
# decay_l = Dropout(0.3)(decay_l)
decay_l = Dense(256)(decay_l)
decay_l = LeakyReLU()(decay_l)
decay_l = Dropout(0.3)(decay_l)
decay_l = Dense(32)(decay_l)
decay_output = LeakyReLU()(decay_l)
# Create joint embedding layer
pdg_embedding = Embedding(
self.num_pdg_codes,
8,
input_length=self.shape_dict['pdg_input'][1],
)
# Network to process individual particles
particle_input = Input(shape=self.shape_dict['particle_input'][1:],
name='particle_input')
# Embed PDG codes
pdg_input = Input(shape=self.shape_dict['pdg_input'][1:],
name='pdg_input')
mother_pdg_input = Input(shape=self.shape_dict['mother_pdg_input'][1:],
name='mother_pdg_input')
pdg_l = pdg_embedding(pdg_input)
mother_pdg_l = pdg_embedding(mother_pdg_input)
# Put all the particle
particle_l = concatenate([particle_input, pdg_l, mother_pdg_l],
axis=-1)
# Node 1
particle_l = self.conv1D_avg_node(
particle_l,
filters=64,
kernel_size=4,
pool='avg',
# dropout=0.3
)
for i in range(2):
particle_l = self.conv1D_avg_node(
particle_l,
filters=64,
kernel_size=3,
# dropout=0.3
)
# Compress
# particle_l = AveragePooling1D(pool_size=2)(particle_l)
# kernel=3, flatten
particle_output = GlobalAveragePooling1D()(particle_l)
# Finally, combine the two networks
comb_l = concatenate([decay_output, particle_output], axis=-1)
comb_l = Dense(512)(comb_l)
comb_l = LeakyReLU()(comb_l)
comb_l = Dropout(0.4)(comb_l)
comb_l = Dense(512)(comb_l)
comb_l = LeakyReLU()(comb_l)
comb_l = Dropout(0.2)(comb_l)
comb_l = Dense(64)(comb_l)
comb_l = LeakyReLU()(comb_l)
comb_output = Dense(1, activation='sigmoid', name='y_output')(comb_l)
# Instantiate the cnn model
model = Model(
inputs=[decay_input, particle_input, pdg_input, mother_pdg_input],
outputs=comb_output,
name='vanilla-LSTM')
# Finally compile the model
model.compile(
loss='binary_crossentropy',
optimizer=self.optimizer,
metrics=['accuracy'],
)
model.summary()
self.model = model
Xtrain1 = Xtrain1[indice1]
Xtrain2 = Xtrain2[indice1]
Ytrain = Ytrain[indice1]
indice2 = np.arange(25000)
np.random.shuffle(indice2)
Xtest1 = Xtest1[indice2]
Xtest2 = Xtest2[indice2]
Ytest = Ytest[indice2]
print('begin to build model ...')
input1 = Input(shape=(max_len, ))
embedding1 = Embedding(num_words,
500,
input_length=max_len,
embeddings_initializer='normal')(input1)
x = GlobalAveragePooling1D()(embedding1)
input2 = Input(shape=((max_len - 1) * 2, ))
embedding2 = Embedding(num_words * 2,
500,
input_length=(max_len - 1) * 2,
embeddings_initializer='normal')(input2)
y = AveragePooling1D(pool_size=2, strides=2)(embedding2)
y = GlobalMaxPooling1D()(y)
z = keras.layers.concatenate([x, y])
#model.add(Dropout(0.5))
output = Dense(1, activation='sigmoid')(z)
model = Model(inputs=[input1, input2], outputs=output)
model.compile(loss='binary_crossentropy',
optimizer='adam',
def bigru_pool_attention(attention=False):
main_input = Input(shape=(maxlen, ),
name='main_input') #, name='main_input'
Ngram_input = Input(shape=(maxlen_char, ),
name='aux_input') #, name='aux_input'
embedded_sequences = Embedding(max_features,
embed_size,
weights=[embedding_matrix],
trainable=False)(main_input)
embedded_sequences_2 = Embedding(weights_char.shape[0],
50,
weights=[weights_char],
trainable=True)(Ngram_input)
#word level
hidden_dim = 128
x = SpatialDropout1D(0.22)(embedded_sequences) #0.1
x_gru_1 = Bidirectional(
CuDNNGRU(hidden_dim,
recurrent_regularizer=regularizers.l2(1e-6),
return_sequences=True))(x)
#char level
hidden_dim = 30
x_2 = SpatialDropout1D(0.21)(embedded_sequences_2) #0.1
x_gru_2 = Bidirectional(
CuDNNGRU(hidden_dim,
recurrent_regularizer=regularizers.l2(1e-8),
return_sequences=True))(x_2)
x_ave_1 = GlobalAveragePooling1D()(x_gru_1)
x_ave_2 = GlobalAveragePooling1D()(x_gru_2)
x_ave = concatenate([x_ave_1, x_ave_2])
x_max_1 = GlobalMaxPool1D()(x_gru_1)
x_max_2 = GlobalMaxPool1D()(x_gru_2)
x_max = concatenate([x_max_1, x_max_2])
x_dense = concatenate([x_max, x_ave])
if attention: #did not use it at the final
x_att_1 = Attention()(x_gru_1)
x_dense = BatchNormalization()(x_dense)
x_dense = Dropout(0.35)(x_dense)
x_dense = Dense(256, activation="elu")(x_dense)
x_dense = Dropout(0.3)(x_dense)
x_dense = Dense(128, activation="elu")(x_dense)
x = Dropout(0.2)(x_dense)
if attention:
x = concatenate([x_att_1, x])
x = Dense(6,
activation="sigmoid",
kernel_regularizer=regularizers.l2(1e-8))(x)
model = Model(inputs=main_input, outputs=x)
nadam = Nadam(lr=0.00225,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
schedule_decay=0.00325)
model.compile(loss='binary_crossentropy',
optimizer=nadam,
metrics=['accuracy', f1_score, auc])
print(model.summary())
return model
def multichannel(nb_classes,
vocab_size,
input_len,
embedding_dim=100,
matrix=None,
activation="sigmoid",
drop=False):
inp = Input(shape=(input_len, ))
if matrix is None:
matrix = np.random.rand(vocab_size + 1, embedding_dim)
assert matrix.shape == (vocab_size + 1, embedding_dim)
embed_layer_1 = Embedding(vocab_size + 1,
embedding_dim,
input_length=input_len)
embed_layer_1.build(input_shape=(input_len, ))
embed_layer_1.set_weights([matrix])
embed_layer_2 = Embedding(vocab_size + 1,
embedding_dim,
input_length=input_len)
embed_layer_2.build(input_shape=(input_len, ))
embed_layer_2.set_weights([matrix])
embed_layer_2.trainable = False
window_sizes = [2, 3, 5]
dic = {}
for size in window_sizes:
#addding all the layer instances to a dic for easy handling
dic[size] = Conv1D(kernel_size=size,
strides=1,
padding="same",
filters=100,
activation="relu")
x1 = embed_layer_1(inp)
x2 = embed_layer_2(inp)
#notice we are sharing the layers , for both x1,x2 we are using the same weights
y1_2 = dic[2](x1)
y1_3 = dic[3](x1)
y1_5 = dic[5](x1)
y2_2 = dic[2](x2)
y2_3 = dic[3](x2)
y2_5 = dic[5](x2)
concat_1 = concatenate([y1_2, y1_3, y1_5])
concat_2 = concatenate([y2_2, y2_3, y2_5])
add_1 = add([concat_1, concat_2])
pool_1 = GlobalAveragePooling1D()(add_1)
if drop == True:
#add dropout
pool_1 = Dropout(0.5)(pool_1)
final_1 = Dense(nb_classes, activation=activation)(pool_1)
model = Model([inp], [final_1])
return model
def c7f4():
"""7 conv1d and 4 dense"""
filters = 128
kernel_size = 25
rate1 = 0.25
rate2 = 0.5
model = Sequential()
# cnn_1
model.add(
Conv1D(filters,
kernel_size,
padding='same',
activation='relu',
input_shape=(None, 12)))
model.add(MaxPooling1D(padding='same'))
model.add(Dropout(rate1))
# cnn_2
model.add(Conv1D(filters, kernel_size, padding='same', activation='relu'))
model.add(MaxPooling1D(padding='same'))
model.add(Dropout(rate1))
# cnn_3
model.add(Conv1D(filters, kernel_size, padding='same', activation='relu'))
model.add(MaxPooling1D(padding='same'))
model.add(Dropout(rate1))
# cnn_4
model.add(Conv1D(filters, kernel_size, padding='same', activation='relu'))
model.add(MaxPooling1D(padding='same'))
model.add(Dropout(rate1))
# cnn_5
model.add(Conv1D(filters, kernel_size, padding='same', activation='relu'))
model.add(MaxPooling1D(padding='same'))
model.add(Dropout(rate1))
# cnn_6
model.add(Conv1D(filters, kernel_size, padding='same', activation='relu'))
model.add(MaxPooling1D(padding='same'))
model.add(Dropout(rate1))
# cnn_7
model.add(Conv1D(filters, kernel_size, padding='same', activation='relu'))
model.add(MaxPooling1D(padding='same'))
model.add(Dropout(rate1))
# Global Average Pooling
model.add(GlobalAveragePooling1D())
# fc_1
model.add(Dense(256, activation='relu'))
model.add(Dropout(rate2))
# fc_2
model.add(Dense(128, activation='relu'))
model.add(Dropout(rate2))
# fc_3
model.add(Dense(64, activation='relu'))
model.add(Dropout(rate2))
# fc_4
model.add(Dense(NUM_CLASSES, activation='softmax'))
return model, sys._getframe().f_code.co_name
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
# words not found in embedding index will be all-zeros.
embedding_matrix[i] = embedding_vector
# build model
inp = Input(shape=(MAX_SEQUENCE_LENGTH, ))
x = Embedding(input_dim=num_words,
output_dim=EMBEDDING_DIM,
embeddings_initializer=Constant(embedding_matrix),
input_length=MAX_SEQUENCE_LENGTH,
trainable=False)(inp)
x = SpatialDropout1D(0.2)(x)
x = Bidirectional(
GRU(256, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)
avg_pool = GlobalAveragePooling1D()(x)
max_pool = GlobalMaxPooling1D()(x)
x = concatenate([avg_pool, max_pool])
outp = Dense(units=6, activation='sigmoid')(x)
model = Model(inp, outp)
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
train_history = model.fit(X_train,
y_train,
batch_size=32,
epochs=4,
verbose=2,
validation_split=0.2)
num_classes = 55
feature_size = 64
epochs = 2
input_data = Input(shape=(5000, 8))
cnn = Conv1D(80, 10, activation='relu')(input_data)
cnn = MaxPooling1D(5)(cnn)
cnn = Dropout(0.3)(cnn)
cnn = Conv1D(80, 10, activation='relu')(cnn)
cnn = MaxPooling1D(5)(cnn)
cnn = Dropout(0.3)(cnn)
cnn = Conv1D(40, 10, activation='relu')(cnn)
cnn = MaxPooling1D(3)(cnn)
cnn = Dropout(0.3)(cnn)
cnn = Conv1D(60, 10, activation='relu')(cnn)
cnn = GlobalAveragePooling1D()(cnn)
cnn = Dropout(0.3)(cnn)
#cnn = Flatten()(cnn)
feature = Dense(feature_size, activation='relu', name='feature')(cnn)
predict = Dense(num_classes, activation='sigmoid',
name='sigmoid')(feature) #至此，得到一个常规的so ftmax分类模型
model_trian = Model(inputs=input_data, outputs=predict)
model_trian.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
#binary_crossentropy
model_trian.summary()
# fit
model_trian.fit(X_train,
#dense_2 = Dense(input_size, activation='relu')(dropout_2)
conc1 = concatenate([atten_1, atten_2])
#conc1 = GlobalMaxPooling1D()(conc1)
conc1 = Dropout(0.1)(conc1)
conc2 = concatenate([max_pool_1, max_pool_2])
#conc2 = GlobalMaxPooling1D()(conc2)
conc2 = Dropout(0.1)(conc2)
conv1 = Conv1D(128,
kernel_size=3,
padding='valid',
kernel_initializer='glorot_uniform')(x)
#conv2 = GlobalMaxPooling1D()(conv1)
conv3 = GlobalAveragePooling1D()(conv1)
conc = concatenate([conc1, conc2, conv3])
conc = Dropout(0.2)(conc)
conc = Dense(input_size, activation='relu')(conc)
conc = Dropout(0.1)(conc)
outp = Dense(1, activation='sigmoid')(conc)
model = Model(inputs=inp, outputs=outp)
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['acc'])
# summarize the model
print(model.summary())
permuted_index = np.random.permutation(len(train))
percentage = 0.2 # percentage of data used for validation
def get_model():
# model:
sequence_input = Input(shape=(maxlen, ))
x = Embedding(max_features, embed_size, trainable=True)(sequence_input)
x = SpatialDropout1D(0.2)(x)
x = Bidirectional(
LSTM(128, return_sequences=True, dropout=0.1,
recurrent_dropout=0.1))(x)
w_conv1 = Conv1D(32,
kernel_size=1,
padding="same",
kernel_initializer="glorot_uniform")(x)
w_conv2 = Conv1D(32,
kernel_size=2,
padding="same",
kernel_initializer="glorot_uniform")(x)
w_conv3 = Conv1D(32,
kernel_size=3,
padding="same",
kernel_initializer="glorot_uniform")(x)
w_conv4 = Conv1D(32,
kernel_size=4,
padding="same",
kernel_initializer="glorot_uniform")(x)
w_conv5 = Conv1D(32,
kernel_size=5,
padding="same",
kernel_initializer="glorot_uniform")(x)
w_conv = concatenate([w_conv1, w_conv2, w_conv3, w_conv4, w_conv5])
w_avg_pool = GlobalAveragePooling1D()(w_conv)
w_max_pool = GlobalMaxPooling1D()(w_conv)
char_input_init = Input(shape=(char_max_len, ))
char_input = Embedding(len(char_index) + 1,
char_embed_size,
trainable=True)(char_input_init)
char_input = SpatialDropout1D(0.2)(char_input)
# char_input = Bidirectional(LSTM(128, return_sequences=True,dropout=0.1,recurrent_dropout=0.1))(char_input)
# do characters have long term dependencies?????
c_conv1 = Conv1D(32,
kernel_size=3,
padding="same",
kernel_initializer="glorot_uniform")(char_input)
c_conv2 = Conv1D(32,
kernel_size=4,
padding="same",
kernel_initializer="glorot_uniform")(char_input)
c_conv3 = Conv1D(32,
kernel_size=5,
padding="same",
kernel_initializer="glorot_uniform")(char_input)
c_conv4 = Conv1D(32,
kernel_size=6,
padding="same",
kernel_initializer="glorot_uniform")(char_input)
c_conv5 = Conv1D(32,
kernel_size=7,
padding="same",
kernel_initializer="glorot_uniform")(char_input)
c_conv = concatenate([c_conv1, c_conv2, c_conv3, c_conv4, c_conv5])
c_avg_pool = GlobalAveragePooling1D()(c_conv)
c_max_pool = GlobalMaxPooling1D()(c_conv)
x = concatenate([w_avg_pool, w_max_pool, c_avg_pool, c_max_pool])
# x = Dense(128, activation='relu')(x)
# x = Dropout(0.1)(x)
preds = Dense(6, activation="sigmoid")(x)
model = Model(inputs=[sequence_input, char_input_init], outputs=preds)
model.compile(loss='binary_crossentropy',
optimizer=Adam(lr=1e-3),
metrics=['accuracy'])
#model.summary()
return model
return x
input_shape = (filtered_ecg_measurements.shape[1],
filtered_ecg_measurements.shape[2])
inputs = Input(shape=input_shape)
layer1 = initial_layer(inputs)
layer2 = Short_cutLayer(layer1, filter_value=64, kernel_value=1, iteration=3)
#layer2 = Dropout(0.2)(layer2)
layer3 = Short_cutLayer(layer2, filter_value=128, kernel_value=1, iteration=4)
layer4 = Short_cutLayer(layer3, filter_value=256, kernel_value=1, iteration=6)
layer5 = Short_cutLayer(layer4, filter_value=512, kernel_value=1, iteration=3)
x = GlobalAveragePooling1D()(layer5)
x = Dropout(dropout_rate)(x)
output = Dense(num_classes, activation='softmax')(x)
model = Model(inputs=inputs, outputs=output)
#model.compile(loss=categorical_focal_loss(gamma=2.0, alpha=0.25), optimizer='adam', metrics=['accuracy']) # -> focal_loss version
#model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
model.compile(loss='categorical_crossentropy',
optimizer=my_optimizer,
metrics=['accuracy'])
model.summary()
################################# Step 7 : Deep learning training #################################
## checkpoint path, batch size. Callbacks setting
embedding_matrix = loadEmbedding('fasttext', config['max_features'],
config['embed_size'], tokenizer)
print "embeddings loaded"
print "Building the model"
inp = Input(shape=(config['maxlen'], ))
x = Embedding(config['max_features'],
config['embed_size'],
weights=[embedding_matrix],
trainable=True)(inp)
x = SpatialDropout1D(0.2)(x)
#x = Bidirectional(GRU(64, return_sequences=True,dropout=0.1, recurrent_dropout=0.3))(x)
#x = Bidirectional(GRU(80, return_sequences=True, recurrent_dropout=0.3))(x)
x = Bidirectional(GRU(80, return_sequences=True))(x)
x_1 = GlobalMaxPool1D()(x)
x_2 = GlobalAveragePooling1D()(x)
x = concatenate([x_1, x_2])
#x = BatchNormalization()(x)
#x = Dense(50, activation="relu")(x)
#x = BatchNormalization()(x)
#x = Dropout(0.1)(x)
x = Dense(6, activation="sigmoid")(x)
model = Model(inputs=inp, outputs=x)
import keras.backend as K
def loss(y_true, y_pred):
return K.binary_crossentropy(y_true, y_pred)
#opt = optimizers.Nadam(lr=0.001)
model.compile(loss=loss, optimizer='adam', metrics=['accuracy'])
def get_test_model_full():
"""Returns a maximally complex test model,
using all supported layer types with different parameter combination.
"""
input_shapes = [
(26, 28, 3),
(4, 4, 3),
(4, 4, 3),
(4, ),
(2, 3),
(27, 29, 1),
(17, 1),
(17, 4),
(2, 3),
(2, 3, 4, 5),
(2, 3, 4, 5, 6),
]
inputs = [Input(shape=s) for s in input_shapes]
outputs = []
outputs.append(Flatten()(inputs[3]))
outputs.append(Flatten()(inputs[4]))
outputs.append(Flatten()(inputs[5]))
outputs.append(Flatten()(inputs[9]))
outputs.append(Flatten()(inputs[10]))
for inp in inputs[6:8]:
for padding in ['valid', 'same']:
for s in range(1, 6):
for out_channels in [1, 2]:
for d in range(1, 4):
outputs.append(
Conv1D(out_channels,
s,
padding=padding,
dilation_rate=d)(inp))
for padding_size in range(0, 5):
outputs.append(ZeroPadding1D(padding_size)(inp))
for crop_left in range(0, 2):
for crop_right in range(0, 2):
outputs.append(Cropping1D((crop_left, crop_right))(inp))
for upsampling_factor in range(1, 5):
outputs.append(UpSampling1D(upsampling_factor)(inp))
for padding in ['valid', 'same']:
for pool_factor in range(1, 6):
for s in range(1, 4):
outputs.append(
MaxPooling1D(pool_factor, strides=s,
padding=padding)(inp))
outputs.append(
AveragePooling1D(pool_factor,
strides=s,
padding=padding)(inp))
outputs.append(GlobalMaxPooling1D()(inp))
outputs.append(GlobalAveragePooling1D()(inp))
for inp in [inputs[0], inputs[5]]:
for padding in ['valid', 'same']:
for h in range(1, 6):
for out_channels in [1, 2]:
for d in range(1, 4):
outputs.append(
Conv2D(out_channels, (h, 1),
padding=padding,
dilation_rate=(d, 1))(inp))
outputs.append(
SeparableConv2D(out_channels, (h, 1),
padding=padding,
dilation_rate=(d, 1))(inp))
for sy in range(1, 4):
outputs.append(
Conv2D(out_channels, (h, 1),
strides=(1, sy),
padding=padding)(inp))
outputs.append(
SeparableConv2D(out_channels, (h, 1),
strides=(sy, sy),
padding=padding)(inp))
for sy in range(1, 4):
outputs.append(
DepthwiseConv2D((h, 1),
strides=(sy, sy),
padding=padding)(inp))
outputs.append(
MaxPooling2D((h, 1), strides=(1, sy),
padding=padding)(inp))
for w in range(1, 6):
for out_channels in [1, 2]:
for d in range(1, 4) if sy == 1 else [1]:
outputs.append(
Conv2D(out_channels, (1, w),
padding=padding,
dilation_rate=(1, d))(inp))
outputs.append(
SeparableConv2D(out_channels, (1, w),
padding=padding,
dilation_rate=(1, d))(inp))
for sx in range(1, 4):
outputs.append(
Conv2D(out_channels, (1, w),
strides=(sx, 1),
padding=padding)(inp))
outputs.append(
SeparableConv2D(out_channels, (1, w),
strides=(sx, sx),
padding=padding)(inp))
for sx in range(1, 4):
outputs.append(
DepthwiseConv2D((1, w),
strides=(sy, sy),
padding=padding)(inp))
outputs.append(
MaxPooling2D((1, w), strides=(1, sx),
padding=padding)(inp))
outputs.append(ZeroPadding2D(2)(inputs[0]))
outputs.append(ZeroPadding2D((2, 3))(inputs[0]))
outputs.append(ZeroPadding2D(((1, 2), (3, 4)))(inputs[0]))
outputs.append(Cropping2D(2)(inputs[0]))
outputs.append(Cropping2D((2, 3))(inputs[0]))
outputs.append(Cropping2D(((1, 2), (3, 4)))(inputs[0]))
for y in range(1, 3):
for x in range(1, 3):
outputs.append(UpSampling2D(size=(y, x))(inputs[0]))
outputs.append(GlobalAveragePooling2D()(inputs[0]))
outputs.append(GlobalMaxPooling2D()(inputs[0]))
outputs.append(AveragePooling2D((2, 2))(inputs[0]))
outputs.append(MaxPooling2D((2, 2))(inputs[0]))
outputs.append(UpSampling2D((2, 2))(inputs[0]))
outputs.append(Dropout(0.5)(inputs[0]))
# same as axis=-1
outputs.append(Concatenate()([inputs[1], inputs[2]]))
outputs.append(Concatenate(axis=3)([inputs[1], inputs[2]]))
# axis=0 does not make sense, since dimension 0 is the batch dimension
outputs.append(Concatenate(axis=1)([inputs[1], inputs[2]]))
outputs.append(Concatenate(axis=2)([inputs[1], inputs[2]]))
outputs.append(BatchNormalization()(inputs[0]))
outputs.append(BatchNormalization(center=False)(inputs[0]))
outputs.append(BatchNormalization(scale=False)(inputs[0]))
outputs.append(Conv2D(2, (3, 3), use_bias=True)(inputs[0]))
outputs.append(Conv2D(2, (3, 3), use_bias=False)(inputs[0]))
outputs.append(SeparableConv2D(2, (3, 3), use_bias=True)(inputs[0]))
outputs.append(SeparableConv2D(2, (3, 3), use_bias=False)(inputs[0]))
outputs.append(DepthwiseConv2D(2, (3, 3), use_bias=True)(inputs[0]))
outputs.append(DepthwiseConv2D(2, (3, 3), use_bias=False)(inputs[0]))
outputs.append(Dense(2, use_bias=True)(inputs[3]))
outputs.append(Dense(2, use_bias=False)(inputs[3]))
shared_conv = Conv2D(1, (1, 1),
padding='valid',
name='shared_conv',
activation='relu')
up_scale_2 = UpSampling2D((2, 2))
x1 = shared_conv(up_scale_2(inputs[1])) # (1, 8, 8)
x2 = shared_conv(up_scale_2(inputs[2])) # (1, 8, 8)
x3 = Conv2D(1, (1, 1), padding='valid')(up_scale_2(inputs[2])) # (1, 8, 8)
x = Concatenate()([x1, x2, x3]) # (3, 8, 8)
outputs.append(x)
x = Conv2D(3, (1, 1), padding='same', use_bias=False)(x) # (3, 8, 8)
outputs.append(x)
x = Dropout(0.5)(x)
outputs.append(x)
x = Concatenate()([MaxPooling2D((2, 2))(x),
AveragePooling2D((2, 2))(x)]) # (6, 4, 4)
outputs.append(x)
x = Flatten()(x) # (1, 1, 96)
x = Dense(4, use_bias=False)(x)
outputs.append(x)
x = Dense(3)(x) # (1, 1, 3)
outputs.append(x)
outputs.append(keras.layers.Add()([inputs[4], inputs[8], inputs[8]]))
outputs.append(keras.layers.Subtract()([inputs[4], inputs[8]]))
outputs.append(keras.layers.Multiply()([inputs[4], inputs[8], inputs[8]]))
outputs.append(keras.layers.Average()([inputs[4], inputs[8], inputs[8]]))
outputs.append(keras.layers.Maximum()([inputs[4], inputs[8], inputs[8]]))
outputs.append(Concatenate()([inputs[4], inputs[8], inputs[8]]))
intermediate_input_shape = (3, )
intermediate_in = Input(intermediate_input_shape)
intermediate_x = intermediate_in
intermediate_x = Dense(8)(intermediate_x)
intermediate_x = Dense(5)(intermediate_x)
intermediate_model = Model(inputs=[intermediate_in],
outputs=[intermediate_x],
name='intermediate_model')
intermediate_model.compile(loss='mse', optimizer='nadam')
x = intermediate_model(x) # (1, 1, 5)
intermediate_model_2 = Sequential()
intermediate_model_2.add(Dense(7, input_shape=(5, )))
intermediate_model_2.add(Dense(5))
intermediate_model_2.compile(optimizer='rmsprop',
loss='categorical_crossentropy')
x = intermediate_model_2(x) # (1, 1, 5)
x = Dense(3)(x) # (1, 1, 3)
shared_activation = Activation('tanh')
outputs = outputs + [
Activation('tanh')(inputs[3]),
Activation('hard_sigmoid')(inputs[3]),
Activation('selu')(inputs[3]),
Activation('sigmoid')(inputs[3]),
Activation('softplus')(inputs[3]),
Activation('softmax')(inputs[3]),
Activation('relu')(inputs[3]),
LeakyReLU()(inputs[3]),
ELU()(inputs[3]),
PReLU()(inputs[2]),
PReLU()(inputs[3]),
PReLU()(inputs[4]),
shared_activation(inputs[3]),
Activation('linear')(inputs[4]),
Activation('linear')(inputs[1]),
x,
shared_activation(x),
]
print('Model has {} outputs.'.format(len(outputs)))
model = Model(inputs=inputs, outputs=outputs, name='test_model_full')
model.compile(loss='mse', optimizer='nadam')
# fit to dummy data
training_data_size = 1
batch_size = 1
epochs = 10
data_in = generate_input_data(training_data_size, input_shapes)
initial_data_out = model.predict(data_in)
data_out = generate_output_data(training_data_size, initial_data_out)
model.fit(data_in, data_out, epochs=epochs, batch_size=batch_size)
return model
def build_model(parameters,
no_of_classes,
input_length,
gpus=1,
optimizer='rmsprop',
loss=keras.losses.categorical_crossentropy):
input_shape = (input_length, )
model = keras.Sequential()
for k, v in parameters.items():
if k != 'accuracy' and k != 'loss':
parameters[k] = np.int64(v)
parameters['kernel'] = (parameters['kernel'], )
parameters['pool'] = (parameters['pool'], )
if parameters['embed_size'] != 0:
model.add(
Embedding(256, parameters['embed_size'],
input_length=input_length))
elif parameters['encoder']: # else use autoencoder
start_time = time.time()
print(f'loading encoder: "{parameters["encoder"]}"...', end='')
try:
encoder = keras.models.load_model(parameters['encoder']).layers[1]
except FileNotFoundError as fnfe:
raise FileNotFoundError(
f'Error loading encoder: "{parameters["encoder"]}"'
f'the "encoder" argument must be a path to a saved encoder model, '
f'example: "./encoder_output/encoder_model.h5"', fnfe)
print('encoder loaded in {:.2f}'.format(time.time() - start_time))
model.add(encoder)
else:
raise ValueError(
'Both "embed_size"={} and "encoder"={} are invalid values. At least one of them must have a value.'
.format(parameters["embed_size"], parameters["encoder"]))
if parameters['layers'] <= 0:
raise ValueError(
'\"layers\" parameter must be a positive integer, got \"layers\"={0}'
.format(parameters['layers']))
for _ in range(parameters['layers']):
model.add(
Conv1D(filters=parameters['filter'],
kernel_size=parameters['kernel']))
model.add(LeakyReLU(alpha=0.3))
model.add(MaxPool1D(parameters['pool']))
model.add(GlobalAveragePooling1D())
#model.add(Dropout(0.3))
model.add(Dropout(0.5))
model.add(Dense(parameters['dense']))
model.add(LeakyReLU(alpha=0.3))
#model.add(Dense(no_of_classes, activation='softmax'))
model.add(
Dense(no_of_classes,
activation='softmax',
kernel_regularizer=keras.regularizers.l2()))
# transform the model to a parallel one if multiple gpus are available.
if gpus != 1:
model = keras.utils.multi_gpu_model(model, gpus=gpus)
if optimizer is not None and loss is not None:
# compiling model
model.compile(optimizer=optimizer, loss=loss, metrics=['acc'])
model.build(input_shape=input_shape)
# model.summary()
return model
def build_model(WINDOW_SIZE):
abits = 16
wbits = 16
kernel_lr_multiplier = 10
def quantized_relu(x):
return quantize_op(x,nb=abits)
def binary_tanh(x):
return binary_tanh_op(x)
# network_type = 'float'
#network_type ='qnn'
# network_type = 'full-qnn'
network_type ='bnn'
# network_type = 'full-bnn'
H = 1.
if network_type =='float':
Conv_ = lambda f, s, c, n: Conv1D(kernel_size=s, filters=f, padding='same', activation='linear',
input_shape = (c,1), name = n)
Conv = lambda f, s, n: Conv1D(kernel_size= s, filters=f, padding='same', activation='linear', name = n)
Dense_ = lambda f, n: Dense(units = f, kernel_initializer='normal', activation='relu', name = n)
Act = lambda: ReLU()
elif network_type=='qnn':
# sys.exit(0)
Conv_ = lambda f, s, c, n: QuantizedConv1D(kernel_size= s, H=1, nb=wbits, filters=f, strides=1,
padding='same', activation='linear',
input_shape = (c,1),name = n)
Conv = lambda f, s, n: QuantizedConv1D(kernel_size=s, H=1, nb=wbits, filters=f, strides= 1,
padding='same', activation='linear',
name = n)
Act = lambda: ReLU()
Dense_ = lambda f, n: QuantizedDense(units = f, nb = wbits, name = n)
elif network_type=='full-qnn':
#sys.exit(0)
Conv_ = lambda f, s, c, n: QuantizedConv1D(kernel_size= s, H=1, nb=wbits, filters=f, strides=1,
padding='same', activation='linear',
input_shape = (c,1),name = n)
Conv = lambda f, s, n: QuantizedConv1D(kernel_size=s, H=1, nb=wbits, filters=f, strides= 1,
padding='same', activation='linear',
name = n)
Act = lambda: Activation(quantized_relu)
Dense_ = lambda f, n: QuantizedDense(units = f, nb = wbits, name = n)
elif network_type=='bnn':
# sys.exit(0)
Conv_ = lambda f,s,c,n: BinaryConv1D(kernel_size= s, H=1, filters=f, strides=1, padding='same',
activation='linear',
input_shape = (c,1),
name = n)
Conv = lambda f,s,n: BinaryConv1D(kernel_size=s, H=1, filters=f, strides=1, padding='same',
activation='linear',
name = n )
Dense_ = lambda f, n: BinaryDense(units = f, name = n)
Act = lambda: ReLU()
elif network_type=='full-bnn':
#sys.exit(0)
Conv_ = lambda f,s,c,n: BinaryConv1D(kernel_size= s, H=1, filters=f, strides=1, padding='same',
activation='linear',
input_shape = (c,1),
name = n)
Conv = lambda f,s,n: BinaryConv1D(kernel_size=s, H=1, filters=f, strides=1, padding='same',
activation='linear',
name = n )
Act = lambda: Activation(binary_tanh)
else:
#sys.exit(0)
print('wrong network type, the supported network types in this repo are float, qnn, full-qnn, bnn and full-bnn')
model = Sequential()
OUTPUT_CLASS = 4 # output classes
#model = Sequential()
model.add(Conv_(64, 55, WINDOW_SIZE, 'conv1') )
model.add(Act())
model.add(MaxPooling1D(10))
model.add(Dropout(0.5))
model.add(Conv(64, 25, 'conv2' ))
model.add(Act())
model.add(MaxPooling1D(5))
model.add(Dropout(0.5))
model.add(Conv(64, 10, 'conv3'))
model.add(Act())
model.add(GlobalAveragePooling1D())
model.add(Dense_(256, 'den6'))
model.add(Dropout(0.5))
model.add(Dense_(128, 'den7'))
model.add(Dropout(0.5))
model.add(Dense_(64, 'den8'))
model.add(Dropout(0.5))
model.add(Dense(OUTPUT_CLASS, kernel_initializer='normal', activation='softmax', name = 'den9'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
print(model.summary())
plot_model(model, to_file='my_model.png')
return model
def create_model(self,
n_lstm_cells=8,
dropout_rate=0.8,
permute_dims=(2, 1),
conv1d_depths=[128, 256, 128],
conv1d_kernels=[8, 5, 3],
local_initializer='he_uniform',
activation_func='relu',
Attention=False,
Squeeze=True,
squeeze_ratio=16,
pre_convolve_rnn=False,
pre_convolve_rnn_stride=2,
squeeze_initializer='he_normal',
logit_output='sigmoid',
use_bias=False,
verbose=False):
input_layer = Input(shape=self.input_shape)
''' Create the LSTM-RNN Tower for the MLSTM-FCN Architecture '''
if pre_convolve_rnn:
''' sabsample timesteps to prevent "Out-of-Memory Errors"
due to the Attention LSTM's size '''
# permute to match Conv1D configuration
rnn_tower = Permute(permute_dims)(input_layer)
rnn_tower = Conv1D(self.input_shape[0] // stride,
conv1d_kernels[0],
strides=stride,
padding='same',
activation=activation_func,
use_bias=use_bias,
kernel_initializer=local_initializer)(rnn_tower)
# re-permute to match LSTM configuration
rnn_tower = Permute(permute_dims)(rnn_tower)
rnn_tower = Masking()(rnn_tower)
else:
# Default behaviour is to mask the input layer itself
rnn_tower = Masking()(input_layer)
if Attention:
rnn_tower = AttentionLSTM(n_lstm_cells)(rnn_tower)
else:
rnn_tower = LSTM(n_lstm_cells)(rnn_tower)
rnn_tower = Dropout(dropout_rate)(rnn_tower)
''' Create the Convolution Tower for the MLSTM-FCN Architecture '''
conv1d_tower = Permute(permute_dims)(input_layer)
zipper = zip(conv1d_depths, conv1d_kernels)
for kl, (conv1d_depth, conv1d_kernel) in enumerate(zipper):
# Loop over all convolution kernel sizes and depths
# to create the Convolution Tower
conv1d_tower = Conv1D_Stack(conv1d_tower,
conv1d_depth=conv1d_depth,
conv1d_kernel=conv1d_kernel,
activation_func=activation_func,
local_initializer=local_initializer)
if Squeeze:
conv1d_tower = squeeze_excite_block(
conv1d_tower,
squeeze_ratio=squeeze_ratio,
activation_func=activation_func,
logit_output=logit_output,
kernel_initializer=squeeze_initializer,
use_bias=use_bias)
# Turn off Squeeze after the second to last layer
# to avoid Squeezing at the last layer
if kl + 2 == len(conv1d_kernels): Squeeze = False
conv1d_tower = GlobalAveragePooling1D()(conv1d_tower)
output_layer = concatenate([rnn_tower, conv1d_tower])
output_layer = Dense(self.num_classes,
activation=logit_output)(output_layer)
self.model = Model(input_layer, output_layer)
if self.verbose or verbose: self.model.summary()
np.average(item),
np.std(item),
np.max(item),
np.min(item),
size,
]).reshape((1, 6))
return sequence, out
EXAMPLES = 15000
model = Sequential()
model.add(Conv1D(8, 5, input_shape=(None, 1)))
model.add(Conv1D(16, 5, activation='relu'))
model.add(GlobalAveragePooling1D())
model.add(Dense(6))
# MUCH slower
# model = Sequential()
# model.add(LSTM(16, input_dim=1))
# model.add(Dense(5))
model.compile(loss="mean_squared_error", optimizer="rmsprop")
model.summary()
for e in trange(EXAMPLES):
sequence, label = datum(
500000 if r.random() < 0.001 else r.randint(50, 100))
model.train_on_batch(sequence, label)
def build_model():
#wrote out all the blocks instead of looping for simplicity
filter_nr = 64
filter_size = 3
max_pool_size = 3
max_pool_strides = 2
dense_nr = 256
spatial_dropout = 0.2
dense_dropout = 0.1
train_embed = False
inp = Input(shape=(max_len,))
emb_comment = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=train_embed)(inp)
emb_comment = SpatialDropout1D(spatial_dropout)(emb_comment)
block1 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear')(emb_comment)
block1 = BatchNormalization()(block1)
block1 = PReLU()(block1)
block1 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear')(block1)
block1 = BatchNormalization()(block1)
block1 = PReLU()(block1)
#we pass embedded comment through conv1d with filter size 1 because it needs to have the same shape as block output
#if you choose filter_nr = embed_size (300 in this case) you don't have to do this part and can add emb_comment directly to block1_output
resize_emb = Conv1D(filter_nr, kernel_size=1, padding='same', activation='linear')(emb_comment)
resize_emb = PReLU()(resize_emb)
block1_output = add([block1, resize_emb])
block1_output = MaxPooling1D(pool_size=max_pool_size, strides=max_pool_strides)(block1_output)
block2 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear')(block1_output)
block2 = BatchNormalization()(block2)
block2 = PReLU()(block2)
block2 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear')(block2)
block2 = BatchNormalization()(block2)
block2 = PReLU()(block2)
block2_output = add([block2, block1_output])
block2_output = MaxPooling1D(pool_size=max_pool_size, strides=max_pool_strides)(block2_output)
block3 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear')(block2_output)
block3 = BatchNormalization()(block3)
block3 = PReLU()(block3)
block3 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear')(block3)
block3 = BatchNormalization()(block3)
block3 = PReLU()(block3)
block3_output = add([block3, block2_output])
block3_output = MaxPooling1D(pool_size=max_pool_size, strides=max_pool_strides)(block3_output)
block4 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear')(block3_output)
block4 = BatchNormalization()(block4)
block4 = PReLU()(block4)
block4 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear')(block4)
block4 = BatchNormalization()(block4)
block4 = PReLU()(block4)
output = add([block4, block3_output])
max1d = GlobalMaxPooling1D()(output)
avg1d = GlobalAveragePooling1D()(output)
output = concatenate([max1d, avg1d])
# output = Dense(dense_nr, activation='linear')(output)
# output = BatchNormalization()(output)
# output = PReLU()(output)
output = Dropout(dense_dropout)(output)
output = Dense(6, activation='sigmoid')(output)
model = Model(inputs = inp, outputs = output)
model.compile(loss = "binary_crossentropy", optimizer = 'adam', metrics = ["accuracy"])
# model.compile(loss = "binary_crossentropy", optimizer = Adam(lr = lr, decay = lr_d), metrics = ["accuracy"])
History = model.fit(X_train, Y_train, batch_size = 128, epochs = 24, validation_data = (X_valid, Y_valid),
verbose = 1, callbacks = [callback, early_stop])
return model, History
def build_discriminator(seq_shape, n_classes, hidden_units=128):
# input (None, 4, 8)
seq_layer = Input(shape=(seq_shape[1], seq_shape[2],))
if method == 'resnet':
b1 = Conv1D(filters=hidden_units, kernel_size=8, padding='same')(seq_layer)
b1 = BatchNormalization()(b1)
b1 = Activation('relu')(b1)
b1 = Conv1D(filters=hidden_units, kernel_size=5, padding='same')(b1)
b1 = BatchNormalization()(b1)
b1 = Activation('relu')(b1)
b1 = Conv1D(filters=hidden_units, kernel_size=3, padding='same')(b1)
b1 = BatchNormalization()(b1)
shortcut = Conv1D(filters=hidden_units, kernel_size=1, padding='same')(seq_layer)
shortcut = BatchNormalization()(shortcut)
b1 = add([shortcut, b1])
b1 = Activation('relu')(b1)
# block 2
b2 = Conv1D(filters=hidden_units*2, kernel_size=8, padding='same')(b1)
b2 = BatchNormalization()(b2)
b2 = Activation('relu')(b2)
b2 = Conv1D(filters=hidden_units*2, kernel_size=5, padding='same')(b2)
b2 = BatchNormalization()(b2)
b2 = Activation('relu')(b2)
b2 = Conv1D(filters=hidden_units*2, kernel_size=3, padding='same')(b2)
b2 = BatchNormalization()(b2)
shortcut = Conv1D(filters=hidden_units*2, kernel_size=1, padding='same')(b1)
shortcut = BatchNormalization()(shortcut)
b2 = add([shortcut, b2])
b2 = Activation('relu')(b2)
# block 3
b3 = Conv1D(filters=hidden_units*2, kernel_size=8, padding='same')(b2)
b3 = BatchNormalization()(b3)
b3 = Activation('relu')(b3)
b3 = Conv1D(filters=hidden_units*2, kernel_size=5, padding='same')(b3)
b3 = BatchNormalization()(b3)
b3 = Activation('relu')(b3)
b3 = Conv1D(filters=hidden_units*2, kernel_size=3, padding='same')(b3)
b3 = BatchNormalization()(b3)
shortcut = BatchNormalization()(b2)
b3 = add([shortcut, b3])
b3 = Activation('relu')(b3)
# output
h = GlobalAveragePooling1D()(b3)
if method == 'cnn':
# downsample (None, 4, 128)
h = Conv1D(filters=hidden_units, kernel_size=3, strides=1, padding='same')(seq_layer)
h = LeakyReLU(alpha=0.2)(h)
# downsample (None, 4, 128)
h = Conv1D(filters=hidden_units, kernel_size=3, strides=1, padding='same')(h)
h = LeakyReLU(alpha=0.2)(h)
# downsample (None, 4, 128)
h = Conv1D(filters=hidden_units, kernel_size=3, strides=1, padding='same')(h)
h = LeakyReLU(alpha=0.2)(h)
# fully connect (None, 512)
h = Flatten()(h)
h = Dropout(0.4)(h)
if method == 'lstm':
h = LSTM(hidden_units, return_sequences=True)(seq_layer)
h = LSTM(hidden_units)(h)
h = Dense(n_classes)(h)
# supervised output
sup_layer = Activation('softmax')(h)
# build supervised discriminator
sup_model = Model(seq_layer, sup_layer)
sup_model.compile(loss='categorical_crossentropy',
optimizer=Adam(lr=learning_rate, beta_1=beta_1),
metrics=['accuracy'])
# unsupervised output
unsup_layer = Lambda(custom_activation)(h)
# build unsupervised discriminator
unsup_model = Model(seq_layer, unsup_layer)
unsup_model.compile(loss='binary_crossentropy',
optimizer=Adam(lr=learning_rate, beta_1=beta_1),
metrics=['accuracy'])
return sup_model, unsup_model
input_length=max_sequence_length,
trainable=True)
inputs = Input(shape=(max_sequence_length, ), dtype='int32', name='input')
embeddings_sequences = embeddings_layer(inputs)
output = Conv1D(32, filter_size, padding='valid', activation='relu',
strides=1)(embeddings_sequences)
# dropout=Dropout(0.25)(output)
output = Conv1D(32, filter_size, padding='valid', activation='relu',
strides=1)(output)
dropout = Dropout(0.25)(output)
output = Conv1D(32, filter_size, padding='valid', activation='relu',
strides=1)(dropout)
dropout = Dropout(0.25)(output)
output = GlobalAveragePooling1D()(dropout)
dropout = Dropout(0.25)(output)
# output=Dense(32,activation='relu')(dropout)
print(output)
output = Dense(1, activation='sigmoid')(output)
model = Model(inputs=inputs, outputs=[output])
model.summary()
model.compile(loss='binary_crossentropy',
optimizer=Adam(0.0001),
metrics=['accuracy'])
checkpoint_filepath = 'D:/DeepLearning/bully_code/diyu/indrnn.h5'
checkpoint = ModelCheckpoint(checkpoint_filepath,
monitor='acc',
