[125] [145] [165] [185]] """ # 将(6,3)转换成3个(6,1) y1 = y[:, 0].reshape((y.shape[0], 1)) y2 = y[:, 1].reshape((y.shape[0], 1)) y3 = y[:, 2].reshape((y.shape[0], 1)) # 定义网络输入和隐藏层的形状，(None,3,3)，第一个None元素代表有N组数据 visible = Input(shape=(n_steps, n_features)) # 卷积层(None,3,3)->(None,2,64) cnn = Conv1D(filters=64, kernel_size=2, activation='relu')(visible) # 池化层(None,2,64)->(None,1,64) cnn = MaxPooling1D(pool_size=2)(cnn) # 平滑层(None,1,64)->(None,64) cnn = Flatten()(cnn) # 密集层(None,64)->(None,50) cnn = Dense(50, activation='relu')(cnn) # 定义多路输出的网络结构 # 密集层(None,50)->(None,1) output1 = Dense(1)(cnn) # 密集层(None,50)->(None,1) output2 = Dense(1)(cnn) # 密集层(None,50)->(None,1) output3 = Dense(1)(cnn) # 将输入结构，和三路输出结构定义到模型中 model = Model(inputs=visible, outputs=[output1, output2, output3])
def Dnn(df, y, encoder, class_wt, trained_w2v, batch_size, n_epoch, \
best_score, feature, weighting, algorithm, repeat, kfold):
y = dict(zip(df.fname, y))
print("Class weight")
class_weight = {}
for i in xrange(len(encoder.classes_)):
if class_wt:
class_weight[i] = len(y) * 1. / sum(x == i for x in y.values())
else:
class_weight[i] = 1
print 'Read corpus for word2vec'
t = time.time()
raw = pd.DataFrame(df[feature]).values.tolist()
fnames = df['fname'].values.tolist()
print "Read %d corpus of documents" % len(raw)
t_e = time.time() - t
print 'Corpus reading time ' + str(t_e) + ' sec'
dict_content = zip(fnames, raw)
tok_dict = dict(dict_content)
del raw, fnames, dict_content
w2vTraining = [item for l in tok_dict.values() for item in l]
print 'Load word2vec model'
if trained_w2v == 'no':
t_w = time.time()
model, featureMatrix = W2v(data=w2vTraining, \
dimension=200, window=5, subsample=1e-5, \
stem=False, concept=True, removeStopwords=True)
t_w2v = time.time() - t_w
print 'Word2vec training time ' + str(t_w2v) + ' sec'
elif trained_w2v == 'umls':
model = word2vec.Word2Vec.load_word2vec_format(
'/Users/weng/_hms_phi/DeVine_etal_200.txt', binary=False)
elif trained_w2v == 'pubmed':
model = word2vec.Word2Vec.load_word2vec_format(
'/Users/weng/_hms_phi/wikipedia-pubmed-and-PMC-w2v.bin',
binary=True)
elif trained_w2v == 'google':
model = word2vec.Word2Vec.load_word2vec_format('.bin', binary=True)
w2v_dict = {}
for k in tok_dict.keys():
w2v_dict[k] = tok_dict[k][0]
w2v_idx, w2v_vec, w2v_emb = create_dictionaries(data=w2v_dict,
model=model,
feature=feature)
vocab_dim = model.vector_size
n_symbols = len(w2v_idx)
embedding_weights = np.zeros((n_symbols + 1, vocab_dim))
for word, index in w2v_idx.items():
try:
embedding_weights[index, :] = w2v_vec[word]
except KeyError:
embedding_weights[index, :] = [0] * model.vector_size
#embedding_weights = embedding_weights[1:]
max_length = max((len(v), k) for k, v in w2v_emb.iteritems())[0]
X = w2v_dict
#https://github.com/fchollet/keras/issues/853
#There are 3 approaches:
#
#Learn embedding from scratch - simply add an Embedding layer to your model
#Fine tune learned embeddings - this involves setting word2vec / GloVe vectors as your Embedding layer's weights.
#Use word word2vec / Glove word vectors as inputs to your model, instead of one-hot encoding.
#The third one is the best option(Assuming the word vectors were obtained from the same domain as the inputs to your models. For e.g, if you are doing sentiment analysis on tweets, you should use GloVe vectors trained on tweets).
#
#In the first option, everything has to be learned from scratch. You dont need it unless you have a rare scenario. The second one is good, but, your model will be unnecessarily big with all the word vectors for words that are not frequently used.
np.random.seed(777)
print("Constrcut the dataset")
tmp = [X, y]
d = {}
for k in X.iterkeys():
d[k] = tuple(d[k] for d in tmp)
X = []
y = []
for k, v in d.iteritems():
X.append(v[0])
y.append(v[1])
print("Split the dataset")
X_train, X_test, y_train, y_test = train_test_split(X,
y,
stratify=y,
test_size=0.3,
random_state=42)
#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)
print("Pad sequences (samples x time)")
X_train = sequence.pad_sequences(X_train, maxlen=max_length)
X_test = sequence.pad_sequences(X_test, maxlen=max_length)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
print("Convert labels to Numpy Sets")
y_train = np.array(y_train)
y_test = np.array(y_test)
print("Convert to categories")
y_train_dummy = to_categorical(y_train)
y_test_dummy = to_categorical(y_test)
print("Modeling")
input_length = max_length
cpu_count = multiprocessing.cpu_count()
pred_table = pd.DataFrame()
score_table = pd.DataFrame()
best_score = 0
best_model = best_feature = best_algorithm = coef = ''
skf = StratifiedKFold(y=y_train,
n_folds=kfold,
shuffle=True,
random_state=None)
for idx, (train, test) in enumerate(skf):
model = Sequential()
model.add(
Embedding(
input_dim=n_symbols + 1,
output_dim=vocab_dim,
#mask_zero = False, # need to be zero otherwise cnn won't work
weights=[embedding_weights],
input_length=input_length,
trainable=True))
model.add(
Convolution1D(nb_filter=64,
filter_length=2,
border_mode='valid',
activation='relu'))
#model.add(Convolution1D(nb_filter=32, filter_length=2, border_mode='valid', activation='relu'))
model.add(MaxPooling1D(pool_length=2))
model.add(Dropout(0.5))
#model.add(Convolution1D(nb_filter=32, filter_length=3, border_mode='valid', activation='relu'))
#model.add(MaxPooling1D(pool_length=2))
#model.add(Dropout(0.5))
#model.add(Bidirectional(LSTM(32)))
#model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(y_train_dummy.shape[1] * 10, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(y_train_dummy.shape[1], activation='softmax'))
#model.compile(loss='binary_crossentropy', optimizer='adadelta', metrics=['accuracy'])
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.summary()
print("Training")
early_stopping = EarlyStopping(monitor='val_loss', patience=3)
t = time.time()
model.fit(X_train, y_train_dummy, batch_size=batch_size, nb_epoch=n_epoch, class_weight=class_weight, \
validation_data=(X_test, y_test_dummy), shuffle = True, callbacks=[early_stopping])
time.sleep(0.1)
t_e = time.time() - t
print "Model: " + feature + ' | ' + str(algorithm) + ' | rep' + str(
repeat + 1) + ' | cv' + str(kfold +
1) + ' | time: ' + str(t_e) + ' sec'
y_pred_prob = model.predict_proba(X_test)
try:
y_pred = model.predict_classes(X_test) # different
except TypeError:
y_pred = model.predict(X_test.values)
t = pd.concat([pd.DataFrame(y_test), pd.DataFrame(y_pred)], axis=1)
t.columns = ['true', 'pred']
pred_tbl = pd.DataFrame(y_pred_prob)
pred_tbl = pd.concat([t, pred_tbl], axis=1)
pred_tbl['rep'] = repeat + 1
pred_tbl['k'] = kfold + 1
pred_tbl['algorithm'] = str(algorithm)
pred_tbl['feature'] = feature
pred_tbl['weighting'] = weighting
acc = accuracy_score(y_test, y_pred)
pr, re, f1, xx = precision_recall_fscore_support(
y_test, y_pred, average='binary') # didn't use 'macro'
auc = multiclass_roc_auc_score(
y_test, y_pred, average='weighted'
) # weighted AUC takes imbalanced label into account
print acc, pr, re, f1, auc
metrics = pd.DataFrame([0])
metrics['time'] = t_e
metrics['accuracy'] = acc
metrics['precision'] = pr
metrics['recall'] = re
metrics['f1'] = f1
metrics['auc'] = auc
metrics['rep'] = repeat + 1
metrics['k'] = idx + 1
metrics['algorithm'] = str(algorithm)
metrics['feature'] = feature
metrics['weighting'] = weighting
if auc > best_score:
best_model = model
best_score = auc
best_feature = feature
best_algorithm = str(algorithm)[0:9]
best_coef = coef
#encoder.inverse_transform(y_pred) #real label name
Y_test = np.argmax(y_test_dummy, axis=1)
Y_pred = np.argmax(y_pred_prob, axis=1)
print(classification_report(Y_test, Y_pred))
pred_table = pd.concat([pred_table, pred_tbl], axis=0)
score_table = pd.concat([score_table, metrics], axis=0)
return best_model, best_feature, best_algorithm, best_coef, score_table, pred_table, metrics
#j = open('best_dnn_model.json', 'r')
#json = j.read()
#j.close()
#model = model_from_json(json)
#model.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy'])
#score = model.evaluate(X_test, y_test_dummy, verbose=0)
#https://github.com/dandxy89/DeepLearning_MachineLearning/blob/master/EmbeddingKeras/imdb_embedding_w2v.py
#http://machinelearningmastery.com/sequence-classification-lstm-recurrent-neural-networks-python-keras/
#https://github.com/fchollet/keras/issues/1629 (consider biridectional lstm)
#http://datascience.stackexchange.com/questions/10048/what-is-the-best-keras-model-for-multi-label-classification (for binary)
#
# '''
# as feature
# '''
# print("Use layer output as features")
# #https://keras.io/getting-started/faq/#how-can-i-visualize-the-output-of-an-intermediate-layer
# print("Extract layer output")
# get_8th_layer_output = K.function([model.layers[0].input, K.learning_phase()],
# [model.layers[8].output])
# output_train = get_8th_layer_output([X_train, 1])[0]
# output_test = get_8th_layer_output([X_test, 0])[0]
#
#
# rf(output_train, y_train, output_test, y_test)
# svc(output_train, y_train, output_test, y_test)
def lstm(print_number):
seed = 4715
i = datetime.now()
orig_stdout = sys.stdout
orig_stderr = sys.stderr
logname = 'log_l_' + print_number + '.txt'
f = open(logname, 'w')
sys.stdout = f
sys.stderr = f
reviews = pickle.load(open('data/reviews.pkl', 'rb'))
prices = pickle.load(open('data/prices.pkl', 'rb'))
reviews_train = pickle.load(open('data/reviews_train.pkl', 'rb'))
reviews_test = pickle.load(open('data/reviews_test.pkl', 'rb'))
prices_train = pickle.load(open('data/prices_train.pkl', 'rb'))
prices_test = pickle.load(open('data/prices_test.pkl', 'rb'))
int_to_word = pickle.load(open('data/int_to_word.pkl', 'rb'))
int_to_char = pickle.load(open('data/int_to_char.pkl', 'rb'))
print('Number of reviews: ', len(reviews))
print('Number of vocab: ', len(int_to_word))
max_review_length = 500
reviews_train = sequence.pad_sequences(reviews_train,
maxlen=max_review_length)
reviews_test = sequence.pad_sequences(reviews_test,
maxlen=max_review_length)
# create the model
embedding_vecor_length = 32
model = Sequential()
model.add(
Embedding(len(reviews),
embedding_vecor_length,
input_length=max_review_length))
model.add(
Convolution1D(nb_filter=32,
filter_length=3,
border_mode='same',
activation='relu'))
model.add(MaxPooling1D(pool_length=2))
model.add(LSTM(100))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print(model.summary())
# define the checkpoint
filepath = 'weights_l_' + str(
print_number
) + '/weights-improvement-e{epoch:02d}-l{loss:.4f}-a{acc:.4f}.hdf5'
mkdir_p(os.path.dirname(filepath))
checkpoint = ModelCheckpoint(filepath, monitor='loss', verbose=1)
callbacks_list = [checkpoint]
model.fit(reviews_train,
prices_train,
nb_epoch=3,
batch_size=64,
callbacks=callbacks_list)
# Final evaluation of the model
predicted = model.predict(reviews_test, verbose=0)
scores = model.evaluate(reviews_test, prices_test, verbose=0)
print("Accuracy: %.2f%%" % (scores[1] * 100))
pathpred = 'data/predicted_classes_l_' + str(print_number) + '.pkl'
pickle.dump(predicted, open(pathpred, 'wb'))
print(predicted)
#report = classification_report(prices_test, predicted)
#print(report)
sys.stderr = orig_stderr
sys.stdout = orig_stdout
f.close()
# reshape from [samples, timesteps] into [samples, timesteps, features] X = X.reshape((X.shape[0], n_seq, n_steps, n_features)) raw_seq.shape X.shape y.shape # split train and test set x_train,x_test,y_train,y_test = train_test_split(X, y, test_size=0.2, random_state=1) # summarize the data for i in range(len(X)): print(X[i], y[i]) # define model model = Sequential() model.add(TimeDistributed(Conv1D(filters=100, kernel_size=4, activation='relu'), input_shape=(None, n_steps, n_features))) model.add(TimeDistributed(MaxPooling1D(pool_size=2))) model.add(TimeDistributed(Flatten())) model.add(TimeDistributed(Dropout(0.25))) model.add(LSTM(50, activation='relu')) model.add(Dense(1, activation='linear')) model.compile(optimizer='adam', loss='mae', metrics=[r_squared]) model.summary() # fit model history = model.fit(x_train, y_train, epochs=10000, verbose=1) X.shape y.shape # demonstrate prediction tr_pred = list(itertools.chain(*model.predict(x_train))) plt.plot(y_train) plt.plot(tr_pred)
# we start off with an efficient embedding layer which maps
# our vocab indices into embedding_dims dimensions
model.add(Embedding(max_features, embedding_dims, input_length=maxlen))
model.add(Dropout(0.25))
# we add a Convolution1D, which will learn nb_filter
# word group filters of size filter_length:
model.add(
Convolution1D(nb_filter=nb_filter,
filter_length=filter_length,
border_mode="valid",
activation="relu",
subsample_length=1))
# we use standard max pooling (halving the output of the previous layer):
model.add(MaxPooling1D(pool_length=2))
# We flatten the output of the conv layer,
# so that we can add a vanilla dense layer:
model.add(Flatten())
# We add a vanilla hidden layer:
model.add(Dense(hidden_dims))
model.add(Dropout(0.25))
model.add(Activation('relu'))
# We project onto a single unit output layer, and squash it with a sigmoid:
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
def create_rnn_model(selection_num):
if (selection_num == 0):
model = Sequential()
model.add(
Embedding(num_words,
word_vec_dim,
weights=[embedding_matrix],
input_length=maxlen,
trainable=False))
model.add((GRU(256,
dropout=drop_out_ratio,
recurrent_dropout=drop_out_ratio,
return_sequences=True)))
model.add((GRU(128,
dropout=drop_out_ratio,
recurrent_dropout=drop_out_ratio)))
model.add(Dense(64, activation='relu'))
model.add(Dropout(drop_out_ratio))
model.add(Dense(64, activation='relu'))
model.add(Dropout(drop_out_ratio))
model.add(Dense(y_train.shape[1], activation='sigmoid'))
if (selection_num == 1):
main_input = Input(shape=(maxlen, ), dtype='int32', name='main_input')
embedding = Embedding(num_words,
word_vec_dim,
weights=[embedding_matrix],
input_length=maxlen,
trainable=False)(main_input)
embedding = Dropout(drop_out_ratio)(embedding)
conv4 = Conv1D(nb_filter=200,
filter_length=4,
border_mode='valid',
activation='relu',
subsample_length=1,
name='conv4')(embedding)
maxConv4 = MaxPooling1D(pool_length=2, name='maxConv4')(conv4)
conv5 = Conv1D(nb_filter=200,
filter_length=5,
border_mode='valid',
activation='relu',
subsample_length=1,
name='conv5')(embedding)
maxConv5 = MaxPooling1D(pool_length=2, name='maxConv5')(conv5)
x = merge([maxConv4, maxConv5], mode='concat')
x = Dropout(drop_out_ratio)(x)
x = merge([maxConv4, maxConv5], mode='concat')
x = Dropout(drop_out_ratio)(x)
x = (GRU(256,
dropout=drop_out_ratio,
recurrent_dropout=drop_out_ratio,
return_sequences=True))(x)
x = (GRU(128, dropout=drop_out_ratio,
recurrent_dropout=drop_out_ratio))(x)
x = Dense(64)(x)
x = Dropout(drop_out_ratio)(x)
x = Dense(64)(x)
x = Dropout(drop_out_ratio)(x)
output = Dense(y_train.shape[1], activation='sigmoid',
name='output')(x)
model = Model(input=main_input, output=output)
return model
def run_neural_network():
print(" == Enter into CNN training step ==")
np.random.seed(0)
x_train = pd.read_pickle("data/pickles/train_after_preprocess.pkl")
x_train = np.array(x_train['tweet'])
x_test = pd.read_pickle("data/pickles/test_after_preprocess.pkl")
x_test = np.array(x_test['tweet'])
y = np.array(int(2500000 / 2) * [0] + int(2500000 / 2) * [1])
print("Data loading finish!")
# Tokenization
tokenizer = Tokenizer(filters='')
tokenizer.fit_on_texts(x_train)
# Turn x_train into sequence form
sequence_train = tokenizer.texts_to_sequences(x_train)
# Turn x_test into sequence form
sequence_test = tokenizer.texts_to_sequences(x_test)
# Transform sequence_train into into a 2D Numpy array
sequence_train = sequence.pad_sequences(sequence_train, maxlen=30)
# Transform sequence_test into into a 2D Numpy array
sequence_test = sequence.pad_sequences(sequence_test, maxlen=30)
# Affect input dimension
input_dim = len(tokenizer.word_index) + 1
input_length = sequence_train.shape[1]
print("Tokenization finish!")
# Shuffle training dataset
new_index = np.arange(sequence_train.shape[0])
np.random.shuffle(new_index)
sequence_train = sequence_train[new_index]
y = y[new_index]
print("Data shuffling finish!")
earlyStopping = EarlyStopping(monitor='val_loss', patience=2)
### Model 1 ###
print("Build model1!")
np.random.seed(1)
model = Sequential()
model.add(Embedding(input_dim, 50, input_length=input_length))
model.add(
Conv1D(padding="same", kernel_size=3, filters=32, activation="relu"))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
model.add(Dense(250, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print("Fit model1!")
model.fit(sequence_train,
y,
validation_split=0.1,
epochs=10,
batch_size=128,
verbose=1,
shuffle=True,
callbacks=[earlyStopping])
print("Generate prediction!")
train_model1 = model.predict(sequence_train, batch_size=128)
pickle.dump(train_model1, open('data/xgboost/train_model1.txt', 'wb'))
test_model1 = model.predict(sequence_test)
pickle.dump(test_model1, open('data/xgboost/test_model1.txt', 'wb'))
print("Model1 finished!")
### Model 2 ###
print("Build model2!")
np.random.seed(2)
model = Sequential()
model.add(Embedding(input_dim, 50, input_length=input_length))
model.add(LSTM(100, recurrent_dropout=0.2, dropout=0.2))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print("Fit model2!")
model.fit(sequence_train,
y,
validation_split=0.1,
epochs=10,
batch_size=128,
verbose=1,
shuffle=True,
callbacks=[earlyStopping])
print("Generate prediction!")
train_model2 = model.predict(sequence_train, batch_size=128)
pickle.dump(train_model2, open('data/xgboost/train_model2.txt', 'wb'))
test_model2 = model.predict(sequence_test)
pickle.dump(test_model2, open('data/xgboost/test_model2.txt', 'wb'))
print("Model2 finished!")
### Model 3 ###
print("Build model1!")
np.random.seed(3)
model = Sequential()
model.add(Embedding(input_dim, 50, input_length=input_length))
model.add(
Conv1D(padding="same", kernel_size=3, filters=32, activation="relu"))
model.add(MaxPooling1D(pool_size=2))
model.add(LSTM(100))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print("Fit model3!")
model.fit(sequence_train,
y,
validation_split=0.1,
epochs=10,
batch_size=128,
verbose=1,
shuffle=True,
callbacks=[earlyStopping])
print("Generate prediction!")
train_model3 = model.predict(sequence_train, batch_size=128)
pickle.dump(train_model3, open('data/xgboost/train_model3.txt', 'wb'))
test_model3 = model.predict(sequence_test)
pickle.dump(test_model3, open('data/xgboost/test_model3.txt', 'wb'))
print("Model3 finished!")
# Helpful resource for building convolutional neural network: # https://machinelearningmastery.com/cnn-models-for-human-activity-recognition-time-series-classification/ # Documentation for keras: # https://keras.io/ # Source for CNN design: # Yildirim O, Baloglu UB, Acharya UR. A Deep Learning Model for Automated Sleep Stages Classification Using PSG Signals. # Int J Environ Res Public Health. 2019;16(4):599. Published 2019 Feb 19. doi:10.3390/ijerph16040599 # Build the CNN n_timesteps, n_features, n_outputs = trainX.shape[1], trainX.shape[2], trainy.shape[1] model = Sequential() model.add(Conv1D(filters=64, kernel_size=5, strides=3, activation='relu', input_shape=(n_timesteps,n_features))) model.add(Conv1D(filters=128, kernel_size=5, strides=1, activation='relu')) model.add(MaxPooling1D(pool_size=2, strides=2)) model.add(Dropout(0.2)) model.add(Conv1D(filters=128, kernel_size=13, strides=1, activation='relu')) model.add(Conv1D(filters=256, kernel_size=7, strides=1, activation='relu')) model.add(MaxPooling1D(pool_size=2, strides=2)) model.add(Conv1D(filters=256, kernel_size=7, strides=1, activation='relu')) model.add(Conv1D(filters=64, kernel_size=4, strides=1, activation='relu')) model.add(MaxPooling1D(pool_size=2, strides=2)) model.add(Conv1D(filters=32, kernel_size=3, strides=1, activation='relu')) model.add(Conv1D(filters=64, kernel_size=6, strides=1, activation='relu')) model.add(MaxPooling1D(pool_size=2, strides=2)) model.add(Conv1D(filters=8, kernel_size=5, strides=1, activation='relu')) model.add(Conv1D(filters=8, kernel_size=2, strides=1, activation='relu')) model.add(MaxPooling1D(pool_size=2, strides=2)) model.add(Flatten()) model.add(Dense(64, activation='relu'))
class EnhancerCnnModel(nn.Module,input_dim=4, input_length=2705, nbfilter = 101):
model = Sequential()
# model.add(brnn)
model.add(Conv1D(input_dim=input_dim, input_length=input_length,
nb_filter=nbfilter,
filter_length=10,
border_mode="valid",
# activation="relu",
subsample_length=1))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(MaxPooling1D(pool_length=3))
model.add(Flatten())
model.add(Dropout(0.5))
model.add(Dense(nbfilter * 2, activation='relu'))
model.add(BatchNormalization())
model.add(Dropout(0.5))
return model
def __init__(self):
super(EnhancerCnnModel, self).__init__()
self.c1_1 = nn.Conv1d(8, CONV1D_FEATURE_SIZE_BLOCK1, CONV1D_KERNEL_SIZE, padding=1)
self.c1_1bn = nn.BatchNorm1d(CONV1D_FEATURE_SIZE_BLOCK1)
self.c1_2 = nn.Conv1d(CONV1D_FEATURE_SIZE_BLOCK1, CONV1D_FEATURE_SIZE_BLOCK1,
CONV1D_KERNEL_SIZE, padding=1)
self.c1_2bn = nn.BatchNorm1d(CONV1D_FEATURE_SIZE_BLOCK1)
self.c1_3 = nn.Conv1d(CONV1D_FEATURE_SIZE_BLOCK1, CONV1D_FEATURE_SIZE_BLOCK1,
CONV1D_KERNEL_SIZE, padding=1)
self.c1_3bn = nn.BatchNorm1d(CONV1D_FEATURE_SIZE_BLOCK1)
self.p1 = nn.MaxPool1d(AVGPOOL1D_KERNEL_SIZE)
self.c2_1 = nn.Conv1d(CONV1D_FEATURE_SIZE_BLOCK1,
CONV1D_FEATURE_SIZE_BLOCK2,
CONV1D_KERNEL_SIZE, padding=1)
self.c2_1bn = nn.BatchNorm1d(CONV1D_FEATURE_SIZE_BLOCK2)
self.c2_2 = nn.Conv1d(CONV1D_FEATURE_SIZE_BLOCK2, CONV1D_FEATURE_SIZE_BLOCK2,
CONV1D_KERNEL_SIZE, padding=1)
self.c2_2bn = nn.BatchNorm1d(CONV1D_FEATURE_SIZE_BLOCK2)
self.c2_3 = nn.Conv1d(CONV1D_FEATURE_SIZE_BLOCK2, CONV1D_FEATURE_SIZE_BLOCK2,
CONV1D_KERNEL_SIZE, padding=1)
self.c2_3bn = nn.BatchNorm1d(CONV1D_FEATURE_SIZE_BLOCK2)
self.p2 = nn.MaxPool1d(AVGPOOL1D_KERNEL_SIZE)
self.fc = nn.Linear(768, FULLY_CONNECTED_LAYER_SIZE)
self.out = nn.Linear(FULLY_CONNECTED_LAYER_SIZE, 1)
self.criterion = nn.BCELoss()
def forward(self, inputs):
batch_size = inputs.size(0)
# Turn (batch_size x seq_len) into (batch_size x input_size x seq_len) for CNN
#inputs = inputs.transpose(1,2)
#print("inputs size: ", inputs.size())
output = F.relu(self.c1_1bn(self.c1_1(inputs)))
output = F.relu(self.c1_2bn(self.c1_2(output)))
output = F.relu(self.c1_3bn(self.c1_3(output)))
output = self.p1(output)
#print("After p1: ", output.shape)
output = F.relu(self.c2_1bn(self.c2_1(output)))
output = F.relu(self.c2_2bn(self.c2_2(output)))
output = F.relu(self.c2_3bn(self.c2_3(output)))
output = self.p2(output)
#print("After p2: ", output.shape)
output = output.view(batch_size, -1)
#print("Reshape : ", output.shape)
output = F.relu(self.fc(output))
#print("After FC layer: ", output.shape)
output = torch.sigmoid(self.out(output))
#print("Final output (After sigmoid): ", output.shape)
#print("Final output: ", output)
return output
def create_cnn(n_timepoints,
n_parameters,
loss='mse',
optimizer='adam',
metrics=['mae'],
print_summary=True):
"""create_cnn
creates and compiles a cnn by adding a nr of Conv1D layers
that depends on the n_timepoints. These are topped off with a pair of
Dense layers that end up leading to the output parameters.
[description]
Parameters
----------
n_timepoints : int
number of timepoints in the data
n_parameters : int
number of parameters in the pRF model to be learned
loss : str, optional
Keras loss function (the default is 'mse', which is good for the learning of continuous outputs)
optimizer : str, optional
Keras optimizer function (the default is 'adam', which is good for the learning of continuous outputs)
metrics : list, optional
Keras metrics used for model performance quantification
(the default is ['mae'], which which is good for the learning of continuous outputs)
Returns
-------
Sequential
Compiled Keras model
"""
n_Conv1D_layers = n_timepoints // 32 - 1
minimal_kernel_size = 4
minimal_pool_size = 2
model = Sequential()
# first convolutional layer here
model.add(
Conv1D(filters=n_timepoints // 2,
kernel_size=minimal_kernel_size,
input_shape=(n_timepoints, 1)))
model.add(MaxPooling1D(pool_size=minimal_pool_size))
# loop over nr of required layers
for l in range(n_Conv1D_layers - 1):
kernel_size = minimal_kernel_size + minimal_kernel_size * l // 2
n_filters = n_timepoints // kernel_size
pool_size = minimal_pool_size + minimal_pool_size * l // 2
if l < n_Conv1D_layers / 2:
model.add(
Conv1D(filters=n_filters,
kernel_size=kernel_size,
activation='relu'))
else:
model.add(
LocallyConnected1D(filters=n_filters,
kernel_size=kernel_size,
activation='relu'))
model.add(MaxPooling1D(pool_size=pool_size))
model.add(Flatten())
model.add(Dense(n_parameters * 2, kernel_initializer='uniform'))
model.add(Dense(n_parameters))
model.compile(loss='mse', optimizer='nadam', metrics=['mse'])
if print_summary:
print(model.summary())
return model
Y_train = Y_train.reshape((-1, 1))
Y_train = onehotencoder.fit_transform(Y_train).toarray()
labelencoder_Y_test = LabelEncoder()
Y_test = labelencoder_Y_test.fit_transform(Y_test)
onehotencoder = OneHotEncoder(categorical_features=[0])
Y_test = Y_test.reshape((-1, 1))
Y_test = onehotencoder.fit_transform(Y_test).toarray()
## #Building the CNN
input_dim = X_train.get_shape()[1]
classifier = Sequential()
classifier.add(Embedding(input_dim, 100))
print(classifier.output_shape)
classifier.add(
Conv1D(filters=100, kernel_size=4, padding='same', activation='relu'))
print(classifier.output_shape)
classifier.add(MaxPooling1D(pool_size=2))
print(classifier.output_shape)
classifier.add(
Conv1D(filters=100, kernel_size=4, padding='same', activation='relu'))
print(classifier.output_shape)
classifier.add(MaxPooling1D(pool_size=2))
print(classifier.output_shape)
classifier.add(Flatten())
print(classifier.output_shape)
classifier.add(Dense(250, activation='relu'))
print(classifier.output_shape)
classifier.add(Dense(len(Y_train[0]), activation='softmax'))
print(classifier.output_shape)
classifier.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
#emb_merge = Merge(mode='concat')([all_word_fea, all_dis_fea1,all_dis_fea2])
emb_merge = layers.concatenate([all_word_fea, all_dis_fea1, all_dis_fea2], axis=-1)
##embed layer dropout
emb_merge = Dropout(0.5)(emb_merge)
cnn_word = Convolution1D(nb_filter=s['nb_filter'],
filter_length=3,
border_mode='same',
activation='tanh',
subsample_length=1)(emb_merge)
print (cnn_word.shape)
cnn_word = MaxPooling1D(pool_length=s['sentence_length'])(cnn_word)
cnn_word = Flatten()(cnn_word)
classify_drop = Dropout(0.5)(cnn_word)
classify_output = Dense(s['class_num'])(classify_drop)
classify_output = Activation('softmax')(classify_output)
##entity sequence dropout
entity_sequence_fea = Dropout(0.5)(entity_sequence_fea)
left_lstm = LSTM(output_dim=s['encoder_hidden_layer'],
init='orthogonal',
activation='tanh',
inner_activation='sigmoid')(entity_sequence_fea)
right_lstm = LSTM(output_dim=s['encoder_hidden_layer'],
init='orthogonal',
def build_model(top_words,
embedding_vecor_length,
max_review_length,
show_summaries=False):
input_layer = Embedding(top_words,
embedding_vecor_length,
input_length=max_review_length)
# --- 3 ---
branch_3 = Sequential()
branch_3.add(input_layer)
branch_3.add(
Conv1D(filters=32,
kernel_size=3,
padding='same',
kernel_regularizer=l2(.01)))
branch_3.add(Activation('relu'))
branch_3.add(MaxPooling1D(pool_size=2))
branch_3.add(Dropout(0.5))
branch_3.add(Dense(24, activation='relu'))
branch_3.add(Dropout(0.5))
branch_3.add(BatchNormalization())
branch_3.add(LSTM(150))
# --- 4 ---
branch_4 = Sequential()
branch_4.add(input_layer)
branch_4.add(
Conv1D(filters=32,
kernel_size=4,
padding='same',
kernel_regularizer=l2(.01)))
branch_4.add(Activation('relu'))
branch_4.add(MaxPooling1D(pool_size=2))
branch_4.add(Dropout(0.5))
branch_3.add(Dense(24, activation='relu'))
branch_3.add(Dropout(0.5))
branch_4.add(BatchNormalization())
branch_4.add(LSTM(150))
# --- 5 ---
branch_5 = Sequential()
branch_5.add(input_layer)
branch_5.add(
Conv1D(filters=32,
kernel_size=5,
padding='same',
kernel_regularizer=l2(.01)))
branch_5.add(Activation('relu'))
branch_5.add(MaxPooling1D(pool_size=2))
branch_5.add(Dropout(0.5))
branch_3.add(Dense(24, activation='relu'))
branch_3.add(Dropout(0.5))
branch_5.add(BatchNormalization())
branch_5.add(LSTM(150))
model = Sequential()
model.add(Merge([branch_3, branch_4, branch_5], mode='concat'))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
if show_summaries:
print(branch_3.summary())
print(branch_4.summary())
print(branch_5.summary())
print(model.summary())
return model
def getModelGivenModelOptionsAndWeightInits(args):
#get the arguments
seed = args.seed
w0 = args.w0
w1 = args.w1
ntasks = args.num_tasks
np.random.seed(seed)
import keras
from keras.layers.core import Dropout, Reshape, Dense, Activation, Flatten
from keras.layers.convolutional import Conv1D, MaxPooling1D
from keras.optimizers import Adadelta, SGD, RMSprop
import keras.losses
from keras.constraints import maxnorm
from keras.layers.normalization import BatchNormalization
from keras.regularizers import l1, l2
from keras import backend as K
from keras.layers import Input, Add
from keras.models import Model
K.set_image_data_format('channels_last')
print(K.image_data_format())
seq = Input(shape=(1000, 4), name='input')
x = Conv1D(filters=300,
kernel_size=19,
input_shape=(1000, 4),
name='conv1')(seq)
x = BatchNormalization(axis=-1, name='bn1')(x)
x = Activation('relu', name='relu1')(x)
x = MaxPooling1D(pool_size=3, name='maxpool1')(x)
x = Conv1D(filters=200, kernel_size=11, name='conv2')(x)
x = BatchNormalization(axis=-1, name='bn2')(x)
x = Activation('relu', name='relu2')(x)
x = MaxPooling1D(pool_size=4, name='maxpool2')(x)
x = Conv1D(filters=200, kernel_size=7, name='conv3')(x)
x = BatchNormalization(axis=-1, name='bn3')(x)
x = Activation('relu', name='relu3')(x)
x = MaxPooling1D(pool_size=4, name='maxpool3')(x)
x = Flatten(name='flatten1')(x)
x = Dense(1000, name='dense1')(x)
x = BatchNormalization(axis=-1, name='bn4')(x)
x = Activation('relu', name='relu4')(x)
x = Dropout(0.3, name='dropout1')(x)
x = Dense(1000, name='dense2')(x)
x = BatchNormalization(axis=-1, name='bn5')(x)
x = Activation('relu', name='relu5')(x)
x = Dropout(0.3, name='dropout2')(x)
outputs = Dense(ntasks, name='final_dense' + str(ntasks))(x)
model = Model(inputs=[seq], outputs=outputs)
adam = keras.optimizers.Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08)
print("compiling!")
loss = ambig_mean_squared_error
model.compile(optimizer=adam, loss=loss)
return model
model_variation = 'CNN-non-static'
np.random.seed(0)
nb_filter = embeddings_dim
main_input = Input(shape=(max_sent_len,))
embedding = Embedding(max_features, embeddings_dim, input_length=max_sent_len, mask_zero=False, weights=[embedding_weights] )(main_input)
Drop1 = Dropout(dropout_prob[0])(embedding)
i=0
conv_name=["" for x in range(len(filter_sizes))]
pool_name=["" for x in range(len(filter_sizes))]
flat_name=["" for x in range(len(filter_sizes))]
for n_gram in filter_sizes:
conv_name[i] = str('conv_' + str(n_gram))
conv_name[i] = Convolution1D(nb_filter=nb_filter, filter_length=n_gram, border_mode='valid', activation='relu', subsample_length=1, input_dim=embeddings_dim, input_length=max_sent_len)(Drop1)
pool_name[i] = str('maxpool_' + str(n_gram))
pool_name[i] = MaxPooling1D(pool_length=max_sent_len - n_gram + 1)(conv_name[i])
flat_name[i] = str('flat_' + str(n_gram))
flat_name[i] = Flatten()(pool_name[i])
i+=1
merged = merge([flat_name[0], flat_name[1], flat_name[2]], mode='concat')
droput_final = Dropout(dropout_prob[1])(merged)
Dense_final = Dense(1, input_dim=nb_filter * len(filter_sizes))(droput_final)
if num_classes ==2:
Out = Dense(1, activation = 'sigmoid')(Dense_final)
else:
Out = Dense(num_classes, activation = 'sigmoid')(Dense_final)
model = Model(inputs=main_input, outputs=Out)
# Print model summary
# ==================================================
embedding = Embedding(max_features,
embeddings_dim,
input_length=max_sent_len,
mask_zero=False,
weights=[embedding_weights])(main_input)
Drop1 = Dropout(dropout_prob[0])(embedding)
convs = []
for n_gram in filter_sizes:
conv = Convolution1D(nb_filter=nb_filter,
filter_length=n_gram,
border_mode='valid',
activation='relu',
subsample_length=1,
input_dim=embeddings_dim,
input_length=max_sent_len)(Drop1)
pool = MaxPooling1D(pool_length=max_sent_len - n_gram + 1)(conv)
flat = Flatten()(pool)
convs.append(flat)
if len(filter_sizes) > 1:
merged = merge(convs, mode='concat')
else:
merged = convs[0]
droput_final = Dropout(dropout_prob[1])(merged)
Dense_final = Dense(1,
input_dim=nb_filter * len(filter_sizes))(droput_final)
if num_classes == 2:
Out = Dense(1, activation='sigmoid')(Dense_final)
else:
Out = Dense(num_classes, activation='sigmoid')(Dense_final)
model = Model(inputs=main_input, outputs=Out)
X_test = data.iloc[test_index, :].values
X_test = X_test[:, 1::3, numpy.newaxis]
y_test = labels.iloc[test_index, :].values
# create the model
model = Sequential()
model.add(
Conv1D(filters=100,
kernel_size=80,
padding='same',
activation='relu',
strides=1,
input_shape=(max_sample_length, 1)))
model.add(MaxPooling1D(pool_size=3, strides=3))
# model.add(Flatten())
# model.add(LSTM(100, input_dim=1, dropout=0.3, recurrent_dropout=0.3, return_sequences=True))
# model.add(LSTM(100, dropout=0.1, recurrent_dropout=0.1, input_shape=(max_sample_length,1)))
model.add(LSTM(80, dropout=0.2, recurrent_dropout=0.1))
model.add(Dropout(0.2))
model.add(Dense(128, activation='sigmoid'))
model.add(Dense(1, activation='sigmoid'))
opt = []
opt.append(optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0))
opt.append(
optimizers.Nadam(lr=0.002,
def createModel(args):
from keras.models import Model
from keras.optimizers import SGD
from keras.layers import Input, Dense, Dropout, Flatten
from keras.layers.convolutional import Convolution1D, MaxPooling1D
from keras.layers.normalization import BatchNormalization
##### PARAMETERS #####
deepFlag = args[0] # for 0) for CNN , 1) for Deep CNN
maxlen = args[1] # 1300 #Max character length of text
nb_epoch = args[2] # 20
batch_size = args[3] # 64
z1 = args[4] # 1024
z2 = args[5] # 1024
path = args[6]
######################
print('Loading data...')
import csv
X_train = []
Y_train = []
with open(path + '/data/amazon_review_polarity_csv/train.csv', 'r') as f:
reader = csv.reader(f)
for line in reader:
X_train.append(line[1].strip() + line[2].strip())
Y_train.append(int(line[0].strip()))
X_train = np.array(X_train, dtype=object)
Y_train = np.array(Y_train)
X_test = []
Y_test = []
with open(path + '/data/amazon_review_polarity_csv/test.csv', 'r') as f:
reader = csv.reader(f)
for line in reader:
X_test.append(line[1].strip() + line[2].strip())
Y_test.append(int(line[0].strip()))
X_test = np.array(X_test, dtype=object)
Y_test = np.array(Y_test)
X_train_clean = []
for text in X_train:
X_train_clean.append(" ".join(makeClean(text, 200)))
X_train = np.array(X_train_clean, dtype=object)
del X_train_clean
X_test_clean = []
for text in X_test:
X_test_clean.append(" ".join(makeClean(text, 200)))
X_test = np.array(X_test_clean, dtype=object)
del X_test_clean
enc = OneHotEncoder()
Y_train = enc.fit_transform(Y_train[:, np.newaxis]).toarray()
Y_test = enc.fit_transform(Y_test[:, np.newaxis]).toarray()
##### PROCESS DATA ####
### create vocab
print('Get characters...')
alphabet = (list(string.ascii_lowercase) + list(string.digits) + [' '] +
list(string.punctuation) + ['\n'])
chars = set(alphabet)
#chars = list(' '.join(X_train) + ' '.join(X_test))
#chars = set(chars)
vocab_size = len(chars)
print('total chars:', len(chars))
char_indices = dict((c, i) for i, c in enumerate(chars))
indices_char = dict((i, c) for i, c in enumerate(chars))
### enocde data
print('Vectorization...')
X_train_char = np.zeros((len(X_train), maxlen, len(chars)), dtype=np.bool)
for i, sentence in enumerate(X_train):
for t, char in enumerate(sentence):
X_train_char[i, t, char_indices[char]] = 1
X_test_char = np.zeros((len(X_test), maxlen, len(chars)), dtype=np.bool)
for i, sentence in enumerate(X_test):
for t, char in enumerate(sentence):
X_test_char[i, t, char_indices[char]] = 1
print('train shape: ', X_train_char.shape)
print('test shape: ', X_test_char.shape)
##### SETUP CNN #####
main_input = Input(shape=(maxlen, vocab_size), name='main_input')
if deepFlag == 0: # Zhang et al. (2015), Character-level CNNs for Text Classification
conv = Convolution1D(nb_filter=256,
filter_length=7,
border_mode='valid',
activation='relu',
input_shape=(maxlen, vocab_size))(main_input)
conv = MaxPooling1D(pool_length=3)(conv)
conv1 = Convolution1D(nb_filter=256,
filter_length=7,
border_mode='valid',
activation='relu')(conv)
conv1 = MaxPooling1D(pool_length=3)(conv1)
conv2 = Convolution1D(nb_filter=256,
filter_length=3,
border_mode='valid',
activation='relu')(conv1)
conv3 = Convolution1D(nb_filter=256,
filter_length=3,
border_mode='valid',
activation='relu')(conv2)
conv4 = Convolution1D(nb_filter=256,
filter_length=3,
border_mode='valid',
activation='relu')(conv3)
conv5 = Convolution1D(nb_filter=256,
filter_length=3,
border_mode='valid',
activation='relu')(conv4)
conv5 = MaxPooling1D(pool_length=3)(conv5)
conv5 = Flatten()(conv5)
#Two dense layers with dropout of .5
z = Dropout(0.5)(Dense(z1, activation='relu')(conv5))
z = Dropout(0.5)(Dense(z2, activation='relu')(z))
else: # Alexis Conneau et al. (2016), Very Deep CNN For NLP.
# Depth is 9 layers + max pooling
#All the convolutional layers...
conv = Convolution1D(nb_filter=64,
filter_length=3,
border_mode='same',
activation='relu',
input_shape=(maxlen, vocab_size))(main_input)
conv = BatchNormalization()(conv)
conv = Dropout(0.1)(conv)
conv1 = Convolution1D(nb_filter=64,
filter_length=3,
border_mode='same',
activation='relu')(conv)
conv1 = Convolution1D(nb_filter=64,
filter_length=3,
border_mode='same',
activation='relu')(conv1)
conv1 = MaxPooling1D(pool_length=3, stride=2)(conv1)
conv1 = BatchNormalization()(conv1)
conv1 = Dropout(0.1)(conv1)
conv2 = Convolution1D(nb_filter=128,
filter_length=3,
border_mode='same',
activation='relu')(conv1)
conv2 = Convolution1D(nb_filter=128,
filter_length=3,
border_mode='same',
activation='relu')(conv2)
conv2 = MaxPooling1D(pool_length=3, stride=2)(conv2)
conv2 = BatchNormalization()(conv2)
conv2 = Dropout(0.1)(conv2)
conv3 = Convolution1D(nb_filter=256,
filter_length=3,
border_mode='same',
activation='relu')(conv2)
conv3 = Convolution1D(nb_filter=256,
filter_length=3,
border_mode='same',
activation='relu')(conv3)
conv3 = MaxPooling1D(pool_length=3, stride=2)(conv3)
conv3 = BatchNormalization()(conv3)
conv3 = Dropout(0.1)(conv3)
conv4 = Convolution1D(nb_filter=512,
filter_length=3,
border_mode='same',
activation='relu')(conv3)
conv4 = Convolution1D(nb_filter=512,
filter_length=3,
border_mode='same',
activation='relu')(conv4)
conv4 = MaxPooling1D(pool_length=3, stride=2)(conv4)
conv4 = BatchNormalization()(conv4)
conv4 = Dropout(0.1)(conv4)
conv4 = Flatten()(conv4)
#Two dense layers with dropout of .5
z = Dense(z1, activation='relu')(conv4) # z1
z = Dense(z2, activation='relu')(z) # z2
#Output dense layer with softmax activation
out = Dense(Y_train.shape[1], activation='softmax')(z)
model = Model(input=main_input, output=out)
sgd = SGD(lr=0.001, decay=1e-6, momentum=0.9)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
##### RUN MODEL ####
history = model.fit(X_train_char,
Y_train,
batch_size=batch_size,
nb_epoch=nb_epoch,
validation_data=(X_test_char, Y_test))
score, acc = model.evaluate(X_test_char, Y_test, batch_size=batch_size)
val_acc = history.history['val_acc']
val_acc_max = np.amax(val_acc)
val_acc_max_ep = np.argmax(val_acc)
print('Best epoch:', val_acc_max_ep + 1)
print('Best test error:', 1 - val_acc_max)
#conv4
model.add(ZeroPadding2D((2,2)))
model.add(Conv2D(16, (5, 5), strides=(1, 1), activation='relu', kernel_initializer=initializer))
model.add(BatchNormalization(axis=-1))
model.add(Dropout(0.25))
#maxpool4
model.add(MaxPooling2D((2,1), strides=(2,1)))
model.add(Dropout(0.50))
#stacking(reshaping)
model.add(Reshape((500, 16)))
#temporal squeeing
model.add(MaxPooling1D((500), strides=(1)))
model.add(Dropout(0.50))
#fully connected layers
model.add(Flatten())
model.add(Dense(196, activation='sigmoid'))
model.add(Dropout(0.50))
model.add(Dense(2, activation='softmax', name='predictions'))
model.summary()
#train_data
classes = 2
feature = np.load('../feature_train.npy')
label = np.load('../label_train.npy')
label = to_categorical(label, 2)
opt = Adam(decay = 1e-6)
test_X = test_X.reshape((test_X.shape[0], n_min, n_features)) # print(train_X.shape, train_y.shape, test_X.shape, test_y.shape) # the model # input layer input_tensor = Input(shape=(n_min, n_features)) # LSTM l1 = LSTM(200)(input_tensor) # (batch, n_min*n_features) l2 = Dropout(0.3)(l1) # CNN c1 = Conv1D(filters=64, kernel_size=5, activation='relu')(input_tensor) c2 = Conv1D(filters=64, kernel_size=5, activation='relu')(c1) c3 = Dropout(0.5)(c2) c4 = MaxPooling1D(pool_size=2)(c3) c5 = Flatten()(c4) # GNN g1 = load_GAT_model() # g2 = g1[None, :] g2 = tf.convert_to_tensor(g1) g2 = g2[None, :] # concat h1 = layers.concatenate([l2, c5, g2], axis=-1) h2 = Dense(300, activation='tanh')(h1) h3 = Dropout(0.3)(h2) h4 = Dense(3, activation='softmax')(h3) # define model
X, y = split_sequence(data, n_steps_in, n_steps_out)
# transform input from [samples, features] to [samples, timesteps, features], a step needed for CNN or LSTM
n_features = 1
X = X.reshape((X.shape[0], X.shape[1], n_features))
print(X.shape, y.shape) # show each sample
for i in range(len(X)):
print(X[i], y[i])
# CNN model
model = Sequential()
model.add(
Conv1D(64, 2, activation='relu', input_shape=(n_steps_in, n_features)))
model.add(MaxPooling1D())
model.add(Conv1D(32, 2, activation='relu'))
model.add(MaxPooling1D())
model.add(Flatten())
model.add(Dense(50, activation='relu'))
model.add(Dense(n_steps_out))
model.compile(optimizer='adam', loss='mse', metrics=['accuracy'])
model.fit(X, y, epochs=1000, verbose=1)
X_test = np.array(data[-n_steps_in:]).reshape(1, n_steps_in, n_features)
y_hat = model.predict(X_test, verbose=0)
print(f'y = {y[-n_steps_out:]}, y_hat = {y_hat}')
def RNN(X_train, y_train, args):
"""
Purpose -> Define and train the proposed LSTM network
Input -> Data, Labels and model hyperparameters
Output -> Trained LSTM network
"""
# Sets the model hyperparameters
# Embedding hyperparameters
max_features = args[0]
maxlen = args[1]
embedding_size = args[2]
# Convolution hyperparameters
filter_length = args[3]
nb_filter = args[4]
pool_length = args[5]
# LSTM hyperparameters
lstm_output_size = args[6]
# Training hyperparameters
batch_size = args[7]
nb_epoch = args[8]
numclasses = args[9]
test_size = args[10]
# Format conversion for y_train for compatibility with Keras
y_train = np_utils.to_categorical(y_train, numclasses)
# Train & Validation data splitting
X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=test_size, random_state=42)
# Build the sequential model
# Model Architecture is:
# Input -> Embedding -> Conv1D+Maxpool1D -> LSTM -> LSTM -> FC-1 -> Softmaxloss
print('Build model...')
model = Sequential()
model.add(Embedding(max_features, embedding_size, input_length=maxlen))
model.add(Convolution1D(nb_filter=nb_filter,
filter_length=filter_length,
border_mode='valid',
activation='relu',
subsample_length=1))
model.add(Convolution1D(nb_filter=nb_filter,
filter_length=filter_length,
border_mode='valid',
activation='relu',
subsample_length=1))
model.add(Convolution1D(nb_filter=nb_filter,
filter_length=filter_length,
border_mode='valid',
activation='relu',
subsample_length=1))
model.add(MaxPooling1D(pool_length=pool_length))
# model.add(Convolution1D(nb_filter=nb_filter,
# filter_length=filter_length,
# border_mode='valid',
# activation='relu',
# subsample_length=1))
# model.add(MaxPooling1D(pool_length=pool_length))
# model.add(Convolution1D(nb_filter=nb_filter,
# filter_length=filter_length,
# border_mode='valid',
# activation='relu',
# subsample_length=1))
# model.add(MaxPooling1D(pool_length=pool_length))
model.add(Bidirectional(LSTM(lstm_output_size, dropout_W=0.2, dropout_U=0.2, return_sequences=True)))
# model.add(Convolution1D(nb_filter=nb_filter,
# filter_length=filter_length,
# border_mode='valid',
# activation='relu',
# subsample_length=1))
# model.add(MaxPooling1D(pool_length=pool_length))
model.add(Bidirectional(LSTM(lstm_output_size, dropout_W=0.2, dropout_U=0.2, return_sequences=False)))
model.add(Dense(128))
model.add(Activation("relu"))
model.add(Dense(64))
model.add(Activation("relu"))
model.add(Dense(32))
model.add(Activation("relu"))
model.add(Dense(16))
model.add(Activation("relu"))
model.add(Dense(numclasses))
model.add(Activation('softmax'))
# Optimizer is Adamax along with categorical crossentropy loss
model.compile(loss='categorical_crossentropy',
optimizer='adamax',
metrics=['accuracy'])
print('Train...')
# Trains model for 50 epochs with shuffling after every epoch for training data and validates on validation data
model.fit(X_train, y_train,
batch_size=batch_size,
shuffle=True,
nb_epoch=nb_epoch,
validation_data=(X_valid, y_valid))
return model
def create_model(x_train, y_train, x_validate, y_validate, x_test, y_test):
import time
import tabulate
start = int(time.time())
np.random.seed(1234567)
try:
'''
b) with and without a hidden fully-connected layer,
c) number of units in the hidden fc layer < 200,
c) different learning rates for adam (explore on a log scale - 0.001, 0.0001, etc),
d) maxpooling widths in the 10-60 range,
e) conv widths in the 10-40 range.
'''
model = Sequential()
kernel_size1 = {{choice([13, 15, 19, 25, 39])}}
kernel_size2 = kernel_size1 - 2
model.add(
Conv1D(filters={{choice([50, 100, 250, 500, 1000])}},
kernel_size=(kernel_size1),
input_shape=(1000, 4)))
model.add(BatchNormalization(axis=-1))
model.add(Activation('relu'))
## a) number of layers between 1 and 4,
#decide on how many conv layers in model
n_conv = {{choice([0, 1, 2, 3])}}
filter_dim = [kernel_size1, kernel_size2, kernel_size2]
for i in range(n_conv):
model.add(
Conv1D(filters={{choice([50, 100, 250, 500, 1000])}},
kernel_size=(filter_dim[i])))
model.add(BatchNormalization(axis=-1))
model.add(Activation('relu'))
model.add(MaxPooling1D(pool_size=({{choice([10, 30, 60])}})))
model.add(Flatten())
n_dense = {{choice([0, 1, 2, 3])}}
for i in range(n_dense):
model.add(Dense({{choice([50, 100, 200])}}))
model.add(BatchNormalization(axis=-1))
model.add(Activation('relu'))
model.add(Dropout({{choice([0.2, 0.4, 0.6])}}))
model.add(Dense(4))
model.add(Activation("sigmoid"))
adam = keras.optimizers.Adam(lr={{choice([0.01, 0.001, 0.0001])}})
model.compile(loss=keras_genomics.losses.ambig_binary_crossentropy,
optimizer=adam,
metrics=['accuracy'])
print("compiled!")
sys.stdout.flush()
# added to collect optimization results
if 'results' not in globals():
global results
results = []
result = model.fit(x_train,
y_train,
batch_size=200,
epochs=20,
verbose=2,
validation_data=(x_validate, y_validate))
print("trained!")
sys.stdout.flush()
loss, acc = model.evaluate(x_validate, y_validate, verbose=2)
print("Validation loss:", loss, "Validation acc:", acc)
sys.stdout.flush()
# added to collect results
valLoss = result.history['val_loss'][-1]
parameters = space
parameters["loss"] = valLoss
parameters["time"] = int(time.time() - start)
results.append(parameters)
print(parameters)
if len(results) % 10 == 0:
tab = tabulate.tabulate(results,
headers="keys",
tablefmt="fancy_grid",
floatfmt=".8f")
print(tab.encode('utf-8'))
else:
tab = tabulate.tabulate(results[-1:],
headers="keys",
tablefmt="fancy_grid",
floatfmt=".8f")
print(tab.encode('utf-8'))
print("model %d done ----------------" % len(results))
sys.stdout.flush()
except:
loss = 1000
acc = 0
print("failed to run model")
sys.stdout.flush()
model = None
return {'loss': loss, 'status': STATUS_OK, 'model': model}
def verylargeCNN(x_train, y_train, x_test, y_test, DROP_OUT, INPUT_SHAPE,
KERNEL_SIZE, NUM_KERNEL, name):
x_val = x_test[range(0, x_test.shape[0], 2), :]
y_val = y_test[range(0, y_test.shape[0], 2)]
x_test = x_test[range(1, x_test.shape[0], 2), :]
y_test = y_test[range(1, y_test.shape[0], 2)]
model = Sequential()
model.add(
Conv1D(NUM_KERNEL,
kernel_size=KERNEL_SIZE,
kernel_initializer=KERNEL_INITIAL,
kernel_constraint=maxnorm(2),
input_shape=INPUT_SHAPE))
model.add(Activation("relu"))
model.add(BatchNormalization())
model.add(Dropout(0.3))
model.add(MaxPooling1D())
model.add(
Conv1D(NUM_KERNEL,
kernel_size=KERNEL_SIZE,
kernel_initializer=KERNEL_INITIAL,
kernel_constraint=maxnorm(2)))
model.add(Activation("relu"))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(MaxPooling1D())
model.add(
Conv1D(NUM_KERNEL,
kernel_size=KERNEL_SIZE,
kernel_initializer=KERNEL_INITIAL,
kernel_constraint=maxnorm(2)))
model.add(Activation("relu"))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(MaxPooling1D())
model.add(
Conv1D(NUM_KERNEL,
kernel_size=KERNEL_SIZE,
kernel_initializer=KERNEL_INITIAL,
kernel_constraint=maxnorm(2)))
model.add(Activation("relu"))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(
Conv1D(NUM_KERNEL,
kernel_size=KERNEL_SIZE,
kernel_initializer=KERNEL_INITIAL,
kernel_constraint=maxnorm(2)))
model.add(Activation("relu"))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(
Conv1D(NUM_KERNEL,
kernel_size=KERNEL_SIZE,
kernel_initializer=KERNEL_INITIAL,
kernel_constraint=maxnorm(2)))
model.add(Activation("relu"))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(
Conv1D(NUM_KERNEL,
kernel_size=KERNEL_SIZE,
kernel_initializer=KERNEL_INITIAL,
kernel_constraint=maxnorm(2)))
model.add(Activation("relu"))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(
Conv1D(NUM_KERNEL,
kernel_size=KERNEL_SIZE,
kernel_initializer=KERNEL_INITIAL,
kernel_constraint=maxnorm(2)))
model.add(Activation("relu"))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(GlobalMaxPooling1D())
model.add(Dense(1))
model.add(Activation("sigmoid"))
filepath = name + "best_veryLargeCNN_dropout_" + str(NUM_KERNEL) + ".hdf5"
checkpoint = ModelCheckpoint(filepath,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='auto')
early_stopping_monitor = EarlyStopping(monitor='val_loss', patience=8)
model.compile(loss=LOSS, optimizer=OPTIMIZER, metrics=METRICS)
print(model.summary())
Tuning = model.fit(
x_train,
y_train,
batch_size=BATCH_SIZE,
epochs=NB_EPOCH,
verbose=VERBOSE,
validation_data=(x_val, y_val),
callbacks=[checkpoint, early_stopping_monitor]
) #,roc_callback(training_data=(x_train, y_train),validation_data=(x_test, y_test))]) #,callbacks=callbacks_list)
#finalmodel = load_model(filepath)
print("train")
GetMetrics(load_model(filepath), x_train, y_train)
print("test")
GetMetrics(load_model(filepath), x_test, y_test)
print("val")
GetMetrics(load_model(filepath), x_val, y_val)
return model, Tuning #,roc_train,roc_test,acc_train,acc_test
mask_zero=False,
weights=[embedding_weights]),
name='embedding',
input='input')
model.add_node(Dropout(0.), name='dropout_embedding', input='embedding')
for n_gram in [3, 5, 7]:
model.add_node(Convolution1D(nb_filter=nb_filter,
filter_length=n_gram,
border_mode='valid',
activation='relu',
subsample_length=1,
input_dim=embeddings_dim,
input_length=maxlen),
name='conv_' + str(n_gram),
input='dropout_embedding')
model.add_node(MaxPooling1D(pool_length=maxlen - n_gram + 1),
name='maxpool_' + str(n_gram),
input='conv_' + str(n_gram))
model.add_node(Flatten(),
name='flat_' + str(n_gram),
input='maxpool_' + str(n_gram))
model.add_node(Dropout(0.),
name='dropout',
inputs=['flat_' + str(n) for n in [3, 5, 7]])
model.add_node(Dense(1, input_dim=nb_filter * len([3, 5, 7])),
name='dense',
input='dropout')
model.add_node(Activation('sigmoid'), name='sigmoid', input='dense')
model.add_output(name='output', input='sigmoid')
model.compile(loss={'output': 'binary_crossentropy'}, optimizer='rmsprop')
model.fit({
def largeCNN_RNN(x_train, y_train, x_test, y_test, DROP_OUT, INPUT_SHAPE,
KERNEL_SIZE, name):
model = Sequential()
model.add(
Conv1D(64,
kernel_size=KERNEL_SIZE,
kernel_initializer=KERNEL_INITIAL,
kernel_constraint=maxnorm(2),
input_shape=INPUT_SHAPE))
model.add(Activation("relu"))
model.add(MaxPooling1D(pool_size=10, strides=5))
model.add(
Conv1D(64,
kernel_size=KERNEL_SIZE,
kernel_initializer=KERNEL_INITIAL,
kernel_constraint=maxnorm(2)))
model.add(Activation("relu"))
model.add(MaxPooling1D(pool_size=10, strides=5))
#model.add(Flatten())
#model.add(Reshape((50,1))) # shape becomes (batch_size,200,1)
HIDDEN_UNITS = 10
if (DROP_OUT == 1):
model.add(
Bidirectional(
LSTM(HIDDEN_UNITS,
activation='tanh',
inner_activation='sigmoid',
kernel_constraint=maxnorm(2),
kernel_initializer=KERNEL_INITIAL,
return_sequences=True,
dropout=0.3)))
else:
model.add(
Bidirectional(
LSTM(HIDDEN_UNITS,
activation='tanh',
inner_activation='sigmoid',
kernel_constraint=maxnorm(2),
kernel_initializer=KERNEL_INITIAL,
return_sequences=True)))
model.add(Flatten())
model.add(Dense(1))
#a soft max classifier
model.add(Activation("sigmoid"))
model.compile(loss=LOSS, optimizer=OPTIMIZER, metrics=METRICS)
filepath = "largeRNN_dropout_" + str(
DROP_OUT) + "_{epoch:02d}_{val_acc:.2f}.hdf5"
filepath = name + "best_largeCNN_RNN_dropout_" + str(DROP_OUT) + ".hdf5"
checkpoint = ModelCheckpoint(filepath,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='max')
early_stopping_monitor = EarlyStopping(monitor='val_loss', patience=8)
#callbacks_list = [checkpoint]
model.compile(loss=LOSS, optimizer='Adam', metrics=METRICS)
print(model.summary())
#Tuning = model.fit(x_train,y_train,batch_size=BATCH_SIZE, epochs = NB_EPOCH, verbose = VERBOSE,
# validation_split = 0.2) #(x_test,y_test),callbacks=[checkpoint,early_stopping_monitor,roc_callback(training_data=(x_train, y_train),validation_data=(x_test, y_test))]) #,callbacks=callbacks_list)
Tuning = model.fit(
x_train,
y_train,
batch_size=BATCH_SIZE,
epochs=NB_EPOCH,
verbose=VERBOSE,
validation_data=(x_test, y_test),
callbacks=[checkpoint, early_stopping_monitor]
) #,roc_callback(training_data=(x_train, y_train),validation_data=(x_test, y_test))]) #,callbacks=callbacks_list)
finalModel = load_model(filepath)
y_pred = finalModel.predict(x_train)
roc_train = roc_auc_score(y_train, y_pred)
y_pred_val = model.predict(x_test)
roc_test = roc_auc_score(y_test, y_pred_val)
print('\rroc-auc: %s - roc-auc_val: %s' %
(str(round(roc_train, 4)), str(round(roc_test, 4))),
end=100 * ' ' + '\n')
score = finalModel.evaluate(x_train, y_train)
acc_train = score[1]
score = finalModel.evaluate(x_test, y_test)
acc_test = score[1]
print(model.summary())
return model, Tuning, roc_train, roc_test, acc_train, acc_test
# univariate cnn example from numpy import array from keras.models import Sequential from keras.layers import Dense from keras.layers import Flatten from keras.layers.convolutional import Conv1D from keras.layers.convolutional import MaxPooling1D # define dataset X = array([[10, 20, 30], [20, 30, 40], [30, 40, 50], [40, 50, 60]]) y = array([40, 50, 60, 70]) # reshape from [samples, timesteps] into [samples, timesteps, features] X = X.reshape((X.shape[0], X.shape[1], 1)) # define model model = Sequential() model.add(Conv1D(filters=64, kernel_size=2, activation='relu', input_shape=(3, 1))) model.add(MaxPooling1D(pool_size=2)) model.add(Flatten()) model.add(Dense(50, activation='relu')) model.add(Dense(1)) model.compile(optimizer='adam', loss='mse') # fit model model.fit(X, y, epochs=1000, verbose=0) # demonstrate prediction x_input = array([50, 60, 70]) x_input = x_input.reshape((1, 3, 1)) yhat = model.predict(x_input, verbose=0) print(yhat)
kernel_size,
strides=1,
padding='valid',
dilation_rate=1,
activation=None,
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None)
Conv2D(filters,
kernel_size,
strides=(1, 1),
padding='valid',
data_format=None,
dilation_rate=(1, 1),
activation=None,
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None)
MaxPooling1D(pool_size=2, strides=None, padding='valid')
MaxPooling2D(pool_size=(2, 2), strides=None, padding='valid', data_format=None)
input_length=max_sent_len,
mask_zero=False,
weights=[embedding_weights2])(main_input)
layer1 = []
for embedding_layer in [embedding1, embedding2]:
conv_pools = []
for n_gram in filter_sizes:
conv = Convolution1D(nb_filter=nb_filter,
filter_length=n_gram,
border_mode='valid',
activation='relu',
subsample_length=1,
input_dim=embeddings_dim,
input_length=max_sent_len)(embedding_layer)
pool = MaxPooling1D(pool_length=max_sent_len - n_gram + 1)(conv)
# flat = Flatten()(pool)
conv_pools.append((n_gram, pool))
layer1.append(conv_pools)
last_layer = []
for layer in layer1: # no of embeddings used
print('first loop')
print(layer)
for (filter_size, input_feature_maps) in layer:
conv = Convolution1D(nb_filter=nb_filter,
filter_length=filter_size - 1,
border_mode='same',
activation='relu')(input_feature_maps)
pool = MaxPooling1D(pool_length=4, border_mode='same')(conv)
last_layer.append(pool)
embedding = Embedding(max_features,
embeddings_dim,
input_length=max_sent_len,
mask_zero=False,
weights=[embedding_weights])(main_input)
Drop1 = Dropout(dropout_prob[0])(embedding)
convs = []
for n_gram in filter_sizes:
conv = Convolution1D(nb_filter=nb_filter,
filter_length=n_gram,
border_mode='valid',
activation='relu',
subsample_length=1,
input_dim=embeddings_dim,
input_length=max_sent_len)(Drop1)
pool = MaxPooling1D(pool_length=max_sent_len - n_gram + 1)(conv)
# flat = Flatten()(pool)
convs.append(pool)
if len(filter_sizes) > 1:
merged = merge(convs, mode='concat')
else:
merged = convs[0]
conv_model = Model(inputs=main_input, outputs=merged)
# add created model grapg in sequential model
model = Sequential()
model.add(conv_model) # add model just like layer
model.add(
Convolution1D(nb_filter=nb_filter,
filter_length=filter_sizes[1],
