def evalModel(dataset, model, modelname):
#################
# Configuration #
#################
#################
X_test = dataset['X_test']
y_test = dataset['y_test']
labels_test = dataset['labels_test']
# split by "continuous variable" and "categorization variable"
X_test_vec = [X_test[:,:,0:-1], X_test[:,:, -1]]
if o.Variables == "Hits":
X_test_vec = X_test
if o.Model == "DenseIP3D":
X_test_vec = [ X_test [:, 0:ntrk_cut, 0:2], X_test [:, 0:ntrk_cut,-1]]
score = model.evaluate(X_test_vec, y_test, batch_size=batch_size)
print('Test score:', score)
classes = model.predict_classes(X_test_vec, batch_size=batch_size)
acc = np_utils.accuracy(classes, y_test)
print('Test accuracy:', acc)
acc = np_utils.accuracy(classes[labels_test[:,0]==5], y_test[labels_test[:,0]==5])
print('Test b accuracy:', acc)
acc = np_utils.accuracy(classes[labels_test[:,0]==0], y_test[labels_test[:,0]==0])
print('Test l accuracy:', acc)
pred = model.predict(X_test_vec, batch_size=batch_size)
return model
def test():
with open("save_weight.pickle", mode="rb") as f:
weights = pickle.load(f)
model = Sequential()
model.add(Dense(output_dim=100, input_dim=28*28))
model.add(Activation("relu"))
model.set_weights(weights)
layey1_value = model.predict(X_test[:5])
y_pred = np_utils.categorical_probas_to_classes(y)
Y = np_utils.categorical_probas_to_classes(y_test)
print np_utils.accuracy(y_pred,Y)
print y_pred.shape
def tune_model(X_train, y_train, X_test, y_test, settings) :
(optimizer, loss_func, activation, nb_epoch, LSTM_in, LSTM_out) = settings
print("Loading data...")
print(len(X_train), 'train sequences')
print('X_train shape:', X_train.shape)
# train LSTM so that we can extract representation
print('Build model...')
model = Sequential()
model.add(Embedding(max_features, LSTM_in))
model.add(LSTM(LSTM_in, LSTM_out))
model.add(Dropout(0.5))
model.add(Dense(LSTM_out, 1))
model.add(Activation(activation))
model.compile(loss=loss_func, optimizer=optimizer, class_mode="binary")
print("Train...")
model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch,
validation_split=0.1, show_accuracy=True, verbose=2)
classes = model.predict_classes(X_test, batch_size=batch_size)
acc = np_utils.accuracy(classes, y_test)
print('LSTM accuracy:', acc)
print('Building partial model...')
# early fusion for testing, average over each application
early_fusion_model = Sequential()
early_fusion_model.add(Embedding(max_features, LSTM_in,
weights=model.layers[0].get_weights()))
early_fusion_model.add(LSTM(LSTM_in, LSTM_out,
weights=model.layers[1].get_weights()))
early_fusion_model.compile(loss=loss_func,
optimizer=optimizer, class_mode="binary")
return early_fusion_model
def on_epoch_end(self, epoch, logs={}):
acc = accuracy(
(y_train),
np.round(
np.array(
model.predict({'input': X_train},
batch_size=self.batch_size)['output'])))
val_acc = accuracy(
(y_test),
np.round(
np.array(
model.predict({'input': X_test},
batch_size=self.batch_size)['output'])))
print('acc:' + acc + ' - val_acc:' + val_acc)
self.accs.append(acc)
self.val_accs.appen(val_acc)
def lstm(trainData,trainMark,testData,testMark,embedding_dim,embedding_matrix,maxlen):
# 填充数据，将每个序列长度保持一致
trainData = list(sequence.pad_sequences(trainData,maxlen=maxlen,dtype='float64')) # sequence返回的是一个numpy数组，pad_sequences用于填充指定长度的序列，长则阶段，短则补0，由于下面序号为0时，对应值也为0，因此可以这样
testData = list(sequence.pad_sequences(testData,maxlen=maxlen,dtype='float64')) # sequence返回的是一个numpy数组，pad_sequences用于填充指定长度的序列，长则阶段，短则补0
# 建立lstm神经网络模型
model = Sequential() # 多个网络层的线性堆叠，可以通过传递一个layer的list来构造该模型，也可以通过.add()方法一个个的加上层
#model.add(Dense(256, input_shape=(train_total_vova_len,))) #使用全连接的输入层
model.add(Embedding(len(embedding_matrix),embedding_dim,weights=[embedding_matrix],mask_zero=True,input_length=maxlen)) # 指定输入层，将高维的one-hot转成低维的embedding表示，第一个参数大或等于0的整数，输入数据最大下标+1，第二个参数大于0的整数，代表全连接嵌入的维度
# lstm层，也是比较核心的层
model.add(LSTM(128)) # 256对应Embedding输出维度，128是输入维度可以推导出来
model.add(Dropout(0.5)) # 每次在参数更新的时候以一定的几率断开层的链接，用于防止过拟合
model.add(Dense(1)) # 全连接，这里用于输出层，1代表输出层维度，128代表LSTM层维度可以自行推导出来
model.add(Activation('sigmoid')) # 输出用sigmoid激活函数
# 编译该模型，binary_crossentropy（亦称作对数损失，logloss），adam是一种优化器，class_mode表示分类模式
model.compile(loss='binary_crossentropy', optimizer='adam', class_mode="binary")
# 正式运行该模型,我知道为什么了，因为没有补0！！每个array的长度是不一样的，因此才会报错
X = np.array(list(trainData)) # 输入数据
print("X:", X)
Y = np.array(list(trainMark)) # 标签
print("Y:", Y)
# batch_size：整数，指定进行梯度下降时每个batch包含的样本数
# nb_epoch：整数，训练的轮数，训练数据将会被遍历nb_epoch次
model.fit(X, Y, batch_size=16, nb_epoch=10) # 该函数的X、Y应该是多个输入：numpy list(其中每个元素为numpy.array)，单个输入：numpy.array
# 进行预测
A = np.array(list(testData)) # 输入数据
print("A:", A)
B = np.array(list(testMark)) # 标签
print("B:", B)
classes = model.predict_classes(A) # 这个是预测的数据
acc = np_utils.accuracy(classes, B) # 计算准确率，使用还不是很清楚
print('Test accuracy:', acc)
def keras_model():
import pandas as pd
import numpy as np
from keras.preprocessing import sequence
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.convolutional import Convolution1D, MaxPooling1D
from keras.callbacks import EarlyStopping
from keras.utils import np_utils
from data_util import load_csvs, load_other
import ml_metrics as metrics
nb_words = 6500
maxlen = 175
filter_length = 10
other_col_dim = 4
X_train, Y_train, X_test, Y_test, nb_classes = load_csvs('data/tpov4/train_1.csv',
'data/tpov4/test_1.csv',
nb_words, maxlen, 'self', w2v=None)
# read _other.csv
other_train = load_other('data/tpov4/train_1_other.csv', maxlen, other_col_dim)
other_test = load_other('data/tpov4/test_1_other.csv', maxlen, other_col_dim)
print('other tensor:', other_train.shape)
pool_length = maxlen - filter_length + 1
model = Sequential()
model.add(Convolution1D(nb_filter=50,
filter_length=filter_length,
border_mode="valid", activation="relu",
input_shape=(maxlen, other_col_dim)))
model.add(MaxPooling1D(pool_length=pool_length))
model.add(Flatten())
model.add(Dropout(0.05))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer={{choice(['rmsprop', 'adam', 'adadelta', 'adagrad'])}})
earlystop = EarlyStopping(monitor='val_loss', patience=1, verbose=1)
model.fit(other_train, Y_train, batch_size=32, nb_epoch=25,
validation_split=0.1, show_accuracy=True, callbacks=[earlystop])
classes = earlystop.model.predict_classes(other_test, batch_size=32)
org_classes = np_utils.categorical_probas_to_classes(Y_test)
acc = np_utils.accuracy(classes, org_classes) # accuracy only supports classes
print('Test accuracy:', acc)
kappa = metrics.quadratic_weighted_kappa(classes, org_classes)
print('Test Kappa:', kappa)
return {'loss': -acc, 'status': STATUS_OK}
def late_fusion (X_train, y_train, X_test, y_test, k_neighbors, seed=107) : # kNN clf = neighbors.KNeighborsClassifier(k_neighbors, weights='distance') clf.fit(X_train, y_train) threshhold = np.mean(y_train) # label applications based on average of sentence predictions app_predictions = [int(np.mean(clf.predict(app)) > threshhold) for app in X_test] acc = np_utils.accuracy(app_predictions, y_test) return acc
def naive_bayes (X_train, y_train, X_test, y_test) : # compute ratios ratios = compute_ratios(X_train, y_train, max_features) pos_percent = sum(y_train) / (len(y_train) * 1.0) # predict on test data based on occurrence ratios avg_sentence_probs = [avg_word_probabilities(sentence, ratios) for sentence in X_test] preds = [int(pos > pos_percent) for pos,neg in avg_sentence_probs] acc = np_utils.accuracy(preds, y_test) return acc, preds
def naive_bayes(X_train, y_train, X_test, y_test):
# compute ratios
ratios = compute_ratios(X_train, y_train, max_features)
pos_percent = sum(y_train) / (len(y_train) * 1.0)
# predict on test data based on occurrence ratios
avg_sentence_probs = [avg_word_probabilities(sentence, ratios) for sentence in X_test]
preds = [int(pos > pos_percent) for pos, neg in avg_sentence_probs]
acc = np_utils.accuracy(preds, y_test)
return acc, preds
def keras_classify(df):
# 预处理，把 text 中的词转成数字编号
from keras.preprocessing.text import Tokenizer
from keras.preprocessing import sequence
from keras.callbacks import EarlyStopping
from sklearn.cross_validation import train_test_split
print "----- Classification by Keras -----"
max_features = 50000 # 只选最重要的词
# Tokenizer 只能处理 str，不能处理 unicode
textraw = map(lambda x: x.encode('utf-8'), df.seg_word.values.tolist())
token = Tokenizer(nb_words=max_features)
# 由于 df.seg_word 以空格相隔，故此这里 Tokenizer 直接按英文方式处理 str 即可完成分词
token.fit_on_texts(textraw)
# token 中记录了每个词的编号和出现次数，这里使用词编号来代替 textraw 中的词文本
# 如 textraw = ['a b c', 'c d e f'] ==> text_seq = [[1, 2, 3], [3, 4, 5, 6]]
text_seq = token.texts_to_sequences(textraw)
nb_classes = len(np.unique(df.label.values))
print "num of features(vocabulary): ", len(token.word_counts)
print "num of labels: ", nb_classes
max_sent_len = np.max([len(s) for s in text_seq])
print "max length or document is: ", max_sent_len
median_sent_len = np.median([len(s) for s in text_seq])
print "median length or document is: ", median_sent_len
# 这里的 df.label.values 中 values 不能忽略，否则后面 np_utils.to_categorical 时会出错
train_X, test_X, train_y, test_y = train_test_split(text_seq, df.label.values, train_size=0.7, random_state=1)
# 目前 train_X & test_X 仍然不是等长的，其每行都是一个 document，需要化为等长的矩阵才能训练
seqlen = int(max_sent_len / 2 + median_sent_len / 2)
X_train = sequence.pad_sequences(train_X, maxlen=seqlen, padding='post', truncating='post')
X_test = sequence.pad_sequences(test_X, maxlen=seqlen, padding='post', truncating='post')
# 把 y 格式展开为 one-hot，目的是在 nn 的最后采用 softmax
Y_train = np_utils.to_categorical(train_y, nb_classes)
Y_test = np_utils.to_categorical(test_y, nb_classes)
model = build_cnn_model(max_features, seqlen, nb_classes)
earlystop = EarlyStopping(monitor='val_loss', patience=1, verbose=1)
# 训练 10 轮，每轮 mini_batch 为 32，训练完调用 earlystop 查看是否已经 ok
model.fit(X_train, Y_train, batch_size=32, nb_epoch=10, validation_split=0.1, callbacks=[earlystop])
evaluate(earlystop.model, X_test, Y_test, test_y)
model = build_lstm_model(max_features, seqlen, nb_classes)
model.fit(X_train, Y_train, batch_size=32, nb_epoch=1, validation_split=0.1)
evaluate(model, X_test, Y_test, test_y)
model = build_mixed_model(max_features, seqlen, nb_classes)
model.fit(X_train, Y_train, batch_size=32, nb_epoch=1, validation_split=0.1)
evaluate(model, X_test, Y_test, test_y)
graph = build_graph_model(max_features, seqlen, nb_classes)
graph.fit({'input': X_train, 'output': Y_train}, nb_epoch=3, batch_size=32, validation_split=0.1)
predict = graph.predict({'input': X_test}, batch_size=32)
predict = predict['output']
classes = predict.argmax(axis=1)
acc = np_utils.accuracy(classes, test_y)
print('Test accuracy: ', acc)
def compute_acc(X, Y, vocab, model, opts):
scores=model.predict({'input': X},batch_size=options.batch_size)['output']
prediction=np.zeros(scores.shape)
for i in range(scores.shape[0]):
l=np.argmax(scores[i])
prediction[i][l]=1.0
assert np.array_equal(np.ones(prediction.shape[0]),np.sum(prediction,axis=1))
plabels=np.argmax(prediction,axis=1)
tlabels=np.argmax(Y,axis=1)
acc = accuracy(tlabels,plabels)
return acc,acc
def mlp():
words_size = 13033
word_vec_len = 100
batch_size = 100
max_sentence_length = 47 # cut texts after this number of words (among top max_features most common words)
X, y = get_data('../Files/yahoo.data.dat')
# 数据shuffle
indices = permutation(X.shape[0]) # shape[0]表示第0轴的长度，通常是训练数据的数量
X = X[indices]
y = y[indices]
X_train, X_test = X[:0.8*len(X)], X[0.8*len(X):]
y_train, y_test = y[:0.8*len(y)], y[0.8*len(y):]
print(len(X_train), 'train sequences')
print(len(X_test), 'test sequences')
print("Pad sequences (samples x time)")
X_train = sequence.pad_sequences(X_train, maxlen=max_sentence_length)
X_test = sequence.pad_sequences(X_test, maxlen=max_sentence_length)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
print('Build model...')
model = Sequential()
model.add(Embedding(words_size, word_vec_len)) # (nb_samples, sequence_length, output_dim)
model.add(Transform((word_vec_len,))) # transform from 3d dimensional input to 2d input for mlp
model.add(Dense(input_dim=word_vec_len, output_dim=25, init='uniform', activation='tanh'))
model.add(Dropout(0.5))
model.add(Dense(input_dim=25, output_dim=15, init='uniform', activation='tanh'))
model.add(Dropout(0.5))
model.add(Dense(input_dim=15, output_dim=8, init='uniform', activation='tanh'))
model.add(Dropout(0.5))
model.add(Dense(input_dim=8, output_dim=1, init='uniform', activation='sigmoid'))
sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='binary_crossentropy', optimizer=sgd, class_mode='binary')
# model.compile(loss='mean_squared_error', optimizer=sgd, class_mode='binary')
print("Train...")
model.fit(X_train, y_train, shuffle=True, batch_size=batch_size, nb_epoch=5, validation_split=0.1, show_accuracy=True)
score = model.evaluate(X_test, y_test, batch_size=10)
print('Test score:', score)
classes = model.predict_classes(X_test, batch_size=batch_size)
acc = np_utils.accuracy(classes, y_test)
print('Test accuracy:', acc) # ('Test accuracy:', 0.64478482859226838)
def late_fusion(X_train, y_train, X_test, y_test, k_neighbors, seed=107):
# kNN
clf = neighbors.KNeighborsClassifier(k_neighbors, weights='distance')
clf.fit(X_train, y_train)
threshhold = np.mean(y_train)
# label applications based on average of sentence predictions
app_predictions = [
int(np.mean(clf.predict(app)) > threshhold) for app in X_test
]
acc = np_utils.accuracy(app_predictions, y_test)
return acc
def compute_acc(X, Y, vocab, model, opts):
scores = model.predict(X, batch_size=options.batch_size)
prediction = np.zeros(scores.shape)
for i in range(scores.shape[0]):
l = np.argmax(scores[i])
prediction[i][l] = 1.0
assert np.array_equal(np.ones(prediction.shape[0]),
np.sum(prediction, axis=1))
plabels = np.argmax(prediction, axis=1)
tlabels = np.argmax(Y, axis=1)
acc = accuracy(tlabels, plabels)
return acc, acc
def cnn1d_selfembd(X_train, Y_train, X_test, Y_test, nb_classes,
maxlen, vocab_size, embd_dim,
nb_filter, filter_length, batch_size, nb_epoch, optm):
"""
- CNN-1d on text input (represented in int)
- MOT
- dropout + L2 softmax
:param <X, Y> train and test sets
:param nb_classes # of classes
:param maxlen max of n char in a sentence
:param vocab_size
:param embd_dim
:param nb_filter
:param filter_length
:param batch_size
:param nb_epoch
:param optm optimizer options, e.g., adam, rmsprop, etc.
:return:
"""
pool_length = maxlen - filter_length + 1
model = Sequential()
model.add(Embedding(vocab_size, embd_dim, input_length=maxlen))
model.add(Dropout(0.25))
model.add(Convolution1D(nb_filter=nb_filter,
filter_length=filter_length,
border_mode="valid",
activation="relu"))
model.add(MaxPooling1D(pool_length=pool_length))
model.add(Flatten())
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer=optm)
earlystop = EarlyStopping(monitor='val_loss', patience=1, verbose=1)
model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch,
validation_split=0.1, show_accuracy=True, callbacks=[earlystop])
classes = earlystop.model.predict_classes(X_test, batch_size=batch_size)
acc = np_utils.accuracy(classes, np_utils.categorical_probas_to_classes(Y_test))
print('Test accuracy:', acc)
# return(acc)
kappa = metrics.quadratic_weighted_kappa(classes, np_utils.categorical_probas_to_classes(Y_test))
print('Test Kappa:', kappa)
return (kappa)
def train_pair(args, train_csv, test_csv):
print('Reading word vectors.')
embeddings_index = read_glove_vectors(args.embedding_file_path)
print('Found {} word vectors.'.format(len(embeddings_index)))
print('Processing input data')
x_train, y_train, x_test, y_test, word_index, = read_input_csv(train_csv,
test_csv,
args.nb_words,
args.max_sequence_len)
print('train tensor {}.'.format(x_train.shape))
print('Preparing embedding matrix.')
# initiate embedding matrix with zero vectors.
nb_words = min(args.nb_words, len(word_index))
embedding_matrix = np.zeros((nb_words + 1, args.embedding_dim))
for word, i in word_index.items():
if i > nb_words:
continue
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
args.nb_words = nb_words
# args.len_labels_index = len(labels_index)
args.len_labels_index = 2 # fixed for sentiment detection.
model = model_selector(args, embedding_matrix)
checkpoint_filepath = os.path.join(args.model_dir, "weights.best.hdf5")
checkpoint = ModelCheckpoint(checkpoint_filepath, monitor='val_loss',
verbose=1, save_best_only=True)
earlystop = EarlyStopping(monitor='val_loss', patience=1, verbose=1)
tsb = TensorBoard(log_dir='./log', histogram_freq=0, write_graph=True, write_images=False)
callbacks_list = [checkpoint, earlystop, tsb]
model_json = model.to_json()
with open(os.path.join(args.model_dir, "model.json"), "w") as json_file:
json_file.write(model_json)
model.fit(x_train, y_train, validation_split=0.1,
nb_epoch=args.num_epochs, batch_size=args.batch_size, callbacks=callbacks_list)
classes = earlystop.model.predict_classes(x_test, batch_size=args.batch_size)
# acc only supports classes
acc = np_utils.accuracy(classes, np_utils.categorical_probas_to_classes(y_test))
print('Test accuracy: {}.'.format(acc))
def compute_acc(X, Y, vocab, model, opts, filename=None):
scores=model.predict(X,batch_size=options.batch_size)
prediction=np.zeros(scores.shape)
for i in range(scores.shape[0]):
l=np.argmax(scores[i])
prediction[i][l]=1.0
assert np.array_equal(np.ones(prediction.shape[0]),np.sum(prediction,axis=1))
plabels=np.argmax(prediction,axis=1)
tlabels=np.argmax(Y,axis=1)
acc = accuracy(tlabels,plabels)
if filename!=None:
f = open(filename,'w')
for i in range(len(X)):
f.write(map_to_txt(X[i],vocab)+ " : "+ str(plabels[i])+ "\n")
f.close()
return acc
def compute_acc(X, Y, vocab, model, opts, filename=None):
scores=model.predict(X,batch_size=options.batch_size)
prediction=np.zeros(scores.shape)
for i in range(scores.shape[0]):
l=np.argmax(scores[i])
prediction[i][l]=1.0
assert np.array_equal(np.ones(prediction.shape[0]),np.sum(prediction,axis=1))
plabels=np.argmax(prediction,axis=1)
tlabels=np.argmax(Y,axis=1)
acc = accuracy(tlabels,plabels)
if filename!=None:
f = open(filename,'w')
for i in range(len(X)):
f.write(map_to_txt(X[i],vocab)+ " : "+ str(plabels[i])+ "\n")
f.close()
return acc
def cnn1d_w2vembd(X_train, Y_train, X_test, Y_test, nb_classes,
maxlen,
nb_filter, filter_length, batch_size, nb_epoch, optm):
"""
- CNN-1d on 3d sensor which uses word2vec embedding
- MOT
:param <X, Y> train and test sets
:param nb_classes # of classes
:param maxlen max of n char in a sentence
:param nb_filter
:param filter_length
:param batch_size
:param nb_epoch
:param optm
:return:
"""
pool_length = maxlen - filter_length + 1
model = Sequential()
model.add(Convolution1D(nb_filter=nb_filter,
filter_length=filter_length,
border_mode="valid",
activation="relu", input_shape=(maxlen, 300)))
model.add(MaxPooling1D(pool_length=pool_length))
model.add(Flatten())
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer=optm)
earlystop = EarlyStopping(monitor='val_loss', patience=1, verbose=1)
model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch,
validation_split=0.1, show_accuracy=True, callbacks=[earlystop])
classes = earlystop.model.predict_classes(X_test, batch_size=batch_size)
acc = np_utils.accuracy(classes, np_utils.categorical_probas_to_classes(Y_test)) # accuracy only supports classes
print('Test accuracy:', acc)
# return(acc)
kappa = metrics.quadratic_weighted_kappa(classes, np_utils.categorical_probas_to_classes(Y_test))
print('Test Kappa:', kappa)
return (kappa)
def lstm_selfembd(X_train, Y_train, X_test, Y_test, nb_classes,
maxlen, vocab_size, embd_dim,
batch_size, nb_epoch, optm):
"""
- LSTM on text input (represented in int)
- fully-connected model
:param <X, Y> train and test sets
:param nb_classes # of classes
:param maxlen max of n char in a sentence
:param vocab_size
:param embd_dim
:param batch_size
:param nb_epoch
:param optm optimizer options, e.g., adam, rmsprop, etc.
:return:
"""
model = Sequential()
model.add(Embedding(vocab_size, embd_dim, input_length=maxlen))
model.add(Dropout(0.25))
# model.add(LSTM(100, return_sequences=True))
model.add(LSTM(50))
model.add(Flatten())
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer=optm)
earlystop = EarlyStopping(monitor='val_loss', patience=2, verbose=1)
model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch,
validation_split=0.1, show_accuracy=True, callbacks=[earlystop])
classes = earlystop.model.predict_classes(X_test, batch_size=batch_size)
acc = np_utils.accuracy(classes, np_utils.categorical_probas_to_classes(Y_test)) # accuracy only supports classes
print('Test accuracy:', acc)
kappa = metrics.quadratic_weighted_kappa(classes, np_utils.categorical_probas_to_classes(Y_test))
print('Test Kappa:', kappa)
return (kappa)
def tune_model(X_train, y_train, X_test, y_test, settings):
(optimizer, loss_func, activation, nb_epoch, LSTM_in, LSTM_out) = settings
print("Loading data...")
print(len(X_train), 'train sequences')
print('X_train shape:', X_train.shape)
# train LSTM so that we can extract representation
print('Build model...')
model = Sequential()
model.add(Embedding(max_features, LSTM_in))
model.add(LSTM(LSTM_in, LSTM_out))
model.add(Dropout(0.5))
model.add(Dense(LSTM_out, 1))
model.add(Activation(activation))
model.compile(loss=loss_func, optimizer=optimizer, class_mode="binary")
print("Train...")
model.fit(X_train,
y_train,
batch_size=batch_size,
nb_epoch=nb_epoch,
validation_split=0.1,
show_accuracy=True,
verbose=2)
classes = model.predict_classes(X_test, batch_size=batch_size)
acc = np_utils.accuracy(classes, y_test)
print('LSTM accuracy:', acc)
print('Building partial model...')
# early fusion for testing, average over each application
early_fusion_model = Sequential()
early_fusion_model.add(
Embedding(max_features, LSTM_in,
weights=model.layers[0].get_weights()))
early_fusion_model.add(
LSTM(LSTM_in, LSTM_out, weights=model.layers[1].get_weights()))
early_fusion_model.compile(loss=loss_func,
optimizer=optimizer,
class_mode="binary")
return early_fusion_model
def cnn_var_selfembd(X_train, Y_train, X_test, Y_test, nb_classes,
maxlen, vocab_size, embd_size,
nb_filter, batch_size, nb_epoches, optm):
ngram_filters = [2, 5, 8]
input = Input(shape=(maxlen,), name='input', dtype='int32')
embedded = Embedding(input_dim=vocab_size, output_dim=embd_size, input_length=maxlen)(input)
convs = [None, None, None]
# three CNNs
for i, n_gram in enumerate(ngram_filters):
pool_length = maxlen - n_gram + 1
convs[i] = Convolution1D(nb_filter=nb_filter,
filter_length=n_gram,
border_mode="valid",
activation="relu")(embedded)
convs[i] = MaxPooling1D(pool_length=pool_length)(convs[i])
convs[i] = Flatten()(convs[i])
merged = merge([convs[0], convs[1], convs[2]], mode='concat', concat_axis=1)
merged = Dropout(0.5)(merged)
output = Dense(nb_classes, activation='softmax', name='output')(merged)
model = Model(input, output)
model.compile(optm, loss={'output': 'categorical_crossentropy'})
earlystop = EarlyStopping(monitor='val_loss', patience=1, verbose=1)
model.fit(X_train, Y_train,
nb_epoch=nb_epoches, batch_size=batch_size,
validation_split=0.1, callbacks=[earlystop])
probs = earlystop.model.predict(X_test, batch_size=batch_size)
classes = np_utils.categorical_probas_to_classes(probs)
acc = np_utils.accuracy(classes,
np_utils.categorical_probas_to_classes(Y_test))
print('Test accuracy:', acc)
kappa = metrics.quadratic_weighted_kappa(classes,
np_utils.categorical_probas_to_classes(Y_test))
print('Test Kappa:', kappa)
return acc
def LSTM_model(X_train, X_test, Y_train, Y_test, test_label, X_val,
Y_val): # 生成LSTM网络
print('Loading embedding successful!')
print('len(X_train):' + str(len(X_train)))
print('len(X_val):' + str(len(X_val)))
print('len(X_test):' + str(len(X_test)))
print('len(Y_train):' + str(len(Y_train)))
print('len(Y_val):') + str(len(Y_val))
print('len(Y_test):' + str(len(Y_test)))
# print(test_label)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
print('Build model...')
model = Sequential() # stacked lstm model
model.add(
LSTM(hidden_dim,
return_sequences=True,
input_shape=(maxlen, embedding_dim)))
model.add(LSTM(hidden_dim, return_sequences=True))
model.add(LSTM(hidden_dim))
model.add(Dense(nb_classes, activation='softmax'))
# model.add(LSTM(lstm_output_dim,input_shape=(maxlen,embedding_dim))) # LSTM 层 100->128
# model.add(Dense(hidden_dim)) # 隐藏层 全连接层 128->64
# model.add(Activation('relu'))
# model.add(Dropout(0.5))
# model.add(Dense(nb_classes))
# model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam')
best_acc = 0
best_pred_label = []
best_pred_prob = []
best_epoch = 0
a0 = 0
b0 = 0
for i in range(nb_epoch):
print('%d epoch...' % (i + 1))
print('Now:best_acc:%.5f \t best_epoch:%d' % (best_acc, best_epoch))
hist = model.fit(
X_train,
Y_train,
batch_size=32,
nb_epoch=1,
verbose=1,
shuffle=True,
show_accuracy=True, #20 10
validation_data=(X_val, Y_val))
acc = max(hist.history['val_acc'])
p_label = model.predict_classes(X_test, batch_size=32, verbose=1)
p_prob = model.predict_proba(X_test, batch_size=32, verbose=1)
print p_label
a1 = np_utils.accuracy(p_label, test_label)
print 'Now epoch test acc:%.5f' % (a1)
if a1 > a0:
print 'a1 better:%.5f' % (a1)
a0 = a1
b0 = (i + 1)
if acc > best_acc:
print('出现更好的acc,正在更新acc和epoch...')
best_acc = acc
best_pred_label = p_label
best_pred_prob = p_prob
best_epoch = (i + 1)
test_acc = np_utils.accuracy(best_pred_label, test_label) # 得到正确率
print(
'the best epoch:%d,and the acc:%.5f.,while best test acc epoch:%d,%.5f'
% (best_epoch, test_acc, b0, a0))
# print('the best pred_class:\n')
# print(best_pred_label)
write2File(best_pred_label, test_label)
return best_pred_label, best_epoch, best_pred_prob
if model_type == "brnn_cnn_multitask":
Y_out = model.predict({'input1': X_test, "input2": X_char_test})
#Y_idx = (Y_test[0][:,:,0] == 0) & (Y_test[0][:,:,5] == 0) # Get indexes of only those tokens which correspond to entitites
Y_idx = Y_test[0][:,:,0] >= 0 # Get all indexes
# Calculate accuracy only based on correct entity identity
logger.info("Evaluation scores on test data:")
scores = {}
Y_pred = []
Y_true = []
score_keys = ["accuracy", "micro_precision", "micro_recall", "micro_f1", "macro_f1", "c_mf1", "c_mp", "c_mr"]
for i, k in enumerate(output_names):
labels = range(Y_out[k].shape[-1])
Y_pred.append(np.argmax(np.array(Y_out[k]), axis=-1)[Y_idx])
Y_true.append(np.argmax(Y_test[i], axis=-1)[Y_idx])
scores[k] = vtu.get_eval_scores(Y_pred[-1], Y_true[-1], labels = labels)
scores[k]["accuracy"] = accuracy(Y_pred[-1], Y_true[-1])
TP, FP, FN = (scores[k][_k][1:-1] for _k in ["TP", "FP", "FN"])
micro_precision = np.sum(TP) * 1. / np.sum(TP + FP)
micro_recall = np.sum(TP) * 1. / np.sum(TP + FN)
micro_f1 = 2*micro_precision*micro_recall / (micro_precision+micro_recall)
scores[k]["c_mf1"] = micro_f1
scores[k]["c_mp"] = micro_precision
scores[k]["c_mr"] = micro_recall
logger.info("%s: %s" % (k, dict((_k, scores[k][_k]) for _k in score_keys)))
all_labels = dict((k, i) for i, k in enumerate((b_i, c_i) for b_i in range(Y_test[0].shape[-1]) for c_i in range(Y_test[1].shape[-1])))
all_true = [all_labels.get(k) for k in zip(Y_true[0], Y_true[1])]
all_pred = [all_labels.get(k) for k in zip(Y_pred[0], Y_pred[1])]
scores_all = vtu.get_eval_scores(all_pred, all_true, labels=range(len(all_labels)))
scores_all["accuracy"] = accuracy(all_pred, all_true)
valid_idx = map(lambda x: x[1], filter(lambda k: (k[0][0] > 0 and k[0][0] < 5 and k[0][1] > 0 and k[0][1] < 95), all_labels.iteritems()))
TP, FP, FN = (scores_all[_k][valid_idx] for _k in ["TP", "FP", "FN"])
def run():
# ======================================= Initialization ======================================= #
all_folds_target_label = np.asarray([])
all_folds_majority_vote_cm = np.zeros(
(Config.NB_CLASSES, Config.NB_CLASSES), dtype=np.int)
all_folds_majority_vote_label = np.asarray([])
all_folds_probability_vote_cm = np.zeros(
(Config.NB_CLASSES, Config.NB_CLASSES), dtype=np.int)
all_folds_probability_vote_label = np.asarray([])
segment_size = sum(Config.LAYERS_ORDER_LIST) * 2 + Config.EXTRA_FRAMES
print('Segment without middle frame:' + str(segment_size))
is_visualization_called_flag = False # visualization is done for first fold only using this variable
# list of paths to the n-fold indices of the Training/Testing/Validation splits
# number of paths should be e.g. 30 for 3x10, where 3 is for the splits and 10 for the 10-folds
# Every 3 files are for one run to train and validate on 9-folds and test on the remaining fold.
folds_index_file_list = glob.glob(
os.path.join(Config.INDEX_PATH, "Fold*.hdf5"))
if len(folds_index_file_list) == 0:
print('Index path is not found = ' + Config.INDEX_PATH)
return
folds_index_file_list.sort()
cross_val_index_list = np.arange(
0, Config.SPLIT_COUNT * Config.CROSS_VALIDATION_FOLDS_COUNT,
Config.SPLIT_COUNT)
# ======================================= Start cross-validation ======================================= #
for j in range(cross_val_index_list[Config.INITIAL_FOLD_ID],
len(folds_index_file_list), Config.SPLIT_COUNT):
test_index_path = folds_index_file_list[j] if folds_index_file_list[
j].lower().endswith('_test.hdf5') else None
train_index_path = folds_index_file_list[
j + 1] if folds_index_file_list[j + 1].lower().endswith(
'_train.hdf5') else None
validation_index_path = folds_index_file_list[
j + 2] if folds_index_file_list[j + 2].lower().endswith(
'_validation.hdf5') else None
if None in [test_index_path, train_index_path, validation_index_path]:
print(
'Train / Validation / Test indices are not correctly assigned')
exit(1)
np.random.seed(0) # for reproducibility
data_loader = DataLoader()
mclnn_trainer = MCLNNTrainer()
# --------------------------------- Load data ----------------------------- #
data_loader.load_data(segment_size, Config.STEP_SIZE,
Config.NB_CLASSES, Config.DATASET_FILE_PATH,
Config.STANDARDIZATION_PATH, train_index_path,
test_index_path, validation_index_path)
# ------------------------------ Weights path ---------------------------- #
train_index_filename = os.path.basename(train_index_path).split('.')[0]
weights_to_store_foldername = train_index_filename + '_' \
+ 'batch' + str(Config.BATCH_SIZE) \
+ 'wait' + str(Config.WAIT_COUNT) \
+ 'order' + str(Config.LAYERS_ORDER_LIST[0]) \
+ 'extra' + str(Config.EXTRA_FRAMES)
fold_weights_path = os.path.join(Config.ALL_FOLDS_WEIGHTS_PATH,
weights_to_store_foldername)
if not os.path.exists(fold_weights_path):
if Config.USE_PRETRAINED_WEIGHTS == False:
os.makedirs(fold_weights_path)
elif Config.USE_PRETRAINED_WEIGHTS == True:
print('Pre-trained weights do not exist in :' +
fold_weights_path)
exit(1)
# -------------------------- Build and Train model ----------------------- #
print('----------- Training param -------------')
print(' batch_size>' + str(Config.BATCH_SIZE) + ' nb_classes>' +
str(Config.NB_CLASSES) + ' nb_epoch>' + str(Config.NB_EPOCH) +
' mclnn_layers>' + str(Config.MCLNN_LAYER_COUNT) +
' dense_layers>' + str(Config.DENSE_LAYER_COUNT) + ' norder>' +
str(Config.LAYERS_ORDER_LIST) + ' extra_frames>' +
str(Config.EXTRA_FRAMES) + ' segment_size>' +
str(segment_size + 1)
+ # plus 1 is for middle frame, considered in segmentation stage
' initial_fold>' +
str(Config.INITIAL_FOLD_ID +
1) + # plus 1 beacuse folds are zero indexed
' wait_count>' + str(Config.WAIT_COUNT) + ' split_count>' +
str(Config.SPLIT_COUNT))
if Config.USE_PRETRAINED_WEIGHTS == False:
model = mclnn_trainer.build_model(
segment_size=data_loader.train_segments.shape[1],
feature_count=data_loader.train_segments.shape[2],
pretrained_weights_path=None)
mclnn_trainer.train_model(model, data_loader, fold_weights_path)
# ------------------ Load trained weights in a new model ------------------ #
# load paths of all weights generated during training
weight_list = glob.glob(os.path.join(fold_weights_path, "*.hdf5"))
if len(weight_list) == 0:
print('Weight path is not found = ' + fold_weights_path)
return
weight_list.sort(key=os.path.getmtime)
if len(weight_list) > 1:
startup_weights = weight_list[-(Config.WAIT_COUNT + 2)]
elif len(weight_list) == 1:
startup_weights = weight_list[0]
print('----------- Weights Loaded ---------------:')
print(startup_weights)
model = mclnn_trainer.build_model(
segment_size=data_loader.train_segments.shape[1],
feature_count=data_loader.train_segments.shape[2],
pretrained_weights_path=startup_weights)
# ------------------------ Visualize a test sample --------------------- #
if is_visualization_called_flag == False and Config.SAVE_TEST_SEGMENT_PREDICTION_IMAGE == True:
visualizer = Visualizer(Config)
visualizer.visualize_weights_and_sample_test_clip(
model=model, data_loader=data_loader)
# mclnn_trainer.visualize_model(model=model, data_loader=data_loader)
is_visualization_called_flag = True
# --------------------------- Evaluate model ------------------------------ #
fold_majority_cm, fold_probability_cm, \
fold_majority_vote_label, fold_probability_vote_label, \
fold_target_label = mclnn_trainer.evaluate_model(segment_size=segment_size,
model=model,
data_loader=data_loader)
all_folds_majority_vote_cm += fold_majority_cm
all_folds_majority_vote_label = np.append(
all_folds_majority_vote_label, fold_majority_vote_label)
all_folds_probability_vote_cm += fold_probability_cm
all_folds_probability_vote_label = np.append(
all_folds_probability_vote_label, fold_probability_vote_label)
all_folds_target_label = np.append(all_folds_target_label,
fold_target_label)
gc.collect()
print('-------------- Cross validation performance --------------')
print(Config.CLASS_NAMES)
print(all_folds_majority_vote_cm)
print(
str(Config.CROSS_VALIDATION_FOLDS_COUNT) +
'-Fold Clip-level majority-vote Accuracy ' + str(
np_utils.accuracy(all_folds_majority_vote_label,
all_folds_target_label)))
print(Config.CLASS_NAMES)
print(all_folds_probability_vote_cm)
print(
str(Config.CROSS_VALIDATION_FOLDS_COUNT) +
'-Fold Clip-level probability-vote Accuracy ' + str(
np_utils.accuracy(all_folds_probability_vote_label,
all_folds_target_label)))
scoref1 = f1score(all_folds_target_label,
all_folds_probability_vote_label,
average='micro')
print('F1 Score micro ' + str(scoref1))
scoref1 = f1score(all_folds_target_label,
all_folds_probability_vote_label,
average='weighted')
print('F1 Score weighted ' + str(scoref1))
model.compile(loss='binary_crossentropy', optimizer='adam', class_mode="binary")
print("Train...")
model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=10, validation_split=0.1, show_accuracy=True)
score = model.evaluate(X_test, y_test, batch_size=batch_size)
print('Test score:', score)
classes = model.predict_classes(X_test, batch_size=batch_size)
acc = np_utils.accuracy(classes, y_test)
print('Test accuracy:', acc)
acc = np_utils.accuracy(classes[y_test==True], y_test[y_test==True])
print('Test b accuracy:', acc)
acc = np_utils.accuracy(classes[y_test==False], y_test[y_test==False])
print('Test non-b accuracy:', acc)
acc = np_utils.accuracy(classes[labels_test[:,0]==4], y_test[labels_test[:,0]==4])
print('Test c accuracy:', acc)
acc = np_utils.accuracy(classes[labels_test[:,0]==0], y_test[labels_test[:,0]==0])
print('Test l accuracy:', acc)
pred = model.predict(X_test, batch_size=batch_size)
else:
print ("Opening model file...")
model = model_from_json(open('my_model_architecture.json').read())
model.load_weights('my_model_weights.h5')
model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=["accuracy"])
score = model.evaluate(X_test_vec, y_test, batch_size=batch_size)
print('Test score:', score)
classes = model.predict_classes(X_test_vec, batch_size=batch_size)
acc = np_utils.accuracy(classes, y_test)
print('Test accuracy:', acc)
acc = np_utils.accuracy(classes[y_test==True], y_test[y_test==True])
print('Test b accuracy:', acc)
acc = np_utils.accuracy(classes[y_test==False], y_test[y_test==False])
print('Test non-b accuracy:', acc)
acc = np_utils.accuracy(classes[labels_test[:,0]==4], y_test[labels_test[:,0]==4])
print('Test c accuracy:', acc)
acc = np_utils.accuracy(classes[labels_test[:,0]==0], y_test[labels_test[:,0]==0])
print('Test l accuracy:', acc)
def cnn_multi_selfembd(X_train, Y_train, X_test, Y_test, nb_classes,
maxlen, vocab_size, embd_size,
pos_train, pos_test, pos_embd_dim,
dp_train, dp_test, dp_embd_dim,
nb_filter, batch_size, nb_epoches, optm):
"""
cnn1d using multi-inputs, i.e., word, POS, DP
word using varying filter lengths
note need using Graph
:param X_train:
:param Y_Train:
:param X_test:
:param Y_test:
:param nb_classes:
:param maxlen:
:param vocab_size:
:param embd_size:
:param batch_size:
:param nb_epoches:
:param optm:
:return:
"""
ngram_filters = [2, 5, 8]
nd_convs = ['conv_' + str(n) for n in ngram_filters]
nd_pools = ['pool_' + str(n) for n in ngram_filters]
nd_flats = ['flat_' + str(n) for n in ngram_filters]
model = Graph()
model.add_input(name='input', input_shape=(maxlen,), dtype=int)
model.add_node(Embedding(vocab_size, embd_size, input_length=maxlen),
name='embedding', input='input')
# three CNNs
for i, n_gram in enumerate(ngram_filters):
pool_length = maxlen - n_gram + 1
model.add_node(Convolution1D(nb_filter=nb_filter,
filter_length=n_gram,
border_mode="valid",
activation="relu"),
name=nd_convs[i], input='embedding')
model.add_node(MaxPooling1D(pool_length=pool_length),
name=nd_pools[i], input=nd_convs[i])
model.add_node(Flatten(), name=nd_flats[i], input=nd_pools[i])
model.add_node(Dropout(0.5), name='dropout', inputs=nd_flats, merge_mode='concat')
# POS CNN
nb_pos = 15
pos_f_len = 3
pos_pool_len = maxlen - pos_f_len + 1
model.add_input(name='posinput', input_shape=(maxlen,), dtype=int)
model.add_node(Embedding(nb_pos, pos_embd_dim, input_length=maxlen),
name='posembd', input='posinput')
model.add_node(Convolution1D(nb_filter=nb_filter,
filter_length=pos_f_len,
border_mode='valid',
activation='relu'),
name='poscnn', input='posembd')
model.add_node(MaxPooling1D(pool_length=pos_pool_len),
name='pospool', input='poscnn')
model.add_node(Flatten(), name='posflat', input='pospool')
model.add_node(Dropout(0.5), name='posdropout', input='posflat')
# DP CNN
nb_dp = vocab_size
dp_f_len = 3
dp_pool_len = maxlen - dp_f_len + 1
model.add_input(name='dpinput', input_shape=(maxlen,), dtype=int)
model.add_node(Embedding(nb_dp, dp_embd_dim, input_length=maxlen),
name='dpembd', input='dpinput')
model.add_node(Convolution1D(nb_filter=nb_filter,
filter_length=dp_f_len,
border_mode='valid',
activation='relu'),
name='dpcnn', input='dpembd')
model.add_node(MaxPooling1D(pool_length=dp_pool_len),
name='dppool', input='dpcnn')
model.add_node(Flatten(), name='dpflat', input='dppool')
model.add_node(Dropout(0.5), name='dpdropout', input='dpflat')
model.add_node(Dense(nb_classes, activation='softmax'), name='softmax',
inputs=['dropout', 'posdropout', 'dpdropout'],
merge_mode='concat')
model.add_output(name='output', input='softmax')
model.compile(optm, loss={'output': 'categorical_crossentropy'}) # note Graph()'s diff syntax
# early stopping
earlystop = EarlyStopping(monitor='val_loss', patience=1, verbose=1)
model.fit({'input': X_train, 'posinput': pos_train, 'dpinput': dp_train,
'output': Y_train},
nb_epoch=nb_epoches, batch_size=batch_size,
validation_split=0.1, callbacks=[earlystop])
# Graph doesn't have several arg/func existing in Sequential()
# - fit no show-accuracy
# - no predict_classes
classes = model.predict({'input': X_test, 'posinput': pos_test, 'dpinput': dp_test}
, batch_size=batch_size)['output'].argmax(axis=1)
acc = np_utils.accuracy(classes, np_utils.categorical_probas_to_classes(Y_test)) # accuracy only supports classes
print('Test accuracy:', acc)
kappa = metrics.quadratic_weighted_kappa(classes, np_utils.categorical_probas_to_classes(Y_test))
print('Test Kappa:', kappa)
return kappa
def Multi_channel(models_all, X_train, X_test, Y_train, Y_test, test_label,
X_val, Y_val): # multi-channel 的神经网络
print('开始Merge所有的通道...')
model = Sequential()
model.add(Merge(models_all, mode='sum')) # 将所有的model合并 merge
model.add(Dropout(0.5))
model.add(Dense(nb_classes)) # 输出层 64->7
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy']) # 配置网络 adam rmsprop 0.49206
# 可以写个for循环 记录做大的acc 或 G-mean
# X_val=[]
# X_train2=[]
# Y_val=[]
# Y_train2=[]
# for j in range(5):
# x_val=[]
# y_val=[]
# x_tra=[]
# y_tra=[]
# for i in range(7):
# x_val.append(X_train[j][288*i+228:288*i+288])
# x_tra.append(X_train[j][288*i:288*i+228])
# y_val.append(Y_train[j][288*i+228:288*i+288])
# y_tra.append(Y_train[j][288*i:288*i+228])
# x_val=np.array(X_val)
# # x_val=np_utils.to_categorical(x_val,nb_classes)
# # y_val=np.array(Y_val)
# y_val=np_utils.to_categorical(y_val,nb_classes)
# x_tra=np.array(x_tra)
# # y_tra=np.array(y_tra)
# y_tra=np_utils.to_categorical(y_tra,nb_classes)
# X_val.append(x_val)
# Y_val.append(y_val)
# X_train2.append(x_tra)
# Y_train2.append(y_tra)
best_acc = 0
best_pred_label = []
best_epoch = 0
a0 = 0
b0 = 0
val_acc_list = []
test_acc_list = []
for i in range(nb_epoch):
print('%d epoch...' % (i + 1))
print('Now:best_acc:%.5f \t best_epoch:%d' % (best_acc, best_epoch))
hist = model.fit(
X_train,
Y_train[0],
batch_size=32,
nb_epoch=1,
verbose=1, #20 10
shuffle=True,
validation_data=(X_val, Y_val[0])) # (X_test,Y_test[0])
acc = max(hist.history['val_acc'])
val_acc_list.append(str(acc))
# acc=model.evaluate(X_test,Y_test[0],batch_size=32,show_accuracy=True,verbose=1)[1]
p_label = model.predict_classes(X_test, batch_size=32, verbose=1)
# acc=np_utils.accuracy(p_label,Y_test) # 得到正确率
print p_label
a1 = np_utils.accuracy(p_label, test_label[0])
test_acc_list.append(str(a1))
print 'Now epoch test acc:%.5f' % (a1)
if a1 > a0:
print 'a1 better:%.5f' % (a1)
a0 = a1
b0 = (i + 1)
# print Y_test[0]
# print test_label[0]
if acc > best_acc:
print('出现更好的acc,正在更新acc和epoch...')
best_acc = acc
best_pred_label = p_label
best_epoch = (i + 1)
test_acc = np_utils.accuracy(best_pred_label, test_label[0]) # 得到正确率
print(
'the best val epoch:%d,and the acc:%.5f,while best test acc epoch:%d,%.5f'
% (best_epoch, test_acc, b0, a0))
# hist=model.fit(X_train,Y_train[0],batch_size=20,nb_epoch=nb_epoch,verbose=1,shuffle=True,
# show_accuracy=True,validation_data=(X_test,Y_test[0])) # validation_data=(X_test,Y_test) 训练网络
# print(hist.history) # 输出历史记录
# best_acc=max(hist.history['val_acc']) # 输出最好的验证正确率
# print('the best epoch:%d,and the acc:%.5f' %(hist.history['val_acc'].index(best_acc)+1,best_acc))
# f=open('acc_record.txt','a') # 将结果保存到文件
# f.write('------------%d epoch---------------\n' %(nb_epoch))
# f.write('val_acc: '+str(hist.history['val_acc'])+'\n')
# f.write('the best epoch:%d,and the acc:%.5f\n' %(hist.history['val_acc'].index(best_acc)+1,best_acc))
# f.write('-----------------------------------\n')
# f.close()
# pred_label=model.predict_classes(X_test,batch_size=20,verbose=1) # 对测试样本进行类别预测
write2File(best_pred_label, test_label[0])
f = open('acc_compare.txt', 'w')
f.write(' '.join(list(val_acc_list)))
f.write('\n')
f.write(' '.join(list(test_acc_list)))
f.write('\n')
f.close()
return best_pred_label, test_label[0], best_epoch
def model():
# set parameters:
AMINO = 'ARNDCEQGHILKMFPSTWYV'
gram = 1
vocabulary_inv = [
''.join(item) for item in itertools.product(AMINO, repeat=gram)
]
vocabulary = {x: i for i, x in enumerate(vocabulary_inv)}
# Load data
print("Loading data...")
DATA_ROOT = 'uniprot-swiss/'
uniprot_file = 'uniprot-swiss-mol-func-yeast'
input_file = DATA_ROOT + uniprot_file + ".txt"
sequences = [line.strip().split('\t') for line in open(input_file, 'rb')]
sequences = [seq[1] for seq in sequences]
go_id = [newd[2] for newd in sequences]
seqdata = list()
maxlen = 700
for seq in range(0, len(sequences)):
ss = sequences[seq]
seq1 = [ss[i:i + gram] for i in range(len(ss) - gram + 1)]
new_seq = seq1[:maxlen]
seqdata.append(new_seq)
data_x = np.zeros((len(sequences), maxlen, 20**gram))
labels = np.zeros((len(sequences), 44))
for i, seq in enumerate(seqdata):
for t, ng in enumerate(seq):
data_x[i, t, vocabulary[ng]] = 1
pdb.set_trace()
# Shuffle data
y = np.array(labels, dtype="float64")
shuffle_indices = np.random.permutation(np.arange(len(y)))
x_shuffled = x[shuffle_indices]
y_shuffled = y[shuffle_indices]
print("Vocabulary Size: {:d}".format(len(vocabulary)))
# Convolution
filter_length = 7
nb_filter = 64
pool_length = 2
k = 7
# LSTM
lstm_output_size = 70
# Training
batch_size = 30
nb_epoch = 12
train, test = train_test_split(labels, data, batch_size=batch_size)
train_label, train_data = train
test_label, test_data = test
test_label_rep = test_label
model = Graph()
model.add_input(name='input', input_shape=((1000, 20)))
model.add_node(Convolution1D(nb_filter=96,
filter_length=7,
border_mode="valid",
activation="relu",
subsample_length=1),
name='conv1',
input='input')
model.add_node(MaxPooling1D(pool_length=3, stride=1),
input='conv1',
name='pool1')
model.add_node(Dropout(0.5), input='conv1', name='drop1')
model.add_node(Dense(100), input='conv1', name='level1')
model.add_node(Dense(26), input='level1', name='GO:0003674')
model.add_node(Dense(100), input='G0:0003674', name='level2')
model.add_node(Dense(7), input='level2', name='G0:0016787')
model.add_node(Dense(100), input='GO:0003674', name='level2a')
model.add_node(Dense(4), input='level2a', name='GO:0016740')
model.add_node(Dense(100), input='GO:0003674', name='level2b')
model.add_node(Dense(5), input='level2b', name='GO:0003723')
model.add_node(Dense(100), input='GO:0003674', name='level2c')
model.add_node(Dense(1), input='level2c', name='GO:0042393')
model.add_node(Dense(100), input='GO:0003674', name='level2d')
model.add_node(Dense(1), input='level2d', name='GO:0005085')
model.add_node(Dense(100), input='GO:0003674', name='level2e')
model.add_node(Dense(1), input='level2e', name='GO:0019899')
model.add_node(Dense(100), input='GO:0003674', name='level2f')
model.add_node(Dense(1), input='level2f', name='GO:0001071')
model.add_node(Dense(100), input='GO:0003674', name='level2g')
model.add_node(Dense(1), input='level2g', name='GO:0004871')
model.add_node(Dense(100), input='GO:0003674', name='level2h')
model.add_node(Dense(1), input='level2h', name='GO:0003677')
model.add_node(Dense(100), input='GO:0003674', name='level2i')
model.add_node(Dense(1), input='level2i', name='GO:0005198')
model.add_node(Dense(100), input='GO:0003674', name='level2j')
model.add_node(Dense(1), input='level2j', name='GO:0030674')
model.add_node(Dense(100), input='GO:0003674', name='level2k')
model.add_node(Dense(1), input='level2k', name='GO:0000988')
model.add_node(Dense(100), input='GO:0003674', name='level2l')
model.add_node(Dense(1), input='level2l', name='GO:0008092')
model.add_node(Dense(100), input='GO:0003674', name='level2m')
model.add_node(Dense(1), input='level2m', name='GO:0016829')
model.add_node(Dense(100), input='GO:0003674', name='level2n')
model.add_node(Dense(1), input='level2n', name='GO:0043167')
model.add_node(Dense(100), input='GO:0003674', name='level2o')
model.add_node(Dense(1), input='level2o', name='GO:0051082')
model.add_node(Dense(100), input='GO:0003674', name='level2p')
model.add_node(Dense(1), input='level2p', name='GO:0016491')
model.add_node(Dense(100), input='GO:0003674', name='level2q')
model.add_node(Dense(1), input='level2q', name='GO:0008134')
model.add_node(Dense(100), input='GO:0003674', name='level2r')
model.add_node(Dense(1), input='level2r', name='GO:0008289')
model.add_node(Dense(100), input='GO:0003674', name='level2s')
model.add_node(Dense(1), input='level2s', name='GO:0016853')
model.add_node(Dense(100), input='GO:0003674', name='level2t')
model.add_node(Dense(1), input='level2t', name='GO:0032182')
model.add_node(Dense(100), input='GO:0003674', name='level2u')
model.add_node(Dense(1), input='level2u', name='GO:0003682')
model.add_node(Dense(100), input='GO:0003674', name='level2v')
model.add_node(Dense(1), input='level2v', name='GO:0008565')
model.add_node(Dense(100), input='GO:0003674', name='level2w')
model.add_node(Dense(1), input='level2w', name='GO:0030234')
model.add_node(Dense(100), input='GO:0003674', name='level2x')
model.add_node(Dense(1), input='level2x', name='GO:0016874')
model.add_node(Dense(100), input='GO:0003674', name='level2y')
model.add_node(Dense(1), input='level2y', name='GO:0022857')
model.add_node(Dense(100), input='GO:0016740', name='level3')
model.add_node(Dense(1), input='level3', name='GO:0016757')
model.add_node(Dense(100), input='GO:0016740', name='level3a')
model.add_node(Dense(1), input='level3a', name='GO:0008168')
model.add_node(Dense(100), input='GO:0016740', name='level3b')
model.add_node(Dense(1), input='level3b', name='GO:0016301')
model.add_node(Dense(100), input='GO:0016740', name='level3c')
model.add_node(Dense(1), input='level3c', name='GO:0016779')
model.add_node(Dense(100), input='GO:0016787', name='level3d')
model.add_node(Dense(1), input='level3d', name='GO:0008233')
model.add_node(Dense(100), input='GO:0016787', name='level3e')
model.add_node(Dense(1), input='level3e', name='GO:0003924')
model.add_node(Dense(100), input='GO:0016787', name='level3f')
model.add_node(Dense(1), input='level3f', name='GO:0016791')
model.add_node(Dense(100), input='GO:0016787', name='level3g')
model.add_node(Dense(1), input='level3g', name='GO:0016798')
model.add_node(Dense(100), input='GO:0016787', name='level3h')
model.add_node(Dense(1), input='level3h', name='GO:0004386')
model.add_node(Dense(100), input='GO:0016787', name='level3i')
model.add_node(Dense(1), input='level3i', name='GO:0016887')
model.add_node(Dense(100), input='GO:0016787', name='level3j')
model.add_node(Dense(1), input='level3j', name='GO:0004518')
model.add_node(Dense(100), input='GO:0003723', name='level3k')
model.add_node(Dense(1), input='level3k', name='GO:0008135')
model.add_node(Dense(100), input='GO:0003723', name='level3l')
model.add_node(Dense(1), input='level3l', name='GO:0030555')
model.add_node(Dense(100), input='GO:0003723', name='level3m')
model.add_node(Dense(1), input='level3m', name='GO:0003729')
model.add_node(Dense(100), input='GO:0003723', name='level3n')
model.add_node(Dense(1), input='level3n', name='GO:0030553')
model.add_node(Dense(100), input='GO:0003723', name='level3o')
model.add_node(Dense(1), input='level3o', name='GO:0019843')
model.add_node(Dense(100), input='GO:0005198', name='level3p')
model.add_node(Dense(1), input='level3p', name='GO:0005198')
model.add_node(Dense(44, activation='softmax'),
name='owl:nothing',
inputs=[
'GO:0003674', 'G0:0016787', 'GO:0016740', 'GO:0003723',
'GO:0042393', 'GO:0005085', 'GO:0019899', 'GO:0001071',
'GO:0004871', 'GO:0003677', 'GO:0005198', 'GO:0030674',
'GO:0000988', 'GO:0008092', 'GO:0016829', 'GO:0043167',
'GO:0051082', 'GO:0016491', 'GO:0008134', 'GO:0008289',
'GO:0016853', 'GO:0032182', 'GO:0003682', 'GO:0008565',
'GO:0030234', 'GO:0016874', 'GO:0022857', 'GO:0016757',
'GO:0008168', 'GO:0016301', 'GO:0016779', 'GO:0008233',
'GO:0003924', 'GO:0016791', 'GO:0016798', 'GO:0004386',
'GO:0016887', 'GO:0004518', 'GO:0008135', 'GO:0030555',
'GO:0003729', 'GO:0030553', 'GO:0019843', 'GO:0005198'
])
model.add_output(name='output', input='owl:nothing')
print 'compiling model'
model.compile('rmsprop', {'output': 'binary_crossentropy'})
print 'running at most 60 epochs'
checkpointer = ModelCheckpoint(filepath="bestmodel.hdf5",
verbose=1,
save_best_only=True)
earlystopper = EarlyStopping(monitor='val_loss', patience=10, verbose=1)
model.fit({
'input': train_data,
'output': train_label
},
batch_size=64,
nb_epoch=60,
validation_split=0.2,
callbacks=[checkpointer, earlystopper])
# # Loading saved weights
# print 'Loading weights'
# model.load_weights(DATA_ROOT + go_id + '.hdf5')
model.load_weights('bestmodel.hdf5')
pred_data = numpy.round(
numpy.array(
model.predict({'input': test_data}, batch_size=10)['output']))
acc = accuracy(test_label, pred_data)
# Saving the model
#tresults = model.evaluate(test_data, test_label,show_accuracy=True)
#print tresults
print acc
return classification_report(list(test_label_rep), pred_data)
Y_test = np_utils.to_categorical(y_test, 10)
model = Graph()
model.add_input(name="input", input_shape=(maxlen,))
model.add_node(Dense(1000, activation="relu"), name="dense1", input="input")
model.add_node(Dropout(0.5), name="dropout1", input="dense1")
model.add_node(Dense(600, activation="relu"), name="dense2", input="dropout1")
model.add_node(Dropout(0.5), name="dropout2", input="dense2")
model.add_node(Dense(10, activation="softmax"), name="soft_max", input="dense2")
model.add_output(name="output", input="soft_max")
model.compile("rmsprop", {"output": "categorical_crossentropy"})
history = model.fit({"input": X_train, "output": Y_train}, nb_epoch=10)
score = model.evaluate({"input": X_test, "output": Y_test})
acc = accuracy(Y_test, np.round(np.array(model.predict({"input": X_test})["output"])))
print "Test score: ", score
print "Test accuracy: ", acc
pred = np.array(model.predict({"input": X_test})["output"])
ac = 0
for i in range(0, len(X_test)):
if np.argmax(Y_test[i]) == np.argmax(pred[i]):
ac += 1
print "Test accuracy: ", float(ac) / float(len(X_test))
# model.add_node(Dropout(0.5), name='dropout', input='dense', )
# model.add_node(Embedding(max_features, 128, input_length=maxlen), name='embedding', input='input')
# model.add_node(LSTM(64), name='forward', input='embedding')
# model.add_node(LSTM(64, go_backwards=True), name='backward', input='embedding')
# model.add_node(Dropout(0.5), name='dropout', inputs=['forward', 'backward'])
def cnn_other(Y_train, Y_test, nb_classes,
Other_train, Other_test, k,
maxlen,
nb_filter, filter_size, batch_size, nb_epoches, optm):
"""
cnn1d using varying filter lengths
note need using Graph
:param Y_Train:
:param Y_test:
:param nb_classes:
:param maxlen:
:param vocab_size:
:param embd_size:
:param batch_size:
:param nb_epoches:
:param optm:
:return:
"""
model = Graph()
# CNN for other
pos_pool_len = maxlen / 2 - filter_size + 1
model.add_input(name='other_input', input_shape=(maxlen, k), dtype='float')
model.add_node(Convolution1D(nb_filter=nb_filter,
filter_length=filter_size,
border_mode='valid',
activation='relu',
input_shape=(maxlen, k)),
name='poscnn', input='other_input')
model.add_node(MaxPooling1D(pool_length=5),
name='pospool', input='poscnn')
# 2nd CNN
model.add_node(Convolution1D(nb_filter=nb_filter * 2,
filter_length=filter_size,
border_mode='valid',
activation='relu'),
name='cnn2', input='pospool')
model.add_node(MaxPooling1D(pool_length=10),
name='cnn2_pool', input='cnn2')
model.add_node(Flatten(), name='posflat', input='cnn2_pool')
model.add_node(Dropout(0.5), name='posdropout', input='posflat')
model.add_node(Dense(nb_classes, activation='softmax'), name='softmax',
input='posdropout')
model.add_output(name='output', input='softmax')
model.compile(optm, loss={'output': 'categorical_crossentropy'}) # note Graph()'s diff syntax
# early stopping
earlystop = EarlyStopping(monitor='val_loss', patience=1, verbose=1)
model.fit({'other_input': Other_train, 'output': Y_train},
nb_epoch=nb_epoches, batch_size=batch_size,
validation_split=0.1, callbacks=[earlystop])
# Graph doesn't have several arg/func existing in Sequential()
# - fit no show-accuracy
# - no predict_classes
classes = model.predict({'other_input': Other_test},
batch_size=batch_size)['output'].argmax(axis=1)
acc = np_utils.accuracy(classes, np_utils.categorical_probas_to_classes(Y_test)) # accuracy only supports classes
print('Test accuracy:', acc)
kappa = metrics.quadratic_weighted_kappa(classes, np_utils.categorical_probas_to_classes(Y_test))
print('Test Kappa:', kappa)
return acc
#训练集
x = np.array(list(pn["sent"]))[::2]
y = np.array(list(pn["mark"]))[::2]
#测试集
x_t = np.array(list(pn["sent"]))[1::2]
y_t = np.array(list(pn["mark"]))[1::2]
# 全集
x_a = np.array(list(pn["sent"]))
y_a = np.array(list(pn["mark"]))
print("Build model ...")
model = Sequential()
model.add(
Embedding(input_dim=len(dict) + 1, output_dim=256,
input_length=maxlen))
# try using a GRU instead, for fun
model.add(LSTM(128))
# model.add(LSTM((256,128)))
# model.add(GRU())
model.add(Dropout(0.5))
model.add(Dense(1))
model.add(Activation("sigmoid"))
model.compile(loss="binary_crossentropy", optimizer="adam")
model.fit(x, y, batch_size=16, epochs=10)
classes = model.predict(x_t)
accuracy = np_utils.accuracy(classes, y_t)
print("test accuracy:", accuracy)
border_mode='valid',
activation='relu'),
name='dpcnn', input='dpembd')
model.add_node(MaxPooling1D(pool_length=dp_pool_len),
name='dppool', input='dpcnn')
model.add_node(Flatten(), name='dpflat', input='dppool')
model.add_node(Dropout(0.5), name='dpdropout', input='dpflat')
# using three CNNs to predict with L1
model.add_node(Dense(nb_classes, activation='softmax', W_regularizer=l2(0.05)), name='softmax',
inputs=['dropout', 'posdropout', 'dpdropout'],
merge_mode='concat')
model.add_output(name='output', input='softmax')
model.compile('rmsprop', loss={'output': 'categorical_crossentropy'})
# early stopping
earlystop = EarlyStopping(monitor='val_loss', patience=1, verbose=1)
model.fit({'input': X_train, 'posinput': pos_train, 'dpinput': dp_train, 'output': Y_train},
nb_epoch=nb_epoch, batch_size=batch_size,
validation_split=0.1, callbacks=[earlystop])
# Graph doesn't have several arg/func existing in Sequential()
# - fit no show-accuracy
# - no predict_classes
classes = model.predict({'input': X_test, 'posinput': pos_test, 'dpinput': dp_test},
batch_size=batch_size)['output'].argmax(axis=1)
acc = np_utils.accuracy(classes, np_utils.categorical_probas_to_classes(Y_test)) # accuracy only supports classes
print('Test accuracy:', acc)
model.add_input(name='input', input_shape=(max_len,), dtype=int)
# model.add_node(Embedding(max_features, 128, input_length=maxlen), name='embedding', input='input')
model.add_node(Embedding(embeddings.shape[0], embeddings.shape[1], input_length=max_len, weights=[embeddings]),
name='embedding', input='input')
model.add_node(LSTM(64), name='forward', input='embedding')
model.add_node(LSTM(64, go_backwards=True), name='backward', input='embedding')
model.add_node(Dropout(0.5), name='dropout', inputs=['forward', 'backward'])
model.add_node(Dense(1, activation='sigmoid'), name='sigmoid', input='dropout')
model.add_output(name='output', input='sigmoid')
# try using different optimizers and different optimizer configs
model.compile('adam', {'output': 'binary_crossentropy'})
print('Train...')
model.fit({'input': X_train, 'output': y_train}, batch_size=batch_size, nb_epoch=nb_epoch)
print('Prediction')
model_predict = model.predict({'input': X_test}, batch_size=batch_size)
predicted_labels = np.round(np.array(model_predict['output']))
# collect wrong predictions
wrong_predictions_ids = []
for i, (a, b) in enumerate(zip(y_test, predicted_labels)):
if a != b:
wrong_predictions_ids.append(ids_test[i])
acc = accuracy(y_test, predicted_labels)
print('Test accuracy:', acc)
print('Wrong predictions:', wrong_predictions_ids)
model.add(Dense(word_vec_len, 100, activation='relu'))
model.add(BatchNormalization((100,)))
model.add(Dense(100,100,activation='relu'))
model.add(BatchNormalization((100,)))
model.add(Dense(100, word_vec_len, activation='relu'))
model.add(Transform((maxseqlen, word_vec_len))) # transform back from 2d to 3d for recurrent input
# Stacked up BiDirectionLSTM layers
model.add(BiDirectionLSTM(word_vec_len, 50, output_mode='concat'))
model.add(BiDirectionLSTM(100, 24, output_mode='sum'))
# MLP layers
model.add(Reshape(24 * maxseqlen))
model.add(BatchNormalization((24 * maxseqlen,)))
model.add(Dense(24 * maxseqlen, 50, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(50, 1, activation='sigmoid'))
# try using different optimizers and different optimizer configs
model.compile(loss='mean_squared_error', optimizer='adam', class_mode="binary")
print("Train...")
model.fit(train_X, train_y, batch_size=batch_size, nb_epoch=5, validation_data=(test_X, test_y), show_accuracy=True)
score = model.evaluate(test_X, test_y, batch_size=batch_size)
print('Test score:', score)
classes = model.predict_classes(test_X, batch_size=batch_size)
acc = np_utils.accuracy(classes, test_y)
print('Test accuracy:', acc)
logger.info("Evaluation scores on test data:")
scores = {}
Y_pred = []
Y_true = []
score_keys = [
"accuracy", "micro_precision", "micro_recall", "micro_f1",
"macro_f1", "c_mf1", "c_mp", "c_mr"
]
for i, k in enumerate(output_names):
labels = range(Y_out[k].shape[-1])
Y_pred.append(np.argmax(np.array(Y_out[k]), axis=-1)[Y_idx])
Y_true.append(np.argmax(Y_test[i], axis=-1)[Y_idx])
scores[k] = vtu.get_eval_scores(Y_pred[-1],
Y_true[-1],
labels=labels)
scores[k]["accuracy"] = accuracy(Y_pred[-1], Y_true[-1])
TP, FP, FN = (scores[k][_k][1:-1] for _k in ["TP", "FP", "FN"])
micro_precision = np.sum(TP) * 1. / np.sum(TP + FP)
micro_recall = np.sum(TP) * 1. / np.sum(TP + FN)
micro_f1 = 2 * micro_precision * micro_recall / (
micro_precision + micro_recall)
scores[k]["c_mf1"] = micro_f1
scores[k]["c_mp"] = micro_precision
scores[k]["c_mr"] = micro_recall
logger.info("%s: %s" %
(k, dict(
(_k, scores[k][_k]) for _k in score_keys)))
all_labels = dict((k, i) for i, k in enumerate(
(b_i, c_i) for b_i in range(Y_test[0].shape[-1])
for c_i in range(Y_test[1].shape[-1])))
all_true = [all_labels.get(k) for k in zip(Y_true[0], Y_true[1])]
def evaluate(model, X_test, Y_test, test_y):
score = model.evaluate(X_test, Y_test, batch_size=32)
print("Test score: ", score)
classes = model.predict_classes(X_test, batch_size=32)
acc = np_utils.accuracy(classes, test_y) # 这里要用未转化为 one-hot 之前的 y
print("Test accuracy: ", acc)
def mlp():
words_size = 13033
word_vec_len = 100
batch_size = 100
max_sentence_length = 47 # cut texts after this number of words (among top max_features most common words)
X, y = get_data('../Files/yahoo.data.dat')
# 数据shuffle
indices = permutation(X.shape[0]) # shape[0]表示第0轴的长度，通常是训练数据的数量
X = X[indices]
y = y[indices]
X_train, X_test = X[:0.8 * len(X)], X[0.8 * len(X):]
y_train, y_test = y[:0.8 * len(y)], y[0.8 * len(y):]
print(len(X_train), 'train sequences')
print(len(X_test), 'test sequences')
print("Pad sequences (samples x time)")
X_train = sequence.pad_sequences(X_train, maxlen=max_sentence_length)
X_test = sequence.pad_sequences(X_test, maxlen=max_sentence_length)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
print('Build model...')
model = Sequential()
model.add(Embedding(
words_size, word_vec_len)) # (nb_samples, sequence_length, output_dim)
model.add(Transform(
(word_vec_len,
))) # transform from 3d dimensional input to 2d input for mlp
model.add(
Dense(input_dim=word_vec_len,
output_dim=25,
init='uniform',
activation='tanh'))
model.add(Dropout(0.5))
model.add(
Dense(input_dim=25, output_dim=15, init='uniform', activation='tanh'))
model.add(Dropout(0.5))
model.add(
Dense(input_dim=15, output_dim=8, init='uniform', activation='tanh'))
model.add(Dropout(0.5))
model.add(
Dense(input_dim=8, output_dim=1, init='uniform', activation='sigmoid'))
sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='binary_crossentropy',
optimizer=sgd,
class_mode='binary')
# model.compile(loss='mean_squared_error', optimizer=sgd, class_mode='binary')
print("Train...")
model.fit(X_train,
y_train,
shuffle=True,
batch_size=batch_size,
nb_epoch=5,
validation_split=0.1,
show_accuracy=True)
score = model.evaluate(X_test, y_test, batch_size=10)
print('Test score:', score)
classes = model.predict_classes(X_test, batch_size=batch_size)
acc = np_utils.accuracy(classes, y_test)
print('Test accuracy:', acc) # ('Test accuracy:', 0.64478482859226838)
logger.info("Evaluation scores on test data:")
scores = {}
Y_pred = []
Y_true = []
score_keys = [
"accuracy", "micro_precision", "micro_recall", "micro_f1",
"macro_f1", "c_mf1", "c_mp", "c_mr"
]
for i, k in enumerate(output_names):
labels = range(Y_out[k].shape[-1])
Y_pred.append(np.argmax(np.array(Y_out[k]), axis=-1)[Y_idx])
Y_true.append(np.argmax(Y_test[i], axis=-1)[Y_idx])
scores[k] = vtu.get_eval_scores(Y_pred[-1],
Y_true[-1],
labels=labels)
scores[k]["accuracy"] = accuracy(Y_pred[-1], Y_true[-1])
TP, FP, FN = (scores[k][_k][1:-1] for _k in ["TP", "FP", "FN"])
micro_precision = np.sum(TP) * 1. / np.sum(TP + FP)
micro_recall = np.sum(TP) * 1. / np.sum(TP + FN)
micro_f1 = 2 * micro_precision * micro_recall / (
micro_precision + micro_recall)
scores[k]["c_mf1"] = micro_f1
scores[k]["c_mp"] = micro_precision
scores[k]["c_mr"] = micro_recall
logger.info("%s: %s" %
(k, dict(
(_k, scores[k][_k]) for _k in score_keys)))
total_time = time.time() - start_time
logger.info(
"Finished training %.3f epochs in %s seconds with %.5f seconds/epoch"
% (save_every, total_time, total_time * 1.0 / save_every))
forwards = LSTM(64)(embedded)
# apply backwards LSTM
backwards = LSTM(64, go_backwards=True)(embedded)
# concatenate the outputs of the 2 LSTMs
merged = merge([forwards, backwards], mode='concat', concat_axis=-1)
after_dp = Dropout(0.5)(merged)
output = Dense(1, activation='sigmoid')(after_dp)
model = Model(input=sequence, output=output)
# try using different optimizers and different optimizer configs
model.compile('adam', 'binary_crossentropy', metrics=['accuracy'])
print('Train...')
model.fit(X_train,
y_train,
batch_size=batch_size,
nb_epoch=4,
validation_data=[X_test, y_test],
verbose=1)
acc = accuracy(
y_test,
np.round(
np.array(
model.predict({'input': X_test},
batch_size=batch_size)['output'])))
print('Test accuracy:', acc)
def evalModel(dataset, model, modelname):
#################
# Configuration #
#################
#################
X_test = dataset['X_test']
y_test = dataset['y_test']
labels_test = dataset['labels_test']
# split by "continuous variable" and "categorization variable"
X_test_vec = [X_test[:,:,0:-1], X_test[:,:, -1]]
if o.Model == "DenseIP3D":
X_test_vec = [ X_test [:, 0:ntrk_cut, 0:2], X_test [:, 0:ntrk_cut,-1]]
if o.Model == "RNNPlusMV2" or o.Model == "RNNPlusSV1":
X_test_vec_dR = [X_test[:,:,0:4], X_test[:,:,-1]]
X_test_vec = None
pred = None
if int(o.nLSTMClass) ==2:
RNNmodel = model_from_json(open( 'V47_LSTM_dR_40epoch_5000kEvts_0nTrackCut_15nMaxTrack_2nLSTMClass_25nLSTMNodes_CMix_architecture.json').read())
RNNmodel.load_weights( 'V47_LSTM_dR_40epoch_5000kEvts_0nTrackCut_15nMaxTrack_2nLSTMClass_25nLSTMNodes_CMix_model_weights.h5' )
RNNmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=["accuracy"])
pred = RNNmodel.predict( X_test_vec_dR, batch_size)
X_test_vec = np.ndarray(shape=( pred[:,0].shape[0], 2))
if int(o.nLSTMClass) ==4:
RNNmodel = model_from_json(open( 'V47_LSTM_dR_40epoch_5000kEvts_0nTrackCut_15nMaxTrack_4nLSTMClass_25nLSTMNodes_CMix_architecture.json').read())
RNNmodel.load_weights( 'V47_LSTM_dR_40epoch_5000kEvts_0nTrackCut_15nMaxTrack_4nLSTMClass_25nLSTMNodes_CMix_model_weights.h5' )
RNNmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=["accuracy"])
pred = RNNmodel.predict( X_test_vec_dR, batch_size)
X_test_vec = np.ndarray(shape=( pred[:, 0].shape[0], 5))
for i in range(X_test_vec.shape[0]):
if o.Model == "RNNPlusMV2":
X_test_vec[i][0] = labels_test[i, 10]
if o.Model == "RNNPlusSV1":
X_test_vec[i][0] = labels_test[i, 9]
for j in range(pred.shape[1]):
X_test_vec[i][j+1] = pred[i, j]
score = model.evaluate(X_test_vec, y_test, batch_size=batch_size)
print('Test score:', score)
classes = model.predict_classes(X_test_vec, batch_size=batch_size)
acc = np_utils.accuracy(classes, y_test)
print('Test accuracy:', acc)
acc = np_utils.accuracy(classes[labels_test[:,0]==5], y_test[labels_test[:,0]==5])
print('Test b accuracy:', acc)
acc = np_utils.accuracy(classes[labels_test[:,0]==0], y_test[labels_test[:,0]==0])
print('Test l accuracy:', acc)
pred = model.predict(X_test_vec, batch_size=batch_size)
return model
sequence = Input(shape=(maxlen,), dtype='int32')
# this embedding layer will transform the sequences of integers
# into vectors of size 128
embedded = Embedding(max_features, 128, input_length=maxlen)(sequence)
# apply forwards LSTM
forwards = LSTM(64)(embedded)
# apply backwards LSTM
backwards = LSTM(64, go_backwards=True)(embedded)
# concatenate the outputs of the 2 LSTMs
merged = merge([forwards, backwards], mode='concat', concat_axis=-1)
after_dp = Dropout(0.5)(merged)
output = Dense(1, activation='sigmoid')(after_dp)
model = Model(input=sequence, output=output)
# try using different optimizers and different optimizer configs
model.compile('adam', 'binary_crossentropy', metrics=['accuracy'])
print('Train...')
model.fit(X_train, y_train,
batch_size=batch_size,
nb_epoch=4,
validation_data=[X_test, y_test], verbose=1)
acc = accuracy(y_test, np.round(np.array(model.predict({'input': X_test},
batch_size=batch_size)['output'])))
print('Test accuracy:', acc)
def evaluate_model(self, segment_size, model, data_loader):
'''
:param segment_size:
:param model:
:param data_loader:
:return:
'''
# ________________ Frame level evaluation for Test/Validation splits ________________________
print('Validation segments = ' +
str(data_loader.validation_segments.shape) +
' one-hot encoded target' +
str(data_loader.validation_one_hot_target.shape))
score = model.evaluate(data_loader.validation_segments,
data_loader.validation_one_hot_target,
verbose=0)
print('Validation score:', score[0])
print('Validation accuracy:', score[1])
print('Test segments = ' + str(data_loader.test_segments.shape) +
' one-hot encoded target' +
str(data_loader.test_one_hot_target.shape))
score = model.evaluate(data_loader.test_segments,
data_loader.test_one_hot_target,
verbose=0)
print('Test score:', score[0])
print('Test accuracy:', score[1])
# ___________________ predict frame-level classes ___________________________________
test_predicted_labels = np_utils.categorical_probas_to_classes(
model.predict(data_loader.test_segments))
test_target_labels = np_utils.categorical_probas_to_classes(
data_loader.test_one_hot_target)
cm_frames = confusion_matrix(test_target_labels, test_predicted_labels)
print('Confusion matrix, frame level')
print(cm_frames)
print(
'Frame level accuracy :' +
str(np_utils.accuracy(test_predicted_labels, test_target_labels)))
# -------------- Voting ------------------------
clip_predicted_probability_mean_vote = []
clip_predicted_majority_vote = []
for i, clip in enumerate(data_loader.test_clips):
segments, segments_target_labels = data_loader.segment_clip(
data=clip,
label=data_loader.test_clips_labels[i],
segment_size=segment_size,
step_size=Config.STEP_SIZE)
segments_predicted_prop = model.predict(segments)
test_predicted_labels = np_utils.categorical_probas_to_classes(
segments_predicted_prop)
labels_histogram = np.bincount(test_predicted_labels)
clip_predicted_majority_vote.append(np.argmax(labels_histogram))
clip_predicted_probability_mean_vote.append(
np.argmax(np.mean(segments_predicted_prop, axis=0)))
cm_majority = confusion_matrix(data_loader.test_clips_labels,
clip_predicted_majority_vote)
print('Fold Confusion matrix - Majority voting - Clip level :')
print(Config.CLASS_NAMES)
print(cm_majority)
print('Clip-level majority-vote Accuracy ' + str(
np_utils.accuracy(clip_predicted_majority_vote,
data_loader.test_clips_labels)))
print('Fold Confusion matrix - Probability MEAN voting - Clip level :')
cm_probability = confusion_matrix(
data_loader.test_clips_labels,
clip_predicted_probability_mean_vote)
print(Config.CLASS_NAMES)
print(cm_probability)
print('Clip-level probability-vote Accuracy ' + str(
np_utils.accuracy(np.asarray(clip_predicted_probability_mean_vote),
np.squeeze(data_loader.test_clips_labels))))
scoref1 = f1score(data_loader.test_clips_labels,
clip_predicted_probability_mean_vote,
average='micro')
print('F1 Score micro ' + str(scoref1))
scoref1 = f1score(data_loader.test_clips_labels,
clip_predicted_probability_mean_vote,
average='weighted')
print('F1 Score weighted ' + str(scoref1))
return cm_majority, cm_probability, clip_predicted_majority_vote, clip_predicted_probability_mean_vote, data_loader.test_clips_labels
word_vec_len))) # transform back from 2d to 3d for recurrent input
# Stacked up BiDirectionLSTM layers
model.add(BiDirectionLSTM(word_vec_len, 50, output_mode='concat'))
model.add(BiDirectionLSTM(100, 24, output_mode='sum'))
# MLP layers
model.add(Reshape(24 * maxseqlen))
model.add(BatchNormalization((24 * maxseqlen, )))
model.add(Dense(24 * maxseqlen, 50, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(50, 1, activation='sigmoid'))
# try using different optimizers and different optimizer configs
model.compile(loss='mean_squared_error', optimizer='adam', class_mode="binary")
print("Train...")
model.fit(train_X,
train_y,
batch_size=batch_size,
nb_epoch=5,
validation_data=(test_X, test_y),
show_accuracy=True)
score = model.evaluate(test_X, test_y, batch_size=batch_size)
print('Test score:', score)
classes = model.predict_classes(test_X, batch_size=batch_size)
acc = np_utils.accuracy(classes, test_y)
print('Test accuracy:', acc)
callbacks=[epochaccuracy])
train_acc = np.array(epochaccuracy.accs)
val_acc = np.array(epochaccuracy.val_accs)
acc_log = open('acc_log.txt', 'a')
acc_log.write('TA-LSTM:' + str(epochaccuracy.val_accs) + '\n')
acc_log.close()
# accs.append(np.max(val_acc))
plt.plot(train_acc, linewidth=3, label='train')
plt.plot(val_acc, linewidth=3, label='val')
plt.grid()
plt.legend()
plt.xlabel('epoch')
plt.ylabel('acc')
plt.savefig('talstm.png', bbox_inches='tight')
plt.close()
acc_train = accuracy((y_train),
np.round(
np.array(
model.predict({'input': X_train},
batch_size=batch_size)['output'])))
yp = np.round(
np.array(
model.predict({'input': X_test}, batch_size=batch_size)['output']))
acc_test = accuracy(y_test, yp)
print('Training accuracy:', acc_train)
print('Test accuracy:', acc_test)
full_report(y_test, yp, ['class 0', 'class 1'])
eval_performance(y_test, yp, ['class 0', 'class 1'])
# print train_y ################### #training svm model ################### print 'training svm model...' clf_svm = SVC(kernel='linear') clf_svm.fit(train_x_reshape, train_y) classes_svm = clf_svm.predict(test_x_reshape) dec = clf_svm.decision_function(test_x_reshape) print clf_svm print classification_report(test_y, classes_svm) ############### # calculate accuracy ############### svm_acc = np_utils.accuracy(classes_svm, test_y) print 'svm accuracy: ', svm_acc ################### #training GaussianNB model ################### print 'training GaussianNB model ...' clf_nb1 = GaussianNB() clf_nb1.fit(train_x_reshape, train_y) classes_nb1 = clf_nb1.predict(test_x_reshape) print classification_report(test_y, classes_nb1) ############### # calculate accuracy ############### nb1_acc = np_utils.accuracy(classes_nb1, test_y) print 'GaussianNB accuracy: ', nb1_acc ###################
print('Train...')
model.fit({'input': X_train, 'output': y_train}, batch_size=batch_size, nb_epoch=nb_epoch)
print('Prediction')
model_predict = model.predict({'input': X_test}, batch_size=batch_size)
proba = np.array(model_predict['output'])
predicted_labels = np.round(proba)
# collect wrong predictions
wrong_predictions_ids = []
for i, (a, b) in enumerate(zip(prefs_test, predicted_labels)):
if a != b:
wrong_predictions_ids.append(ids_test[i])
acc = accuracy(prefs_test, predicted_labels)
print('Test accuracy:', acc)
print('Wrong predictions:', wrong_predictions_ids)
import pickle
all_proba[foldidx] = proba
all_predictions[foldidx] = predicted_labels
results = (all_proba, all_predictions)
import os
resultsfile = os.path.expanduser('~/data/personalised_argumentation/outputdata/habernal_separatescript_BILSTM_test.pkl')
with open(resultsfile, 'w') as fh:
pickle.dump(results, fh)
if model_type == "brnn_cnn_multitask":
model.fit({"input1": X_train, "input2": X_char_train, output_names[0]: Y_train[0]},\
validation_data={"input1": X_test, "input2": X_char_test, output_names[0]: Y_test[0]}, nb_epoch=save_every, verbose=verbosity)
Y_out = model.predict({'input1': X_test, "input2": X_char_test})
#Y_idx = (Y_test[0][:,:,0] == 0) & (Y_test[0][:,:,5] == 0) # Get indexes of only those tokens which correspond to entitites
Y_idx = Y_test[0][:,:,0] >= 0 # Get all indexes
# Calculate accuracy only based on correct entity identity
logger.info("Evaluation scores on test data:")
scores = {}
Y_pred = []
Y_true = []
score_keys = ["accuracy", "micro_precision", "micro_recall", "micro_f1", "macro_f1", "c_mf1", "c_mp", "c_mr"]
for i, k in enumerate(output_names):
labels = range(Y_out[k].shape[-1])
Y_pred.append(np.argmax(np.array(Y_out[k]), axis=-1)[Y_idx])
Y_true.append(np.argmax(Y_test[i], axis=-1)[Y_idx])
scores[k] = vtu.get_eval_scores(Y_pred[-1], Y_true[-1], labels = labels)
scores[k]["accuracy"] = accuracy(Y_pred[-1], Y_true[-1])
TP, FP, FN = (scores[k][_k][1:-1] for _k in ["TP", "FP", "FN"])
micro_precision = np.sum(TP) * 1. / np.sum(TP + FP)
micro_recall = np.sum(TP) * 1. / np.sum(TP + FN)
micro_f1 = 2*micro_precision*micro_recall / (micro_precision+micro_recall)
scores[k]["c_mf1"] = micro_f1
scores[k]["c_mp"] = micro_precision
scores[k]["c_mr"] = micro_recall
logger.info("%s: %s" % (k, dict((_k, scores[k][_k]) for _k in score_keys)))
total_time = time.time() - start_time
logger.info("Finished training %.3f epochs in %s seconds with %.5f seconds/epoch" % (save_every, total_time, total_time * 1.0/ save_every))
model.save_weights("%s/%s_%s_h%s-%s.h5" % (SAVE_MODEL_DIR, MODEL_PREFIX, model_type, num_hidden_layers, epoch), overwrite=True)
def train_cnn_lstm():
traindf, testdf = getJobData()
jobs = codecs.open('../data/pos_name.txt', 'rb', encoding = 'utf-8').readlines()
jobs = [ job.strip() for job in jobs]
traindf = traindf.loc[traindf['label'].isin(jobs)]
trainraw = traindf.txt.values.tolist()
trainraw = [line.encode('utf-8') for line in trainraw]
testraw = testdf.txt.values.tolist()
testraw = [line.encode('utf-8') for line in testraw]
maxfeatures = 50000
from keras.preprocessing.text import Tokenizer
token = Tokenizer(nb_words=maxfeatures)
token.fit_on_texts(trainraw + testraw)
train_seq = token.texts_to_sequences(trainraw)
test_seq = token.texts_to_sequences(testraw)
np.median([len(x) for x in train_seq])
from sklearn.preprocessing import LabelEncoder
le= LabelEncoder()
y = le.fit_transform(traindf['label'])
nb_classes =32
from sklearn.cross_validation import train_test_split
train_X, val_X, train_y, val_y = train_test_split(train_seq, y , train_size=0.8, random_state=1)
test_X = np.array(test_seq)
from keras.optimizers import RMSprop
from keras.preprocessing import sequence
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.embeddings import Embedding
from keras.layers.convolutional import Convolution1D, MaxPooling1D
from keras.layers.recurrent import SimpleRNN, GRU, LSTM
from keras.callbacks import EarlyStopping
from keras.utils import np_utils
maxlen = 80
batch_size = 16
word_dim = 50
nb_filter = 100
filter_length = 10
hidden_dims = 100
nb_epoch = 10
pool_length = 40
print("Pad sequences (samples x time)")
X_train = sequence.pad_sequences(train_X, maxlen=maxlen,padding='post', truncating='post')
X_val = sequence.pad_sequences(val_X, maxlen=maxlen,padding='post', truncating='post')
X_test = sequence.pad_sequences(test_X, maxlen=maxlen,padding='post', truncating='post')
print('X_train shape:', X_train.shape)
print('X_val shape:', X_val.shape)
Y_train = np_utils.to_categorical(train_y, nb_classes)
Y_val = np_utils.to_categorical(val_y, nb_classes)
print('Build model...')
model = Sequential()
model.add(Embedding(maxfeatures, word_dim,input_length=maxlen))
model.add(Dropout(0.2))
model.add(Convolution1D(nb_filter=nb_filter,
filter_length=filter_length,
border_mode="valid",
activation="relu"))
model.add(MaxPooling1D(pool_length=pool_length))
model.add(Flatten())
model.add(Dense(hidden_dims))
model.add(Dropout(0.3))
model.add(Activation('relu'))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
earlystop = EarlyStopping(monitor='val_loss', patience=1, verbose=1)
result = model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch,
validation_split=0.1, show_accuracy=True,callbacks=[earlystop])
score = earlystop.model.evaluate(X_val, Y_val, batch_size=batch_size)
print('val score:', score)
classes = earlystop.model.predict_classes(X_val, batch_size=batch_size)
acc = np_utils.accuracy(classes, val_y) # 要用没有转换前的y
print('Validation accuracy:', acc)
pred_cnn = earlystop.model.predict(X_test, batch_size = batch_size)
print('2 LSTM...')
model = Sequential()
model.add(Embedding(maxfeatures, word_dim,input_length=maxlen))
#model.add(Dropout(0.25))
model.add(LSTM(100))
model.add(Flatten())
model.add(Dense(hidden_dims))
model.add(Dropout(0.25))
model.add(Activation('relu'))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
earlystop = EarlyStopping(monitor='val_loss', patience=1, verbose=1)
result = model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch,
validation_split=0.1, show_accuracy=True,callbacks=[earlystop])
score = earlystop.model.evaluate(X_val, Y_val, batch_size=batch_size)
print('val score:', score)
classes = earlystop.model.predict_classes(X_val, batch_size=batch_size)
acc = np_utils.accuracy(classes, val_y)
print('Validation accuracy:', acc)
pred_lstm = earlystop.model.predict(X_test, batch_size = batch_size)
print('3 CNN + LSTM model...')
model = Sequential()
model.add(Embedding(maxfeatures, word_dim,input_length=maxlen))
model.add(Dropout(0.1))
model.add(Convolution1D(nb_filter=nb_filter,
filter_length=filter_length,
border_mode="valid",
activation="relu"))
model.add(MaxPooling1D(pool_length=pool_length))
model.add(LSTM(100))
#model.add(Flatten())
model.add(Dropout(0.2))
model.add(Activation('relu'))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
from keras.models import Graph
fw = [2,10, 5]
pool_length = [2,50, 10]
print('Build model...')
graph = Graph()
graph.add_input(name='input', input_shape=(maxlen,), dtype=int)
graph.add_node(Embedding(maxfeatures, word_dim, input_length=maxlen),
name='embedding', input='input')
graph.add_node(Convolution1D(nb_filter=nb_filter,filter_length=fw[0],
activation="relu"),
name='conv1', input='embedding')
graph.add_node(MaxPooling1D(pool_length =pool_length[0], ignore_border = False), name='pool1', input = 'conv1')
graph.add_node(Flatten(), name='flat1', input='conv1')
graph.add_node(Convolution1D(nb_filter=nb_filter,filter_length=fw[1],
activation="relu"),
name='conv2', input='embedding')
graph.add_node(MaxPooling1D(pool_length =pool_length[1], ignore_border = False), name='pool2', input = 'conv2')
graph.add_node(Flatten(), name='flat2', input='conv2')
graph.add_node(Convolution1D(nb_filter=nb_filter,filter_length=fw[2],
activation="relu"),
name='conv3', input='embedding')
graph.add_node(MaxPooling1D(pool_length =pool_length[2], ignore_border = False), name='pool3', input = 'conv3')
graph.add_node(Flatten(), name='flat3', input='conv3')
graph.add_node(Dense(hidden_dims,activation='relu'), name='dense1',
inputs=['flat1', 'flat2', 'flat3'], merge_mode='concat')
graph.add_node(Dropout(0.4), name='drop1', input='dense1')
graph.add_node(Dense(nb_classes, activation='softmax'), name='softmax', input='drop1')
graph.add_output(name='output', input='softmax')
graph.compile('Adam', loss = {'output': 'categorical_crossentropy'})
result = graph.fit({'input':X_train, 'output':Y_train},
nb_epoch=3,batch_size=batch_size,
validation_split=0.1)
predict = graph.predict({'input':X_test}, batch_size=batch_size)
pred_cnn_lstm = predict['output']
classes = pred_cnn_lstm.argmax(axis=1)
return pred_cnn, pred_lstm, pred_cnn_lstm
self.accs.append(acc)
self.val_accs.appen(val_acc)
# logs.info("Accuracy after epoch {}: {}".format(epoch, acc_val))
epochaccuracy = EpochAccuracy(batch_size)
# earlystop = EarlyStopping(monitor='val_loss', patience=3, verbose=2)
history = model.fit({'input': X_train, 'output': y_train}, verbose=2, validation_data={'input':X_valid,'output':y_valid}, batch_size=batch_size, nb_epoch=nb_epoch,show_accuracy=True,callbacks=[epochaccuracy])
train_acc = np.array(epochaccuracy.accs)
val_acc = np.array(epochaccuracy.val_accs)
acc_log = open('acc_log.txt','a')
acc_log.write('TA-LSTM:'+str(epochaccuracy.val_accs)+'\n')
acc_log.close()
# accs.append(np.max(val_acc))
plt.plot(train_acc,linewidth=3,label='train')
plt.plot(val_acc,linewidth=3,label='val')
plt.grid()
plt.legend()
plt.xlabel('epoch')
plt.ylabel('acc')
plt.savefig('talstm.png', bbox_inches='tight')
plt.close()
acc_train = accuracy((y_train), np.round(np.array(model.predict({'input': X_train}, batch_size=batch_size)['output'])))
yp = np.round(np.array(model.predict({'input': X_test}, batch_size=batch_size)['output']))
acc_test = accuracy(y_test,yp)
print('Training accuracy:', acc_train)
print('Test accuracy:', acc_test)
full_report(y_test, yp,['class 0','class 1'])
eval_performance(y_test, yp,['class 0','class 1'])
class_mode="binary")
model.fit(x_data, y_data, batch_size=batchsize, nb_epoch=epoch)
return model
w, pn = process_data()
dict = pd.DataFrame(pd.Series(w).value_counts()) #统计词的出现次数
x = np.array(list(pn['sent']))[::2] #训练集
y = np.array(list(pn['mark']))[::2]
xt = np.array(list(pn['sent']))[1::2] #测试集
yt = np.array(list(pn['mark']))[1::2]
xa = np.array(list(pn['sent'])) #全集
ya = np.array(list(pn['mark']))
#del w,d2v_train
dict['id'] = list(range(1, len(dict) + 1))
get_sent = lambda x: list(dict['id'][x])
pn['sent'] = pn['words'].apply(get_sent)
maxlen = 50
print("Pad sequences (samples x time)")
pn['sent'] = list(sequence.pad_sequences(pn['sent'], maxlen=maxlen))
model = buil_model(x, y)
classes = model.predict_classes(xt)
acc = np_utils.accuracy(classes, yt)
print('Test Accuray is:%s' % acc)
def on_epoch_end(self, epoch, logs={}):
acc = accuracy((y_train), np.round(np.array(model.predict({'input': X_train}, batch_size=self.batch_size)['output'])))
val_acc = accuracy((y_test), np.round(np.array(model.predict({'input': X_test}, batch_size=self.batch_size)['output'])))
print('acc:'+acc+' - val_acc:'+val_acc)
self.accs.append(acc)
self.val_accs.appen(val_acc)
X_test = tokenizer.sequences_to_matrix(X_test, mode="binary")
print 'X_train shape:', X_train.shape
print 'X_test shape:', X_test.shape
print "Convert class vector to binary class matrix (for use with categorical_crossentropy)"
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)
print 'Y_train shape:', Y_train.shape
print 'Y_test shape:', Y_test.shape
print "Building model..."
model = Sequential()
model.add(Dense(max_words, 256, init='normal'))
model.add(Activation('relu'))
model.add(BatchNormalization(input_shape=(256,))) # try without batch normalization (doesn't work as well!)
model.add(Dropout(0.5))
model.add(Dense(256, nb_classes, init='normal'))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adadelta')
print "Training..."
model.fit(X_train, Y_train, nb_epoch=5, batch_size=batch_size)
score = model.evaluate(X_test, Y_test, batch_size=batch_size)
print 'Test score:', score
classes = model.predict_classes(X_test, batch_size=batch_size)
acc = np_utils.accuracy(classes, y_test)
print 'Test accuracy:', acc
model.add(Dense(128, 1))
model.add(Activation('sigmoid'))
# try using different optimizers and different optimizer configs
model.compile(loss='binary_crossentropy',
optimizer='adam',
class_mode="binary")
print("Train...")
model.fit(X_train,
labels_train,
batch_size=batch_size,
nb_epoch=15,
validation_split=0.1,
show_accuracy=True)
X_test = tokens_to_word_vectors(reviews_tokens_test, w2vmodel)
X_test = np.array(X_test)
labels_test = np.array(labels_test[:NUM_ELEMENTS_TEST])
print "Padding test sequences"
X_test = pad_sequence_word_vectors(X_test, maxlen=maxlen)
print('X_test shape:', X_test.shape)
score = model.evaluate(X_test, labels_test, batch_size=batch_size)
print('Test score:', score)
classes = model.predict_classes(X_test, batch_size=batch_size)
acc = np_utils.accuracy(classes, labels_test)
print('Test accuracy:', acc)
#model.save_weights("GRU_128_DROPOUT_0.1_RELU_50_epochs_moredata")
print("Pad sequences (samples x time)")
X_train = sequence.pad_sequences(X_train, maxlen=maxlen)
X_test = sequence.pad_sequences(X_test, maxlen=maxlen)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
y_train = np.array(y_train)
y_test = np.array(y_test)
print('Build model...')
model = Graph()
model.add_input(name='input', input_shape=(1,), dtype=int)
model.add_node(Embedding(max_features, 128, input_length=maxlen),
name='embedding', input='input')
model.add_node(LSTM(64), name='forward', input='embedding')
model.add_node(LSTM(64, go_backwards=True), name='backward', input='embedding')
model.add_node(Dropout(0.5), name='dropout', inputs=['forward', 'backward'])
model.add_node(Dense(1, activation='sigmoid'), name='sigmoid', input='dropout')
model.add_output(name='output', input='sigmoid')
# try using different optimizers and different optimizer configs
model.compile('adam', {'output': 'binary_crossentropy'})
print("Train...")
model.fit({'input': X_train, 'output': y_train},
batch_size=batch_size,
nb_epoch=4)
acc = accuracy(y_test,
np.round(np.array(model.predict({'input': X_test},
batch_size=batch_size)['output'])))
print('Test accuracy:', acc)
def evaluate(model, X_test, Y_test, test_y):
score = model.evaluate(X_test, Y_test, batch_size=32)
print("Test score: ", score)
classes = model.predict_classes(X_test, batch_size=32)
acc = np_utils.accuracy(classes, test_y) # 这里要用未转化为 one-hot 之前的 y
print("Test accuracy: ", acc)
