def build_hybrid_net(x):
inputs = tflearn.input_data(placeholder=x)
with tf.name_scope('1d-cnn'):
network_array = []
for p in xrange(S_INFO):
branch = tflearn.conv_1d(inputs[:, :, p:p + 1],
FEATURE_NUM,
3,
activation='relu',
regularizer="L2")
branch = tflearn.flatten(branch)
network_array.append(branch)
out_cnn = tflearn.merge(network_array, 'concat')
with tf.name_scope('gru'):
net = tflearn.gru(inputs, FEATURE_NUM, return_seq=True)
out_gru = tflearn.gru(net, FEATURE_NUM)
header = tflearn.merge([out_cnn, out_gru], 'concat')
dense_net = tflearn.fully_connected(header,
FEATURE_NUM * 2,
activation='relu',
regularizer="L2")
dense_net = tflearn.fully_connected(dense_net,
FEATURE_NUM * 1,
activation='relu',
regularizer="L2")
out = tflearn.fully_connected(dense_net,
1,
activation='sigmoid',
regularizer="L2")
return out, header
def hybrid_header(x, reuse=False):
# size = 3
# inputs_shape = x.get_shape().as_list()
# with tf.variable_scope('1d-cnn'):
# split_array = []
# for t in xrange(S_LEN - 1):
# tmp_split = tflearn.conv_1d(
# x[:, t:t + 1, :], FEATURE_NUM, size, activation='relu')
# tmp_split_flat = tflearn.flatten(tmp_split)
# tmp_split_flat = tflearn.layers.normalization.batch_normalization(tmp_split_flat)
# split_array.append(tmp_split_flat)
# merge_net = tflearn.merge(split_array, 'concat')
# _count = merge_net.get_shape().as_list()[1]
# out_cnn = tf.reshape(out_cnn
# merge_net, [-1, inputs_shape[1], _count / inputs_shape[1]])
# with tf.variable_scope('gru'):
# net = tflearn.gru(out_cnn, FEATURE_NUM, return_seq=True)
# out_gru = tflearn.gru(net, FEATURE_NUM)
# out_gru = tf.expand_dims(out_gru, 1)
#conv_1d_net = tflearn.conv_1d(out_gru, FEATURE_NUM, size, activation='relu')
#conv_1d_net_flattern = tflearn.flatten(conv_1d_net)
with tf.name_scope('gru'):
net = tflearn.gru(x, FEATURE_NUM, return_seq=True)
net = tflearn.gru(x, FEATURE_NUM)
#out_gru = tflearn.fully_connected(
# net, FEATURE_NUM, activation='relu')
#out_gru = tflearn.dropout(out_gru, 0.5)
return net
def vqn_model(x):
with tf.variable_scope('vqn'):
inputs = tflearn.input_data(placeholder=x)
_split_array = []
_cnn_array = []
for i in range(INPUT_SEQ):
tmp_network = tf.reshape(inputs[:, i:i + 1, :, :, :],
[-1, INPUT_H, INPUT_W, INPUT_D])
if i == 0:
_tmp_split, _tmp_cnn = CNN_Core(tmp_network)
else:
_tmp_split, _tmp_cnn = CNN_Core(tmp_network, True)
_split_array.append(_tmp_split)
_cnn_array.append(_tmp_cnn)
merge_net = tflearn.merge(_split_array, 'concat')
merge_net = tflearn.flatten(merge_net)
_count = merge_net.get_shape().as_list()[1]
with tf.variable_scope('full-lstm'):
net = tf.reshape(merge_net, [-1, INPUT_SEQ, _count / INPUT_SEQ])
net = tflearn.gru(net, HIDDEN_UNIT, return_seq=True)
net = tflearn.gru(net, HIDDEN_UNIT, return_seq=True)
net, alphas = attention(net, HIDDEN_UNIT)
out = tflearn.fully_connected(net,
OUTPUT_DIM,
activation='sigmoid')
return out, tf.stack(_cnn_array, axis=0), alphas
def baseline_model(inputlen, outputlen): #第二个参数为输出分类数目
# Build the network for classification
# Each input has length of 15
net = tflearn.input_data([None, inputlen
]) #每个样本整数向量的长度，也就是每行单词的排序索引 长度是每句话的最大单词数
# The 15 input word integers are then casted out into 256 dimensions each creating a word embedding.
# We assume the dictionary has 10000 words maximum
net = tflearn.embedding(net, input_dim=10000, output_dim=256)
# Each input would have a size of 15x256 and each of these 256 sized vectors are fed into the LSTM layer one at a time.
# All the intermediate outputs are collected and then passed on to the second LSTM layer.
net = tflearn.gru(net, 256, dropout=0.9, return_seq=True)
# Using the intermediate outputs, we pass them to another LSTM layer and collect the final output only this time
net = tflearn.gru(net, 256, dropout=0.9)
#net = tflearn.lstm(net, 128, dropout=0.8)
# The output is then sent to a fully connected layer that would give us our final 11 classes
net = tflearn.fully_connected(net, outputlen, activation='softmax')
# We use the adam optimizer instead of standard SGD since it converges much faster
net = tflearn.regression(net,
optimizer='adam',
learning_rate=0.001,
loss='categorical_crossentropy')
#模型初始化
model = tflearn.DNN(net, tensorboard_verbose=0)
return model
def vqn_model(x):
with tf.variable_scope('vqn'):
inputs = tflearn.input_data(placeholder=x)
_split_array = []
for i in range(INPUT_SEQ):
tmp_network = tf.reshape(inputs[:, i:i + 1, :, :, :],
[-1, INPUT_H, INPUT_W, INPUT_D])
if i == 0:
_split_array.append(CNN_Core(tmp_network))
else:
_split_array.append(CNN_Core(tmp_network, True))
merge_net = tflearn.merge(_split_array, 'concat')
merge_net = tflearn.flatten(merge_net)
_count = merge_net.get_shape().as_list()[1]
with tf.variable_scope('full-lstm'):
net = tf.reshape(merge_net, [-1, INPUT_SEQ, _count / INPUT_SEQ])
net = tflearn.gru(net, DENSE_SIZE, return_seq=True)
out_gru = tflearn.gru(net, DENSE_SIZE, dropout=0.8)
gru_result = tflearn.fully_connected(out_gru,
DENSE_SIZE,
activation='relu')
out = tflearn.fully_connected(gru_result,
OUTPUT_DIM,
activation='sigmoid')
return out
def prop_net(prop_statement_input, prop_hyps_input):
with tf.variable_scope('statement'):
prop_statement_net = tflearn.gru(prop_statement_input, output_dim, return_seq=False, dynamic=True)
print(prop_statement_net)
with tf.variable_scope('hyps'):
prop_net = tflearn.gru(prop_hyps_input, output_dim, return_seq=False, initial_state=prop_statement_net,
dynamic=True)
print(prop_net)
return prop_net
def _build_regression_lstm_net_gru(self, n_timesteps=1, n_inputdim=None, n_hidden=None,
n_outputdim=None, dropout=1.0):
net = tflearn.input_data([None, n_timesteps, n_inputdim],dtype=tf.float32, name='input_data')
output_mask = tflearn.input_data([None, n_timesteps, n_outputdim], dtype=tf.float32, name='output_mask')
net, hidden_states_1 = tflearn.gru(net, n_hidden, weights_init='xavier', return_seq=True, return_state=True, dropout=dropout, name="gru_1")
net, hidden_states_2 = tflearn.gru(net, n_outputdim, weights_init='xavier', activation='sigmoid', return_seq=True, return_state=True, dropout=dropout, name="gru_2")
net = tf.stack(net, axis=1)
net = net * output_mask
net = tflearn.regression(net, optimizer='adam', learning_rate=0.0005,
loss='mean_square') # mean square works
return net, hidden_states_1, hidden_states_2
def DecisionNetwork():
tflearn.init_graph(soft_placement=True)
with tf.device('/gpu:0'):
network = tflearn.input_data(shape=[None, FRAME_KEEP, FEATURES_LENGTH],
name='input')
network = tflearn.gru(network,
256,
return_seq=True,
name='DBFull_layer1')
network = tflearn.dropout(network, 0.6, name='DBFull_layer2')
network = tflearn.gru(network,
256,
return_seq=True,
name='DBFull_layer3')
network = tflearn.dropout(network, 0.6, name='DBFull_layer4')
movement_network = tflearn.gru(network,
256,
return_seq=False,
name='DBMove_layer1')
movement_network = tflearn.dropout(movement_network,
0.6,
name='DBMove_layer2')
movement_network = tflearn.fully_connected(movement_network,
OUTPUT_MOVE,
activation='softmax',
name='DBMove_layer3')
movement_network = tflearn.regression(movement_network,
optimizer='adam',
loss='categorical_crossentropy',
learning_rate=LR,
name='DBMove_layer4')
action_network = tflearn.gru(network,
256,
return_seq=False,
name='DBAct_layer1')
action_network = tflearn.dropout(action_network,
0.6,
name='DBAct_layer2')
action_network = tflearn.fully_connected(action_network,
OUTPUT_ACT,
activation='softmax',
name='DBAct_layer3')
action_network = tflearn.regression(action_network,
optimizer='adam',
loss='categorical_crossentropy',
learning_rate=LR,
name='DBAct_layer4')
network = tflearn.merge([movement_network, action_network],
mode='concat',
name='DBFull_layer5')
return tflearn.DNN(network,
max_checkpoints=5,
tensorboard_verbose=0,
checkpoint_path='full_model/full_model.tfl.ckpt')
def train(maxlen=100, embedding_dim=128): # 主训练/测试代码
start = time.time()
l_trainX, r_trainX, ret_labels, l_topredictX, r_topredictX = do.load_data_bi_word2vec(maxlen=maxlen,
words_keep=50000,
validation_portion=0.,
embedding_dim=embedding_dim,
ma="A")
trainY = to_categorical(ret_labels, nb_classes=3)
del ret_labels
lnet = tflearn.input_data([None, maxlen, embedding_dim])
rnet = tflearn.input_data([None, maxlen, embedding_dim])
lnet = tflearn.gru(lnet, embedding_dim, dropout=0.8, return_seq=False, dynamic=True)
rnet = tflearn.gru(rnet, embedding_dim, dropout=0.8, return_seq=False, dynamic=True)
net = tflearn.layers.merge_outputs([lnet, rnet])
net = tflearn.fully_connected(net, 3, activation='softmax')
net = tflearn.regression(net, optimizer='adam', learning_rate=0.001,
loss='categorical_crossentropy')
# Training
model = tflearn.DNN(net, tensorboard_verbose=0)
model.fit([l_trainX, r_trainX], trainY, validation_set=0.1, show_metric=True,
batch_size=32)
model.save('MODELS/E_W2V_GRU_TC{}_{}.dy'.format(embedding_dim, maxlen))
# model.load('MODELS/E_W2V_GRU_TC{}_{}.dy'.format(embedding_dim, maxlen))
del l_trainX
del r_trainX
del trainY
idx2cla = {0: 'neu', 1: 'pos', 2: 'neg'}
filename = "Result/result_{}.csv".format(datetime.datetime.now().strftime("%Y%m%d%H%M"))
prefix = list(open('Result/A_AFTER_NRP_200', 'r').readlines())
f = open(filename, 'w')
f.write('SentenceId,View,Opinion\n')
a = [0, 5000, 10000, 15000, 20000, 25000, 30000, 35000, 40000, 45000, 50000, 55000]
b = [5000, 10000, 15000, 20000, 25000, 30000, 35000, 40000, 45000, 50000, 55000, 65000]
ANS = []
for i in range(12):
ans = model.predict([l_topredictX[a[i]:b[i]], r_topredictX[a[i]:b[i]]])
ANS.extend([s for s in ans])
print("ANS.LENGTH: {}".format(len(ans)))
for i, r in enumerate(ANS):
f.write(prefix[i].strip())
idx = int(np.argmax(r))
f.write(idx2cla[idx])
k = ""
for l in r:
k += ',{:.4f}'.format(l)
f.write(k)
f.write('\n')
f.close()
end = time.time()
print("TIME COST: {}".format(end-start))
outf = vote_by_score(filename)
add(outf)
def get_network_wide(historyLength, input_feat_dim, num_classes):
net = tflearn.input_data(shape=[None, historyLength , input_feat_dim])
print("Input Layer created")
net = tflearn.gru(net, 128, return_seq = True)
print("Recurrent layer 1 created")
net = tflearn.gru(net, 64)
print("Recurrent layer 2 created")
net = tflearn.fully_connected(net, num_classes, activation = 'softmax', regularizer = 'L2')
print("FC layer 1 created")
net = tflearn.regression(net, optimizer = 'adam', loss = 'mean_square', name = 'output1', learning_rate = 0.001)
print("Regression Layer created")
return net
def hybrid_header(x, reuse=False):
size = 3
inputs_shape = x.get_shape().as_list()
with tf.variable_scope('1d-cnn'):
split_array = []
for t in xrange(S_LEN):
tmp_split = tflearn.conv_1d(
x[:, t:t + 1, :], FEATURE_NUM, size, activation='relu')
tmp_split_flat = tflearn.flatten(tmp_split)
tmp_split_flat = tflearn.layers.normalization.batch_normalization(tmp_split_flat)
split_array.append(tmp_split_flat)
merge_net = tflearn.merge(split_array, 'concat')
_count = merge_net.get_shape().as_list()[1]
out_cnn = tf.reshape(
merge_net, [-1, inputs_shape[1], _count / inputs_shape[1]])
with tf.variable_scope('gru'):
net = tflearn.gru(out_cnn, FEATURE_NUM, return_seq=True)
out_gru = tflearn.gru(net, FEATURE_NUM)
out_gru = tf.expand_dims(out_gru, 1)
conv_1d_net = tflearn.conv_1d(out_gru, FEATURE_NUM, size, activation='relu')
conv_1d_net_flattern = tflearn.flatten(conv_1d_net)
# with tf.name_scope('1d-cnn'):
# network_array = []
# for p in xrange(S_INFO - 1):
# branch_array = []
# for i in xrange(2,4):
# sp_branch = tflearn.conv_1d(x[:, :, p:p+1], FEATURE_NUM, i, padding='valid', activation='relu', regularizer="L2")
# branch_array.append(sp_branch)
# branch = tflearn.merge(branch_array, mode='concat', axis=1)
# branch = tf.expand_dims(branch, 2)
# branch = global_max_pool(branch)
# branch = tflearn.dropout(branch, 0.5)
# network_array.append(branch)
# out_cnn = tflearn.merge(network_array, 'concat')
#with tf.name_scope('gru'):
# #net = tflearn.gru(x, FEATURE_NUM, return_seq=True)
# net = tflearn.gru(x, FEATURE_NUM)
# out_gru = tflearn.fully_connected(
# net, FEATURE_NUM, activation='relu')
# out_gru = tflearn.dropout(out_gru, 0.5)
#merge_net = tflearn.merge([out_cnn, out_gru], 'concat')
return conv_1d_net_flattern
def setup(self):
self.load_dataset()
X, Y = self.create_dataset()
# Build neural network
net = tflearn.input_data(shape=[None, self.steps_of_history, 1])
# LSTMは時間かかるのでGRU
# http://dhero.hatenablog.com/entry/2016/12/02/%E6%9C%80%E5%BC%B1SE%E3%81%A7%E3%82%82%E6%A9%9F%E6%A2%B0%E5%AD%A6%E7%BF%92%E3%81%A7%E3%81%8A%E9%87%91%E3%81%8C%E7%A8%BC%E3%81%8E%E3%81%9F%E3%81%84%E3%80%905%E6%97%A5%E7%9B%AE%E3%83%BBTFLearn%E3%81%A8
net = tflearn.gru(net, n_units=6)
net = tflearn.fully_connected(net, 1, activation='linear')
# 回帰の設定
# Adam法で測定
# http://qiita.com/TomokIshii/items/f355d8e87d23ee8e0c7a
# 時系列分析での予測精度の指標にmean_squareを使っている
# mapeが一般的なようだ
# categorical_crossentropy
# mean_square : 二乗平均平方根
net = tflearn.regression(net, optimizer='adam', learning_rate=0.001,
loss='mean_square')
# Define model
self.model = tflearn.DNN(net, tensorboard_verbose=0)
# 今回は80%を訓練データセット、20%をテストデータセットとして扱う。
pos = round(len(X) * (1 - 0.2))
trainX, trainY = X[:pos], Y[:pos]
testX, testY = X[pos:], Y[pos:]
return trainX, trainY, testX
def build_hybrid_net(x, sess=None):
inputs = tflearn.input_data(placeholder=x)
with tf.name_scope('1d-cnn'):
network_array = []
for p in xrange(S_INFO - 1):
branch = tflearn.conv_1d(
inputs[:, :, p:p+1], FEATURE_NUM, 3, activation='relu')
branch = tflearn.flatten(branch)
network_array.append(branch)
out_cnn = tflearn.merge(network_array, 'concat')
with tf.name_scope('gru'):
net = tflearn.gru(x, FEATURE_NUM, return_seq=True)
out_gru = tflearn.gru(x, FEATURE_NUM)
header = tflearn.merge([out_cnn, out_gru], 'concat')
dense_net = tflearn.fully_connected(inputs, FEATURE_NUM, activation='relu')
out = tflearn.fully_connected(header, 1, activation='linear')
return out, header
def RNN_GRU(max_length,n_words,n_classes,n_units,dynamic=True):
"""define RNN with GRU units"""
net = tflearn.input_data([None, max_length])
net = tflearn.embedding(net, input_dim=n_words, output_dim=n_units)
net = tflearn.gru(net, n_units, dropout=0.8, dynamic=True)
net = tflearn.fully_connected(net, n_classes, activation='softmax')
net = tflearn.regression(net, optimizer='adam', learning_rate=0.001,
loss='categorical_crossentropy')
return net
def create_network(self):
with tf.variable_scope('actor'):
inputs = tflearn.input_data(shape=[None, self.S_INFO, self.S_LEN])
split_0 = tflearn.fully_connected(inputs[:, 0:1, -1],
FEATURE_NUM,
activation='relu')
split_1 = tflearn.fully_connected(inputs[:, 1:2, -1],
FEATURE_NUM,
activation='relu')
split_2 = tflearn.conv_1d(inputs[:, 2:3, :],
FEATURE_NUM,
4,
activation='relu')
split_3 = tflearn.conv_1d(inputs[:, 3:4, :],
FEATURE_NUM,
4,
activation='relu')
split_4 = tflearn.conv_1d(inputs[:, 4:5, :self.A_DIM],
FEATURE_NUM,
4,
activation='relu')
split_5 = tflearn.conv_1d(inputs[:, 5:6, :self.A_DIM],
FEATURE_NUM,
4,
activation='relu')
split_6 = tflearn.fully_connected(inputs[:, 6:7, -1],
FEATURE_NUM,
activation='relu')
split_2_flat = tflearn.flatten(split_2)
split_3_flat = tflearn.flatten(split_3)
split_4_flat = tflearn.flatten(split_4)
split_5_flat = tflearn.flatten(split_5)
merge_net = tf.stack([
split_0, split_1, split_2_flat, split_3_flat, split_4_flat,
split_5_flat, split_6
],
axis=-1)
# shuffle to fit gru layer
merge_net = tf.transpose(merge_net, [0, 2, 1])
dense_net_0 = tflearn.gru(merge_net,
FEATURE_NUM,
activation='relu')
out = tflearn.fully_connected(dense_net_0,
self.A_DIM,
activation='softmax')
return inputs, out
def _build_regression_gru_net2(self, n_timesteps=1, n_inputdim=None, n_hidden=None,
n_outputdim=None, dropout=1.0, output_dropout=1.0):
# don't have 2 lstms, just have a shared output layer
# this alternative doesn't seem to work as well
net = tflearn.input_data([None, n_timesteps, n_inputdim],dtype=tf.float32, name='input_data')
output_mask = tflearn.input_data([None, n_timesteps, n_outputdim], dtype=tf.float32, name='output_mask')
net, hidden_states_1 = tflearn.gru(net, n_hidden, weights_init='xavier', return_seq=True, return_state=True, dropout=dropout, name="gru_1")
# only add dropout to the outputs
net = tflearn.dropout(net, output_dropout)
net = [tflearn.fully_connected(net[i], n_outputdim, activation='sigmoid', weights_init='xavier', scope='output_shared', reuse=(i>0)) for i in six.moves.range(n_timesteps)]
net = tf.stack(net, axis=1)
net = net * output_mask
net = tflearn.regression(net, optimizer='adam', learning_rate=0.0005,
loss='mean_square') # mean square works
return net, hidden_states_1, None
def setup(self):
# Build neural network
# 入力層
net = tflearn.input_data(shape=[None, self.steps_of_history, 1])
# LSTMは時間かかるのでGRU
# 隠れ層
net = tflearn.gru(net, n_units=6)
# dropoutを入れると過学習を防いでくれる
net = tflearn.dropout(net, 0.9995, name = "dropout_1")
# 出力層
net = tflearn.fully_connected(net, 1, activation='linear')
# 回帰の設定、Adam法で測定
# 時系列分析での予測精度の指標にmean_squareを使っている
# mapeが一般的なようだ
# mean_square : 二乗平均平方根
net = tflearn.regression(net, optimizer='adam', learning_rate=0.001,
loss='mean_square')
# Define model
self.model = tflearn.DNN(net, tensorboard_verbose=0)
def setup(self):
self.load_dataset()
X, Y = self.create_dataset()
# Build neural network
net = tflearn.input_data(shape=[None, self.steps_of_history, 1])
net = tflearn.gru(net, n_units=6)
net = tflearn.fully_connected(net, 1, activation='linear')
net = tflearn.regression(net,
optimizer='adam',
learning_rate=0.01,
loss='mean_square')
# Define model
self.model = tflearn.DNN(net, tensorboard_verbose=1)
pos = round(len(X) * (1 - 0.2))
# 8割を訓練データ
trainX, trainY = X[:pos], Y[:pos]
# 2割をテストデータにする
testX, testY = X[pos:], Y[pos:]
return trainX, trainY, testX, testY
# pickle.dump (X_train, open ("xtrain.p", b))
# pickle.dump (X_test, open ("xtest.p", b))
# X_train = pickle.load (open ("xtrain.p", rb))
# X_test = pickle.load (open ("xtest.p", rb))
### Models
print('Build model')
net = tflearn.input_data([None, model_size])
net = tflearn.embedding(net, input_dim=n_words, output_dim=lstm_size[0])
for i in range(len(lstm_size)):
if i < len(lstm_size) - 1:
net = tflearn.gru(net,
lstm_size[i],
activation=activation_function,
return_seq=True)
net = tflearn.dropout(net, dropout_ratio)
else:
net = tflearn.gru(net, lstm_size[i], activation=activation_function)
net = tflearn.dropout(net, dropout_ratio)
net = tflearn.fully_connected(net, len(qualities), activation='softmax')
net = tflearn.regression(net,
optimizer='adam',
learning_rate=0.001,
loss='categorical_crossentropy')
print('Train model')
model = tflearn.DNN(net, tensorboard_verbose=0, tensorboard_dir="logdir/gru")
pos = round(len(x) * (1 - test_size))
trainX, trainY = x[:pos], y[:pos]
testX, testY = x[pos:], y[pos:]
return trainX, trainY, testX, testY
steps_of_history = 1
steps_in_future = 1
X, Y = create_dataset(dataset, steps_of_history, steps_in_future)
trainX, trainY, testX, testY = split_data(X, Y, 0.33)
print(X)
net = tflearn.input_data(shape=[None, steps_of_history, 1])
#net = tflearn.lstm(net, n_units=6)
#net = tflearn.gru(net, n_units=6, return_seq=True)
#net = tflearn.gru(net, n_units=6, return_seq=True)
net = tflearn.gru(net, n_units=6)
net = tflearn.fully_connected(net, 1, activation='linear')
net = tflearn.regression(net, optimizer='adam', learning_rate=0.001,
loss='mean_square')
model = tflearn.DNN(net, tensorboard_verbose=0)
with tf.device('/cpu:0'):
model.fit(trainX, trainY, validation_set=0.1, batch_size=1, n_epoch=50)
train_predict = model.predict(trainX)
test_predict = model.predict(testX)
totalY = np.array(list(vocab_proc2.fit_transform(totalY))) - 1
# Convert the indices into 11 dimensional vectors
totalY = to_categorical(totalY, nb_classes=11)
# Split into training and testing data
trainX, testX, trainY, testY = train_test_split(totalX, totalY, test_size=0.1)
# Build the network for classification
# Each input has length of 15
net = tflearn.input_data([None, 15])
# The 15 input word integers are then casted out into 256 dimensions each creating a word embedding.
# We assume the dictionary has 10000 words maximum
net = tflearn.embedding(net, input_dim=10000, output_dim=256)
# Each input would have a size of 15x256 and each of these 256 sized vectors are fed into the LSTM layer one at a time.
# All the intermediate outputs are collected and then passed on to the second LSTM layer.
net = tflearn.gru(net, 256, dropout=0.9, return_seq=True)
# Using the intermediate outputs, we pass them to another LSTM layer and collect the final output only this time
net = tflearn.gru(net, 256, dropout=0.9)
# The output is then sent to a fully connected layer that would give us our final 11 classes
net = tflearn.fully_connected(net, 11, activation='softmax')
# We use the adam optimizer instead of standard SGD since it converges much faster
net = tflearn.regression(net,
optimizer='adam',
learning_rate=0.001,
loss='categorical_crossentropy')
# Train the network
model = tflearn.DNN(net, tensorboard_verbose=0)
if load_model == 1:
model.load('gamemodel.tfl')
print('Data preprocessing finished')
tf.reset_default_graph()
main_statement_time_steps = get_max_main_statement_length()
main_hyps_time_steps = get_max_main_hyps_length()
prop_statement_time_steps = get_max_prop_statement_length()
prop_hyps_time_steps = get_max_prop_hyps_length()
main_statement_input = tflearn.input_data([None, main_statement_time_steps, input_dim], name='main_statement_input')
main_hyps_input = tflearn.input_data([None, main_hyps_time_steps, input_dim], name='main_hyps_input')
with tf.variable_scope('main') as scope:
main_statement_net = tflearn.gru(main_statement_input, output_dim, return_seq=False, dynamic=True,
scope='main_statement')
print(main_statement_net)
main_net = tflearn.gru(main_hyps_input, output_dim, return_seq=False, initial_state=main_statement_net,
dynamic=True, scope='main_hyps')
print(main_net)
main_net = tflearn.dropout(main_net, p)
prop_statement_input_1 = tflearn.input_data([None, prop_statement_time_steps, input_dim], name='prop_statement_input_1')
prop_hyps_input_1 = tflearn.input_data([None, prop_hyps_time_steps, input_dim], name='prop_hyps_input_1')
prop_statement_input_2 = tflearn.input_data([None, prop_statement_time_steps, input_dim], name='prop_statement_input_2')
prop_hyps_input_2 = tflearn.input_data([None, prop_hyps_time_steps, input_dim], name='prop_hyps_input_2')
prop_statement_input_3 = tflearn.input_data([None, prop_statement_time_steps, input_dim], name='prop_statement_input_3')
prop_hyps_input_3 = tflearn.input_data([None, prop_hyps_time_steps, input_dim], name='prop_hyps_input_3')
def _create(self, prefix=''):
shape = [None] + [i for i in self.i_dim]
inputs = tflearn.input_data(shape=shape)
layer = inputs
# init_state = tf.get_variable('{}Initstate'.format(prefix), [1, 6],
# initializer=tf.constant_initializer(0.0))
# init_state = tf.tile(init_state, [2, 1])
for i in range(self.num_hidden_layers):
weights_init = tflearn.initializations.uniform(
minval=-1 / sqrt(self.layer_size[i]),
maxval=1 / sqrt(self.layer_size[i]))
if 'dropout' in self.layer_other[i + 1]:
dropout = self.dropout
else:
dropout = None
if self.layer_type[i + 1] == 'fc':
new_layer = tflearn.fully_connected(layer,
self.layer_size[i + 1],
name="{}Layer{}".format(
prefix, i),
weights_init=weights_init)
elif self.layer_type[i + 1] == 'rnn':
new_layer = tflearn.simple_rnn(
layer,
self.layer_size[i + 1],
name="{}Layer{}".format(prefix, i),
weights_init=weights_init,
return_seq=False,
activation='linear',
dropout=dropout,
#initial_state=init_state,
dynamic=True)
elif self.layer_type[i + 1] == 'gru':
new_layer = tflearn.gru(
layer,
self.layer_size[i + 1],
name="{}Layer{}".format(prefix, i),
weights_init=weights_init,
return_seq=False,
activation='linear',
dropout=dropout,
#initial_state=init_state,
dynamic=True)
elif self.layer_type[i + 1] == 'lstm':
new_layer = tflearn.lstm(layer,
self.layer_size[i + 1],
name="{}Layer{}".format(prefix, i),
weights_init=weights_init,
return_seq=False,
activation='linear',
dropout=dropout,
dynamic=True)
else:
raise ValueError('Unsupported layer {}'.format(i))
if self.batch_norm:
new_layer = tflearn.layers.normalization.batch_normalization(
new_layer, name="{}Layer{}_norm".format(prefix, i))
if self.layer_activation[i + 1] == 'linear':
new_layer = tflearn.activations.linear(new_layer)
elif self.layer_activation[i + 1] == 'relu':
new_layer = tflearn.activations.relu(new_layer)
elif self.layer_activation[i + 1] == 'tanh':
new_layer = tflearn.activations.tanh(new_layer)
elif self.layer_activation[i + 1] == 'sigmoid':
new_layer = tflearn.activations.sigmoid(new_layer)
if i < self.num_hidden_layers - 1:
layer = new_layer
return inputs, new_layer
test_IC50 = read_labels("./data/test_ic50")
print(test_compound_max)
## separating train,dev, test data
compound_train_ver, compound_dev_ver, compound_train_adj, compound_dev_adj, IC50_train, IC50_dev, protein_train, protein_dev = train_dev_split(
train_protein, train_compound_ver, train_compound_adj, train_IC50,
dev_perc, comp_MAX_size, protein_MAX_size, batch_size)
## RNN for protein
prot_data = input_data(shape=[None, protein_MAX_size])
prot_embd = tflearn.embedding(prot_data,
input_dim=vocab_size_protein,
output_dim=GRU_size_prot)
prot_gru_1 = tflearn.gru(prot_embd,
GRU_size_prot,
initial_state=prot_init_state_1,
trainable=True,
return_seq=True,
restore=False)
prot_gru_1 = tf.stack(prot_gru_1, axis=1)
prot_gru_2 = tflearn.gru(prot_gru_1,
GRU_size_prot,
initial_state=prot_init_state_2,
trainable=True,
return_seq=True,
restore=False)
prot_gru_2 = tf.stack(prot_gru_2, axis=1)
W_prot = tflearn.variables.variable(name="Attn_W_prot",
shape=[GRU_size_prot, GRU_size_prot],
initializer=tf.random_normal(
[GRU_size_prot, GRU_size_prot],
stddev=0.1),
def train_model(train, test, vocab_size, n_epoch=5, n_units=128):
'''Method to load the dataset and train the RNN model.'''
train_x = train['sub_seqs']
train_y = train['sub_label']
test_x = test['sub_seqs']
test_y = test['sub_label']
sequence_chunk_size = 15
learning_rate = 0.0001
dropout = 0.6
# Sequence padding
train_x = pad_sequences(train_x,
maxlen=sequence_chunk_size,
value=0.,
padding='post')
test_x = pad_sequences(test_x,
maxlen=sequence_chunk_size,
value=0.,
padding='post')
# Converting labels to binary vectors
train_y = to_categorical(train_y, nb_classes=vocab_size)
test_y = to_categorical(test_y, nb_classes=vocab_size)
print("Building network topology...")
# Network building
net = tflearn.input_data([None, 15])
net = tflearn.embedding(net,
input_dim=vocab_size,
output_dim=128,
trainable=True)
net = tflearn.gru(net,
n_units=n_units,
dropout=dropout,
weights_init=tflearn.initializations.xavier(),
return_seq=False)
net = tflearn.fully_connected(
net,
vocab_size,
activation='softmax',
weights_init=tflearn.initializations.xavier())
net = tflearn.regression(net,
optimizer='adam',
learning_rate=learning_rate,
loss='categorical_crossentropy')
# Training
model = tflearn.DNN(net, tensorboard_verbose=2)
print("Training model...")
model.fit(train_x,
train_y,
validation_set=(test_x, test_y),
show_metric=False,
batch_size=256,
n_epoch=n_epoch)
# For visualizations
embedding = tflearn.get_layer_variables_by_name("Embedding")[0]
return [model, embedding]
comp_MAX_size, 0)
test0_IC50 = read_labels("./data/test_ic50")
## separating train,dev, test data
compound_train, compound_dev, IC50_train, IC50_dev, protein_train, protein_dev = train_dev_split(
train_protein, train_compound, train_IC50, dev_perc, comp_MAX_size,
protein_MAX_size, batch_size)
## RNN for protein
prot_data = input_data(shape=[None, protein_MAX_size])
prot_embd = tflearn.embedding(prot_data,
input_dim=vocab_size_protein,
output_dim=GRU_size_prot)
prot_gru_1 = tflearn.gru(prot_embd,
GRU_size_prot,
trainable=False,
return_seq=True,
restore=True)
prot_gru_1 = tf.stack(prot_gru_1, axis=1)
prot_gru_2 = tflearn.gru(prot_gru_1,
GRU_size_prot,
trainable=False,
return_seq=True,
restore=True)
prot_gru_2 = tf.stack(prot_gru_2, axis=1)
drug_data = input_data(shape=[None, comp_MAX_size])
drug_embd = tflearn.embedding(drug_data,
input_dim=vocab_size_compound,
output_dim=GRU_size_drug)
drug_gru_1 = tflearn.gru(drug_embd,
def run_on_ign():
df_dataset = pd.read_csv(ign_dataset_path)
df_dataset.set_index(['index'], inplace=True)
# fill null values with empty strings
df_dataset.fillna(value='', inplace=True)
# extract the required columns for inputs and outputs
data_X = df_dataset.title
# data_X[0]
label_Y = df_dataset.score_phrase
# label_Y[5]
# convert the strings in the input into integers corresponding to the dictionary positions
# maps documents to sequences of word ids
# data is automatically padded so we need to pad_sequences manually
vocab_proc = VocabularyProcessor(15)
total_X = np.array(list(vocab_proc.fit_transform(data_X)))
# total_X[0]
# we will have 11 classes in total for prediction, indices from 0 to 10
# vocabulary processor for single word
vocab_proc2 = VocabularyProcessor(1)
total_Y = np.array(list(vocab_proc2.fit_transform(label_Y))) - 1
# total_Y[5]
# len(total_Y)
# as we have 11 unique score_phrase
# convert the indices into 11 dimensional vectors
# This is wrong as it generate same array for different score_phrase
# total_Y = to_categorical(total_Y, nb_classes=11)
array_list = []
for array in total_Y:
array_list.append(get_categorical(array, 11))
total_Y = np.array(array_list)
# total_Y[4]
# split into training and testing data
train_X, test_X, train_Y, test_Y = train_test_split(total_X,
total_Y,
test_size=0.1)
# build the network for classification
# each input has length of 15
net = tflearn.input_data([None, 15])
# the 15 input word integers are then casted out into 256 dimensions each creating a word embedding.
# we assume the dictionary has 10000 words maximum
net = tflearn.embedding(net, input_dim=10000, output_dim=256)
# each input would have a size of 15x256 and each of these 256 sized vectors are fed into the LSTM layer one at a time.
# all the intermediate outputs are collected and then passed on to the second LSTM layer.
net = tflearn.gru(net, 256, dropout=0.9, return_seq=True)
# using the intermediate outputs, we pass them to another LSTM layer and collect the final output only this time
net = tflearn.gru(net, 256, dropout=0.9)
# the output is then sent to a fully connected layer that would give us our final 11 classes
net = tflearn.fully_connected(net, 11, activation='softmax')
# we use the adam optimizer instead of standard SGD since it converges much faster
net = tflearn.regression(net,
optimizer='adam',
learning_rate=0.001,
loss='categorical_crossentropy')
model = tflearn.DNN(net, tensorboard_verbose=0)
if check_file_exist(ign_model_path):
model.load(ign_model_path)
model.fit(train_X,
train_Y,
validation_set=(test_X, test_Y),
show_metric=True,
batch_size=32,
n_epoch=20)
if save_model:
print("Saving model as './ign_model.tfl'")
model.save(ign_model_path)
return 0
def train_model(train,
test,
vocab_size,
max_seq_size,
chunks=1024,
num_epochs=10,
learning_rate=0.001,
n_units=256,
dropout=0.5):
trainX = train['sub_seqs']
trainY = train['sub_label']
testX = test['sub_seqs']
testY = test['sub_label']
# Sequence padding
trainX = pad_sequences(trainX,
maxlen=max_seq_size,
value=0.,
padding='post')
testX = pad_sequences(testX, maxlen=max_seq_size, value=0., padding='post')
trainX = da.from_array(np.asarray(trainX), chunks=chunks)
trainY = da.from_array(np.asarray(trainY), chunks=chunks)
testX = da.from_array(np.asarray(testX), chunks=chunks)
testY = da.from_array(np.asarray(testY), chunks=chunks)
# Converting labels to binary vectors
trainY = to_categorical(trainY, nb_classes=vocab_size)
testY = to_categorical(testY, nb_classes=vocab_size)
# Network building
net = tflearn.input_data([None, max_seq_size])
net = tflearn.embedding(net,
input_dim=vocab_size,
output_dim=128,
trainable=True)
net = tflearn.gru(net,
n_units=n_units,
dropout=dropout,
weights_init=tflearn.initializations.xavier(),
return_seq=False)
net = tflearn.fully_connected(
net,
vocab_size,
activation='softmax',
weights_init=tflearn.initializations.xavier())
net = tflearn.regression(net,
optimizer='adam',
learning_rate=learning_rate,
loss='categorical_crossentropy')
# Training
model = tflearn.DNN(net,
tensorboard_dir='/tmp/tflearn_logs/shallow_gru/',
tensorboard_verbose=2)
#checkpoint_path='/tmp/tflearn_logs/shallow_lstm/',
#best_checkpoint_path="C:/Users/macle/Desktop/UPC Masters/Semester 2/CI/SubRecommender/models/")
model.fit(trainX,
trainY,
validation_set=(testX, testY),
show_metric=False,
snapshot_epoch=True,
batch_size=256,
n_epoch=num_epochs,
run_id=str(learning_rate) + "-" + str(n_units) + "-" +
str(dropout))
return model
#train_compound += test_compound + ER_compound + GPCR_compound + kinase_compound + channel_compound
#train_IC50 += test_IC50 + ER_IC50 + GPCR_IC50 + kinase_IC50 + channel_IC50
## separating train,dev, test data
compound_train, compound_dev, IC50_train, IC50_dev, protein_train, protein_dev = train_dev_split(
train_protein, train_compound, train_IC50, dev_perc, comp_MAX_size,
protein_MAX_size, batch_size)
## RNN for protein
prot_data = input_data(shape=[None, protein_MAX_size])
prot_embd = tflearn.embedding(prot_data,
input_dim=vocab_size_protein,
output_dim=GRU_size_prot)
prot_gru_1 = tflearn.gru(prot_embd,
GRU_size_prot,
initial_state=prot_init_state_1,
trainable=False,
return_seq=True)
prot_gru_1 = tf.stack(prot_gru_1, axis=1)
prot_gru_2 = tflearn.gru(prot_gru_1,
GRU_size_prot,
initial_state=prot_init_state_2,
trainable=False,
return_seq=True)
prot_gru_2 = tf.stack(prot_gru_2, axis=1)
drug_data = input_data(shape=[None, comp_MAX_size])
drug_embd = tflearn.embedding(drug_data,
input_dim=vocab_size_compound,
output_dim=GRU_size_drug)
drug_gru_1 = tflearn.gru(drug_embd,
import tensorflow as tf
from data_analysis import data_pre
import sys
sys.path.insert(0, '/home/lpy/tflearn/')
import tflearn
train_data, train_label = data_pre('train')
test_data, test_label = data_pre('test')
print train_data.shape
print test_data.shape
days = 35
net = tflearn.input_data(shape=[None, days, 21])
net = tflearn.gru(net, 40, return_seq=False) #,dropout=0.8)
net = tflearn.dropout(net, 0.8)
#net = tflearn.lstm(net, 512)#,dropout=0.8)
#net = tflearn.dropout(net, 0.8)
net = tflearn.fully_connected(net, 1, activation='linear')
net = tflearn.regression(net,
optimizer='rmsprop',
loss='mean_square',
metric='R2',
name="target")
model = tflearn.DNN(net)
#model.fit(train_data, train_label, n_epoch=10000,run_id='lstm',snapshot_epoch=True,validation_set=(test_data,test_label), show_metric=True, batch_size=20)
#model.save("model/bitcoin_lstm.tfl")
raise ValueError("Incorrect argument provided.")
# Setting batches
validation_batch = batch_generator.mfcc_batch_generator(
gv.validation_batch_size, gv.height)
x, y, z = next(validation_batch)
testX, testY = x, y
training_batch = batch_generator.mfcc_batch_generator(gv.training_batch_size,
gv.height,
exclude=z)
# Network building
net = tflearn.input_data([None, gv.width, gv.height])
if mode == 'gru':
net = tflearn.gru(net, 128, dropout=0.8)
elif mode == 'rnn':
net = tflearn.simple_rnn(net, 128, dropout=0.8)
else:
net = tflearn.lstm(net, 128, dropout=0.8)
net = tflearn.fully_connected(net, gv.classes, activation='softmax')
net = tflearn.regression(net,
optimizer='adam',
learning_rate=gv.learning_rate,
loss='categorical_crossentropy')
# Training
model = tflearn.DNN(net, tensorboard_verbose=gv.tensorboard_verbosity)
for times in range(gv.training_iters):
X, Y, Z = next(training_batch)
trainX, trainY = X, Y
totalY = np.array(list(vocab_proc2.fit_transform(totalY))) - 1
# Convert the indices into 11 dimensional vectors
totalY = to_categorical(totalY, nb_classes=11)
# Split into training and testing data
trainX, testX, trainY, testY = train_test_split(totalX, totalY, test_size=0.1)
# Build the network for classification
# Each input has length of 15
net = tflearn.input_data([None, 15])
# The 15 input word integers are then casted out into 256 dimensions each creating a word embedding.
# We assume the dictionary has 10000 words maximum
net = tflearn.embedding(net, input_dim=10000, output_dim=256)
# Each input would have a size of 15x256 and each of these 256 sized vectors are fed into the LSTM layer one at a time.
# All the intermediate outputs are collected and then passed on to the second LSTM layer.
net = tflearn.gru(net, 256, dropout=0.9, return_seq=True)
# Using the intermediate outputs, we pass them to another LSTM layer and collect the final output only this time
net = tflearn.gru(net, 256, dropout=0.9)
# The output is then sent to a fully connected layer that would give us our final 11 classes
net = tflearn.fully_connected(net, 11, activation='softmax')
# We use the adam optimizer instead of standard SGD since it converges much faster
net = tflearn.regression(net, optimizer='adam', learning_rate=0.001,
loss='categorical_crossentropy')
# Train the network
model = tflearn.DNN(net, tensorboard_verbose=0)
if load_model == 1:
model.load('gamemodel.tfl')
model.fit(trainX, trainY, validation_set=(testX, testY), show_metric=True, batch_size=32, n_epoch=20)
