def build_model(vocabFile, model_type='bilstm'):
processor = VocabularyProcessor.restore(vocabFile)
n_words = len(processor.vocabulary_)
net = tflearn.input_data([None, 300])
net = tflearn.embedding(net, input_dim=n_words, output_dim=200)
if model_type == 'bilstm':
net = tflearn.bidirectional_rnn(net, tflearn.BasicLSTMCell(200),
tflearn.BasicLSTMCell(200))
net = dropout(net, 0.5)
elif model_type == 'lstm':
net = tflearn.lstm(net, 200, dropout=0.5)
net = dropout(net, 0.5)
elif model_type == 'cnn':
net = conv_model(net)
net = tflearn.fully_connected(net, 2, activation='softmax')
net = tflearn.regression(net,
optimizer='adam',
learning_rate=0.05,
loss='categorical_crossentropy')
return net
def model(self,
type=None,
mode="train",
num_layers=2,
state_size=32,
learning_rate=0.001,
tensorboard_verbose=3):
net = tflearn.input_data(shape=[None, 9])
net = tflearn.embedding(net,
input_dim=21,
output_dim=32,
weights_init='xavier')
if type == 'bi_rnn':
out_rnn = tflearn.bidirectional_rnn(net, tflearn.BasicLSTMCell(32),
tflearn.BasicLSTMCell(32))
elif type == 'basic_lstm':
for i in range(4):
net = tflearn.lstm(net, n_units=40, return_seq=True)
#net = tflearn.lstm(net, n_units=40, return_seq=True)
out_rnn = tflearn.lstm(net, n_units=40, return_seq=False)
elif type == 'basic_rnn':
out_rnn = tflearn.simple_rnn(net, 40)
else:
out_rnn = net
net = tflearn.fully_connected(out_rnn, 100, activation='prelu')
net = tflearn.layers.normalization.batch_normalization(net)
net = tflearn.dropout(net, 0.1)
net = tflearn.fully_connected(net, 1, activation='sigmoid')
"""
single_cell = getattr(tf.contrib.rnn, cell_type)(cell_size, state_is_tuple=True)
if num_layers == 1:
cell = single_cell
else:
cell = tf.contrib.rnn.MultiRNNCell([single_cell] * num_layers)
"""
with tf.name_scope(
"TargetsData"
): # placeholder for target variable (i.e. trainY input)
targetY = tf.placeholder(shape=[None, 1],
dtype=tf.float32,
name="Y")
network = tflearn.regression(net,
placeholder=targetY,
optimizer=self.optimizer(learning_rate),
learning_rate=learning_rate,
loss=tflearn.mean_square(net, targetY),
metric=self.accuracy(net, targetY),
name='no rnn')
model = tflearn.DNN(network, tensorboard_verbose=tensorboard_verbose)
return model
def build_net3():
"""Bidirectional RNN"""
net = tflearn.input_data(shape=[None, time_window, 1])
lstm1 = tflearn.BasicLSTMCell(num_units=lstm_units)
lstm2 = tflearn.BasicLSTMCell(num_units=lstm_units)
#net = tflearn.fully_connected(net, 10, activation='sigmoid')
#net = tflearn.reshape(net, [-1, 10, 1])
#net = tflearn.lstm(net, dropout=(1.0, 0.5), n_units=lstm_units, return_seq=True)
net = tflearn.bidirectional_rnn(net, lstm1, lstm2, return_seq=False)
#net = tflearn.fully_connected(net, 5, activation='linear')
#net = tflearn.dropout(net, 0.5)
#net = tflearn.fully_connected(net, lstm_units, activation='relu')
#net = tflearn.dropout(net, 0.5)
#For model 2
net = tflearn.fully_connected(net, predict_steps+1, activation='linear')
#For model 1 and 3
#net = tflearn.fully_connected(net, 1, activation='linear')
#For model 4
#net = tflearn.fully_connected(net, 3, activation='softmax')
#net = tflearn.regression(net, metric='accuracy', optimizer=optimizer,
#loss='categorical_crossentropy',
#learning_rate=learn_rate, shuffle_batches=False)
net = tflearn.regression(net, metric=None, optimizer=optimizer, loss='mean_square',
learning_rate=learn_rate, shuffle_batches=False)
model = tflearn.DNN(net, clip_gradients=0.0, tensorboard_verbose=0)
return model
def simple_blstm(): input_layer = tf.input_data(shape=[None, 5, 19]) model = tf.bidirectional_rnn(input_layer, tf.BasicLSTMCell(91), tf.BasicLSTMCell(91)) model = tf.dropout(model, 0.5) model = tf.fully_connected(model, 11, activation='sigmoid') sgd = tf.SGD(learning_rate=0.01, lr_decay=0.96, decay_step=1000) model = tf.regression(model, optimizer=sgd, loss='categorical_crossentropy') return tf.DNN(model, clip_gradients=0., tensorboard_verbose=0)
def run():
net = tflearn.input_data([None, 100])
net = tflearn.embedding(net, input_dim=20000, output_dim=128)
net = tflearn.bidirectional_rnn(
net, tflearn.BasicLSTMCell(128), tflearn.BasicLSTMCell(128))
net = tflearn.dropout(net, 0.5)
net = tflearn.fully_connected(net, 2, activation='softmax')
net = tflearn.regression(
net, optimizer='adam', loss='categorical_crossentropy')
m = tflearn.DNN(net, clip_gradients=0., tensorboard_verbose=2)
m.fit(trainX, trainY, validation_set=0.1, show_metric=True, batch_size=64)
m.save('models/bidirectional_rnn.tfl')
def create_a3c_lstm_network(input_tensor, output_num):
l_hid1 = tflearn.conv_2d(input_tensor, 16, 8, strides=4, activation='relu', scope='conv1', padding='valid')
l_hid2 = tflearn.conv_2d(l_hid1, 32, 4, strides=2, activation='relu', scope='conv2', padding='valid')
l_hid3 = tflearn.fully_connected(l_hid2, 256, activation='relu', scope='dense3')
# reshape l_hid3 to lstm usable shape (1, batch_size, 256)
l_hid3_reshape = tf.reshape(l_hid3, [1, -1, 256])
# have to custom make the lstm output here to use tf.nn.dynamic_rnn
l_lstm = tflearn.BasicLSTMCell(256)
# BasicLSTMCell lists state size as tuple so we need to pass tuple into dynamic_rnn
lstm_state_size = tuple([[1, x] for x in l_lstm.state_size])
# has to specifically be the same type tf.python.ops.rnn_cell.LSTMStateTuple
from tensorflow.python.ops.nn import rnn_cell as _rnn_cell
initial_lstm_state = _rnn_cell.LSTMStateTuple(tf.placeholder(tf.float32, shape=lstm_state_size[0], name='initial_lstm_state1'),
tf.placeholder(tf.float32, shape=lstm_state_size[1], name='initial_lstm_state2'))
# dynamically get the sequence length
sequence_length = tf.reshape(tf.shape(l_hid3)[0], [1])
l_lstm4, new_lstm_state = tf.nn.dynamic_rnn(l_lstm, l_hid3_reshape,
initial_state=initial_lstm_state, sequence_length=sequence_length,
time_major=False, scope='lstm4')
# reshape lstm back to (batch_size, 256)
l_lstm4_reshape = tf.reshape(l_lstm4, [-1, 256])
actor_out = tflearn.fully_connected(l_lstm4_reshape, output_num, activation='softmax', scope='actorout')
critic_out = tflearn.fully_connected(l_lstm4_reshape, 1, activation='linear', scope='criticout')
return actor_out, critic_out, initial_lstm_state, new_lstm_state
def add_deep_layers(self, net, model_type, out_embedding_dim, layer_size, n_layers):
if model_type == 'embedding_rnn':
out_rnn = tflearn.embedding(net, input_dim=21, output_dim=out_embedding_dim, weights_init='xavier')
elif model_type == 'bi_rnn':
out_rnn = tflearn.bidirectional_rnn(net, tflearn.BasicLSTMCell(layer_size), tflearn.BasicLSTMCell(layer_size))
elif model_type == 'deep_rnn':
for i in range(n_layers):
net = tflearn.lstm(net, n_units=layer_size, return_seq=True)
out_rnn = tflearn.lstm(net, layer_size)
elif model_type == 'basic_rnn':
out_rnn = tflearn.simple_rnn(net, layer_size)
else:
out_rnn = net
return out_rnn
def create_basic_lstm_layer(self,
input_data,
input_size,
num_of_units,
scope=None):
batch_size = tf.shape(input_data)[0]
input_data_reshape = tf.reshape(input_data, [1, -1, input_size])
lstm_layer = tflearn.BasicLSTMCell(num_units=num_of_units,
state_is_tuple=True)
lstm_state_size = tuple([[1, x] for x in lstm_layer.state_size])
initial_lstm_state = _rnn_cell.LSTMStateTuple(
tf.placeholder(tf.float32,
shape=lstm_state_size[0],
name='initial_lstm_state1'),
tf.placeholder(tf.float32,
shape=lstm_state_size[1],
name='initial_lstm_state2'))
sequence_length = tf.reshape(batch_size, [1])
lstm_output, new_lstm_state = tf.nn.dynamic_rnn(
lstm_layer,
input_data_reshape,
initial_state=initial_lstm_state,
sequence_length=sequence_length,
time_major=False,
scope=scope)
lstm_output_reshape = tf.reshape(lstm_output, [-1, num_of_units])
self.hiddens.append(lstm_output_reshape)
self.num_of_hidden_layers += 1
return lstm_output_reshape, initial_lstm_state, new_lstm_state
# X, Y, char_idx = \
# textfile_to_semi_redundant_sequences(path, seq_maxlen=maxlen, redun_step=3)
# TODO - What do these look like?
# print(X)
# print(Y)
# Saves our dictionary for later use
pickle.dump(char_idx, open(char_idx_file, 'wb'))
# New
g = tflearn.input_data([None, maxlen, len(char_idx)])
# g = tflearn.fully_connected(g, len(char_idx), activation='softmax')
# g = tflearn.embedding(g, 10000, 342)
g = tflearn.bidirectional_rnn(g,
tflearn.BasicLSTMCell(342),
tflearn.BasicLSTMCell(342),
dynamic=True)
g = tflearn.dropout(g, 0.5)
g = tflearn.fully_connected(g, len(char_idx), activation='softmax')
g = tflearn.regression(g,
optimizer='adam',
loss='categorical_crossentropy',
learning_rate=0.001)
# Old
# g = tflearn.input_data([None, maxlen, len(char_idx)])
# g = tflearn.lstm(g, 512, return_seq=True)
# g = tflearn.dropout(g, 0.5)
# g = tflearn.lstm(g, 512, return_seq=True)
# g = tflearn.dropout(g, 0.5)