def __init__(self, sequence_length, num_classes, embeddings, num_filters, l2_reg_lambda=0.0, dropout=None):
self.input_text = layers.input_data( (None, sequence_length), dtype=tf.int32)
with tf.variable_scope('Embedding'):
embeddings_var = tf.Variable(embeddings, name='W', dtype=tf.float32)
embeddings_var = tf.concat([np.zeros((1, embeddings.shape[1]) ), embeddings_var[1:] ] , axis = 0)
self.embeded_text = tf.gather(embeddings_var, self.input_text)
net = self.embeded_text
self.mask = tf.expand_dims(tf.cast(tf.not_equal(self.input_text, 0), tf.float32), axis = 2)
if dropout is not None:
dropout = map(float, dropout.split(',') )
for num_filter in num_filters:
net = layers.lstm(net, num_filter, return_seq=True, dropout=dropout)
net = tf.transpose(tf.stack(net), (1, 0, 2) )
features = tf.reduce_sum(net * self.mask, axis=1) / (tf.reduce_sum(self.mask, axis=1) + 1e-5)
self.probas = layers.fully_connected(features, num_classes, activation='softmax', regularizer='L2', weight_decay=l2_reg_lambda)
optimizer = tflearn.optimizers.Adam(learning_rate=0.001)
self.train_op = layers.regression(
self.probas,
optimizer=optimizer,
batch_size=128)
def __init__(self, sequence_length, num_classes, embeddings, num_filters, l2_reg_lambda=0.0, dropout=None, bn=False):
self.input_text = layers.input_data( (None, sequence_length), dtype=tf.int32)
with tf.variable_scope('Embedding'):
embeddings_var = tf.Variable(embeddings, name='W', dtype=tf.float32)
embeddings_var = tf.concat([np.zeros((1, embeddings.shape[1]) ), embeddings_var[1:] ] , axis = 0)
self.embeded_text = tf.gather(embeddings_var, self.input_text)
net = self.embeded_text
for num_filter in num_filters:
if bn:
# , weights_init=tflearn.initializations.uniform(minval=-0.001, maxval=0.001)
net = layers.conv_1d(net, num_filter, 3, padding='valid', activation='linear', bias=False)
net = layers.batch_normalization(net)
net = layers.activation(net, 'relu')
else:
net = layers.conv_1d(net, num_filter, 3, padding='valid', activation='relu', bias=True, regularizer='L2', weight_decay=l2_reg_lambda)
if dropout is not None:
net = layers.dropout(net, float(dropout) )
features = layers.flatten( layers.max_pool_1d(net, net.shape.as_list()[1], padding='valid') )
self.probas = layers.fully_connected(features, num_classes, activation='softmax', regularizer='L2', weight_decay=l2_reg_lambda)
#optimizer = tflearn.optimizers.Momentum(learning_rate=0.1, momentum=0.9, lr_decay=0.2, decay_step=1000, staircase=True)
optimizer = tflearn.optimizers.Adam(learning_rate=0.001)
self.train_op = layers.regression(
self.probas,
optimizer=optimizer,
batch_size=128)
def nn_model(input_size):
# same implementation with keras
# model = Sequential()
# model.add(Dense(128, input_shape=size, activation='relu'))
network = input_data(shape=[None, input_size, 1], name='input')
network = fully_connected(network, 128, activation='relu')
network = dropout(network,
0.8) # meaning 0.8 will be kept, opposite in keras
network = fully_connected(network, 256, activation='relu')
network = dropout(network, 0.8)
network = fully_connected(network, 512, activation='relu')
network = dropout(network, 0.8)
network = fully_connected(network, 256, activation='relu')
network = dropout(network, 0.8)
network = fully_connected(network, 512, activation='relu')
network = dropout(network, 0.8)
network = fully_connected(network, 2, activation='softmax')
network = regression(network,
optimizer='adam',
learning_rate=LR,
loss='categorical_crossentropy',
name='targets')
model = tflearn.DNN(network, tensorboard_dir='log')
return model
def __init__(self,
max_document_length,
num_classes=2,
num_characters=71,
num_blocks=None,
char_vec_size=16,
weight_decay=2e-4):
self.input_text = layers.input_data((None, max_document_length))
self.target_label = tf.placeholder(shape=(None, num_classes),
dtype=tf.float32)
embeded_text = layers.embedding(self.input_text, num_characters,
char_vec_size)
top_feature = embeded_text
filters = 64
if num_blocks[0] == 0:
self.block = (2, 2, 2, 2)
else:
self.block = num_blocks
for i, num_block in enumerate(self.block):
if i > 0:
filters *= 2
top_feature = layers.max_pool_1d(top_feature,
3,
strides=2,
padding='same')
for block_i in range(num_block):
top_feature = self.conv_block(top_feature, filters)
pooled_feature = layers.flatten(
layers.custom_layer(top_feature, self.kmax_pool_1d))
fc1 = layers.fully_connected(pooled_feature,
2048,
activation='relu',
regularizer='L2',
weight_decay=weight_decay)
fc2 = layers.fully_connected(fc1,
2048,
activation='relu',
regularizer='L2',
weight_decay=weight_decay)
self.probas = layers.fully_connected(fc2,
num_classes,
activation='softmax',
regularizer='L2',
weight_decay=weight_decay)
self.train_op = layers.regression(self.probas,
placeholder=self.target_label)
"""
#TFLearnwith tensorflow - Deep Learning with Neural Networks 14
#https://pythonprogramming.net/tflearn-machine-learning-tutorial/
import tflearn
from tflearn.layers.conv import conv_2d, max_pool_2d
from tflearn.layers import input_data, dropout, fully_connected
from tflearn.layers.estimator import regression
import tflearn.datasets.mnist as mnist
X, Y, test_x, test_y = mnist.load_data(one_hot=True)
X = X.reshape([-1, 28, 28, 1])
test_x = test_x.reshape([-1, 28, 28, 1])
convnet = input_data(shape=[None, 28, 28, 1], name='input')
convnet = conv_2d(convnet, 32, 2, activation='relu')
convnet = max_pool_2d(convnet, 2)
convnet = conv_2d(convnet, 64, 2, activation='relu')
convnet = max_pool_2d(convnet, 2)
convnet = fully_connected(convnet, 1024, activation='relu')
convnet = dropout(convnet, 0.8)
convnet = fully_connected(convnet, 10, activation='softmax')
convnet = regression(convnet,
optimizer='adam',
learning_rate=0.01,
def __init__(self,
max_document_length,
num_classes=2,
num_characters=71,
char_vec_size=16,
weight_decay=2e-4,
optimizer='sgd',
dropout=None,
num_blocks=None):
self.input_text = layers.input_data((None, max_document_length))
self.target_label = tf.placeholder(shape=(None, num_classes),
dtype=tf.float32)
embeded_text = layers.embedding(self.input_text, num_characters,
char_vec_size)
mask = tf.cast(tf.not_equal(self.input_text, 0), tf.float32)
embeded_text = embeded_text * tf.expand_dims(mask, 2)
self.embeded_text = embeded_text
top_feature = embeded_text
filters = 64
if num_blocks[0] == 0:
self.block = (1, 1, 1, 1)
else:
self.block = num_blocks
for i, num_block in enumerate(self.block):
if i > 0:
filters *= 2
top_feature = layers.max_pool_1d(top_feature,
3,
strides=2,
padding='same')
for block_i in range(num_block):
top_feature = self.conv_block(top_feature, filters)
pooled_feature = layers.flatten(
layers.custom_layer(top_feature, self.kmax_pool_1d))
if dropout is not None:
pooled_feature = layers.dropout(pooled_feature, dropout)
fc1 = layers.fully_connected(pooled_feature,
2048,
activation='relu',
regularizer='L2',
weight_decay=weight_decay)
if dropout is not None:
fc1 = layers.dropout(fc1, dropout)
fc2 = layers.fully_connected(fc1,
2048,
activation='relu',
regularizer='L2',
weight_decay=weight_decay)
self.probas = layers.fully_connected(fc2,
num_classes,
activation='softmax',
regularizer='L2',
weight_decay=weight_decay)
def build_sgd(learning_rate):
step_tensor = tf.Variable(0.,
name="Training_step",
trainable=False)
steps = [-1.0, 16000.0, 24000.0]
lrs = [1e-1, 1e-2, 1e-3]
lr = tf.reduce_min(
tf.cast(tf.less(step_tensor, steps), tf.float32) + lrs)
tflearn.helpers.summarizer.summarize(
lr, 'scalar', 'lr', 'Optimizer_training_summaries')
return tf.train.MomentumOptimizer(learning_rate=lr,
momentum=0.9), step_tensor
if optimizer == 'sgd':
optimizer = build_sgd
self.train_op = layers.regression(self.probas,
optimizer=optimizer,
learning_rate=0.001,
placeholder=self.target_label)