def transform_embedded_sequences(self, embedded_sequences):
drop_1, drop_2 = self.dropout_rates
net = dropout(embedded_sequences, drop_1)
conv_blocks = []
for sz in self.filter_sizes:
conv = conv_1d(net,
nb_filter=self.num_filters,
filter_size=sz,
padding="valid",
activation="relu",
regularizer="L2")
conv_blocks.append(conv)
net = merge(conv_blocks, mode='concat',
axis=1) if len(conv_blocks) > 1 else conv_blocks[0]
net = tf.expand_dims(net, 2)
net = global_max_pool(net)
net = dropout(net, drop_2)
model_output = fully_connected(net,
self.class_count,
activation="softmax")
return model_output
def code_classifier_forward(config,
incoming=None,
image=None,
scope="code_classifier",
name=None,
reuse=False):
with tf.variable_scope(scope, name, reuse=reuse):
output = relu(fully_connected(incoming, 512))
output1 = dropout(output, 0.8)
print(config.batch_size, image.shape)
output = relu(
fully_connected(tf.reshape(image, [config.batch_size, 28 * 28]),
512))
output2 = dropout(output, 0.8)
output = tf.concat([output1, output2], axis=-1)
output = relu(fully_connected(output, 1024))
output = dropout(output, 0.5)
output = relu(fully_connected(output, 512))
output = dropout(output, 0.8)
output = fully_connected(output, 10)
return output
def code_classifier_forward(config,
incoming=None,
image=None,
scope="code_classifier",
name=None,
reuse=False):
with tf.variable_scope(scope, name, reuse=reuse):
code_output = leaky_relu(fully_connected(incoming, 512))
output = conv_2d(image, 32, 5, 2, name="conv1")
output = residual_block(output,
2,
32,
downsample=True,
batch_norm=True,
name="rb1")
output = residual_block(output,
1,
64,
downsample=True,
batch_norm=True,
name="rb2")
output = leaky_relu(
fully_connected(
tf.reshape(output, [config.batch_size, 4 * 4 * 64]), 1024))
prod = tf.matmul(code_output[:, :, None], output[:, None, :])
prob = tf.nn.softmax(prod)
prob2 = tf.nn.softmax(tf.transpose(prod, perm=[0, 2, 1]))
output = tf.concat([
code_output,
tf.matmul(prob, output[:, :, None])[:, :, 0],
tf.matmul(prob2, code_output[:, :, None])[:, :, 0]
],
axis=-1)
output = relu(fully_connected(output, 1024))
output = dropout(output, 0.6)
output = relu(fully_connected(output, 512))
output = dropout(output, 0.6)
output = relu(fully_connected(output, 256))
output = dropout(output, 0.8)
output = fully_connected(output, 5)
return output
def __init__(self, sequence_length, num_classes, embeddings, num_filters, l2_reg_lambda=0.0, dropout=None, bn=False):
self.input_text = layers.input_data( (None, sequence_length), dtype=tf.int32)
with tf.variable_scope('Embedding'):
embeddings_var = tf.Variable(embeddings, name='W', dtype=tf.float32)
embeddings_var = tf.concat([np.zeros((1, embeddings.shape[1]) ), embeddings_var[1:] ] , axis = 0)
self.embeded_text = tf.gather(embeddings_var, self.input_text)
net = self.embeded_text
for num_filter in num_filters:
if bn:
# , weights_init=tflearn.initializations.uniform(minval=-0.001, maxval=0.001)
net = layers.conv_1d(net, num_filter, 3, padding='valid', activation='linear', bias=False)
net = layers.batch_normalization(net)
net = layers.activation(net, 'relu')
else:
net = layers.conv_1d(net, num_filter, 3, padding='valid', activation='relu', bias=True, regularizer='L2', weight_decay=l2_reg_lambda)
if dropout is not None:
net = layers.dropout(net, float(dropout) )
features = layers.flatten( layers.max_pool_1d(net, net.shape.as_list()[1], padding='valid') )
self.probas = layers.fully_connected(features, num_classes, activation='softmax', regularizer='L2', weight_decay=l2_reg_lambda)
#optimizer = tflearn.optimizers.Momentum(learning_rate=0.1, momentum=0.9, lr_decay=0.2, decay_step=1000, staircase=True)
optimizer = tflearn.optimizers.Adam(learning_rate=0.001)
self.train_op = layers.regression(
self.probas,
optimizer=optimizer,
batch_size=128)
def get_sentiment_score(self, rnn_output, query):
"""Linear softmax answer module"""
rnn_output = dropout(rnn_output, self.args['dropout'])
output = tf.layers.dense(tf.concat([rnn_output, query], 1),
1,
activation=tf.sigmoid)
return output
def conv(classes, input_shape):
model = input_data(input_shape, name="input")
model = conv_2d(model, 32, (3, 3), activation='relu')
model = conv_2d(model, 64, (3, 3), activation='relu')
model = max_pool_2d(model, (2, 2))
model = dropout(model, 0.25)
model = flatten(model)
model = fully_connected(model, 128, activation='relu')
model = dropout(model, 0.5)
model = fully_connected(model, classes, activation='softmax')
model = regression(model,
optimizer='adam',
learning_rate=0.001,
loss='categorical_crossentropy',
name='target')
# Training
model = tflearn.DNN(model, tensorboard_verbose=3)
return model
def build_model(self, metadata_path=None, embedding_weights=None):
self.embedding_weights, self.config = ops.embedding_layer(
metadata_path[0], embedding_weights[0])
self.pos_embedding_weights, self.config = ops.embedding_layer(
metadata_path[1], embedding_weights[1], name='pos_embedding')
self.embedded_input = tf.nn.embedding_lookup(self.embedding_weights,
self.input)
self.embedded_pos = tf.nn.embedding_lookup(self.pos_embedding_weights,
self.pos)
self.merged_input = tf.concat([self.embedded_input, self.embedded_pos],
axis=-1)
cells_fw, cells_bw = [], []
for layer in range(self.args['rnn_layers']):
cells_fw.append(
tf.contrib.rnn.LSTMCell(self.args['hidden_units'],
state_is_tuple=True))
cells_bw.append(
tf.contrib.rnn.LSTMCell(self.args['hidden_units'],
state_is_tuple=True))
self.rnn_output, _, _ = stack_bidirectional_rnn(
cells_fw,
cells_bw,
tf.unstack(tf.transpose(self.merged_input, perm=[1, 0, 2])),
dtype=tf.float32,
sequence_length=self.input_lengths)
weight, bias = self.weight_and_bias(2 * self.args['hidden_units'],
self.args['n_classes'])
self.rnn_output = tf.reshape(
tf.transpose(tf.stack(self.rnn_output), perm=[1, 0, 2]),
[-1, 2 * self.args['hidden_units']])
self.rnn_output = dropout(self.rnn_output,
keep_prob=self.args['dropout'])
logits = tf.matmul(self.rnn_output, weight) + bias
prediction = tf.nn.softmax(logits)
self.prediction = tf.reshape(
prediction,
[-1, self.args.get("sequence_length"), self.args['n_classes']])
open_targets = tf.reshape(self.output, [-1, self.args['n_classes']])
with tf.name_scope("loss"):
#self.loss = self.cost()
self.loss = tf.losses.softmax_cross_entropy(open_targets, logits)
if self.args["l2_reg_beta"] > 0.0:
self.regularizer = ops.get_regularizer(
self.args["l2_reg_beta"])
self.loss = tf.reduce_mean(self.loss + self.regularizer)
with tf.name_scope('accuracy'):
self.correct_prediction = tf.equal(tf.argmax(prediction, 1),
tf.argmax(open_targets, 1))
self.accuracy = tf.reduce_mean(
tf.cast(self.correct_prediction, tf.float32))
def res18_forward(incoming, scope=None, name="resnet_18", reuse=False):
with tf.variable_scope(scope, default_name=name, reuse=reuse):
network = conv_2d(incoming, 32, 5, 2, name="conv1",)
network = residual_block(network, 2, 32, downsample=True, batch_norm=True, name="rb1")
network = residual_block(network, 2, 64, downsample=True, batch_norm=True, name="rb2")
network = residual_block(network, 2, 128, downsample=True, batch_norm=True, name="rb3")
network = residual_block(network, 2, 256, downsample=True, batch_norm=True, name="rb4")
network = dropout(network, 0.6)
network = relu(batch_normalization(fully_connected(network, 128, name="fc1")))
network = fully_connected(network, 1, name="fc2")
return network
def build_model(self, metadata_path=None, embedding_weights=None):
self.embedding_weights, self.config = ops.embedding_layer(
metadata_path, embedding_weights)
self.embedded = tf.nn.embedding_lookup(self.embedding_weights,
self.input)
self.lstm_out = ops.lstm_block(
self.embedded,
self.args["hidden_units"],
dropout=self.args["dropout"],
layers=self.args["rnn_layers"],
dynamic=False,
bidirectional=self.args["bidirectional"])
self.dense1 = fully_connected(self.lstm_out, 128)
dropped_out = dropout(self.dense1, keep_prob=0.8)
self.dense2 = fully_connected(dropped_out, 128)
dropped_out = dropout(self.dense2, keep_prob=0.8)
self.out = tf.squeeze(fully_connected(dropped_out, 1))
with tf.name_scope("loss"):
#self.loss = self.cost()
self.loss = losses.mean_squared_error(self.input_sim, self.out)
if self.args["l2_reg_beta"] > 0.0:
self.regularizer = ops.get_regularizer(
self.args["l2_reg_beta"])
self.loss = tf.reduce_mean(self.loss + self.regularizer)
# Compute some Evaluation Measures to keep track of the training process
with tf.name_scope("Pearson_correlation"):
self.pco, self.pco_update = tf.contrib.metrics.streaming_pearson_correlation(
self.out, self.input_sim, name="pearson")
# Compute some Evaluation Measures to keep track of the training process
with tf.name_scope("MSE"):
self.mse, self.mse_update = tf.metrics.mean_squared_error(
self.input_sim, self.out, name="mse")
X, Y, test_x, test_y = mnist.load_data(one_hot=True)
X = X.reshape([-1, 28, 28, 1])
test_x = test_x.reshape([-1, 28, 28, 1])
convnet = input_data(shape=[None, 28, 28, 1], name='input')
convnet = conv_2d(convnet, 32, 2, activation='relu')
convnet = max_pool_2d(convnet, 2)
convnet = conv_2d(convnet, 64, 2, activation='relu')
convnet = max_pool_2d(convnet, 2)
convnet = fully_connected(convnet, 1024, activation='relu')
convnet = dropout(convnet, 0.8)
convnet = fully_connected(convnet, 10, activation='softmax')
convnet = regression(convnet,
optimizer='adam',
learning_rate=0.01,
loss='categorical_crossentropy',
name='targets')
model = tflearn.DNN(convnet)
model.fit({'input': X}, {'targets': Y},
n_epoch=3,
validation_set=({
'input': test_x
def __init__(self,
max_document_length,
num_classes=2,
num_characters=71,
char_vec_size=16,
weight_decay=2e-4,
optimizer='sgd',
dropout=None,
num_blocks=None):
self.input_text = layers.input_data((None, max_document_length))
self.target_label = tf.placeholder(shape=(None, num_classes),
dtype=tf.float32)
embeded_text = layers.embedding(self.input_text, num_characters,
char_vec_size)
mask = tf.cast(tf.not_equal(self.input_text, 0), tf.float32)
embeded_text = embeded_text * tf.expand_dims(mask, 2)
self.embeded_text = embeded_text
top_feature = embeded_text
filters = 64
if num_blocks[0] == 0:
self.block = (1, 1, 1, 1)
else:
self.block = num_blocks
for i, num_block in enumerate(self.block):
if i > 0:
filters *= 2
top_feature = layers.max_pool_1d(top_feature,
3,
strides=2,
padding='same')
for block_i in range(num_block):
top_feature = self.conv_block(top_feature, filters)
pooled_feature = layers.flatten(
layers.custom_layer(top_feature, self.kmax_pool_1d))
if dropout is not None:
pooled_feature = layers.dropout(pooled_feature, dropout)
fc1 = layers.fully_connected(pooled_feature,
2048,
activation='relu',
regularizer='L2',
weight_decay=weight_decay)
if dropout is not None:
fc1 = layers.dropout(fc1, dropout)
fc2 = layers.fully_connected(fc1,
2048,
activation='relu',
regularizer='L2',
weight_decay=weight_decay)
self.probas = layers.fully_connected(fc2,
num_classes,
activation='softmax',
regularizer='L2',
weight_decay=weight_decay)
def build_sgd(learning_rate):
step_tensor = tf.Variable(0.,
name="Training_step",
trainable=False)
steps = [-1.0, 16000.0, 24000.0]
lrs = [1e-1, 1e-2, 1e-3]
lr = tf.reduce_min(
tf.cast(tf.less(step_tensor, steps), tf.float32) + lrs)
tflearn.helpers.summarizer.summarize(
lr, 'scalar', 'lr', 'Optimizer_training_summaries')
return tf.train.MomentumOptimizer(learning_rate=lr,
momentum=0.9), step_tensor
if optimizer == 'sgd':
optimizer = build_sgd
self.train_op = layers.regression(self.probas,
optimizer=optimizer,
learning_rate=0.001,
placeholder=self.target_label)