def discriminator_forward(config,
labels,
incoming,
scope="discriminator",
name=None,
reuse=False):
with tf.variable_scope(scope, name, reuse=reuse):
output = leaky_relu(
batch_normalization(
conv_2d(incoming, config.dim, 5, 2, name="conv1"), 0.2))
output = leaky_relu(
batch_normalization(
conv_2d(output, 2 * config.dim, 5, 2, name="conv2"), 0.2))
output = leaky_relu(
batch_normalization(
conv_2d(output, 4 * config.dim, 5, 2, name="Conv3"), 0.2))
output = tf.reshape(output, [-1, 4 * 4 * 4 * config.dim])
output = fully_connected(output, 56 * config.dim, name="fc1_1")
embed = fully_connected(labels, 8 * config.dim, name="fc1_2")
output = leaky_relu(
batch_normalization(tf.concat([output, embed], axis=-1)), 0.2)
output = fully_connected(output, 8 * config.dim, name="fc2")
output = batch_normalization(output)
output = leaky_relu(output, 0.2)
output = tf.reshape(fully_connected(output, 1, bias=False, name="fc3"),
[-1])
return output
def code_classifier_forward(config,
incoming=None,
image=None,
scope="code_classifier",
name=None,
reuse=False):
with tf.variable_scope(scope, name, reuse=reuse):
output = relu(fully_connected(incoming, 512))
output1 = dropout(output, 0.8)
print(config.batch_size, image.shape)
output = relu(
fully_connected(
tf.reshape(image, [config.batch_size, 32 * 32 * 3]), 512))
output2 = dropout(output, 0.8)
output = tf.concat([output1, output2], axis=-1)
output = relu(fully_connected(output, 1024))
output = dropout(output, 0.5)
output = relu(fully_connected(output, 512))
output = dropout(output, 0.8)
output = fully_connected(output, 10)
return output
def disc_func(args, reuse=False):
# import pdb; pdb.set_trace()
"""Discriminator function """
with tf.variable_scope("discriminator", reuse=reuse):
inp = tf.concat(args, axis=2)
inp = fully_connected(inp, 10, activation='elu')
inp = fully_connected(inp, n_samples, activation='sigmoid')
return [inp]
def gen_func(args):
"""Generator function"""
with tf.variable_scope("generator", reuse=False):
inp = tf.concat(args, axis=1)
inp = fully_connected(inp, 1, activation='elu')
inp = batch_normalization(inp)
inp = fully_connected(inp, 1, activation='elu')
inp = batch_normalization(inp)
return [inp]
def disc_func(args, reuse=False):
# import pdb; pdb.set_trace()
"""Discriminator function """
with tf.variable_scope("discriminator", reuse=reuse):
inp = tf.concat(args, axis=2)
# inp = tf.Print(inp, [inp[0]], message="inp to disc", summarize=100)
# inp = fully_connected(inp, 20, activation='elu')
inp = fully_connected(inp, 10, activation='elu')
inp = fully_connected(inp, n_samples, activation='sigmoid')
return [inp]
def gen_func(args):
"""Generator function"""
with tf.variable_scope("generator", reuse=False):
# inp = tf.concat(args, axis=1
shapes = [info[port]['shape'] for port in inv.param_ports()]
inp = args[0]
inp = fully_connected(inp, 2, activation='elu')
return [
fully_connected(inp, shape[1], activation='elu')
for shape in shapes
]
def res18_forward(incoming, scope=None, name="resnet_18", reuse=False):
with tf.variable_scope(scope, default_name=name, reuse=reuse):
network = conv_2d(incoming, 32, 5, 2, name="conv1",)
network = residual_block(network, 2, 32, downsample=True, batch_norm=True, name="rb1")
network = residual_block(network, 2, 64, downsample=True, batch_norm=True, name="rb2")
network = residual_block(network, 2, 128, downsample=True, batch_norm=True, name="rb3")
network = residual_block(network, 2, 256, downsample=True, batch_norm=True, name="rb4")
network = relu(batch_normalization(fully_connected(network, 256, name="fc1")))
network = fully_connected(network, 5, name="fc2")
return network
def disc_func(args):
"""Discriminator function"""
with tf.variable_scope("discriminator", reuse=False):
assert len(args) == 1
inp = args[0]
inp = fully_connected(inp, 5, activation='elu')
# inp = batch_normalization(inp)
inp = fully_connected(inp, 5, activation='elu')
# inp = batch_normalization(inp)
inp = args[0]
inp = fully_connected(inp, n_samples, activation='sigmoid')
return [inp]
def __init__(self,
max_document_length,
num_classes=2,
num_characters=71,
num_blocks=None,
char_vec_size=16,
weight_decay=2e-4):
self.input_text = layers.input_data((None, max_document_length))
self.target_label = tf.placeholder(shape=(None, num_classes),
dtype=tf.float32)
embeded_text = layers.embedding(self.input_text, num_characters,
char_vec_size)
top_feature = embeded_text
filters = 64
if num_blocks[0] == 0:
self.block = (2, 2, 2, 2)
else:
self.block = num_blocks
for i, num_block in enumerate(self.block):
if i > 0:
filters *= 2
top_feature = layers.max_pool_1d(top_feature,
3,
strides=2,
padding='same')
for block_i in range(num_block):
top_feature = self.conv_block(top_feature, filters)
pooled_feature = layers.flatten(
layers.custom_layer(top_feature, self.kmax_pool_1d))
fc1 = layers.fully_connected(pooled_feature,
2048,
activation='relu',
regularizer='L2',
weight_decay=weight_decay)
fc2 = layers.fully_connected(fc1,
2048,
activation='relu',
regularizer='L2',
weight_decay=weight_decay)
self.probas = layers.fully_connected(fc2,
num_classes,
activation='softmax',
regularizer='L2',
weight_decay=weight_decay)
self.train_op = layers.regression(self.probas,
placeholder=self.target_label)
def g(y, z):
"""Generator"""
with tf.name_scope("generator"):
with tf.variable_scope("generator"):
# y = tf.expand_dims(y, 1)
# inp = tf.concat([y, z], axis=1)
inp = y
inp = fully_connected(inp, 10, activation='elu')
# inp = batch_normalization(inp)
inp = fully_connected(inp, 10, activation='elu')
# inp = batch_normalization(inp)
inp = fully_connected(inp, x_len, activation='elu')
# inp = batch_normalization(inp)
return inp
def __init__(self, sequence_length, num_classes, embeddings, num_filters, l2_reg_lambda=0.0, dropout=None):
self.input_text = layers.input_data( (None, sequence_length), dtype=tf.int32)
with tf.variable_scope('Embedding'):
embeddings_var = tf.Variable(embeddings, name='W', dtype=tf.float32)
embeddings_var = tf.concat([np.zeros((1, embeddings.shape[1]) ), embeddings_var[1:] ] , axis = 0)
self.embeded_text = tf.gather(embeddings_var, self.input_text)
net = self.embeded_text
self.mask = tf.expand_dims(tf.cast(tf.not_equal(self.input_text, 0), tf.float32), axis = 2)
if dropout is not None:
dropout = map(float, dropout.split(',') )
for num_filter in num_filters:
net = layers.lstm(net, num_filter, return_seq=True, dropout=dropout)
net = tf.transpose(tf.stack(net), (1, 0, 2) )
features = tf.reduce_sum(net * self.mask, axis=1) / (tf.reduce_sum(self.mask, axis=1) + 1e-5)
self.probas = layers.fully_connected(features, num_classes, activation='softmax', regularizer='L2', weight_decay=l2_reg_lambda)
optimizer = tflearn.optimizers.Adam(learning_rate=0.001)
self.train_op = layers.regression(
self.probas,
optimizer=optimizer,
batch_size=128)
def transform_embedded_sequences(self, embedded_sequences):
drop_1, drop_2 = self.dropout_rates
net = dropout(embedded_sequences, drop_1)
conv_blocks = []
for sz in self.filter_sizes:
conv = conv_1d(net,
nb_filter=self.num_filters,
filter_size=sz,
padding="valid",
activation="relu",
regularizer="L2")
conv_blocks.append(conv)
net = merge(conv_blocks, mode='concat',
axis=1) if len(conv_blocks) > 1 else conv_blocks[0]
net = tf.expand_dims(net, 2)
net = global_max_pool(net)
net = dropout(net, drop_2)
model_output = fully_connected(net,
self.class_count,
activation="softmax")
return model_output
def __init__(self, sequence_length, num_classes, embeddings, num_filters, l2_reg_lambda=0.0, dropout=None, bn=False):
self.input_text = layers.input_data( (None, sequence_length), dtype=tf.int32)
with tf.variable_scope('Embedding'):
embeddings_var = tf.Variable(embeddings, name='W', dtype=tf.float32)
embeddings_var = tf.concat([np.zeros((1, embeddings.shape[1]) ), embeddings_var[1:] ] , axis = 0)
self.embeded_text = tf.gather(embeddings_var, self.input_text)
net = self.embeded_text
for num_filter in num_filters:
if bn:
# , weights_init=tflearn.initializations.uniform(minval=-0.001, maxval=0.001)
net = layers.conv_1d(net, num_filter, 3, padding='valid', activation='linear', bias=False)
net = layers.batch_normalization(net)
net = layers.activation(net, 'relu')
else:
net = layers.conv_1d(net, num_filter, 3, padding='valid', activation='relu', bias=True, regularizer='L2', weight_decay=l2_reg_lambda)
if dropout is not None:
net = layers.dropout(net, float(dropout) )
features = layers.flatten( layers.max_pool_1d(net, net.shape.as_list()[1], padding='valid') )
self.probas = layers.fully_connected(features, num_classes, activation='softmax', regularizer='L2', weight_decay=l2_reg_lambda)
#optimizer = tflearn.optimizers.Momentum(learning_rate=0.1, momentum=0.9, lr_decay=0.2, decay_step=1000, staircase=True)
optimizer = tflearn.optimizers.Adam(learning_rate=0.001)
self.train_op = layers.regression(
self.probas,
optimizer=optimizer,
batch_size=128)
def disc_func(args):
"""Discriminator function"""
with tf.variable_scope("discriminator", reuse=False):
assert len(args) == 1
inp = args[0]
l1 = fully_connected(inp, n_samples, activation='sigmoid')
return [l1]
def discriminator_forward(config,
incoming,
labels,
scope="discriminator",
name=None,
reuse=False):
with tf.variable_scope(scope, name, reuse=reuse):
output = leaky_relu(
batch_normalization(
conv_2d(incoming, config.dim, 5, 2, name="conv1")), 0.2)
output = leaky_relu(
batch_normalization(
conv_2d(output, 2 * config.dim, 5, 2, name="conv2")), 0.2)
output_shared = conv_2d(output,
2 * config.dim,
5,
2,
name="conv3_shared")
output_cs = [
conv_2d(output, 2 * config.dim, 5, 2, name="conv3_cs")
for _ in xrange(5)
]
output = tf.concat([output_cs, output_shared])
output = tf.reshape(output, [-1, 4 * 4 * 4 * config.dim])
output = tf.reshape(fully_connected(output, 1, bias=False), [-10])
def gen_func(args, reuse=False):
# import pdb; pdb.set_trace()
"""Generator function"""
with tf.variable_scope("generator", reuse=reuse):
inp = tf.concat(args, axis=1)
inp = fully_connected(inp, nitems, activation='elu')
inps = tf.split(inp, axis=1, num_or_size_splits=nitems)
return inps
def g_pi(y, z):
"""Parametric Inverse Generator"""
with tf.name_scope("generator"):
with tf.variable_scope("generator"):
theta_len = 1
# the neural network will take as input z, and output
# the two parameters for
inp = z
inp = fully_connected(inp, 20, activation='elu')
inp = batch_normalization(inp)
inp = fully_connected(inp, 20, activation='elu')
inp = batch_normalization(inp)
theta = fully_connected(inp, theta_len, activation='elu')
theta = batch_normalization(theta)
x_1 = tf.expand_dims(y, 1) - theta
x_2 = theta
x = tf.concat([x_1, x_2], 1)
return x
def score_net(inputs):
state = inputs[0]
curr_layer = state
layers = []
curr_layer = conv_2d_layer(curr_layer, 8, 1)
curr_layer = batch_normalization(curr_layer)
curr_layer = conv_2d_layer(curr_layer, 2, 4)
curr_layer = batch_normalization(curr_layer)
curr_layer = fully_connected(curr_layer, 1, activation='elu')
curr_layer = batch_normalization(curr_layer)
return [curr_layer]
def discriminator_net(inputs):
field = inputs[0]
curr_layer = field
layers = []
curr_layer = tf.reshape(curr_layer, (-1, 16, 16, 1))
curr_layer = conv_2d_layer(curr_layer, 16, 1)
curr_layer = batch_normalization(curr_layer)
curr_layer = fully_connected(curr_layer, 1)
curr_layer = batch_normalization(curr_layer)
curr_layer = tflearn.activations.sigmoid(curr_layer)
return [curr_layer]
def disc(x, y, reuse, use_y=False):
"""Discriminator"""
with tf.name_scope("discriminator"):
with tf.variable_scope("discriminator", reuse=reuse):
if use_y:
inp = tf.concat([x, tf.expand_dims(y, 1)], 1)
else:
inp = x
# import pdb; pdb.set_trace()
# inp = fully_connected(inp, 3, activation='elu')
out = fully_connected(inp, 1, activation='sigmoid')
return out
def gen_func(args, reuse=False):
# import pdb; pdb.set_trace()
"""Generator function"""
with tf.variable_scope("generator", reuse=reuse):
inp = tf.concat(args, axis=1)
# inp = fully_connected(inp, 10, activation='elu')
inp = fully_connected(inp, inv.num_param_ports(), activation='elu')
inps = tf.split(inp,
axis=1,
num_or_size_splits=inv.num_param_ports())
# inps = [tf.Print(inp, [inp[0]], message="Generated!", summarize=100) for inp in inps]
return inps
def nn_model(input_size):
# same implementation with keras
# model = Sequential()
# model.add(Dense(128, input_shape=size, activation='relu'))
network = input_data(shape=[None, input_size, 1], name='input')
network = fully_connected(network, 128, activation='relu')
network = dropout(network,
0.8) # meaning 0.8 will be kept, opposite in keras
network = fully_connected(network, 256, activation='relu')
network = dropout(network, 0.8)
network = fully_connected(network, 512, activation='relu')
network = dropout(network, 0.8)
network = fully_connected(network, 256, activation='relu')
network = dropout(network, 0.8)
network = fully_connected(network, 512, activation='relu')
network = dropout(network, 0.8)
network = fully_connected(network, 2, activation='softmax')
network = regression(network,
optimizer='adam',
learning_rate=LR,
loss='categorical_crossentropy',
name='targets')
model = tflearn.DNN(network, tensorboard_dir='log')
return model
def code_classifier_forward(config, incoming=None, image=None,
scope="code_classifier", name=None, reuse=False):
with tf.variable_scope(scope, name, reuse=reuse):
code_output = leaky_relu(fully_connected(incoming, 512))
output = leaky_relu(fully_connected(tf.reshape(image, [config.batch_size, 28 * 28]), 512))
prod = tf.matmul(code_output[:, :, None], output[:, None, :])
prob = tf.nn.softmax(prod)
prob2 = tf.nn.softmax(tf.transpose(prod, perm=[0, 2, 1]))
output = tf.concat([code_output,
tf.matmul(prob, output[:, :, None])[:, :, 0],
tf.matmul(prob2, code_output[:, :, None])[:, :, 0]], axis=-1)
output = relu(fully_connected(output, 1024))
output = dropout(output, 0.6)
output = relu(fully_connected(output, 512))
output = dropout(output, 0.6)
output = relu(fully_connected(output, 256))
output = dropout(output, 0.8)
output = fully_connected(output, 10)
return output
def generator_forward(config,
labels,
noise=None,
scope="generator",
name=None,
reuse=False,
num_samples=-1):
with tf.variable_scope(scope, name, reuse=reuse):
if noise is None:
noise = tf.random_normal(
[config.batch_size if num_samples == -1 else num_samples, 128],
name="noise")
embed = fully_connected(labels, 8 * config.dim)
noise = fully_connected(noise, 56 * config.dim)
cat = relu(batch_normalization(tf.concat([embed, noise], axis=-1)))
output = fully_connected(cat, 4 * 4 * 4 * config.dim)
output = batch_normalization(output)
output = tf.nn.relu(output)
output = tf.reshape(output, [-1, 4, 4, 4 * config.dim])
output = conv_2d_transpose(output,
2 * config.dim,
5, [8, 8],
strides=2)
output = output[:, :7, :7, :]
output = batch_normalization(output)
output = relu(output)
output = conv_2d_transpose(output, config.dim, 5, [14, 14], strides=2)
output = batch_normalization(output)
output = tf.nn.relu(output)
output = conv_2d_transpose(output, 1, 5, [28, 28], strides=2)
output = tf.tanh(output)
return output
def code_classifier_forward(config,
incoming=None,
image=None,
scope="code_classifier",
name=None,
reuse=False):
with tf.variable_scope(scope, name, reuse=reuse):
code_output = leaky_relu(fully_connected(incoming, 512))
output = conv_2d(image, 32, 5, 2, name="conv1")
output = residual_block(output,
2,
32,
downsample=True,
batch_norm=True,
name="rb1")
output = residual_block(output,
1,
64,
downsample=True,
batch_norm=True,
name="rb2")
output = leaky_relu(
fully_connected(
tf.reshape(output, [config.batch_size, 4 * 4 * 64]), 1024))
prod = tf.matmul(code_output[:, :, None], output[:, None, :])
prob = tf.nn.softmax(prod)
prob2 = tf.nn.softmax(tf.transpose(prod, perm=[0, 2, 1]))
output = tf.concat([
code_output,
tf.matmul(prob, output[:, :, None])[:, :, 0],
tf.matmul(prob2, code_output[:, :, None])[:, :, 0]
],
axis=-1)
output = relu(fully_connected(output, 1024))
output = dropout(output, 0.6)
output = relu(fully_connected(output, 512))
output = dropout(output, 0.6)
output = relu(fully_connected(output, 256))
output = dropout(output, 0.8)
output = fully_connected(output, 5)
return output
import tflearn.datasets.mnist as mnist
X, Y, test_x, test_y = mnist.load_data(one_hot=True)
X = X.reshape([-1, 28, 28, 1])
test_x = test_x.reshape([-1, 28, 28, 1])
convnet = input_data(shape=[None, 28, 28, 1], name='input')
convnet = conv_2d(convnet, 32, 2, activation='relu')
convnet = max_pool_2d(convnet, 2)
convnet = conv_2d(convnet, 64, 2, activation='relu')
convnet = max_pool_2d(convnet, 2)
convnet = fully_connected(convnet, 1024, activation='relu')
convnet = dropout(convnet, 0.8)
convnet = fully_connected(convnet, 10, activation='softmax')
convnet = regression(convnet,
optimizer='adam',
learning_rate=0.01,
loss='categorical_crossentropy',
name='targets')
model = tflearn.DNN(convnet)
model.fit({'input': X}, {'targets': Y},
n_epoch=3,
validation_set=({
def __init__(self,
max_document_length,
num_classes=2,
num_characters=71,
char_vec_size=16,
weight_decay=2e-4,
optimizer='sgd',
dropout=None,
num_blocks=None):
self.input_text = layers.input_data((None, max_document_length))
self.target_label = tf.placeholder(shape=(None, num_classes),
dtype=tf.float32)
embeded_text = layers.embedding(self.input_text, num_characters,
char_vec_size)
mask = tf.cast(tf.not_equal(self.input_text, 0), tf.float32)
embeded_text = embeded_text * tf.expand_dims(mask, 2)
self.embeded_text = embeded_text
top_feature = embeded_text
filters = 64
if num_blocks[0] == 0:
self.block = (1, 1, 1, 1)
else:
self.block = num_blocks
for i, num_block in enumerate(self.block):
if i > 0:
filters *= 2
top_feature = layers.max_pool_1d(top_feature,
3,
strides=2,
padding='same')
for block_i in range(num_block):
top_feature = self.conv_block(top_feature, filters)
pooled_feature = layers.flatten(
layers.custom_layer(top_feature, self.kmax_pool_1d))
if dropout is not None:
pooled_feature = layers.dropout(pooled_feature, dropout)
fc1 = layers.fully_connected(pooled_feature,
2048,
activation='relu',
regularizer='L2',
weight_decay=weight_decay)
if dropout is not None:
fc1 = layers.dropout(fc1, dropout)
fc2 = layers.fully_connected(fc1,
2048,
activation='relu',
regularizer='L2',
weight_decay=weight_decay)
self.probas = layers.fully_connected(fc2,
num_classes,
activation='softmax',
regularizer='L2',
weight_decay=weight_decay)
def build_sgd(learning_rate):
step_tensor = tf.Variable(0.,
name="Training_step",
trainable=False)
steps = [-1.0, 16000.0, 24000.0]
lrs = [1e-1, 1e-2, 1e-3]
lr = tf.reduce_min(
tf.cast(tf.less(step_tensor, steps), tf.float32) + lrs)
tflearn.helpers.summarizer.summarize(
lr, 'scalar', 'lr', 'Optimizer_training_summaries')
return tf.train.MomentumOptimizer(learning_rate=lr,
momentum=0.9), step_tensor
if optimizer == 'sgd':
optimizer = build_sgd
self.train_op = layers.regression(self.probas,
optimizer=optimizer,
learning_rate=0.001,
placeholder=self.target_label)
