def generator(hparams, z, train, reuse):
with tf.variable_scope("generator") as scope:
if reuse:
tf.get_variable_scope().reuse_variables()
s_h, s_w = hparams.output_height, hparams.output_width
s_h2, s_h4, s_h8, s_h16 = int(s_h/2), int(s_h/4), int(s_h/8), int(s_h/16)
s_w2, s_w4, s_w8, s_w16 = int(s_w/2), int(s_w/4), int(s_w/8), int(s_w/16)
g_bn0 = ops.batch_norm(name='g_bn0')
g_bn1 = ops.batch_norm(name='g_bn1')
g_bn2 = ops.batch_norm(name='g_bn2')
g_bn3 = ops.batch_norm(name='g_bn3')
# project `z` and reshape
h0 = tf.reshape(ops.linear(z, hparams.gf_dim*8*s_h16*s_w16, 'g_h0_lin'), [-1, s_h16, s_w16, hparams.gf_dim * 8])
h0 = tf.nn.relu(g_bn0(h0, train=train))
h1 = ops.deconv2d(h0, [hparams.batch_size, s_h8, s_w8, hparams.gf_dim*4], name='g_h1')
h1 = tf.nn.relu(g_bn1(h1, train=train))
h2 = ops.deconv2d(h1, [hparams.batch_size, s_h4, s_w4, hparams.gf_dim*2], name='g_h2')
h2 = tf.nn.relu(g_bn2(h2, train=train))
h3 = ops.deconv2d(h2, [hparams.batch_size, s_h2, s_w2, hparams.gf_dim*1], name='g_h3')
h3 = tf.nn.relu(g_bn3(h3, train=train))
h4 = ops.deconv2d(h3, [hparams.batch_size, s_h, s_w, hparams.c_dim], name='g_h4')
x_gen = tf.nn.tanh(h4)
return x_gen
def __call__(self, x, is_reuse=False, is_train=True):
with tf.variable_scope('generator') as scope:
if is_reuse:
scope.reuse_variables()
unit_size = self.img_size[0] // (2 ** self.layer_n)
unit_n = self.smallest_hidden_unit_n * (2 ** (self.layer_n - 1))
batch_size = int(x.shape[0])
with tf.variable_scope('pre'):
x = linear(x, unit_size * unit_size * unit_n)
x = tf.reshape(x, (batch_size, unit_size, unit_size, unit_n))
if self.is_bn:
x = batch_norm(x, is_train)
x = tf.nn.relu(x)
for i in range(self.layer_n):
with tf.variable_scope('layer{}'.format(i)):
if i == self.layer_n - 1:
unit_n = self.img_dim
else:
unit_n = self.smallest_hidden_unit_n * (2 ** (self.layer_n - i - 2))
x_shape = x.get_shape().as_list()
x = tf.image.resize_bilinear(x, (x_shape[1] * 2, x_shape[2] * 2))
x = conv2d(x, unit_n, self.k_size, 1, 'SAME')
if i != self.layer_n - 1:
if self.is_bn:
x = batch_norm(x, is_train)
x = tf.nn.relu(x)
x = tf.nn.tanh(x)
return x
def discriminator(hparams, x, train, reuse):
if reuse:
tf.get_variable_scope().reuse_variables()
d_bn1 = ops.batch_norm(name='d_bn1')
d_bn2 = ops.batch_norm(name='d_bn2')
d_bn3 = ops.batch_norm(name='d_bn3')
h0 = ops.lrelu(ops.conv2d(x, hparams.df_dim, name='d_h0_conv'))
h1 = ops.conv2d(h0, hparams.df_dim * 2, name='d_h1_conv')
h1 = ops.lrelu(d_bn1(h1, train=train))
h2 = ops.conv2d(h1, hparams.df_dim * 4, name='d_h2_conv')
h2 = ops.lrelu(d_bn2(h2, train=train))
h3 = ops.conv2d(h2, hparams.df_dim * 8, name='d_h3_conv')
h3 = ops.lrelu(d_bn3(h3, train=train))
h4 = ops.linear(tf.reshape(h3, [hparams.batch_size, -1]), 1, 'd_h3_lin')
d_logit = h4
d = tf.nn.sigmoid(d_logit)
return d, d_logit
def eve_model(self, data_input=None, reuse=False):
####### Eve's network #######
with tf.variable_scope("eve") as scope:
if reuse:
scope.reuse_variables()
self.eve_input = data_input
print(self.eve_input)
self.eve0 = lrelu(
conv2d(self.eve_input, self.output_size, name='eve_h0_conv'))
self.eve_bn1 = batch_norm(name='eve_bn1')
self.eve1 = lrelu(
self.eve_bn1(
conv2d(self.eve0, self.output_size * 2,
name='eve_h1_conv')))
self.eve_bn2 = batch_norm(name='eve_bn2')
self.eve2 = lrelu(
self.eve_bn2(
conv2d(self.eve1, self.output_size * 4,
name='eve_h2_conv')))
self.eve_bn3 = batch_norm(name='eve_bn3')
self.eve3 = lrelu(
self.eve_bn3(
conv2d(self.eve2, self.output_size * 8,
name='eve_h3_conv')))
self.eve4 = linear(tf.reshape(self.eve0, [self.batch_size, -1]), 1,
'eve_h3_lin')
return self.eve4, self.eve4
def bob_model(self, data_input_image=None):
####### Bob's network #######
# bob's input
self.bob_input = data_input_image
print(self.bob_input)
self.bob0 = lrelu(
conv2d(self.bob_input, self.output_size, name='bob_h0_conv'))
self.bob_bn1 = batch_norm(name='bob_bn1')
self.bob1 = lrelu(
self.bob_bn1(
conv2d(self.bob0, self.output_size * 2, name='bob_h1_conv')))
self.bob_bn2 = batch_norm(name='bob_bn2')
self.bob2 = lrelu(
self.bob_bn2(
conv2d(self.bob1, self.output_size * 4, name='bob_h2_conv')))
self.bob_bn3 = batch_norm(name='bob_bn3')
self.bob3 = lrelu(
self.bob_bn3(
conv2d(self.bob2, self.output_size * 8, name='bob_h3_conv')))
self.bob4 = linear(tf.reshape(self.bob3, [self.batch_size, -1]),
self.msg_len, 'bob_h3_lin')
return tf.nn.tanh(self.bob4)
def generator(input_z,
t_txt=None,
is_train=True,
reuse=False,
batch_size=batch_size):
g_bn0 = ops.batch_norm(name='g_bn0')
g_bn1 = ops.batch_norm(name='g_bn1')
g_bn2 = ops.batch_norm(name='g_bn2')
g_bn3 = ops.batch_norm(name='g_bn3')
s = image_size # output image size [64]
s2, s4, s8, s16 = int(s / 2), int(s / 4), int(s / 8), int(s / 16)
gf_dim = 128
with tf.variable_scope("generator", reuse=reuse):
tl.layers.set_name_reuse(reuse)
z_concat = tf.concat([input_z, t_txt], 1)
z_ = ops.linear(z_concat, gf_dim * 8 * s16 * s16, 'g_h0_lin')
h0 = tf.reshape(z_, [-1, s16, s16, gf_dim * 8])
h0 = tf.nn.relu(g_bn0(h0))
h1 = ops.deconv2d(h0, [batch_size, s8, s8, gf_dim * 4], name='g_h1')
h1 = tf.nn.relu(g_bn1(h1))
h2 = ops.deconv2d(h1, [batch_size, s4, s4, gf_dim * 2], name='g_h2')
h2 = tf.nn.relu(g_bn2(h2))
h3 = ops.deconv2d(h2, [batch_size, s2, s2, gf_dim * 1], name='g_h3')
h3 = tf.nn.relu(g_bn3(h3))
h4 = ops.deconv2d(h3, [batch_size, s, s, 3], name='g_h4')
return h4, tf.tanh(h4)
def out_block(input_anc,
input_pos,
channels,
laxer_idx,
stride_input=1,
k_size=8,
padding_type='VALID'):
# Last conv layer, flatten the output
weights = ops.weight([k_size, k_size, k_size, channels[0], channels[1]],
layer_name='wcnn' + str(laxer_idx + 1))
bias = ops.bias([1, 1, 1, channels[1]],
layer_name='bcnn' + str(laxer_idx + 1))
conv_output_anc = tf.add(
ops.conv3d(input_anc,
weights,
stride=[stride_input, stride_input, stride_input],
padding=padding_type), bias)
conv_output_pos = tf.add(
ops.conv3d(input_pos,
weights,
stride=[stride_input, stride_input, stride_input],
padding=padding_type), bias)
conv_output_anc = ops.batch_norm(conv_output_anc)
conv_output_pos = ops.batch_norm(conv_output_pos)
conv_output_anc = tf.contrib.layers.flatten(conv_output_anc)
conv_output_pos = tf.contrib.layers.flatten(conv_output_pos)
return conv_output_anc, conv_output_pos
def generator(self, z, y, is_training=True, reuse=False):
with tf.variable_scope('generator', values=[z], reuse=reuse):
z = tf.concat([z, y], axis=1)
net = tf.layers.dense(z,
units=self.s_size * self.s_size *
self.g_depth)
net = tf.reshape(net, [-1, self.s_size, self.s_size, self.g_depth])
net = tf.nn.relu(batch_norm(net, is_training))
shortcut = net
#define 16 res block, do not change the feature map size and number
for i in range(16):
net = self.g_block(net, is_training)
net = tf.nn.relu(batch_norm(net, is_training))
net = tf.add(net, shortcut)
#sub-pixel, convolution, upsampling
for i in range(self.num_layers):
net = conv2d(net, self.g_depth * 4, 3, 1, use_bias=False)
net = tf.depth_to_space(
net,
2) #change from imgH * imgW * 256 to 2*imgH * 2*imgW * 64
net = tf.nn.relu(batch_norm(net, is_training))
net = tf.layers.conv2d(net,
filters=self.c_dim,
kernel_size=5,
strides=1,
padding='SAME',
activation=tf.nn.tanh,
use_bias=True)
return net
def batch_norm_test():
with tf.device('/' + FLAGS.device + ":0"):
input_tensor = tf.Variable(initial_value=tf.truncated_normal([4, 3, 3, 2], mean=0.1, dtype=dtype))
mean = tf.Variable(initial_value=tf.truncated_normal([2], mean=0.1, dtype=dtype))
variance = tf.Variable(initial_value=tf.square(tf.truncated_normal([2], mean=0.1, dtype=dtype)))
offset = tf.Variable(initial_value=tf.truncated_normal([2], mean=0.1, dtype=dtype))
scale = tf.Variable(initial_value=tf.truncated_normal([2], mean=0.1, dtype=dtype))
param_1 = tf.constant(np.random.rand(4, 3, 3, 2), dtype=dtype)
param_2 = tf.constant(np.random.rand(4 * 3 * 3, 2), dtype=dtype)
output_tensor = ops.batch_norm(input_tensor, mean, variance, offset, scale, 1e-6)
output_tensor = tf.reshape(output_tensor * param_1, [-1, 2])
output_tensor = ops.batch_norm(output_tensor, mean, variance, offset, scale, 1e-6)
recv = tf.reduce_sum(output_tensor * param_2)
grads = tf.gradients(recv, [input_tensor, mean, variance, offset, scale])
output_tensor_ = tf.nn.batch_normalization(input_tensor, mean, variance, offset, scale, 1e-6)
output_tensor_ = tf.reshape(output_tensor_ * param_1, [-1, 2])
output_tensor_ = ops.batch_norm(output_tensor_, mean, variance, offset, scale, 1e-6)
recv_ = tf.reduce_sum(output_tensor_ * param_2)
grads_ = tf.gradients(recv_, [input_tensor, mean, variance, offset, scale])
with tf.Session() as sess:
tf.global_variables_initializer().run()
assert np.max(np.abs(sess.run(output_tensor) - sess.run(output_tensor_))) < 1e-5
for g, g_ in zip(grads, grads_):
assert np.max(np.abs(sess.run(g) - sess.run(g_))) < 2e-4
def get_discriminator_net(self, name, images, reuse=False):
batch_size = self.get_batch_size()
with tf.variable_scope("discriminator", reuse=reuse) as scope:
# concat
yb = tf.reshape(self.labels, shape=[batch_size, 1, 1, -1])
net = ops.conv_cond_concat(images, yb)
# TODO: why is output_dim is self.data.get_number_of_labels() here??
net, w1 = ops.conv2d(
net,
self.data.get_number_of_labels(),
4,
4,
2,
2,
scope='dis_conv1',
weights_initializer=self.get_weights_initializer())
tf.add_to_collection('weight_1', w1)
net = ops.lrelu(net)
net = ops.conv_cond_concat(net, yb)
tf.add_to_collection('ac_1', net)
net, w2 = ops.conv2d(
net,
batch_size,
4,
4,
2,
2,
scope='dis_conv2',
weights_initializer=self.get_weights_initializer())
tf.add_to_collection('weight_2', w2)
net = ops.lrelu(
ops.batch_norm(net, self.is_training, scope='dis_bn1'))
tf.add_to_collection('ac_2', net)
net = tf.reshape(net, [batch_size, -1])
net = tf.concat([net, self.labels], 1)
# TODO: figure out what implications the change in "output_size" will have.
net = ops.linear(
net,
output_size=1024,
scope='dis_fully1',
weights_initializer=self.get_weights_initializer())
net = ops.batch_norm(net,
self.is_training,
scope='dis_bn2',
reuse=reuse)
net = ops.lrelu(net)
net = tf.concat([net, self.labels], 1)
out = ops.linear(net,
output_size=1,
scope='dis_fully2',
weights_initializer=xavier_initializer())
return tf.nn.sigmoid(out, name=name), out
def naive_discriminator(self, image, y=None, reuse=False):
with tf.variable_scope("discriminator") as scope:
# image is 128 x 128 x (input_c_dim + output_c_dim)
if reuse:
tf.get_variable_scope().reuse_variables()
else:
assert tf.get_variable_scope().reuse == False
h0 = lrelu(conv2d(image, self.df_dim, name='adv_d_h0_conv'))
# h0 is (128 x 128 x self.df_dim)
h1 = lrelu(
batch_norm((conv2d(h0, self.df_dim * 2, name='adv_d_h1_conv')),
name="adv_d_bn1"))
# h1 is (64 x 64 x self.df_dim*2)
h2 = lrelu(
batch_norm(conv2d(h1, self.df_dim * 4, name='adv_d_h2_conv'),
name="adv_d_bn2"))
# h2 is (32x 32 x self.df_dim*4)
h3 = lrelu(
batch_norm(conv2d(h2,
self.df_dim * 8,
d_h=1,
d_w=1,
name='adv_d_h3_conv'),
name="adv_d_bn3"))
# h3 is (16 x 16 x self.df_dim*8)
h4 = linear(tf.reshape(h3, [self.batch_size, -1]), 1,
'adv_d_h3_lin')
return tf.nn.sigmoid(h4), h4
def discriminator(self, opts, input_, is_training,
prefix='DISCRIMINATOR', reuse=False):
"""Encoder function, suitable for simple toy experiments.
"""
num_filters = opts['d_num_filters']
with tf.variable_scope(prefix, reuse=reuse):
h0 = ops.conv2d(opts, input_, num_filters / 8, scope='h0_conv')
h0 = ops.batch_norm(opts, h0, is_training, reuse, scope='bn_layer1')
h0 = tf.nn.relu(h0)
h1 = ops.conv2d(opts, h0, num_filters / 4, scope='h1_conv')
h1 = ops.batch_norm(opts, h1, is_training, reuse, scope='bn_layer2')
h1 = tf.nn.relu(h1)
h2 = ops.conv2d(opts, h1, num_filters / 2, scope='h2_conv')
h2 = ops.batch_norm(opts, h2, is_training, reuse, scope='bn_layer3')
h2 = tf.nn.relu(h2)
h3 = ops.conv2d(opts, h2, num_filters, scope='h3_conv')
h3 = ops.batch_norm(opts, h3, is_training, reuse, scope='bn_layer4')
h3 = tf.nn.relu(h3)
# Already has NaNs!!
latent_mean = ops.linear(opts, h3, opts['latent_space_dim'], scope='h3_lin')
log_latent_sigmas = ops.linear(opts, h3, opts['latent_space_dim'], scope='h3_lin_sigma')
return latent_mean, log_latent_sigmas
def __init__(self,
sess,
batch_size=32,
image_size=256,
lam=0.8,
checkpoint_dir=None):
self.sess = sess
self.batch_size = batch_size
self.image_size = image_size
self.image_shape = [image_size, image_size, 3]
self.lam = lam
self.checkpoint_dir = checkpoint_dir
self.global_step = tf.Variable(0, trainable=False)
self.images = tf.placeholder(tf.float32, [batch_size] + self.image_shape, name='images')
self.images_summary = tf.summary.image("image", self.images)
self.d_bns = [batch_norm(name='d_bn{}'.format(i)) for i in range(5)]
self.local_d_bns = [batch_norm(name='d_local_bn{}'.format(i)) for i in range(4)]
self.g_bns = [batch_norm(name='g_bn{}'.format(i, )) for i in range(15)]
self.D, self.D_logits = self.discriminator(self.images, self.image_size)
self.d_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(
logits=self.D_logits, labels=tf.ones_like(self.D)))
self.D_summary = tf.summary.histogram("d", self.D)
self.d_loss_real_summary = tf.summary.scalar("d_loss_real", self.d_loss_real)
self.masks = tf.placeholder(tf.float32, [batch_size] + self.image_shape, name='masks')
self.MG = tf.multiply(self.images, self.masks)
self.G = self.generator(self.MG)
self.MG_summary = tf.summary.image("mg", self.MG)
self.G_summary = tf.summary.image("g", self.G)
self.D_fake, self.D_fake_logits = self.discriminator(self.G, self.image_size, reuse=True)
self.d_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(
logits=self.D_fake_logits, labels=tf.zeros_like(self.D_fake)))
self.D_fake_summary = tf.summary.histogram("d_fake", self.D_fake)
self.d_loss_fake_summary = tf.summary.scalar("d_loss_fake", self.d_loss_fake)
self.d_loss = self.d_loss_real + self.d_loss_fake
self.g_loss_d = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(
logits=self.D_fake_logits, labels=tf.ones_like(self.D_fake)))
self.g_loss_l = tf.reduce_mean(tf.contrib.layers.flatten(
tf.multiply(self.G - self.images, self.G - self.images)))
self.g_loss = (1 - self.lam) * self.g_loss_d + self.lam * self.g_loss_l
self.g_loss_d_summary = tf.summary.scalar("g_loss_d", self.g_loss_d)
self.g_loss_l_summary = tf.summary.scalar("g_loss_l", self.g_loss_l)
self.g_loss_summary = tf.summary.scalar("g_loss", self.g_loss)
t_vars = tf.trainable_variables()
self.d_vars = [var for var in t_vars if 'd_' in var.name]
self.g_vars = [var for var in t_vars if 'g_' in var.name]
self.saver = tf.train.Saver(max_to_keep=10)
self.g_summary = tf.summary.merge([
self.G_summary, self.MG_summary, self.D_fake_summary, self.d_loss_fake_summary,
self.g_loss_summary, self.g_loss_d_summary, self.g_loss_l_summary])
self.d_summary = tf.summary.merge([
self.images_summary, self.D_summary, self.d_loss_real_summary])
self.writer = tf.summary.FileWriter(os.path.join(self.checkpoint_dir, "logs"), self.sess.graph)
def _build_model(self):
with tf.variable_scope('discriminator', reuse=self.reuse):
df_dim = self.df_dim
h0 = ops.lrelu(ops.conv2d(
self.image, df_dim, name='d_h0_conv')) # 32
h1 = ops.lrelu(ops.batch_norm(ops.conv2d(
h0, df_dim * 2, name='d_h1_conv'))) # 16
h2 = ops.lrelu(ops.batch_norm(ops.conv2d(
h1, df_dim * 4, name='d_h2_conv'))) # 8
h3 = ops.lrelu(ops.batch_norm(ops.conv2d(
h2, df_dim * 8, name='d_h3_conv'))) # 4
# ADD TEXT EMBEDDING TO THE NETWORK
reduced_text_embeddings = ops.lrelu(ops.linear(
self.text, self.hparas['TEXT_DIM'], 'd_embedding'))
reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings, 1)
reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings, 2)
tiled_embeddings = tf.tile(reduced_text_embeddings, [
1, 4, 4, 1], name='tiled_embeddings')
h3_concat = tf.concat([h3, tiled_embeddings], 3, name='h3_concat')
h3_new = ops.lrelu(ops.batch_norm(ops.conv2d(
h3_concat, df_dim * 8, 1, 1, 1, 1, name='d_h3_conv_new'))) # 4
h4 = ops.linear(tf.reshape(
h3_new, [self.hparas['BATCH_SIZE'], -1]), 1, 'd_h3_lin')
self.logits = h4
self.discriminator_net = tf.nn.sigmoid(h4)
self.outputs = tf.nn.sigmoid(h4)
def generator(hparams, z, scope_name, train, reuse):
with tf.variable_scope(scope_name) as scope:
if reuse:
scope.reuse_variables()
output_size = 64
s = output_size
s2, s4, s8, s16 = int(s/2), int(s/4), int(s/8), int(s/16)
g_bn0 = ops.batch_norm(name='g_bn0')
g_bn1 = ops.batch_norm(name='g_bn1')
g_bn2 = ops.batch_norm(name='g_bn2')
g_bn3 = ops.batch_norm(name='g_bn3')
# project `z` and reshape
h0 = tf.reshape(ops.linear(z, hparams.gf_dim*8*s16*s16, 'g_h0_lin'), [-1, s16, s16, hparams.gf_dim * 8])
h0 = tf.nn.relu(g_bn0(h0, train=train))
h1 = ops.deconv2d(h0, [hparams.batch_size, s8, s8, hparams.gf_dim*4], name='g_h1')
h1 = tf.nn.relu(g_bn1(h1, train=train))
h2 = ops.deconv2d(h1, [hparams.batch_size, s4, s4, hparams.gf_dim*2], name='g_h2')
h2 = tf.nn.relu(g_bn2(h2, train=train))
h3 = ops.deconv2d(h2, [hparams.batch_size, s2, s2, hparams.gf_dim*1], name='g_h3')
h3 = tf.nn.relu(g_bn3(h3, train=train))
h4 = ops.deconv2d(h3, [hparams.batch_size, s, s, hparams.c_dim], name='g_h4')
x_gen = tf.nn.tanh(h4)
return x_gen
def _build_model(self):
with tf.variable_scope('generator', reuse=self.reuse):
s = self.hparas['IMAGE_SIZE'][0]
s2, s4, s8, s16 = int(s / 2), int(s / 4), int(s / 8), int(s / 16)
gf_dim = self.gf_dim
reduced_text_embedding = ops.lrelu(ops.linear(
self.text, self.hparas['TEXT_DIM'], 'g_embedding'))
z_concat = tf.concat([self.z, reduced_text_embedding], 1)
z_ = ops.linear(z_concat, gf_dim * 8 * s16 * s16, 'g_h0_lin')
h0 = tf.reshape(z_, [-1, s16, s16, gf_dim * 8])
h0 = tf.nn.relu(ops.batch_norm(h0))
h1 = ops.deconv2d(h0, [self.hparas['BATCH_SIZE'],
s8, s8, gf_dim * 4], name='g_h1')
h1 = tf.nn.relu(ops.batch_norm(h1))
h2 = ops.deconv2d(h1, [self.hparas['BATCH_SIZE'],
s4, s4, gf_dim * 2], name='g_h2')
h2 = tf.nn.relu(ops.batch_norm(h2))
h3 = ops.deconv2d(h2, [self.hparas['BATCH_SIZE'],
s2, s2, gf_dim * 1], name='g_h3')
h3 = tf.nn.relu(ops.batch_norm(h3))
h4 = ops.deconv2d(h3, [self.hparas['BATCH_SIZE'], s, s, 3], name='g_h4')
self.generator_net = tf.tanh(h4) / 2.0 + 0.5
self.outputs = tf.tanh(h4) / 2.0 + 0.5
def generator(z, is_training):
# Firstly let's reshape input vector into 3-d tensor.
z_ = ops.linear(z, GENERATOR_DENSE_SIZE*4*4, 'g_h0_lin')
h_in = tf.reshape(z_, [-1, 4, 4, GENERATOR_DENSE_SIZE])
g_batch_norm_in=ops.batch_norm(name='g_batch_norm_in')
h_in_bn = g_batch_norm_in(h_in,is_training)
h_in_z=ops.lrelu(x=h_in_bn,name='g_lr_1')
h_1=ops.deconv2d(h_in_z,output_shape=[BATCH_SIZE,8,8,512],k_h=5,k_w=5,d_h=2, d_w=2,name="g_deconv_1")
g_batch_norm_1=ops.batch_norm(name='g_batch_norm_1')
h_1_bn = g_batch_norm_1(h_1,is_training)
h_1_z=ops.lrelu(x=h_1_bn,name='g_lr_2')
h_1_z_dr=tf.nn.dropout(h_1_z,0.3)
h_2=ops.deconv2d(h_1_z_dr,output_shape=[BATCH_SIZE,16,16,256],k_h=5,k_w=5,d_h=2, d_w=2,name="g_deconv_2")
g_batch_norm_2=ops.batch_norm(name='g_batch_norm_2')
h_2_bn = g_batch_norm_2(h_2,is_training)
h_2_z=ops.lrelu(x=h_2_bn,name='g_lr_3')
h_2_z_dr=tf.nn.dropout(h_2_z,0.3)
h_3=ops.deconv2d(h_2_z_dr,output_shape=[BATCH_SIZE,32,32,128],k_h=5,k_w=5,d_h=2, d_w=2,name="g_deconv_3")
g_batch_norm_3=ops.batch_norm(name='g_batch_norm_3')
h_3_bn = g_batch_norm_3(h_3,is_training)
h_3_z=ops.lrelu(x=h_3_bn,name='g_lr_4')
h_out = ops.deconv2d(h_3_z, [BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE, N_CHANNELS],
name='g_out')
return tf.nn.tanh(h_out)
def discriminator(self, image, is_training, reuse=False):
with tf.variable_scope("discriminator"):
if reuse:
tf.get_variable_scope().reuse_variables()
h0 = lrelu(conv2d(image, self.discriminator_dim,
scope="d_h0_conv"))
h1 = lrelu(
batch_norm(conv2d(h0,
self.discriminator_dim * 2,
scope="d_h1_conv"),
is_training,
scope="d_bn_1"))
h2 = lrelu(
batch_norm(conv2d(h1,
self.discriminator_dim * 4,
scope="d_h2_conv"),
is_training,
scope="d_bn_2"))
h3 = lrelu(
batch_norm(conv2d(h2,
self.discriminator_dim * 8,
sh=1,
sw=1,
scope="d_h3_conv"),
is_training,
scope="d_bn_3"))
# real or fake binary loss
fc1 = fc(tf.reshape(h3, [self.batch_size, -1]), 1, scope="d_fc1")
# category loss
fc2 = fc(tf.reshape(h3, [self.batch_size, -1]),
self.embedding_num,
scope="d_fc2")
return tf.nn.sigmoid(fc1), fc1, fc2
def generator(inputs):
"""
Defines the graph for the generator.
Takes input (random samples from uniform distribution) and returns a Spectrogram.
:param inputs: A vector with random entries (either from a uniform or a gaussian distribution).
:return: A generated audio sample, that hopefully fools the discriminator.
"""
with tf.variable_scope("g_") as scope:
net = fc(inputs, 28672, 'fc1')
net = tf.reshape(net, [-1, 4, 7, 1024])
net = tf.nn.relu(net)
net = deconv(net, [FLAGS.kernel_size,FLAGS.kernel_size,1024, 1024], 'deconv1')
net = batch_norm(True, net, 'bn2')
net = tf.nn.relu(net)
net = deconv(net, [FLAGS.kernel_size, FLAGS.kernel_size, 512, 1024], 'deconv2')
net = batch_norm(True, net, 'bn3')
net = tf.nn.relu(net)
net = deconv(net, [FLAGS.kernel_size, FLAGS.kernel_size, 256, 512], 'deconv3')
net = batch_norm(True, net, 'bn4')
net = tf.nn.relu(net)
net = deconv(net, [FLAGS.kernel_size, FLAGS.kernel_size, 128, 256], 'deconv4')
net = batch_norm(True, net, 'bn5')
net = tf.nn.relu(net)
net = deconv(net, [FLAGS.kernel_size, FLAGS.kernel_size, 2, 128], 'deconv5')
net = net[:, :98, :201, :]
# Map logmag and phase to [-1,1]
logmag, phase = tf.unstack(net, axis=3)
phase = tf.nn.tanh(phase)
logmag = tf.nn.tanh(logmag)
net = tf.stack([logmag, phase], axis=3)
return net
def g_block(self, inputs, is_training):
h = tf.nn.relu(
batch_norm(conv2d(inputs, self.g_depth, 3, 1, use_bias=False),
is_training))
h = batch_norm(conv2d(h, self.g_depth, 3, 1, use_bias=False),
is_training)
h = tf.add(inputs, h)
return h
def _initialize_params(self):
all_weights = {}
batch_norms = {}
gen_vars = []
disc_vars = []
# init generator weights
prev_layer_dim = self.z_dim
for layer_i in xrange(len(self.generator_params['dim'])):
name = 'gen_w' + str(layer_i)
all_weights[name] = ops.variable(name,
[self.generator_params['ksize'][layer_i],
self.generator_params['ksize'][layer_i],
self.generator_params['dim'][layer_i],
prev_layer_dim],
self.initializer)
gen_vars.append(all_weights[name])
if layer_i+1==len(self.generator_params['dim']):
name = 'gen_b' + str(layer_i)
all_weights[name] = ops.variable(name,
[self.generator_params['dim'][layer_i]],
)
gen_vars.append(all_weights[name])
else:
name = 'gen_bn' + str(layer_i)
batch_norms[name] = ops.batch_norm(self.generator_params['dim'][layer_i], name=name)
prev_layer_dim = self.generator_params['dim'][layer_i]
# init discriminator weights
for disc_i in xrange(len(self.discriminators_params)):
prev_layer_dim = self.image_dim
cur_params = self.discriminators_params[disc_i]
for layer_i in xrange(len(cur_params['dim'])):
name = 'disc' + str(disc_i) + '_w' + str(layer_i)
all_weights[name] = ops.variable(name,
[cur_params['ksize'][layer_i],
cur_params['ksize'][layer_i],
prev_layer_dim,
cur_params['dim'][layer_i]],
self.initializer)
disc_vars.append(all_weights[name])
if layer_i+1==len(cur_params['dim']):
name = 'disc' + str(disc_i) + '_b' + str(layer_i)
all_weights[name] = ops.variable(name,
[cur_params['dim'][layer_i]],
tf.constant_initializer(0.0))
disc_vars.append(all_weights[name])
else:
name = 'disc_' + str(disc_i) + '_bn' + str(layer_i)
batch_norms[name] = ops.batch_norm(cur_params['dim'][layer_i], name=name)
prev_layer_dim = cur_params['dim'][layer_i]
return all_weights, batch_norms, gen_vars, disc_vars
def GeneratorCNN(z, config, reuse=None):
'''
maps z to a 64x64 images with values in [-1,1]
uses batch normalization internally
'''
#trying to get around batch_size like this:
batch_size = tf.shape(z)[0]
#batch_size=tf.placeholder_with_default(64,[],'bs')
with tf.variable_scope("generator", reuse=reuse) as vs:
g_bn0 = batch_norm(name='g_bn0')
g_bn1 = batch_norm(name='g_bn1')
g_bn2 = batch_norm(name='g_bn2')
g_bn3 = batch_norm(name='g_bn3')
s_h, s_w = config.gf_dim, config.gf_dim #64,64
s_h2, s_w2 = conv_out_size_same(s_h, 2), conv_out_size_same(s_w, 2)
s_h4, s_w4 = conv_out_size_same(s_h2, 2), conv_out_size_same(s_w2, 2)
s_h8, s_w8 = conv_out_size_same(s_h4, 2), conv_out_size_same(s_w4, 2)
s_h16, s_w16 = conv_out_size_same(s_h8, 2), conv_out_size_same(s_w8, 2)
# project `z` and reshape
z_, self_h0_w, self_h0_b = linear(z,
config.gf_dim * 8 * s_h16 * s_w16,
'g_h0_lin',
with_w=True)
self_h0 = tf.reshape(z_, [-1, s_h16, s_w16, config.gf_dim * 8])
h0 = tf.nn.relu(g_bn0(self_h0))
h1, h1_w, h1_b = deconv2d(h0,
[batch_size, s_h8, s_w8, config.gf_dim * 4],
name='g_h1',
with_w=True)
h1 = tf.nn.relu(g_bn1(h1))
h2, h2_w, h2_b = deconv2d(h1,
[batch_size, s_h4, s_w4, config.gf_dim * 2],
name='g_h2',
with_w=True)
h2 = tf.nn.relu(g_bn2(h2))
h3, h3_w, h3_b = deconv2d(h2,
[batch_size, s_h2, s_w2, config.gf_dim * 1],
name='g_h3',
with_w=True)
h3 = tf.nn.relu(g_bn3(h3))
h4, h4_w, h4_b = deconv2d(h3, [batch_size, s_h, s_w, config.c_dim],
name='g_h4',
with_w=True)
out = tf.nn.tanh(h4)
variables = tf.contrib.framework.get_variables(vs)
return out, variables
def testReuseUpdateOps(self):
height, width = 3, 3
with self.test_session():
images = tf.random_uniform((5, height, width, 3), seed=1)
ops.batch_norm(images, scope='bn')
self.assertEquals(
len(tf.get_collection(ops.UPDATE_OPS_COLLECTION)), 2)
ops.batch_norm(images, scope='bn', reuse=True)
self.assertEquals(
len(tf.get_collection(ops.UPDATE_OPS_COLLECTION)), 4)
def testMovingAverageVariables(self):
height, width = 3, 3
with self.test_session():
images = tf.random_uniform((5, height, width, 3), seed=1)
ops.batch_norm(images, scale=True)
moving_mean = tf.moving_average_variables()[0]
moving_variance = tf.moving_average_variables()[1]
self.assertEquals(moving_mean.op.name, 'BatchNorm/moving_mean')
self.assertEquals(moving_variance.op.name,
'BatchNorm/moving_variance')
def res_build_block_v1(X,
increase=False,
init=1.0,
stddev=1.0,
training=False,
projection=None,
name=None):
print('input shape for {} :: {}'.format(name, X.get_shape().as_list()))
shortcut_projection = X
stride = 1
in_channels = X.get_shape().as_list()[-1]
out_channels = in_channels
if increase:
out_channels = 2 * in_channels
stride = 2
if projection:
name_ = name + 'projection_batch_norm'
shortcut_projection = projection(shortcut_projection,
out_channels=out_channels,
training=training,
name=name_)
else:
shortcut_projection = maxpool(shortcut_projection,
filter_size=[1, 2, 2, 1],
stride_size=[1, 2, 2, 1],
padding='VALID')
pad = (out_channels - in_channels)
shortcut_projection = tf.pad(shortcut_projection,
[[0, 0], [0, 0], [0, 0], [0, pad]])
with tf.name_scope('conv1layer'):
name_ = name + 'conv1layer_batch_norm'
X = conv(X,
filter_size=3,
out_channels=out_channels,
stride_size=stride,
padding='SAME',
init_bias=init,
stddev=stddev)
X = batch_norm(X, training, name=name_)
X = tf.nn.relu(X)
with tf.name_scope('conv2layer'):
name_ = name + 'conv2layer_batch_norm'
X = conv(X,
filter_size=3,
out_channels=out_channels,
stride_size=1,
padding='SAME',
init_bias=init,
stddev=stddev)
X = batch_norm(X, training, name=name_)
with tf.name_scope('residual'):
X += shortcut_projection
X = tf.nn.relu(X)
print('output shape for {} :: {}'.format(name, X.get_shape().as_list()))
return X
def __call__(self,
x,
is_reuse=False,
is_train=True,
is_intermediate=False):
with tf.variable_scope('generator') as scope:
if is_reuse:
scope.reuse_variables()
unit_size = self.img_size[0] // (2**self.layer_n)
unit_n = self.smallest_hidden_unit_n * (2**(self.layer_n - 1))
batch_size = int(x.shape[0])
if is_intermediate:
intermediate_xs = list()
with tf.variable_scope('pre'):
x = linear(x, unit_size * unit_size * unit_n)
x = tf.reshape(x, (batch_size, unit_size, unit_size, unit_n))
if self.is_bn:
x = batch_norm(x, is_train)
x = tf.nn.relu(x)
if is_intermediate:
y = avgpool2d(x, unit_size, 1, 'VALID')
y = tf.reshape(y, (batch_size, -1))
intermediate_xs.append(y)
for i in range(self.layer_n):
with tf.variable_scope('layer{}'.format(i)):
if i == self.layer_n - 1:
unit_n = self.img_dim
else:
unit_n = self.smallest_hidden_unit_n * (2**(
self.layer_n - i - 2))
x_shape = x.get_shape().as_list()
if self.is_transpose:
x = deconv2d(x, [
x_shape[0], x_shape[1] * 2, x_shape[1] * 2, unit_n
], self.k_size, 2, 'SAME')
else:
x = tf.image.resize_bilinear(
x, (x_shape[1] * 2, x_shape[2] * 2))
x = conv2d(x, unit_n, self.k_size, 1, 'SAME')
if i != self.layer_n - 1:
if self.is_bn:
x = batch_norm(x, is_train)
x = tf.nn.relu(x)
if is_intermediate:
y = tf.reshape(x, (batch_size, -1))
intermediate_xs.append(y)
x = tf.nn.tanh(x)
if is_intermediate:
return x, intermediate_xs
return x
def testReuseVariables(self):
height, width = 3, 3
with self.test_session():
images = tf.random_uniform((5, height, width, 3), seed=1)
ops.batch_norm(images, scale=True, scope='bn')
ops.batch_norm(images, scale=True, scope='bn', reuse=True)
beta = variables.get_variables_by_name('beta')
gamma = variables.get_variables_by_name('gamma')
self.assertEquals(len(beta), 1)
self.assertEquals(len(gamma), 1)
moving_vars = tf.get_collection('moving_vars')
self.assertEquals(len(moving_vars), 2)
def testUpdateOps(self):
height, width = 3, 3
with self.test_session():
images = tf.random_uniform((5, height, width, 3), seed=1)
ops.batch_norm(images)
update_ops = tf.get_collection(ops.UPDATE_OPS_COLLECTION)
update_moving_mean = update_ops[0]
update_moving_variance = update_ops[1]
self.assertEquals(update_moving_mean.op.name,
'BatchNorm/AssignMovingAvg')
self.assertEquals(update_moving_variance.op.name,
'BatchNorm/AssignMovingAvg_1')
def discriminator(image, reuse=False):
d_bn1 = ops.batch_norm(FLAGS.batch_size, name='d_bn1')
d_bn2 = ops.batch_norm(FLAGS.batch_size, name='d_bn2')
d_bn3 = ops.batch_norm(FLAGS.batch_size, name='d_bn3')
image = tf.reshape(image, [FLAGS.batch_size, FLAGS.image_size, FLAGS.image_size, 1])
if reuse: tf.get_variable_scope().reuse_variables()
h0 = ops.lrelu(ops.conv2d(image, FLAGS.df_dim, name='d_h0_conv'))
h1 = ops.lrelu(d_bn1(ops.conv2d(h0, FLAGS.df_dim * 2, name='d_h1_conv')))
h2 = ops.lrelu(d_bn2(ops.conv2d(h1, FLAGS.df_dim * 4, name='d_h2_conv')))
h3 = ops.lrelu(d_bn3(ops.conv2d(h2, FLAGS.df_dim * 8, name='d_h3_conv')))
h4 = ops.linear(tf.reshape(h3, [FLAGS.batch_size, -1]), 1, 'd_h3_lin')
return tf.nn.sigmoid(h4)
def __init__(self,
sess,
image_size,
batch_size=64,
sample_size=64,
lowres=8,
z_dim=100,
gf_dim=64,
df_dim=64,
gfc_dim=1024,
dfc_dim=1024,
c_dim=3,
checkpoint_dir=None,
lamda=0.01,
center_scale=0.25):
# Currently, image size must be a (power of 2) and (8 or higher).
assert(image_size & (image_size - 1) == 0 and image_size >= 8)
print("========INITIALIZING THE WENGER DCGAN============")
# INIT from train-dcgan!
self.sess = sess
self.is_crop = False
self.batch_size = batch_size
self.image_size = image_size
self.sample_size = sample_size
self.image_shape = [image_size, image_size, c_dim]
self.lowres = lowres
self.lowres_size = image_size // lowres
self.lowres_shape = [self.lowres_size, self.lowres_size, c_dim]
self.z_dim = z_dim
self.gf_dim = gf_dim
self.df_dim = df_dim
self.gfc_dim = gfc_dim
self.dfc_dim = dfc_dim
self.lamda = lamda
self.c_dim = c_dim
self.center_scale = center_scale
# batch normalization : deals with poor initialization helps gradient flow
self.d_bns = [
batch_norm(name='d_bn{}'.format(i,)) for i in range(4)]
log_size = int(math.log(image_size) / math.log(2))
self.g_bns = [
batch_norm(name='g_bn{}'.format(i,)) for i in range(log_size)]
self.checkpoint_dir = checkpoint_dir
print("========CALLING SETUP TO BUILD MODEL!============")
self.setup()
self.model_name = "DCGAN.model"
def block_without_condition(x, out_channels, is_training, name):
with tf.variable_scope(name):
bn0 = ops.batch_norm(name='bn_0')
bn1 = ops.batch_norm(name='bn_1')
# bn = ops.batch_norm()
x = bn0(x, is_training)
x = tf.nn.relu(x)
x = usample(x)
x = ops.snconv2d(x, out_channels, 3, 3, 1, 1, name='snconv1')
x = bn1(x, is_training)
x = tf.nn.relu(x)
x = ops.snconv2d(x, out_channels, 3, 3, 1, 1, name='snconv2')
return x
def discriminator(image, reuse=False):
d_bn1 = ops.batch_norm(FLAGS.batch_size, name='d_bn1')
d_bn2 = ops.batch_norm(FLAGS.batch_size, name='d_bn2')
d_bn3 = ops.batch_norm(FLAGS.batch_size, name='d_bn3')
image = tf.reshape(
image, [FLAGS.batch_size, FLAGS.image_size, FLAGS.image_size, 1])
if reuse: tf.get_variable_scope().reuse_variables()
h0 = ops.lrelu(ops.conv2d(image, FLAGS.df_dim, name='d_h0_conv'))
h1 = ops.lrelu(d_bn1(ops.conv2d(h0, FLAGS.df_dim * 2, name='d_h1_conv')))
h2 = ops.lrelu(d_bn2(ops.conv2d(h1, FLAGS.df_dim * 4, name='d_h2_conv')))
h3 = ops.lrelu(d_bn3(ops.conv2d(h2, FLAGS.df_dim * 8, name='d_h3_conv')))
h4 = ops.linear(tf.reshape(h3, [FLAGS.batch_size, -1]), 1, 'd_h3_lin')
return tf.nn.sigmoid(h4)
def __init__(self,
sess,
batch_size=32,
image_size=256,
lam=0.8,
checkpoint_dir=None):
self.sess = sess
self.batch_size = batch_size
self.image_size = image_size
self.image_shape = [image_size, image_size, 3]
self.lam = lam
self.checkpoint_dir = checkpoint_dir
self.global_step = tf.Variable(0, trainable=False)
self.images = tf.placeholder(tf.float32, [batch_size] + self.image_shape, name='images')
self.d_bns = [batch_norm(name='d_bn{}'.format(i)) for i in range(5)]
self.g_bns = [batch_norm(name='g_bn{}'.format(i, )) for i in range(15)]
self.D_real = self.discriminator(self.images, self.image_size)
self.masks = tf.placeholder(tf.float32, [batch_size] + self.image_shape, name='masks')
self.MG = tf.multiply(self.images, self.masks)
self.G = self.generator(self.MG)
self.D_fake = self.discriminator(self.G, self.image_size, reuse=True)
self.D_loss = -tf.reduce_mean(self.D_real) + tf.reduce_mean(self.D_fake)
self.G_loss_d = -tf.reduce_mean(self.D_fake)
self.G_loss_l = tf.reduce_mean(tf.contrib.layers.flatten(
tf.multiply(self.G - self.images, self.G - self.images)))
self.G_loss = (1 - self.lam) * self.G_loss_d + self.lam * self.G_loss_l
self.images_summary = tf.summary.image("image", self.images)
self.D_loss_summary = tf.summary.scalar("d_loss", self.D_loss)
self.G_loss_d_summary = tf.summary.scalar("g_loss_d", self.G_loss_d)
self.G_loss_l_summary = tf.summary.scalar("g_loss_l", self.G_loss_l)
self.G_loss_summary = tf.summary.scalar("g_loss", self.G_loss)
self.MG_summary = tf.summary.image("mg", self.MG)
self.G_summary = tf.summary.image("g", self.G)
t_vars = tf.trainable_variables()
self.d_vars = [var for var in t_vars if 'd_' in var.name]
self.g_vars = [var for var in t_vars if 'g_' in var.name]
self.clip_D = [p.assign(tf.clip_by_value(p, -0.01, 0.01)) for p in self.d_vars]
self.saver = tf.train.Saver(max_to_keep=10)
self.g_summary = tf.summary.merge([
self.G_summary, self.MG_summary,
self.G_loss_summary, self.G_loss_l_summary, self.G_loss_d_summary])
self.d_summary = tf.summary.merge([
self.images_summary, self.G_summary, self.MG_summary,
self.D_loss_summary])
self.writer = tf.summary.FileWriter(os.path.join(self.checkpoint_dir, "logs"), self.sess.graph)
def prepare_model(self):
with tf.variable_scope("LSTMTDNN"):
self.char_inputs = []
self.word_inputs = []
self.cnn_outputs = []
if self.use_char:
char_W = tf.get_variable("char_embed",
[self.char_vocab_size, self.char_embed_dim])
if self.use_word:
word_W = tf.get_variable("word_embed",
[self.word_vocab_size, self.word_embed_dim])
with tf.variable_scope("CNN") as scope:
self.char_inputs = tf.placeholder(tf.int32, [self.batch_size, self.seq_length, self.max_word_length])
self.word_inputs = tf.placeholder(tf.int32, [self.batch_size, self.seq_length])
char_indices = tf.split(1, self.seq_length, self.char_inputs)
word_indices = tf.split(1, self.seq_length, tf.expand_dims(self.word_inputs, -1))
for idx in xrange(self.seq_length):
char_index = tf.reshape(char_indices[idx], [-1, self.max_word_length])
word_index = tf.reshape(word_indices[idx], [-1, 1])
if idx != 0:
scope.reuse_variables()
if self.use_char:
# [batch_size x word_max_length, char_embed]
char_embed = tf.nn.embedding_lookup(char_W, char_index)
char_cnn = TDNN(char_embed, self.char_embed_dim, self.feature_maps, self.kernels)
if self.use_word:
word_embed = tf.nn.embedding_lookup(word_W, word_index)
cnn_output = tf.concat(1, [char_cnn.output, tf.squeeze(word_embed, [1])])
else:
cnn_output = char_cnn.output
else:
cnn_output = tf.squeeze(tf.nn.embedding_lookup(word_W, word_index))
if self.use_batch_norm:
bn = batch_norm()
norm_output = bn(tf.expand_dims(tf.expand_dims(cnn_output, 1), 1))
cnn_output = tf.squeeze(norm_output)
if highway:
#cnn_output = highway(input_, input_dim_length, self.highway_layers, 0)
cnn_output = highway(cnn_output, cnn_output.get_shape()[1], self.highway_layers, 0)
self.cnn_outputs.append(cnn_output)
with tf.variable_scope("LSTM") as scope:
self.cell = tf.nn.rnn_cell.BasicLSTMCell(self.rnn_size)
self.stacked_cell = tf.nn.rnn_cell.MultiRNNCell([self.cell] * self.layer_depth)
outputs, _ = tf.nn.rnn(self.stacked_cell,
self.cnn_outputs,
dtype=tf.float32)
self.lstm_outputs = []
self.true_outputs = tf.placeholder(tf.int64,
[self.batch_size, self.seq_length])
loss = 0
true_outputs = tf.split(1, self.seq_length, self.true_outputs)
for idx, (top_h, true_output) in enumerate(zip(outputs, true_outputs)):
if self.dropout_prob > 0:
top_h = tf.nn.dropout(top_h, self.dropout_prob)
if self.hsm > 0:
self.lstm_outputs.append(top_h)
else:
if idx != 0:
scope.reuse_variables()
proj = tf.nn.rnn_cell._linear(top_h, self.word_vocab_size, 0)
self.lstm_outputs.append(proj)
loss += tf.nn.sparse_softmax_cross_entropy_with_logits(self.lstm_outputs[idx], tf.squeeze(true_output))
self.loss = tf.reduce_mean(loss) / self.seq_length
tf.scalar_summary("loss", self.loss)
tf.scalar_summary("perplexity", tf.exp(self.loss))
