def decoder(self, dinput, scope_name='decoder'):
with tf.variable_scope(scope_name):
d1 = op.deconv2d(dinput,
kernel_size=5,
stride=2,
num_filter=256,
scope_name='dconv1')
d1 = op.batch_norm(d1, scope_name='bn4')
d1 = tf.nn.relu(d1)
self.d1.append(d1)
d2 = op.deconv2d(d1,
kernel_size=5,
stride=1,
num_filter=128,
scope_name='dconv2')
d2 = op.batch_norm(d2, scope_name='bn5')
d2 = tf.nn.relu(d2)
self.d2.append(d2)
d3 = op.deconv2d(d2,
kernel_size=5,
stride=2,
num_filter=64,
scope_name='dconv3')
d3 = op.batch_norm(d3, scope_name='bn6')
d3 = tf.nn.relu(d3)
self.d3.append(d3)
d4 = op.deconv2d(d3,
kernel_size=5,
stride=1,
num_filter=3,
scope_name='dconv4')
self.d3.append(d4)
print(d1.shape, d2.shape, d3.shape, d4.shape)
return d4
def encoder(self, input_image, scope_name="encoder", reuse=False):
with tf.variable_scope(scope_name):
c1 = op.conv2d(input_image,
64,
kernel_h=5,
kernel_w=5,
k_stride=1,
scope_name='conv1')
c1 = op.batch_norm(c1, scope_name='bn1')
self.c1.append(c1)
c2 = op.conv2d(c1,
128,
kernel_h=5,
kernel_w=5,
k_stride=2,
scope_name='conv2')
c2 = op.batch_norm(c2, scope_name='bn2')
self.c2.append(c2)
c3 = op.conv2d(c2,
256,
kernel_h=5,
kernel_w=5,
k_stride=1,
scope_name='conv3')
c3 = op.batch_norm(c3, scope_name='bn3')
self.c3.append(c3)
c4 = op.conv2d(c3,
512,
kernel_h=5,
kernel_w=5,
k_stride=2,
scope_name='conv4')
self.c3.append(c4)
print(c1.shape, c2.shape, c3.shape, c4.shape)
return c4
def build(self, z, train):
with tf.variable_scope(self.scope_name, reuse=tf.AUTO_REUSE):
batch_size = tf.shape(z)[0]
layers = [z]
with tf.variable_scope("layer0"):
layers.append(linear(layers[-1], 128))
layers.append(tf.nn.relu(layers[-1]))
layers.append(batch_norm()(layers[-1], train=train))
with tf.variable_scope("layer1"):
layers.append(linear(layers[-1], 4 * 4 * 64))
layers.append(tf.nn.relu(layers[-1]))
layers.append(batch_norm()(layers[-1], train=train))
layers.append(tf.reshape(layers[-1], [-1, 4, 4, 64]))
with tf.variable_scope("layer2"):
layers.append(
deconv2d(layers[-1], [batch_size, 8, 8, 64],
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel))
layers.append(lrelu(layers[-1]))
layers.append(batch_norm()(layers[-1], train=train))
with tf.variable_scope("layer3"):
layers.append(
deconv2d(layers[-1], [batch_size, 16, 16, 32],
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel))
layers.append(lrelu(layers[-1]))
layers.append(batch_norm()(layers[-1], train=train))
with tf.variable_scope("layer4"):
layers.append(
deconv2d(layers[-1], [batch_size, 32, 32, 32],
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel))
layers.append(lrelu(layers[-1]))
layers.append(batch_norm()(layers[-1], train=train))
with tf.variable_scope("layer5"):
layers.append(
deconv2d(layers[-1], [
batch_size, self.output_height, self.output_width,
self.output_depth
],
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel))
layers.append(tf.nn.sigmoid(layers[-1]))
return layers[-1], layers
def build(self, images, train):
with tf.variable_scope(self.scope_name, reuse=tf.AUTO_REUSE):
layers = [images]
with tf.variable_scope("layer0"):
layers.append(conv2d(
layers[-1],
32,
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel,
sn_op=None))
layers.append(lrelu(layers[-1]))
with tf.variable_scope("layer1"):
layers.append(conv2d(
layers[-1],
32,
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel,
sn_op=None))
layers.append(lrelu(layers[-1]))
layers.append(batch_norm()(layers[-1], train=train))
with tf.variable_scope("layer2"):
layers.append(conv2d(
layers[-1],
64,
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel,
sn_op=None))
layers.append(lrelu(layers[-1]))
layers.append(batch_norm()(layers[-1], train=train))
with tf.variable_scope("layer3"):
layers.append(conv2d(
layers[-1],
64,
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel,
sn_op=None))
layers.append(lrelu(layers[-1]))
layers.append(batch_norm()(layers[-1], train=train))
with tf.variable_scope("layer4"):
layers.append(linear(layers[-1], 128, sn_op=None))
layers.append(lrelu(layers[-1]))
layers.append(batch_norm()(layers[-1], train=train))
with tf.variable_scope("layer5"):
layers.append(linear(layers[-1], self.output_length))
return layers[-1], layers
def build(self, image, train):
with tf.variable_scope(self.scope_name, reuse=tf.AUTO_REUSE):
layers = [image]
with tf.variable_scope("layer0"):
layers.append(
conv2d(layers[-1],
self.start_depth,
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel))
layers.append(lrelu(layers[-1]))
with tf.variable_scope("layer1"):
layers.append(
conv2d(layers[-1],
self.start_depth * 2,
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel))
layers.append(batch_norm()(layers[-1], train=train))
layers.append(lrelu(layers[-1]))
with tf.variable_scope("layer2"):
layers.append(
conv2d(layers[-1],
self.start_depth * 4,
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel))
layers.append(batch_norm()(layers[-1], train=train))
layers.append(lrelu(layers[-1]))
with tf.variable_scope("layer3"):
layers.append(
conv2d(layers[-1],
self.start_depth * 8,
d_h=self.stride,
d_w=self.stride,
k_h=self.kernel,
k_w=self.kernel))
layers.append(batch_norm()(layers[-1], train=train))
layers.append(lrelu(layers[-1]))
with tf.variable_scope("layer4"):
layers.append(linear(layers[-1], self.output_length))
return layers[-1], layers
def build(self,
attribute_input_noise,
addi_attribute_input_noise,
feature_input_noise,
feature_input_data,
train,
attribute=None):
with tf.variable_scope(self.scope_name, reuse=tf.AUTO_REUSE):
batch_size = tf.shape(feature_input_noise)[0]
if attribute is None:
all_attribute = []
all_discrete_attribute = []
if len(self.addi_attribute_outputs) > 0:
all_attribute_input_noise = \
[attribute_input_noise,
addi_attribute_input_noise]
all_attribute_outputs = \
[self.real_attribute_outputs,
self.addi_attribute_outputs]
all_attribute_part_name = \
[self.STR_REAL, self.STR_ADDI]
all_attribute_out_dim = \
[self.real_attribute_out_dim,
self.addi_attribute_out_dim]
else:
all_attribute_input_noise = [attribute_input_noise]
all_attribute_outputs = [self.real_attribute_outputs]
all_attribute_part_name = [self.STR_REAL]
all_attribute_out_dim = [self.real_attribute_out_dim]
else:
all_attribute = [attribute]
all_discrete_attribute = [attribute]
if len(self.addi_attribute_outputs) > 0:
all_attribute_input_noise = \
[addi_attribute_input_noise]
all_attribute_outputs = \
[self.addi_attribute_outputs]
all_attribute_part_name = \
[self.STR_ADDI]
all_attribute_out_dim = [self.addi_attribute_out_dim]
else:
all_attribute_input_noise = []
all_attribute_outputs = []
all_attribute_part_name = []
all_attribute_out_dim = []
for part_i in range(len(all_attribute_input_noise)):
with tf.variable_scope("attribute_{}".format(
all_attribute_part_name[part_i]),
reuse=tf.AUTO_REUSE):
if len(all_discrete_attribute) > 0:
layers = [
tf.concat([all_attribute_input_noise[part_i]] +
all_discrete_attribute,
axis=1)
]
else:
layers = [all_attribute_input_noise[part_i]]
for i in range(self.attribute_num_layers - 1):
with tf.variable_scope("layer{}".format(i)):
layers.append(
linear(layers[-1], self.attribute_num_units))
layers.append(tf.nn.relu(layers[-1]))
layers.append(batch_norm()(layers[-1],
train=train))
with tf.variable_scope(
"layer{}".format(self.attribute_num_layers - 1),
reuse=tf.AUTO_REUSE):
part_attribute = []
part_discrete_attribute = []
for i in range(len(all_attribute_outputs[part_i])):
with tf.variable_scope("output{}".format(i),
reuse=tf.AUTO_REUSE):
output = all_attribute_outputs[part_i][i]
sub_output_ori = linear(layers[-1], output.dim)
if (output.type_ == OutputType.DISCRETE):
sub_output = tf.nn.softmax(sub_output_ori)
sub_output_discrete = tf.one_hot(
tf.argmax(sub_output, axis=1),
output.dim)
elif (output.type_ == OutputType.CONTINUOUS):
if (output.normalization ==
Normalization.ZERO_ONE):
sub_output = tf.nn.sigmoid(
sub_output_ori)
elif (output.normalization ==
Normalization.MINUSONE_ONE):
sub_output = tf.nn.tanh(sub_output_ori)
else:
raise Exception("unknown normalization"
" type")
sub_output_discrete = sub_output
else:
raise Exception("unknown output type")
part_attribute.append(sub_output)
part_discrete_attribute.append(
sub_output_discrete)
part_attribute = tf.concat(part_attribute, axis=1)
part_discrete_attribute = tf.concat(
part_discrete_attribute, axis=1)
part_attribute = tf.reshape(
part_attribute,
[batch_size, all_attribute_out_dim[part_i]])
part_discrete_attribute = tf.reshape(
part_discrete_attribute,
[batch_size, all_attribute_out_dim[part_i]])
# batch_size * dim
part_discrete_attribute = tf.stop_gradient(
part_discrete_attribute)
all_attribute.append(part_attribute)
all_discrete_attribute.append(part_discrete_attribute)
all_attribute = tf.concat(all_attribute, axis=1)
all_discrete_attribute = tf.concat(all_discrete_attribute, axis=1)
all_attribute = tf.reshape(all_attribute,
[batch_size, self.attribute_out_dim])
all_discrete_attribute = tf.reshape(
all_discrete_attribute, [batch_size, self.attribute_out_dim])
with tf.variable_scope("feature", reuse=tf.AUTO_REUSE):
all_cell = []
for i in range(self.feature_num_layers):
with tf.variable_scope("unit{}".format(i),
reuse=tf.AUTO_REUSE):
cell = tf.nn.rnn_cell.LSTMCell(
num_units=self.feature_num_units,
state_is_tuple=True)
all_cell.append(cell)
rnn_network = tf.nn.rnn_cell.MultiRNNCell(all_cell)
feature_input_data_dim = \
len(feature_input_data.get_shape().as_list())
if feature_input_data_dim == 3:
feature_input_data_reshape = tf.transpose(
feature_input_data, [1, 0, 2])
feature_input_noise_reshape = tf.transpose(
feature_input_noise, [1, 0, 2])
# time * batch_size * ?
if self.initial_state == RNNInitialStateType.ZERO:
initial_state = rnn_network.zero_state(
batch_size, tf.float32)
elif self.initial_state == RNNInitialStateType.RANDOM:
initial_state = tf.random_normal(
shape=(self.feature_num_layers, 2, batch_size,
self.feature_num_units),
mean=0.0,
stddev=1.0)
initial_state = tf.unstack(initial_state, axis=0)
initial_state = tuple([
tf.nn.rnn_cell.LSTMStateTuple(initial_state[idx][0],
initial_state[idx][1])
for idx in range(self.feature_num_layers)
])
elif self.initial_state == RNNInitialStateType.VARIABLE:
initial_state = []
for i in range(self.feature_num_layers):
sub_initial_state1 = tf.get_variable(
"layer{}_initial_state1".format(i),
(1, self.feature_num_units),
initializer=tf.random_normal_initializer(
stddev=self.initial_stddev))
sub_initial_state1 = tf.tile(sub_initial_state1,
(batch_size, 1))
sub_initial_state2 = tf.get_variable(
"layer{}_initial_state2".format(i),
(1, self.feature_num_units),
initializer=tf.random_normal_initializer(
stddev=self.initial_stddev))
sub_initial_state2 = tf.tile(sub_initial_state2,
(batch_size, 1))
sub_initial_state = tf.nn.rnn_cell.LSTMStateTuple(
sub_initial_state1, sub_initial_state2)
initial_state.append(sub_initial_state)
initial_state = tuple(initial_state)
else:
return NotImplementedError
time = feature_input_noise.get_shape().as_list()[1]
if time is None:
time = tf.shape(feature_input_noise)[1]
def compute(i, state, last_output, all_output, gen_flag,
all_gen_flag, all_cur_argmax, last_cell_output):
input_all = [all_discrete_attribute]
if self.noise:
input_all.append(feature_input_noise_reshape[i])
if self.feed_back:
if feature_input_data_dim == 3:
input_all.append(feature_input_data_reshape[i])
else:
input_all.append(last_output)
input_all = tf.concat(input_all, axis=1)
cell_new_output, new_state = rnn_network(input_all, state)
new_output_all = []
id_ = 0
for j in range(self.sample_len):
for k in range(len(self.feature_outputs)):
with tf.variable_scope("output{}".format(id_),
reuse=tf.AUTO_REUSE):
output = self.feature_outputs[k]
sub_output = linear(cell_new_output,
output.dim)
if (output.type_ == OutputType.DISCRETE):
sub_output = tf.nn.softmax(sub_output)
elif (output.type_ == OutputType.CONTINUOUS):
if (output.normalization ==
Normalization.ZERO_ONE):
sub_output = tf.nn.sigmoid(sub_output)
elif (output.normalization ==
Normalization.MINUSONE_ONE):
sub_output = tf.nn.tanh(sub_output)
else:
raise Exception("unknown normalization"
" type")
else:
raise Exception("unknown output type")
new_output_all.append(sub_output)
id_ += 1
new_output = tf.concat(new_output_all, axis=1)
for j in range(self.sample_len):
all_gen_flag = all_gen_flag.write(
i * self.sample_len + j, gen_flag)
cur_gen_flag = tf.to_float(
tf.equal(
tf.argmax(new_output_all[(
j * len(self.feature_outputs) +
self.gen_flag_id)],
axis=1), 0))
cur_gen_flag = tf.reshape(cur_gen_flag, [-1, 1])
all_cur_argmax = all_cur_argmax.write(
i * self.sample_len + j,
tf.argmax(
new_output_all[(j * len(self.feature_outputs) +
self.gen_flag_id)],
axis=1))
gen_flag = gen_flag * cur_gen_flag
return (i + 1, new_state, new_output,
all_output.write(i, new_output), gen_flag,
all_gen_flag, all_cur_argmax, cell_new_output)
(i, state, _, feature, _, gen_flag, cur_argmax,
cell_output) = \
tf.while_loop(
lambda a, b, c, d, e, f, g, h:
tf.logical_and(a < time,
tf.equal(tf.reduce_max(e), 1)),
compute,
(0,
initial_state,
feature_input_data if feature_input_data_dim == 2
else feature_input_data_reshape[0],
tf.TensorArray(tf.float32, time),
tf.ones((batch_size, 1)),
tf.TensorArray(tf.float32, time * self.sample_len),
tf.TensorArray(tf.int64, time * self.sample_len),
tf.zeros((batch_size, self.feature_num_units))))
def fill_rest(i, all_output, all_gen_flag, all_cur_argmax):
all_output = all_output.write(
i, tf.zeros((batch_size, self.feature_out_dim)))
for j in range(self.sample_len):
all_gen_flag = all_gen_flag.write(
i * self.sample_len + j, tf.zeros((batch_size, 1)))
all_cur_argmax = all_cur_argmax.write(
i * self.sample_len + j,
tf.zeros((batch_size, ), dtype=tf.int64))
return (i + 1, all_output, all_gen_flag, all_cur_argmax)
_, feature, gen_flag, cur_argmax = tf.while_loop(
lambda a, b, c, d: a < time, fill_rest,
(i, feature, gen_flag, cur_argmax))
feature = feature.stack()
# time * batch_size * (dim * sample_len)
gen_flag = gen_flag.stack()
# (time * sample_len) * batch_size * 1
cur_argmax = cur_argmax.stack()
gen_flag = tf.transpose(gen_flag, [1, 0, 2])
# batch_size * (time * sample_len) * 1
cur_argmax = tf.transpose(cur_argmax, [1, 0])
# batch_size * (time * sample_len)
length = tf.reduce_sum(gen_flag, [1, 2])
# batch_size
feature = tf.transpose(feature, [1, 0, 2])
# batch_size * time * (dim * sample_len)
gen_flag_t = tf.reshape(gen_flag,
[batch_size, time, self.sample_len])
# batch_size * time * sample_len
gen_flag_t = tf.reduce_sum(gen_flag_t, [2])
# batch_size * time
gen_flag_t = tf.to_float(gen_flag_t > 0.5)
gen_flag_t = tf.expand_dims(gen_flag_t, 2)
# batch_size * time * 1
gen_flag_t = tf.tile(gen_flag_t, [1, 1, self.feature_out_dim])
# batch_size * time * (dim * sample_len)
# zero out the parts after sequence ends
feature = feature * gen_flag_t
feature = tf.reshape(feature, [
batch_size, time * self.sample_len,
self.feature_out_dim / self.sample_len
])
# batch_size * (time * sample_len) * dim
return feature, all_attribute, gen_flag, length, cur_argmax