def build(self):
with tf.variable_scope(self.name):
stacker = Stacker(self.x)
stacker.conv2d(1, CONV_FILTER_3311)
self.recon = stacker.sigmoid()
return self
def build(self):
with tf.variable_scope(self.name, reuse=self.reuse):
self.stacker = Stacker(self.x, verbose=self.verbose)
self.stacker.resize_image((224, 224))
self.stacker = self.head(self.stacker)
self.stacker = self.body(self.stacker)
self.foot(self.stacker, self.n_channel, self.n_classes)
def build(self):
with tf.variable_scope(self.name, reuse=self.reuse):
self.stacker = Stacker(self.x)
self.stacker.conv_block(self.capacity * 8, CONV_FILTER_3311, relu)
self.stacker.conv_block(self.capacity * 8, CONV_FILTER_3311, relu)
self.stacker.upscale_2x_block(self.capacity * 8, CONV_FILTER_3322,
relu)
self.stacker.conv_block(self.capacity * 4, CONV_FILTER_3311, relu)
self.stacker.conv_block(self.capacity * 4, CONV_FILTER_3311, relu)
self.stacker.upscale_2x_block(self.capacity * 4, CONV_FILTER_3322,
relu)
self.stacker.conv_block(self.capacity * 2, CONV_FILTER_3311, relu)
self.stacker.conv_block(self.capacity * 2, CONV_FILTER_3311, relu)
self.stacker.upscale_2x_block(self.capacity * 2, CONV_FILTER_3322,
relu)
self.stacker.conv_block(self.capacity, CONV_FILTER_3311, relu)
self.stacker.conv_block(self.capacity, CONV_FILTER_3311, relu)
self.stacker.conv_block(self.capacity, CONV_FILTER_3311, relu)
self.stacker.conv_block(self.capacity, CONV_FILTER_3311, relu)
self.stacker.conv2d(self.output_channel, CONV_FILTER_3311)
self.stacker.relu()
self.decode = self.stacker.last_layer
return self
def build(self):
with tf.variable_scope(self.name):
self.stacker = Stacker(self.x)
self.stacker.resize_image(self.resize_shape)
stacker = self.stem(self.stacker)
for i in range(4):
stacker.add_layer(self.inception_A)
stacker.add_layer(self.reduction_A)
for i in range(7):
stacker.add_layer(self.inception_B)
self.aux_logit, self.aux_proba = self.aux(stacker.last_layer,
self.n_classes)
stacker.add_layer(self.reduction_B)
for i in range(3):
stacker.add_layer(self.inception_C)
stacker.max_pooling((8, 8, 8, 8))
# dropout
self.flatten_layer = stacker.flatten()
stacker.linear_block(self.n_channel * 64, relu)
self.logit = stacker.linear(self.n_classes)
self.proba = stacker.softmax()
def generator(zs, Ys, cs, net_shapes, flatten_size, output_shape, reuse=False, name='generator'):
with tf.variable_scope(name, reuse=reuse):
layer = Stacker(concat((zs, Ys, cs), axis=1))
for shape in net_shapes:
layer.linear_block(shape, lrelu)
layer.linear(flatten_size)
layer.sigmoid()
layer.reshape(output_shape)
return layer.last_layer
def build(self):
def _Unet_recursion(stacker, n_channel, level, dropout_tensor=None):
if level == 0:
for i in range(self.depth):
stacker.conv_block(n_channel, CONV_FILTER_3311, relu)
if dropout_tensor:
stacker.dropout(dropout_tensor)
else:
# encode
for i in range(self.depth):
stacker.conv_block(n_channel, CONV_FILTER_3311, relu)
if dropout_tensor:
stacker.dropout(dropout_tensor)
concat = stacker.last_layer
stacker.max_pooling(CONV_FILTER_2222)
stacker = _Unet_recursion(stacker, n_channel * 2, level - 1)
# decode
stacker.upscale_2x_block(n_channel, CONV_FILTER_2211, relu)
stacker.concat(concat, axis=3)
for i in range(self.depth):
stacker.conv_block(n_channel, CONV_FILTER_3311, relu)
if dropout_tensor:
stacker.dropout(dropout_tensor)
return stacker
with tf.variable_scope(self.name, reuse=self.reuse):
self.DynamicDropoutRate = DynamicDropoutRate(self.dropout_rate)
self.dropout_tensor = self.DynamicDropoutRate.tensor
self.stacker = Stacker(self.x, verbose=self.verbose)
self.stacker = _Unet_recursion(self.stacker,
n_channel=self.n_channel,
level=self.level,
dropout_tensor=self.dropout_tensor)
self.stacker.conv2d(self.n_classes, CONV_FILTER_3311)
self.logit = self.stacker.last_layer
self.stacker.pixel_wise_softmax()
self.proba = self.stacker.last_layer
return self
def build(self):
with tf.variable_scope(self.name):
self.skip_tensors = []
self.stacker = Stacker(self.x)
stacker = self.stem(self.stacker)
stacker.add_layer(self.module_A, self.a_channels0)
stacker.add_layer(self.module_A, self.a_channels1)
stacker.add_layer(self.module_A, self.a_channels2)
self.skip_tensors += [stacker.last_layer]
stacker.add_layer(self.multi_pool_A, self.a_b_multi_pool_channels)
stacker.add_layer(self.module_B, self.b_channels0)
stacker.add_layer(self.module_B, self.b_channels1)
stacker.add_layer(self.module_B, self.b_channels2)
stacker.add_layer(self.module_B, self.b_channels3)
# self.aux_logit, self.aux_proba = self.aux(stacker.last_layer, self.n_classes)
self.skip_tensors += [stacker.last_layer]
stacker.add_layer(self.multi_pool_B, self.b_c_multi_pool_channels)
stacker.add_layer(self.module_C, self.c_channels0)
stacker.add_layer(self.module_C, self.c_channels1)
self.last_layer = stacker.last_layer
def bottom_layer(self, x, n_channel, depth, dropout_rate_tensor):
stacker = Stacker(x, name='bottom')
stacker = self.conv_seq(stacker, n_channel, depth, dropout_rate_tensor)
stacker = self.conv_seq(stacker, n_channel, depth, dropout_rate_tensor)
self.bottom_layer_stacker = stacker
return self.bottom_layer_stacker.last_layer
def discriminator(Xs, net_shapes, reuse=False, name='discriminator'):
with tf.variable_scope(name, reuse=reuse):
layer = Stacker(flatten(Xs))
for shape in net_shapes:
layer.linear_block(shape, relu)
layer.linear(1)
layer.sigmoid()
return layer.last_layer
def common_AE_decoder_tail(stack: Stacker,
flatten_size,
output_shape,
bn=False,
activation='sigmoid') -> Stacker:
stack.linear(flatten_size)
if bn:
stack.bn()
stack.activation(activation)
stack.reshape(output_shape)
return stack
def _residual_atrous_block(x, n_channel, filter_, activation):
name = 'residual_block'
with tf.variable_scope(name):
x_in = x
stack = Stacker(x)
stack.atrous_conv2d_block(n_channel, filter_, 2, activation)
stack.atrous_conv2d_block(n_channel, filter_, 2, activation)
stack.atrous_conv2d_block(n_channel, filter_, 2, activation)
stack.residual_add(x_in)
return stack.last_layer
def encoder(self, Xs, Ys, net_shapes, reuse=False, name='encoder'):
with tf.variable_scope(name, reuse=reuse):
stack = Stacker(concat((flatten(Xs), Ys), axis=1))
for shape in net_shapes:
stack.linear(shape)
stack.lrelu()
stack.linear(self.z_size * 2)
return stack.last_layer
def build(self):
with tf.variable_scope(self.name):
self.stacker = Stacker(self.x, verbose=self.verbose)
# resize to 224 * 224
self.stacker.resize_image((224, 224))
# 224
self.stacker.conv_block(self.n_channel, CONV_FILTER_3311, relu)
self.stacker.conv_block(self.n_channel, CONV_FILTER_3311, relu)
self.stacker.max_pooling(CONV_FILTER_2222)
# 112
self.stacker.conv_block(self.n_channel * 2, CONV_FILTER_3311, relu)
self.stacker.conv_block(self.n_channel * 2, CONV_FILTER_3311, relu)
self.stacker.max_pooling(CONV_FILTER_2222)
# 52
self.stacker.conv_block(self.n_channel * 4, CONV_FILTER_3311, relu)
self.stacker.conv_block(self.n_channel * 4, CONV_FILTER_3311, relu)
self.stacker.conv_block(self.n_channel * 4, CONV_FILTER_3311, relu)
self.stacker.max_pooling(CONV_FILTER_2222)
# 28
self.stacker.conv_block(self.n_channel * 8, CONV_FILTER_3311, relu)
self.stacker.conv_block(self.n_channel * 8, CONV_FILTER_3311, relu)
self.stacker.conv_block(self.n_channel * 8, CONV_FILTER_3311, relu)
self.stacker.max_pooling(CONV_FILTER_2222)
# 14
self.stacker.conv_block(self.n_channel * 8, CONV_FILTER_3311, relu)
self.stacker.conv_block(self.n_channel * 8, CONV_FILTER_3311, relu)
self.stacker.conv_block(self.n_channel * 8, CONV_FILTER_3311, relu)
self.stacker.max_pooling(CONV_FILTER_2222)
# 7 512 to fc
self.stacker.flatten()
self.stacker.linear_block(self.n_channel * 64, relu)
self.stacker.linear_block(self.n_channel * 64, relu)
self.stacker.linear(self.n_classes)
self.logit = self.stacker.last_layer
self.stacker.softmax()
self.proba = self.stacker.last_layer
def test_TFL1Normalize():
import numpy as np
x = np.random.normal(size=[100, 10])
y = np.random.normal(size=[100, 1])
x_ph = placeholder(tf.float32, [-1, 10], name='ph_x')
with tf.variable_scope('net'):
stack = Stacker(x_ph)
stack.linear_block(100, relu)
stack.linear_block(100, relu)
logit = stack.linear(1)
proba = stack.softmax()
loss = (proba - y)**2
var_list = collect_vars('net')
l1_norm = TFL1Normalize(var_list)
l1_norm.build()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
val = sess.run(l1_norm.penalty, feed_dict={x_ph: x})
rate = sess.run(l1_norm.rate_var)
print(f'l1_norm = {val}, rate = {rate}')
l1_norm.update_rate(sess, 0.5)
val = sess.run(l1_norm.penalty, feed_dict={x_ph: x})
rate = sess.run(l1_norm.rate_var)
print(f'l1_norm = {val}, rate = {rate}')
l1_norm.update_rate(sess, 0.1)
val = sess.run(l1_norm.penalty, feed_dict={x_ph: x})
rate = sess.run(l1_norm.rate_var)
print(f'l1_norm = {val}, rate = {rate}')
def build(self):
with tf.variable_scope(self.name, reuse=self.reuse):
self.stacker = Stacker(self.x)
self.stacker.conv_block(self.capacity, CONV_FILTER_3311, relu)
self.stacker.conv_block(self.capacity, CONV_FILTER_3311, relu)
self.stacker.max_pooling(CONV_FILTER_2222)
self.stacker.conv_block(self.capacity * 2, CONV_FILTER_3311, relu)
self.stacker.conv_block(self.capacity * 2, CONV_FILTER_3311, relu)
self.stacker.max_pooling(CONV_FILTER_2222)
self.stacker.conv_block(self.capacity * 4, CONV_FILTER_3311, relu)
self.stacker.conv_block(self.capacity * 4, CONV_FILTER_3311, relu)
self.stacker.max_pooling(CONV_FILTER_2222)
self.stacker.conv_block(self.capacity * 8, CONV_FILTER_3311, relu)
self.stacker.conv_block(self.capacity * 8, CONV_FILTER_3311, relu)
self.stacker.conv_block(self.capacity * 8, CONV_FILTER_3311, relu)
self.encode = self.stacker.last_layer
return self
def encoder_tail(stack: Stacker,
latent_code_size,
bn=False,
activation='none') -> Stacker:
stack.linear(latent_code_size * 2)
if bn:
stack.bn()
stack.activation(activation)
return stack
def common_AE_encoder_tail(stack: Stacker,
latent_code_size,
bn=False,
activation='relu') -> Stacker:
stack.linear(latent_code_size)
if bn:
stack.bn()
stack.activation(activation)
return stack
def common_linear_stack(stack: Stacker,
net_shapes,
bn=True,
activation='relu') -> Stacker:
for shape in net_shapes:
stack.linear(shape)
if bn:
stack.bn()
stack.activation(activation)
return stack
def classifier(self, Xs, net_shapes, name='classifier'):
with tf.variable_scope(name):
layer = Stacker(flatten(Xs))
for net_shape in net_shapes:
layer.linear_block(net_shape, relu)
layer.linear(self.Y_flatten_size)
logit = layer.last_layer
h = softmax(logit)
return logit, h
def encoder_tail(stack: Stacker,
Y_flatten_size,
z_size,
bn=False,
activation='none'):
stack.linear(z_size + Y_flatten_size)
if bn:
stack.bn()
stack.activation(activation)
zs = stack.last_layer[:, :z_size]
Ys_gen = stack.last_layer[:, z_size:]
hs = softmax(Ys_gen)
return zs, Ys_gen, hs
def Q_function(Xs_gen, net_shapes, discrete_code_size, continuous_code_size, reuse=False, name='Q_function'):
with tf.variable_scope(name, reuse=reuse):
layer = Stacker(Xs_gen)
for shape in net_shapes:
layer.linear_block(shape, relu)
code_logit = layer.linear(discrete_code_size + continuous_code_size)
code = layer.softmax()
discrete_code = code[:, :discrete_code_size]
continuous_code = code[:, discrete_code_size:]
discrete_code_logit = code_logit[:, :discrete_code_size]
continuous_code_logit = code_logit[:, discrete_code_size:]
return discrete_code, discrete_code_logit, continuous_code, continuous_code_logit
def test_TFDynamicLearningRate():
import numpy as np
x = np.random.normal(size=[100, 10])
y = np.random.normal(size=[100, 1])
x_ph = placeholder(tf.float32, [-1, 10], name='ph_x')
stack = Stacker(x_ph)
stack.linear_block(100, relu)
stack.linear_block(100, relu)
logit = stack.linear(1)
proba = stack.softmax()
loss = (proba - y) ** 2
dlr = TFDynamicLearningRate(0.01)
dlr.build()
lr_var = dlr.learning_rate
var_list = None
train_op = tf.train.AdamOptimizer(learning_rate=lr_var, beta1=0.9).minimize(loss, var_list=var_list)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
print(f'lr = {dlr.lr_tensor(sess)}')
dlr.update(sess, 0.1)
print(f'lr = {dlr.lr_tensor(sess)}')
sess.run(train_op, feed_dict={x_ph: x})
print(f'loss = {np.mean(sess.run(loss, feed_dict={x_ph: x}))}')
dlr.update(sess, 0.05)
print(f'lr = {dlr.lr_tensor(sess)}')
sess.run(train_op, feed_dict={x_ph: x})
print(f'loss = {np.mean(sess.run(loss, feed_dict={x_ph: x}))}')
dlr.update(sess, 0.02)
print(f'lr = {dlr.lr_tensor(sess)}')
sess.run(train_op, feed_dict={x_ph: x})
print(f'loss = {np.mean(sess.run(loss, feed_dict={x_ph: x}))}')
def multi_pool_B(x, channels, name='multi_pool_B'):
with tf.variable_scope(name):
a = Stacker(x, name='branch_A')
a.conv_block(channels['a0'], CONV_FILTER_1111, relu)
a.conv_block(channels['a1'], CONV_FILTER_3322, relu)
b = Stacker(x, name='branch_B')
b.conv_block(channels['b0'], CONV_FILTER_1111, relu)
b.conv_block(channels['b1'], CONV_FILTER_7111, relu)
b.conv_block(channels['b2'], CONV_FILTER_1711, relu)
b.conv_block(channels['b3'], CONV_FILTER_3322, relu)
c = Stacker(x, name='branch_C')
c.max_pooling(CONV_FILTER_3322)
return concat([a.last_layer, b.last_layer, c.last_layer], axis=3)
def module_B(x, channels, name='module_B'):
# mixed4: 17 x 17 x 768.
with tf.variable_scope(name):
a = Stacker(x, name='branch_A')
a.conv_block(channels['a0'], CONV_FILTER_1111, relu)
b = Stacker(x, name='branch_B')
b.conv_block(channels['b0'], CONV_FILTER_1111, relu)
b.conv_block(channels['b1'], CONV_FILTER_7111, relu)
b.conv_block(channels['b2'], CONV_FILTER_1711, relu)
c = Stacker(x, name='branch_C')
c.conv_block(channels['c0'], CONV_FILTER_1111, relu)
c.conv_block(channels['c1'], CONV_FILTER_7111, relu)
c.conv_block(channels['c2'], CONV_FILTER_1711, relu)
c.conv_block(channels['c3'], CONV_FILTER_7111, relu)
c.conv_block(channels['c4'], CONV_FILTER_1711, relu)
d = Stacker(x, name='branch_D')
d.max_pooling(CONV_FILTER_3311)
d.conv_block(channels['d1'], CONV_FILTER_3311, relu)
return concat(
[a.last_layer, b.last_layer, c.last_layer, d.last_layer],
axis=3)
def decoder_head(zs, Ys) -> Stacker:
stack = Stacker(concat((zs, Ys), axis=1))
return stack
def encoder_head(Xs):
stack = Stacker(Xs)
stack.flatten()
return stack
def build(self):
self.DynamicDropoutRate = DynamicDropoutRate(self.dropout_rate)
self.drop_out_tensor = self.DynamicDropoutRate.tensor
with tf.variable_scope(self.name, reuse=self.reuse):
self.encoder = InceptionV2UNetEncoderModule(self.x,
None,
resize_shape=(201,
201),
capacity=self.capacity)
self.encoder.build()
encode = self.encoder.last_layer
skip_tensors = self.encoder.skip_tensors[::-1]
bottom_layer = self.bottom_layer(encode, self.capacity * 128,
self.depth, self.drop_out_tensor)
pprint(skip_tensors)
stacker = Stacker(bottom_layer)
stacker.layers_conv2d_transpose(self.n_channel * 16 * 2, (3, 3),
(2, 2), 'SAME')
stacker.bn()
stacker.relu()
stacker.concat(
[stacker.last_layer[:, :13, :13, :], skip_tensors[0]], 3)
stacker.layers_dropout(self.drop_out_tensor)
stacker.layers_conv2d(self.n_channel * 16 * 2, (3, 3), (1, 1),
'SAME')
stacker.bn()
stacker.relu()
stacker.add_layer(_residual_block_pre_activation,
self.n_channel * 16 * 2)
stacker.add_layer(_residual_block_pre_activation,
self.n_channel * 16 * 2)
stacker.add_layer(_residual_block_pre_activation,
self.n_channel * 16 * 2)
stacker.bn()
stacker.relu()
# 12 to 25
stacker.layers_conv2d_transpose(self.n_channel * 8 * 2, (3, 3),
(2, 2), 'SAME')
stacker.bn()
stacker.relu()
stacker.concat([stacker.last_layer, skip_tensors[1]], 3)
stacker.layers_dropout(self.drop_out_tensor)
stacker.layers_conv2d(self.n_channel * 8 * 2, (3, 3), (1, 1),
'SAME')
stacker.bn()
stacker.relu()
stacker.add_layer(_residual_block_pre_activation,
self.n_channel * 8 * 2)
stacker.add_layer(_residual_block_pre_activation,
self.n_channel * 8 * 2)
stacker.add_layer(_residual_block_pre_activation,
self.n_channel * 8 * 2)
stacker.bn()
stacker.relu()
# 25 to 50
stacker.layers_conv2d_transpose(self.n_channel * 4 * 2, (3, 3),
(2, 2), 'SAME')
stacker.bn()
stacker.relu()
stacker.concat(
[stacker.last_layer[:, :51, :51, :], skip_tensors[2]], 3)
stacker.layers_dropout(self.drop_out_tensor)
stacker.layers_conv2d(self.n_channel * 4 * 2, (3, 3), (1, 1),
'SAME')
stacker.bn()
stacker.relu()
stacker.add_layer(_residual_block_pre_activation,
self.n_channel * 4 * 2)
stacker.add_layer(_residual_block_pre_activation,
self.n_channel * 4 * 2)
stacker.add_layer(_residual_block_pre_activation,
self.n_channel * 4 * 2)
stacker.bn()
stacker.relu()
# 50 to 101
stacker.layers_conv2d_transpose(self.n_channel * 2 * 2, (3, 3),
(2, 2), 'SAME')
stacker.bn()
stacker.relu()
stacker.concat(
[stacker.last_layer[:, :101, :101, :], skip_tensors[3]], 3)
stacker.layers_dropout(self.drop_out_tensor)
stacker.layers_conv2d(self.n_channel * 2 * 2, (3, 3), (1, 1),
'SAME')
stacker.bn()
stacker.relu()
stacker.add_layer(_residual_block_pre_activation,
self.n_channel * 2 * 2)
stacker.add_layer(_residual_block_pre_activation,
self.n_channel * 2 * 2)
stacker.add_layer(_residual_block_pre_activation,
self.n_channel * 2 * 2)
stacker.bn()
stacker.relu()
decode = stacker.last_layer
stacker = Stacker(decode, name='to_match')
stacker.conv2d(self.n_classes, CONV_FILTER_3311)
self.logit = stacker.last_layer
self.proba = stacker.sigmoid()
def module_C(x, channels, name='module_C'):
# type b?
with tf.variable_scope(name):
a = Stacker(x, name='branch_A')
a.conv_block(channels['a0'], CONV_FILTER_1111, relu)
b = Stacker(x, name='branch_B')
b.conv_block(channels['b0'], CONV_FILTER_1111, relu)
b0 = Stacker(b.last_layer, name='branch_B0')
b0.conv_block(channels['b1-0'], CONV_FILTER_3111, relu)
b1 = Stacker(b.last_layer, name='branch_B1')
b1.conv_block(channels['b1-1'], CONV_FILTER_1311, relu)
c = Stacker(x, name='branch_C')
c.conv_block(channels['c0'], CONV_FILTER_1111, relu)
c.conv_block(channels['c1'], CONV_FILTER_3311, relu)
c0 = Stacker(c.last_layer, name='branch_C0')
c0.conv_block(channels['c2-0'], CONV_FILTER_1311, relu)
c1 = Stacker(c.last_layer, name='branch_C1')
c1.conv_block(channels['c2-1'], CONV_FILTER_3111, relu)
d = Stacker(x, name='branch_D')
d.max_pooling(CONV_FILTER_3311)
d.conv_block(channels['d1'], CONV_FILTER_3311, relu)
return concat([
a.last_layer, b0.last_layer, b1.last_layer, c0.last_layer,
c1.last_layer, d.last_layer
],
axis=3)
def _residual_block_pre_activation(x,
n_channel,
filter_=(3, 3),
name='residual_block_pre_activation'):
with tf.variable_scope(name):
stack = Stacker(x)
stack.bn()
stack.relu()
stack.layers_conv2d(n_channel, filter_, (1, 1), 'SAME')
stack.bn()
stack.relu()
stack.layers_conv2d(n_channel, filter_, (1, 1), 'SAME')
stack.residual_add(x)
return stack.last_layer
def aux(self, x, n_classes, name='aux'):
with tf.variable_scope(name):
stack = Stacker(x)
stack.avg_pooling(CONV_FILTER_5533)
stack.conv_block(self.n_channel * 8, CONV_FILTER_1111, relu)
filter_ = list(stack.last_layer.shape[1:3]) + [1, 1]
stack.conv_block(self.n_channel * 48, filter_, relu)
stack.flatten()
logit = stack.linear(n_classes)
proba = stack.softmax()
return logit, proba