def feature_discriminator(x, phase, C=1, reuse=None):
with tf.variable_scope('disc/feat', reuse=reuse):
with arg_scope([dense], activation=tf.nn.relu): # Switch to leaky?
x = dense(x, 100)
x = dense(x, C, activation=None)
return x
def __call__(self, x, is_training = True):
with tf.variable_scope(self.name) as scope:
with arg_scope([tcl.batch_norm], is_training=is_training, scale=True):
with arg_scope([tcl.conv2d, tcl.conv2d_transpose], activation_fn=tf.nn.relu,
normalizer_fn=tcl.batch_norm,
biases_initializer=None,
padding='SAME',
weights_regularizer=tcl.l2_regularizer(0.0002)):
size = 16
# x: s x s x 3
se = tcl.conv2d(x, num_outputs=size, kernel_size=4, stride=1) # 256 x 256 x 16
se = resBlock(se, num_outputs=size * 2, kernel_size=4, stride=2) # 128 x 128 x 32
se = resBlock(se, num_outputs=size * 2, kernel_size=4, stride=1) # 128 x 128 x 32
se = resBlock(se, num_outputs=size * 4, kernel_size=4, stride=2) # 64 x 64 x 64
se = resBlock(se, num_outputs=size * 4, kernel_size=4, stride=1) # 64 x 64 x 64
se = resBlock(se, num_outputs=size * 8, kernel_size=4, stride=2) # 32 x 32 x 128
se = resBlock(se, num_outputs=size * 8, kernel_size=4, stride=1) # 32 x 32 x 128
se = resBlock(se, num_outputs=size * 16, kernel_size=4, stride=2) # 16 x 16 x 256
se = resBlock(se, num_outputs=size * 16, kernel_size=4, stride=1) # 16 x 16 x 256
se = resBlock(se, num_outputs=size * 32, kernel_size=4, stride=2) # 8 x 8 x 512
se = resBlock(se, num_outputs=size * 32, kernel_size=4, stride=1) # 8 x 8 x 512
pd = tcl.conv2d_transpose(se, size * 32, 4, stride=1) # 8 x 8 x 512
pd = tcl.conv2d_transpose(pd, size * 16, 4, stride=2) # 16 x 16 x 256
pd = tcl.conv2d_transpose(pd, size * 16, 4, stride=1) # 16 x 16 x 256
pd = tcl.conv2d_transpose(pd, size * 16, 4, stride=1) # 16 x 16 x 256
pd = tcl.conv2d_transpose(pd, size * 8, 4, stride=2) # 32 x 32 x 128
pd = tcl.conv2d_transpose(pd, size * 8, 4, stride=1) # 32 x 32 x 128
pd = tcl.conv2d_transpose(pd, size * 8, 4, stride=1) # 32 x 32 x 128
pd = tcl.conv2d_transpose(pd, size * 4, 4, stride=2) # 64 x 64 x 64
pd = tcl.conv2d_transpose(pd, size * 4, 4, stride=1) # 64 x 64 x 64
pd = tcl.conv2d_transpose(pd, size * 4, 4, stride=1) # 64 x 64 x 64
pd = tcl.conv2d_transpose(pd, size * 2, 4, stride=2) # 128 x 128 x 32
pd = tcl.conv2d_transpose(pd, size * 2, 4, stride=1) # 128 x 128 x 32
pd = tcl.conv2d_transpose(pd, size, 4, stride=2) # 256 x 256 x 16
pd = tcl.conv2d_transpose(pd, size, 4, stride=1) # 256 x 256 x 16
pd = tcl.conv2d_transpose(pd, 3, 4, stride=1) # 256 x 256 x 3
pd = tcl.conv2d_transpose(pd, 3, 4, stride=1) # 256 x 256 x 3
pos = tcl.conv2d_transpose(pd, 3, 4, stride=1, activation_fn = tf.nn.sigmoid)#, padding='SAME', weights_initializer=tf.random_normal_initializer(0, 0.02))
return pos
def Batch_Normalization(x, training, scope):
with arg_scope([batch_norm],
scope=scope,
updates_collections=None,
decay=0.9,
center=True,
scale=True,
zero_debias_moving_mean=True) :
return tf.cond(training,
lambda : batch_norm(inputs=x, is_training=training, reuse=None),
lambda : batch_norm(inputs=x, is_training=training, reuse=True))
def classifier(x, phase, enc_phase=1, trim=0, scope='class', reuse=None, internal_update=False, getter=None):
with tf.variable_scope(scope, reuse=reuse, custom_getter=getter):
with arg_scope([leaky_relu], a=0.1), \
arg_scope([conv2d, dense], activation=leaky_relu, bn=True, phase=phase), \
arg_scope([batch_norm], internal_update=internal_update):
preprocess = instance_norm if args.inorm else tf.identity
layout = [
(preprocess, (), {}),
(conv2d, (96, 3, 1), {}),
(conv2d, (96, 3, 1), {}),
(conv2d, (96, 3, 1), {}),
(max_pool, (2, 2), {}),
(dropout, (), dict(training=phase)),
(noise, (1,), dict(phase=phase)),
(conv2d, (192, 3, 1), {}),
(conv2d, (192, 3, 1), {}),
(conv2d, (192, 3, 1), {}),
(max_pool, (2, 2), {}),
(dropout, (), dict(training=phase)),
(noise, (1,), dict(phase=phase)),
(conv2d, (192, 3, 1), {}),
(conv2d, (192, 3, 1), {}),
(conv2d, (192, 3, 1), {}),
(avg_pool, (), dict(global_pool=True)),
(dense, (args.Y,), dict(activation=None))
]
if enc_phase:
start = 0
end = len(layout) - trim
else:
start = len(layout) - trim
end = len(layout)
for i in xrange(start, end):
with tf.variable_scope('l{:d}'.format(i)):
f, f_args, f_kwargs = layout[i]
x = f(x, *f_args, **f_kwargs)
return x
def _model_graph_def(model_type='resnet_v1_50', arg_sc=None):
"""Constructs a model graph, returning GraphDef and end-points.
Args:
model_type: Type of model to be used.
arg_sc: Optional arg scope to use in constructing the graph.
Returns:
graph_def: GraphDef of constructed graph.
end_points: A dictionary from components of the network to the corresponding
activations.
"""
if arg_sc is None:
arg_sc = {}
g = ops.Graph()
with g.as_default():
with framework.arg_scope(arg_sc):
end_points = _construct_model(model_type)
return g.as_graph_def(), end_points
def q_net(x,
observed=None,
n_z=None,
is_training=False,
is_initializing=False):
net = spt.BayesianNet(observed=observed)
normalizer_fn = None if not config.act_norm else functools.partial(
spt.layers.act_norm,
axis=-1 if config.channels_last else -3,
initializing=is_initializing,
value_ndims=3,
)
# compute the hidden features
with arg_scope([spt.layers.resnet_conv2d_block],
kernel_size=config.kernel_size,
shortcut_kernel_size=config.shortcut_kernel_size,
activation_fn=tf.nn.leaky_relu,
normalizer_fn=normalizer_fn,
kernel_regularizer=spt.layers.l2_regularizer(config.l2_reg),
channels_last=config.channels_last):
h_x = tf.to_float(x)
h_x = spt.layers.resnet_conv2d_block(h_x, 16) # output: (16, 28, 28)
h_x = spt.layers.resnet_conv2d_block(h_x, 32,
strides=2) # output: (32, 14, 14)
h_x = spt.layers.resnet_conv2d_block(h_x, 32) # output: (32, 14, 14)
h_x = spt.layers.resnet_conv2d_block(h_x, 64,
strides=2) # output: (64, 7, 7)
h_x = spt.layers.resnet_conv2d_block(h_x, 64) # output: (64, 7, 7)
# sample z ~ q(z|x)
h_x = spt.ops.reshape_tail(h_x, ndims=3, shape=[-1])
z_mean = spt.layers.dense(h_x, config.z_dim, name='z_mean')
z_logstd = spt.layers.dense(h_x, config.z_dim, name='z_logstd')
z = net.add('z',
spt.Normal(mean=z_mean, logstd=z_logstd),
n_samples=n_z,
group_ndims=1)
return net
def pix2pix_arg_scope():
"""Returns a default argument scope for isola_net.
Returns:
An arg scope.
"""
# These parameters come from the online port, which don't necessarily match
# those in the paper.
# TODO(nsilberman): confirm these values with Philip.
instance_norm_params = {
'center': True,
'scale': True,
'epsilon': 0.00001,
}
with contrib_framework.arg_scope(
[layers.conv2d, layers.conv2d_transpose],
normalizer_fn=layers.instance_norm,
normalizer_params=instance_norm_params,
weights_initializer=tf.random_normal_initializer(0, 0.02)) as sc:
return sc
def predict(self, input_tensor, scope=None):
"""
Args:
input_tensor: a float tensor with shape [batch_size, image_height, image_width, 3]
Returns:
predictions_dict: a diction of predicted tensors
"""
with tf.variable_scope(scope, 'CTCModel', [input_tensor]):
# CRNN特征提取
with tf.variable_scope('Feature_extractor', [input_tensor]):
preprocessed_inputs = self._feature_extractor.preprocess(input_tensor)
feature_maps = self._feature_extractor.extract_feature(preprocessed_inputs) # [B, 1, w, C]
if len(feature_maps) != 1:
raise ValueError('CTCModel only accepts single feature sequence')
feature_sequence = tf.squeeze(feature_maps[0], axis=1) # [B, W, C]
# 全连接层
with tf.variable_scope('Predictor', [feature_sequence]),arg_scope(self._fc_hyperparams):
logits = fully_connected(feature_sequence, self._label_map.num_classes + 1, activation_fn=None) # [B, W, num_classes+1]
return {'logits': logits}
def q_net(config, x, observed=None, n_z=None, is_training=True):
net = BayesianNet(observed=observed)
# compute the hidden features
with arg_scope([dense],
activation_fn=tf.nn.leaky_relu,
kernel_regularizer=l2_regularizer(config.l2_reg)):
h_x = tf.to_float(x)
h_x = dense(h_x, 500)
h_x = dense(h_x, 500)
# sample z ~ q(z|x)
z_mean = dense(h_x, config.z_dim, name='z_mean')
z_logstd = dense(h_x, config.z_dim, name='z_logstd')
z = net.add('z',
Normal(mean=z_mean, logstd=z_logstd),
n_samples=n_z,
group_ndims=1,
flow=posterior_flow())
return net
def default_arg_scope(is_training, **kwargs):
""" arg_scope
Args:
is_training: boolean tensor, Whether outputs of the model is used to train or inference
kwargs: keyword arguments
Returns:
arg_scope: A arg_scope context manager
"""
default_kwargs = dict(normalizer_fn=batch_norm,
normalizer_params={
'is_training': is_training,
'center': True,
'scale': True,
'fused': True
},
weights_initializer=tf.random_normal_initializer(
mean=0.0, stddev=0.02))
default_kwargs.update(kwargs)
with arg_scope([conv2d, conv2d_transpose, fully_connected],
**default_kwargs) as sc:
return sc
def testShareParams(self):
# Tests reuse option.
first_outputs = 2
alternate_num_outputs = 12
parameterization = {'first/Conv2D': first_outputs}
decorator = ops.ConfigurableOps(parameterization=parameterization)
explicit = layers.conv2d(self.inputs, first_outputs, 3, scope='first')
with arg_scope([layers.conv2d], reuse=True):
decorated = decorator.conv2d(self.inputs,
num_outputs=alternate_num_outputs,
kernel_size=3,
scope='first')
with self.cached_session():
tf.global_variables_initializer().run()
# verifies that parameters are shared.
self.assertAllClose(explicit.eval(), decorated.eval())
conv_ops = sorted([
op.name for op in tf.get_default_graph().get_operations()
if op.type == 'Conv2D'
])
self.assertAllEqual(['first/Conv2D', 'first_1/Conv2D'], conv_ops)
def h_for_p_x(z, is_training, channels_last):
with arg_scope([deconv_resnet_block],
shortcut_kernel_size=config.shortcut_kernel_size,
activation_fn=tf.nn.leaky_relu,
kernel_regularizer=l2_regularizer(config.l2_reg),
channels_last=channels_last):
h_z, s1, s2 = flatten(z, 2)
h_z = tf.reshape(dense(h_z, 64 * 7 * 7),
[-1, 7, 7, 64] if channels_last else [-1, 64, 7, 7])
h_z = deconv_resnet_block(h_z, 64) # output: (64, 7, 7)
h_z = deconv_resnet_block(h_z, 32, strides=2) # output: (32, 14, 14)
h_z = deconv_resnet_block(h_z, 32) # output: (32, 14, 14)
h_z = deconv_resnet_block(h_z, 16, strides=2) # output: (16, 28, 28)
h_z = conv2d(h_z,
1, (1, 1),
padding='same',
name='feature_map_to_pixel',
channels_last=channels_last) # output: (1, 28, 28)
h_z = tf.reshape(h_z, [-1, config.x_dim])
x_logits = unflatten(h_z, s1, s2)
return {'logits': x_logits}
def __init__(self, model: CVAE1):
self.model = model
self.iaf_layers = model.layers
with arg_scope([conv2d, deconv2d], init=(self.model.mode == "init")):
self.latent_layers = [LatentLayer(lower, upper)
for lower, upper in
zip(self.iaf_layers[:-1], self.iaf_layers[1:])]
self.top_context_inputs = create_context_inputs(self.model.hps, prefix='top_')
input = self.model.initial_input_down()
h_det, posterior, prior, _ = self.iaf_layers[-1].down_split(input,
*self.top_context_inputs)
top_inputs = (h_det, input)
self.top_posterior_params_and_inputs = (posterior.mean, posterior.std), top_inputs
self.top_prior_params_and_inputs = (prior.mean, prior.std), top_inputs
self.bottom_down_inputs = create_down_inputs(self.model.hps, prefix='bottom_')
input = self.iaf_layers[0].down_merge(*self.bottom_down_inputs)
self.outputs = self.model.upsample_and_postprocess(input), self.model.dec_log_stdv
def forward_relax3(self):
'''
:return: the hidden state with shape: [time_step, batch_size, K * n_hidden],
parameters: A dict with A, gama, lambda1, lambda2, h_0
'''
# define variables and constants in the sista_rnn model
Us = []
Vs = []
with tf.variable_scope('sista_rnn_trainable_variables'):
for i in range(self.K):
Vs.append(
self.__get_variable('V_{}'.format(i + 1),
[self.n_hidden, self.n_input]))
for i in range(self.K - 1):
Us.append(
self.__get_variable('U_{}'.format(i + 1),
[self.n_hidden, self.n_hidden]))
gama = self.__get_variable('gama', None,
tf.constant(self.gama, tf.float32))
lambda1 = self.__get_variable(
'lambda1', None, tf.constant(self.lambda1, tf.float32))
parameters = OrderedDict([('U_{}'.format(i + 1), Us[i])
for i in range(self.K - 1)] +
[('V_{}'.format(i + 1), Vs[i])
for i in range(self.K)] +
[('gama', gama), ('lambda1', lambda1)])
with arg_scope([matmul], transpose_b=True):
h = self.__soft(matmul(self.now_input[0], Vs[0]), lambda1 / gama)
#h = tf.nn.tanh(matmul(self.now_input[0], Vs[0]))
for k in range(self.K - 1):
h = self.__soft(
matmul(self.now_input[0], Vs[k + 1]) + matmul(h, Us[k]),
lambda1 / gama)
#h = tf.nn.tanh(matmul(self.now_input[0], Vs[k + 1]) + matmul(h, Us[k]))
h = tf.expand_dims(h, 0)
return h, parameters
def build(self):
endpoints = self.endpoints
y = self.inputs['images']
with arg_scope([layers.conv2d], activation_fn=tf.nn.relu6,
weights_regularizer=layers.l2_regularizer(self.weight_decay),
biases_regularizer=layers.l2_regularizer(self.weight_decay)):
y = layers.conv2d(y, 96, (11, 11), 4, 'VALID', scope='conv1')
endpoints['conv1'] = y
y = tf.nn.lrn(y, 5, 1, 0.0001, 0.75)
y = layers.max_pool2d(y, (3, 3), 2, 'VALID', scope='pool1')
y1, y2 = tf.split(y, 2, 3)
y1 = layers.conv2d(y1, 128, (5, 5), 1, 'SAME', scope='conv2_1')
y2 = layers.conv2d(y2, 128, (5, 5), 1, 'SAME', scope='conv2_2')
endpoints['conv2_1'] = y1
endpoints['conv2_2'] = y2
y = tf.concat((y1, y2), 3)
endpoints['conv2'] = y
y = tf.nn.lrn(y, 5, 1, 0.0001, 0.75)
y = layers.max_pool2d(y, (3, 3), 2, 'VALID', scope='pool2')
y = layers.conv2d(y, 384, (3, 3), 1, 'SAME', scope='conv3')
endpoints['conv3'] = y
y1, y2 = tf.split(y, 2, 3)
y1 = layers.conv2d(y1, 192, (3, 3), 1, 'SAME', scope='conv4_1')
y2 = layers.conv2d(y2, 192, (3, 3), 1, 'SAME', scope='conv4_2')
endpoints['conv4_1'] = y1
endpoints['conv4_2'] = y2
y1 = layers.conv2d(y1, 128, (3, 3), 1, 'SAME', scope='conv5_1')
y2 = layers.conv2d(y2, 128, (3, 3), 1, 'SAME', scope='conv5_2')
endpoints['conv5_1'] = y1
endpoints['conv5_2'] = y2
y = tf.concat([y1, y2], 3)
endpoints['conv5'] = y
y = layers.max_pool2d(y, (3, 3), 2, 'VALID', scope='pool5')
y = layers.conv2d(y, 4096, (6, 6), 1, 'VALID', scope='fc6')
endpoints['fc6'] = y
y = layers.conv2d(y, 4096, (1, 1), 1, 'VALID', scope='fc7')
endpoints['fc7'] = y
y = layers.conv2d(y, 1000, (1, 1), 1, 'VALID', scope='fc8', activation_fn=None)
endpoints['fc8'] = y
self.outputs['logits'] = tf.squeeze(y)
def model(x, is_training, channels_last, k=4, n=2):
with arg_scope([spt.layers.resnet_conv2d_block],
kernel_size=config.kernel_size,
activation_fn=tf.nn.leaky_relu,
normalizer_fn=functools.partial(
tf.layers.batch_normalization,
axis=-1 if channels_last else -3,
training=is_training,
),
dropout_fn=functools.partial(tf.layers.dropout,
rate=config.dropout,
training=is_training),
kernel_regularizer=spt.layers.l2_regularizer(config.l2_reg),
channels_last=channels_last):
if not channels_last:
h_x = x
else:
h_x = tf.transpose(x, [0, 2, 3, 1])
h_x = spt.layers.conv2d(h_x,
16 * k, (1, 1),
channels_last=channels_last)
# 1st group, (16 * k, 32, 32)
for i in range(n):
h_x = spt.layers.resnet_conv2d_block(h_x, 16 * k)
# 2nd group, (32 * k, 16, 16)
h_x = spt.layers.resnet_conv2d_block(h_x, 32 * k, strides=2)
for i in range(n):
h_x = spt.layers.resnet_conv2d_block(h_x, 32 * k)
# 3rd group, (64 * k, 8, 8)
h_x = spt.layers.resnet_conv2d_block(h_x, 64 * k, strides=2)
for i in range(n):
h_x = spt.layers.resnet_conv2d_block(h_x, 64 * k)
h_x = spt.layers.global_avg_pool2d(
h_x, channels_last=channels_last) # output: (64 * k,)
logits = spt.layers.dense(h_x, 10, name='logits')
return logits
def hybrid(T):
def build_l2_loss():
l2 = []
bcde_variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
'bcde')
for bcde_v in bcde_variables:
bjde_v_name = bcde_v.name.replace('bcde/', 'bjde/')
bjde_v_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
bjde_v_name)
assert len(bjde_v_list) == 1
print "Pairing {:s}".format(bcde_v.name)
l2 += [tf.nn.l2_loss(bcde_v - bjde_v_list[0])]
l2_loss = tf.add_n(l2)
return l2_loss
with arg_scope([dense], bn=True, phase=T.phase):
with tf.variable_scope('bjde') as sc:
run_marginal = args.n_label < args.n_total
print "Building BJDE"
T.bjde_xu = bjde_x(T.xu) if run_marginal else constant(0)
T.bjde_yu = bjde_y(T.yu) if run_marginal else constant(0)
T.bjde_x = bjde_x(T.x, reuse=run_marginal)
T.bjde_xy = bjde_xy(T.x, T.y, reuse_x=True, reuse_y=run_marginal)
with tf.variable_scope('bcde'):
print "Building BCDE"
T.bcde = bcde(T.x, T.y, T.iw)
with tf.name_scope('l2_loss'):
print "Building l2_loss"
T.l2 = build_l2_loss()
with tf.name_scope('loss'):
# Eq. 13 from BCDE paper
T.loss = (args.l2 * T.l2 + T.u * (T.bjde_xu + T.bjde_yu) + 0.5 *
(1 - T.u) * (T.bjde_xy + T.bjde_x + T.bcde))
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
T.train_step = build_optimizer(T.loss, update_ops)
def __init__(self, hidden_size, batch_size, learning_rate):
self.input_tensor = tf.placeholder(tf.float32, [None, 28 * 28])
with arg_scope([layers.conv2d, layers.conv2d_transpose],
activation_fn=concat_elu,
normalizer_fn=layers.batch_norm,
normalizer_params={'scale': True}):
with tf.variable_scope("model"):
D1 = discriminator(self.input_tensor) # positive examples
D_params_num = len(tf.trainable_variables())
G = decoder(tf.random_normal([batch_size, hidden_size]))
self.sampled_tensor = G
with tf.variable_scope("model", reuse=True):
D2 = discriminator(G) # generated examples
D_loss = self.__get_discrinator_loss(D1, D2)
G_loss = self.__get_generator_loss(D2)
params = tf.trainable_variables()
D_params = params[:D_params_num]
G_params = params[D_params_num:]
# train_discrimator = optimizer.minimize(loss=D_loss, var_list=D_params)
# train_generator = optimizer.minimize(loss=G_loss, var_list=G_params)
global_step = tf.contrib.framework.get_or_create_global_step()
self.train_discrimator = layers.optimize_loss(D_loss,
global_step,
learning_rate / 10,
'Adam',
variables=D_params,
update_ops=[])
self.train_generator = layers.optimize_loss(G_loss,
global_step,
learning_rate,
'Adam',
variables=G_params,
update_ops=[])
self.sess = tf.Session()
self.sess.run(tf.global_variables_initializer())
def inference(feats, is_training, num_classes, reuse=False):
normalizer_params = {
'scale': False,
'is_training': is_training,
'reuse': reuse
}
with arg_scope(
[tf.contrib.layers.fully_connected],
activation_fn=tf.nn.relu,
normalizer_fn=tf.contrib.layers.batch_norm,
# normalizer_params={'scale':False,'is_training':is_training,'reuse':reuse},
reuse=reuse):
normalizer_params['scope'] = 'fc1/bn'
fc1 = tf.contrib.layers.fully_connected(
feats, 64, scope='fc1', normalizer_params=normalizer_params)
normalizer_params['scope'] = 'fc2/bn'
fc2 = tf.contrib.layers.fully_connected(
fc1, 128, scope='fc2', normalizer_params=normalizer_params)
normalizer_params['scope'] = 'fc3/bn'
fc3 = tf.contrib.layers.fully_connected(
fc2, 128, scope='fc3', normalizer_params=normalizer_params)
normalizer_params['scope'] = 'fc4/bn'
fc4 = tf.contrib.layers.fully_connected(
fc3, 256, scope='fc4', normalizer_params=normalizer_params)
normalizer_params['scope'] = 'fc5/bn'
fc4 = tf.contrib.layers.dropout(fc4, 0.7, is_training=is_training)
logits = tf.contrib.layers.fully_connected(fc4,
num_classes,
activation_fn=None,
normalizer_fn=None,
scope='fc5')
# print tf.trainable_variables()
return logits
def model(x, is_training, channels_last, k=4, n=2):
with arg_scope([resnet_block],
activation_fn=tf.nn.leaky_relu,
normalizer_fn=functools.partial(
batch_norm_2d,
channels_last=channels_last,
training=is_training,
),
dropout_fn=functools.partial(
tf.layers.dropout,
rate=config.dropout,
training=is_training
),
kernel_regularizer=l2_regularizer(config.l2_reg),
channels_last=channels_last):
if not channels_last:
h_x = x
else:
h_x = tf.transpose(x, [-1, 2, 3, 1])
h_x = conv2d(h_x, 16 * k, (1, 1), channels_last=channels_last)
# 1st group, (16 * k, 32, 32)
for i in range(n):
h_x = resnet_block(h_x, 16 * k)
# 2nd group, (32 * k, 16, 16)
h_x = resnet_block(h_x, 32 * k, strides=2)
for i in range(n):
h_x = resnet_block(h_x, 32 * k)
# 3rd group, (64 * k, 8, 8)
h_x = resnet_block(h_x, 64 * k, strides=2)
for i in range(n):
h_x = resnet_block(h_x, 64 * k)
h_x = global_average_pooling(
h_x, channels_last=channels_last) # output: (64 * k, 1, 1)
h_x = tf.reshape(h_x, [-1, 64 * k])
logits = dense(h_x, 10, name='logits')
return logits
def decoder(self, z_tensor, is_training):
""" Generator which maps a latent point to a data point
Args:
z_tensor: Tensor of shape [batch, latent_dim], a batch of latent points
is_training: Boolean, whether the output of this function will be used for training or inference
Returns:
output_tensor: Tensor of shape [batch, height, width, channel]
"""
with arg_scope(default_arg_scope(is_training)):
with tf.variable_scope('generator', reuse=tf.AUTO_REUSE):
h = fully_connected(z_tensor,
num_outputs=4 * 4 * 256,
scope='transform')
h = tf.reshape(h, shape=[-1, 4, 4, 256], name='reshape')
h = conv2d_transpose(h,
num_outputs=128,
kernel_size=3,
stride=2,
scope='conv1_transpose')
h = conv2d_transpose(h,
num_outputs=64,
kernel_size=3,
stride=2,
scope='conv2_transpose')
h = conv2d_transpose(h,
num_outputs=32,
kernel_size=3,
stride=2,
scope='conv3_transpose')
h = conv2d_transpose(h,
num_outputs=3,
kernel_size=3,
stride=1,
activation_fn=None,
normalizer_fn=None,
scope='conv4_transpose')
output_tensor = tf.math.tanh(h, name='tanh')
return output_tensor
def discriminator(self, x_tensor, z_tensor, is_training):
""" Build discriminator to compute probabilities that given samples (x, z) are from the encoder
Args:
x_tensor: Tensor of shape [batch, height, width, channel], a batch of data points
z_tensor: Tensor of shape [batch, latent_dim], a batch of latent points
is_training: Boolean, whether the output of this function will be used for training or inference
Returns:
probs: Tensor of shape [batch, 1], probability that given data points are from a true data distribution
disc_features: Tensor of shape [batch, feature_dim], discriminative feature vectors
"""
with arg_scope(kdd_arg_scope(is_training, activation_fn=leaky_relu)):
with tf.variable_scope('discriminator', reuse=tf.AUTO_REUSE):
with tf.variable_scope('network_x'):
x_features = fully_connected(x_tensor,
num_outputs=128,
scope='fc')
with tf.variable_scope('network_z'):
z_features = fully_connected(z_tensor,
num_outputs=128,
scope='fc')
with tf.variable_scope('network_relation'):
total_features = tf.concat([x_features, z_features],
axis=-1)
h = dropout(total_features,
keep_prob=0.2,
is_training=is_training,
scope='dropout1')
h = fully_connected(h, num_outputs=128, scope='fc')
disc_features = dropout(h,
keep_prob=0.2,
is_training=is_training,
scope='dropout2')
logits = fully_connected(disc_features,
num_outputs=1,
activation_fn=None,
scope='logit')
probs = tf.math.sigmoid(logits, name='sigmoid')
return probs, disc_features
def build_inception(inputs,
num_classes=1000,
params=params_inception(),
is_training=True,
scope=''):
"""Build Inception v4 architectures.
See here for reference: http://arxiv.org/pdf/1602.07261v1.pdf
"""
residual = params['residual']
version = params['version']
dropout_keep_prob = params['dropout_keep_prob']
residual_scale = params['residual_scale']
with arg_scope(inception_arg_scope()) as sc:
# fetch the activation_fn from scope for direct use in build fn
key = getattr(layers.conv2d, '_key_op', str(layers.conv2d))
activation_fn = sc[key]['activation_fn']
if residual:
assert version == 1 or version == 2
logits, endpoints = _build_inception_resnet(
inputs,
num_classes=num_classes,
ver=version,
res_scale=residual_scale,
activation_fn=
activation_fn, # activation_fn used directly in res blocks
dropout_keep_prob=dropout_keep_prob,
is_training=is_training,
scope=scope)
else:
assert version == 4
logits, endpoints = _build_inception_v4(
inputs,
num_classes=num_classes,
dropout_keep_prob=dropout_keep_prob,
is_training=is_training,
scope=scope)
return logits, endpoints
def G_omega(z, output_dim):
normalizer_fn = None
# compute the hidden features
with arg_scope([spt.layers.resnet_deconv2d_block],
kernel_size=config.kernel_size,
shortcut_kernel_size=config.shortcut_kernel_size,
activation_fn=tf.nn.leaky_relu,
normalizer_fn=normalizer_fn,
kernel_regularizer=spt.layers.l2_regularizer(
config.l2_reg)):
h_z = spt.layers.dense(z,
256 * config.x_shape[0] // 8 *
config.x_shape[1] // 8,
scope='level_0',
normalizer_fn=None)
h_z = spt.ops.reshape_tail(h_z,
ndims=1,
shape=(config.x_shape[0] // 8,
config.x_shape[1] // 8, 256))
h_z = spt.layers.resnet_deconv2d_block(
h_z, 256, scope='level_1') # output: (7, 7, 64)
h_z = spt.layers.resnet_deconv2d_block(
h_z, 256, scope='level_2') # output: (7, 7, 64)
h_z = spt.layers.resnet_deconv2d_block(
h_z, 128, strides=2, scope='level_3') # output: (14, 14, 32)
h_z = spt.layers.resnet_deconv2d_block(h_z, 64,
scope='level_4') # output:
h_z = spt.layers.resnet_deconv2d_block(h_z, 32,
scope='level_5') # output:
h_z = spt.layers.resnet_deconv2d_block(
h_z, 16, scope='level_6') # output: (28, 28, 16)
x_mean = spt.layers.conv2d(h_z,
output_dim, (1, 1),
padding='same',
scope='feature_map_mean_to_pixel',
kernel_initializer=tf.zeros_initializer(),
activation_fn=None)
return x_mean
def test_factor_out_layer_conv(self):
images_np = np.random.rand(8, 32, 32, 16)
images = tf.to_float(images_np)
flow = fl.InputLayer(images)
layer = fl.FactorOutLayer()
self.forward_inverse(layer, flow)
with tf_framework.arg_scope([fl.FlowLayer.__call__], forward=True):
new_flow = layer(flow)
x, logdet, z = new_flow
self.assertEqual([8, 32, 32, 8], x.shape.as_list())
self.assertEqual([8, 32, 32, 8], z.shape.as_list())
with self.test_session() as sess:
new_flow_np = sess.run(new_flow)
# x
self.assertAllClose(new_flow_np[0], images_np[:, :, :, 8:])
# z
self.assertAllClose(new_flow_np[2], images_np[:, :, :, :8])
# logdet
self.assertAllClose(new_flow_np[1], np.zeros_like(new_flow_np[1]))
def model_fn(features, labels, mode, params):
model_args = params['model_args'] if 'model_args' in params else {}
add_arg_scope(tf.layers.dropout)
with tf.variable_scope('GAN', values=[features]):
with arg_scope([tf.layers.dropout],
training=mode == tf.estimator.ModeKeys.TRAIN):
if mode == tf.estimator.ModeKeys.TRAIN:
train_args = params['train_args']
batch_size = train_args['train_batch_size']
model_args.update(batch_size=batch_size, **train_args)
return GAN.train(features, **model_args)
elif mode == tf.estimator.ModeKeys.EVAL:
eval_args = params['eval_args']
batch_size = eval_args['eval_batch_size']
model_args.update(batch_size=batch_size, **eval_args)
return GAN.evaluate(features, **model_args)
elif mode == tf.estimator.ModeKeys.PREDICT:
return GAN.predict(features, **model_args)
else:
raise ValueError
def create_towers(model_fn,
devices,
custom_variable_getter=None,
*args,
**kwargs):
""" Create towers for the passed in devices """
towers = []
scope_params = {'reuse': None, 'custom_getter': custom_variable_getter}
with framework.arg_scope([framework.model_variable, framework.variable],
device='/cpu:0'):
for index, device in enumerate(devices):
with tf.name_scope('tower{0}'.format(index)) as tower_scope:
with tf.device(device_fn(device)):
with tf.variable_scope(tf.get_variable_scope(),
**scope_params):
towers.append(
Tower(device, model_fn(*args, **kwargs),
tower_scope))
scope_params['reuse'] = True
return towers
def __init__(self, hidden_size, batch_size, learning_rate, generate_size):
self.input_tensor = tf.placeholder(tf.float32, [None, 28 * 28])
with arg_scope([layers.conv2d, layers.conv2d_transpose],
activation_fn=tf.nn.elu,
normalizer_fn=layers.batch_norm,
normalizer_params={'scale': True}):
with tf.variable_scope("model") as scope:
encoded = encoder(self.input_tensor, hidden_size * 2)
mean = encoded[:, :hidden_size]
stddev = tf.sqrt(tf.exp(encoded[:, hidden_size:]))
epsilon = tf.random_normal([tf.shape(mean)[0], hidden_size])
input_sample = mean + epsilon * stddev
output_tensor = decoder(input_sample)
with tf.variable_scope("model", reuse=True) as scope:
self.sampled_tensor = decoder(
tf.random_normal([batch_size, hidden_size]))
with tf.variable_scope("model", reuse=True) as scope1:
self.sampled_tensor_gener = decoder(
tf.random_normal([generate_size, hidden_size]))
vae_loss = self.__get_vae_cost(mean, stddev)
rec_loss = self.__get_reconstruction_cost(output_tensor,
self.input_tensor)
loss = vae_loss + rec_loss
self.train = layers.optimize_loss(
loss,
tf.contrib.framework.get_or_create_global_step(),
learning_rate=learning_rate,
optimizer='Adam',
update_ops=[])
self.sess = tf.Session()
self.sess.run(tf.global_variables_initializer())
def _block_b_reduce(net, endpoints, scope='BlockReduceB'):
# 17 x 17 -> 8 x 8 reduce
with arg_scope([layers.conv2d, layers.max_pool2d, layers.avg_pool2d],
padding='VALID'):
with tf.variable_scope(scope):
with tf.variable_scope('Br1_Pool'):
br1 = layers.max_pool2d(net, [3, 3],
stride=2,
scope='Pool1_3x3/2')
with tf.variable_scope('Br2_3x3'):
br2 = layers.conv2d(net,
192, [1, 1],
padding='SAME',
scope='Conv1_1x1')
br2 = layers.conv2d(br2,
192, [3, 3],
stride=2,
scope='Conv2_3x3/2')
with tf.variable_scope('Br3_7x7x3'):
br3 = layers.conv2d(net,
256, [1, 1],
padding='SAME',
scope='Conv1_1x1')
br3 = layers.conv2d(br3,
256, [1, 7],
padding='SAME',
scope='Conv2_1x7')
br3 = layers.conv2d(br3,
320, [7, 1],
padding='SAME',
scope='Conv3_7x1')
br3 = layers.conv2d(br3,
320, [3, 3],
stride=2,
scope='Conv4_3x3/2')
net = tf.concat(3, [br1, br2, br3], name='Concat1')
endpoints[scope] = net
print('%s output shape: %s' % (scope, net.get_shape()))
return net
def encoder(self,
x_tensor,
is_training):
""" Encoder which maps a data point to a latent point
Args:
x_tensor: Tensor of shape [batch, height, width, channel], a batch of data points
is_training: Boolean, whether the output of this function will be used for training or inference
Returns:
z_enc: Tensor of shape [batch, latent_dim]
"""
with arg_scope(default_arg_scope(is_training)):
with tf.variable_scope('encoder', reuse=tf.AUTO_REUSE):
h = conv2d(x_tensor, num_outputs=64, kernel_size=3, stride=2, scope='conv1')
h = conv2d(h, num_outputs=128, kernel_size=3, stride=2, scope='conv2')
h = conv2d(h, num_outputs=256, kernel_size=3, stride=2, scope='conv3')
h = conv2d(h, num_outputs=64, kernel_size=3, stride=1, scope='conv4')
conv_features = tf.layers.flatten(h, name='flatten')
z_enc = fully_connected(conv_features, num_outputs=self.latent_dim,
activation_fn=None, scope='encoding')
return z_enc
def heatmap(inputs, mode):
"""
inputs: output of uNet
mode[str]:'topLeft','bottomRight'
return [?,h,w,1] sigmoided
"""
normalizer_fn = tf_layers.batch_norm
activation = tf.nn.leaky_relu
with framework.arg_scope([tf_layers.conv2d],
kernel_size=3,
stride=1,
normalizer_fn=normalizer_fn,
activation_fn=tf.nn.leaky_relu,
padding="same"):
net = utils.corner_pool(inputs, mode=mode)
net = tf_layers.conv2d(net, num_outputs=128)
net = tf.add(net, inputs)
net = tf_layers.conv2d(net, num_outputs=64)
net = tf_layers.conv2d(net, num_outputs=32)
net = tf_layers.conv2d(net, num_outputs=1, activation_fn=tf.sigmoid)
return net
def coupling_layer_shift_and_scale(x1, n2):
# compute the hidden features
with arg_scope([spt.layers.dense],
activation_fn=tf.nn.leaky_relu,
kernel_regularizer=spt.layers.l2_regularizer(
config.l2_reg)):
h = x1
for _ in range(config.n_flow_hidden_layers):
h = spt.layers.dense(h, 500)
# compute shift and scale
shift = spt.layers.dense(h,
n2,
kernel_initializer=tf.zeros_initializer(),
bias_initializer=tf.zeros_initializer(),
name='shift')
scale = spt.layers.dense(h,
n2,
kernel_initializer=tf.zeros_initializer(),
bias_initializer=tf.zeros_initializer(),
name='scale')
return shift, scale
def p_net(config,
observed=None,
n_z=None,
is_training=True,
channels_last=False):
net = BayesianNet(observed=observed)
# sample z ~ p(z)
z = net.add('z',
Normal(mean=tf.zeros([1, config.z_dim]),
logstd=tf.zeros([1, config.z_dim])),
group_ndims=1,
n_samples=n_z)
# compute the hidden features
with arg_scope([deconv_resnet_block],
shortcut_kernel_size=config.shortcut_kernel_size,
activation_fn=tf.nn.leaky_relu,
kernel_regularizer=l2_regularizer(config.l2_reg),
channels_last=channels_last):
h_z, s1, s2 = flatten(z, 2)
h_z = tf.reshape(dense(h_z, 64 * 7 * 7),
[-1, 7, 7, 64] if channels_last else [-1, 64, 7, 7])
h_z = deconv_resnet_block(h_z, 64) # output: (64, 7, 7)
h_z = deconv_resnet_block(h_z, 32, strides=2) # output: (32, 14, 14)
h_z = deconv_resnet_block(h_z, 32) # output: (32, 14, 14)
h_z = deconv_resnet_block(h_z, 16, strides=2) # output: (16, 28, 28)
h_z = conv2d(h_z,
1, (1, 1),
padding='same',
name='feature_map_to_pixel',
channels_last=channels_last) # output: (1, 28, 28)
h_z = tf.reshape(h_z, [-1, config.x_dim])
# sample x ~ p(x|z)
x_logits = unflatten(h_z, s1, s2)
x = net.add('x', Bernoulli(logits=x_logits), group_ndims=1)
return net
def fire_module(inputs,
squeeze_depth,
expand_depth,
scope=None,
training=True):
'''
Description:
Creates 'a fire module' (defined in the SqueezeNet paper: arxiv.org/abs/1602.07360)
Composed of 3 1x1 conv2d layers followed by 4 1x1 conv2d layers concatenated with 4 3x3 conv2d layers
Input:
inputs: [rank 4 tf tensor] Conv2D input tensor
squeeze_depth: [int] Downsample filter number
expand_depth: [int] Sets output 4th dimension size
scope: [string] Name for tf.graph
training: [bool] Training flag for disabling batch norm at test time
Output:
net: [rank 4 tf tensor] Tensor layer output
'''
with tf.name_scope(scope):
with arg_scope([conv2d, max_pool2d]):
# Squeeze Layer
net = conv2d(inputs,
squeeze_depth, [1, 1],
name='squeeze',
training=training)
# Expand layer
e1x1 = conv2d(net,
expand_depth, [1, 1],
name='e1x1',
training=training)
e3x3 = conv2d(net,
expand_depth, [3, 3],
name='e3x3',
training=training)
# Concatenate operation
net = tf.concat([e1x1, e3x3], 3)
return net
def _block_stem_res(net, endpoints, scope='Stem'):
# Simpler _stem for inception-resnet-v1 network
# NOTE observe endpoints of first 3 layers
# default padding = VALID
# default stride = 1
with arg_scope([layers.conv2d, layers.max_pool2d, layers.avg_pool2d],
padding='VALID'):
with tf.variable_scope(scope):
# 299 x 299 x 3
net = layers.conv2d(net, 32, [3, 3], stride=2, scope='Conv1_3x3/2')
endpoints[scope + '/Conv1'] = net
# 149 x 149 x 32
net = layers.conv2d(net, 32, [3, 3], scope='Conv2_3x3')
endpoints[scope + '/Conv2'] = net
# 147 x 147 x 32
net = layers.conv2d(net,
64, [3, 3],
padding='SAME',
scope='Conv3_3x3')
endpoints[scope + '/Conv3'] = net
# 147 x 147 x 64
net = layers.max_pool2d(net, [3, 3], stride=2, scope='Pool1_3x3/2')
# 73 x 73 x 64
net = layers.conv2d(net,
80, [1, 1],
padding='SAME',
scope='Conv4_1x1')
# 73 x 73 x 80
net = layers.conv2d(net, 192, [3, 3], scope='Conv5_3x3')
# 71 x 71 x 192
net = layers.conv2d(net,
256, [3, 3],
stride=2,
scope='Conv6_3x3/2')
# 35 x 35 x 256
endpoints[scope] = net
print('%s output shape: %s' % (scope, net.get_shape()))
return net
hidden_output_size = 512
final_output_size = 10
xavier_init = tf.contrib.layers.xavier_initializer()
bn_params = {
'is_training': train_mode,
'decay': 0.9,
'updates_collections': None
}
# We can build short code using 'arg_scope' to avoid duplicate code
# same function with different arguments
with arg_scope([fully_connected],
activation_fn=tf.nn.relu,
weights_initializer=xavier_init,
biases_initializer=None,
normalizer_fn=batch_norm,
normalizer_params=bn_params
):
hidden_layer1 = fully_connected(X, hidden_output_size, scope="h1")
h1_drop = dropout(hidden_layer1, keep_prob, is_training=train_mode)
hidden_layer2 = fully_connected(h1_drop, hidden_output_size, scope="h2")
h2_drop = dropout(hidden_layer2, keep_prob, is_training=train_mode)
hidden_layer3 = fully_connected(h2_drop, hidden_output_size, scope="h3")
h3_drop = dropout(hidden_layer3, keep_prob, is_training=train_mode)
hidden_layer4 = fully_connected(h3_drop, hidden_output_size, scope="h4")
h4_drop = dropout(hidden_layer4, keep_prob, is_training=train_mode)
hypothesis = fully_connected(h4_drop, final_output_size, activation_fn=None, scope="hypothesis")
# define cost/loss & optimizer
