def __init__(self,
decay=1e-6,
affine_w_initializer=None,
affine_b_initializer=None,
acti_func='relu',
name='inet-affine'):
"""
This network estimates affine transformations from
a pair of moving and fixed image:
Hu et al., Label-driven weakly-supervised learning for
multimodal deformable image registration, arXiv:1711.01666
https://arxiv.org/abs/1711.01666
:param decay:
:param affine_w_initializer:
:param affine_b_initializer:
:param acti_func:
:param name:
"""
BaseNet.__init__(self, name=name)
self.fea = [4, 8, 16, 32, 64]
self.k_conv = 3
self.affine_w_initializer = affine_w_initializer
self.affine_b_initializer = affine_b_initializer
self.res_param = {
'w_initializer': GlorotUniform.get_instance(''),
'w_regularizer': regularizers.l2_regularizer(decay),
'acti_func': acti_func}
self.affine_param = {
'w_regularizer': regularizers.l2_regularizer(decay),
'b_regularizer': None}
def test_3d_reg_shape(self):
x = self.get_3d_input()
unet_block_op = UNetBlock(
'DOWNSAMPLE', (32, 64), (3, 3), with_downsample_branch=True,
w_regularizer=regularizers.l2_regularizer(0.3))
out_1, out_2 = unet_block_op(x, is_training=True)
print(unet_block_op)
print(out_1)
print(out_2)
unet_block_op = UNetBlock(
'UPSAMPLE', (32, 64), (3, 3), with_downsample_branch=False,
w_regularizer=regularizers.l2_regularizer(0.3))
out_3, _ = unet_block_op(x, is_training=True)
print(unet_block_op)
print(out_3)
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
out_1 = sess.run(out_1)
self.assertAllClose((2, 8, 8, 8, 64), out_1.shape)
out_2 = sess.run(out_2)
self.assertAllClose((2, 16, 16, 16, 64), out_2.shape)
out_3 = sess.run(out_3)
self.assertAllClose((2, 32, 32, 32, 64), out_3.shape)
def test_3d_reg_shape(self):
x = self.get_3d_data()
vnet_block_op = VNetBlock('DOWNSAMPLE', 2, 16, 8,
w_regularizer=regularizers.l2_regularizer(
0.2))
out_1, out_2 = vnet_block_op(x, x)
print(vnet_block_op)
vnet_block_op = VNetBlock('UPSAMPLE', 2, 16, 8,
w_regularizer=regularizers.l2_regularizer(
0.2))
out_3, out_4 = vnet_block_op(x, x)
print(vnet_block_op)
vnet_block_op = VNetBlock('SAME', 2, 16, 8,
w_regularizer=regularizers.l2_regularizer(
0.2))
out_5, out_6 = vnet_block_op(x, x)
print(vnet_block_op)
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
out_1 = sess.run(out_1)
self.assertAllClose((2, 16, 16, 16, 16), out_1.shape)
out_2 = sess.run(out_2)
self.assertAllClose((2, 8, 8, 8, 8), out_2.shape)
out_3 = sess.run(out_3)
self.assertAllClose((2, 16, 16, 16, 16), out_3.shape)
out_4 = sess.run(out_4)
self.assertAllClose((2, 32, 32, 32, 8), out_4.shape)
out_5 = sess.run(out_5)
self.assertAllClose((2, 16, 16, 16, 16), out_5.shape)
out_6 = sess.run(out_6)
self.assertAllClose((2, 16, 16, 16, 8), out_6.shape)
def test_fc_2d_bias_reg_shape(self):
input_param = {'n_output_chns': 10,
'with_bias': True,
'w_regularizer': regularizers.l2_regularizer(0.5),
'b_regularizer': regularizers.l2_regularizer(0.5)}
self._test_fc_output_shape(rank=2,
param_dict=input_param,
output_shape=(2, 10))
def test_fclayer_3d_bias_reg_shape(self):
input_param = {'n_output_chns': 10,
'with_bn': False,
'w_regularizer': regularizers.l2_regularizer(0.5),
'b_regularizer': regularizers.l2_regularizer(0.5)}
self._test_fc_layer_output_shape(rank=3,
param_dict=input_param,
output_shape=(2, 10))
def test_fclayer_2d_bn_reg_shape(self):
input_param = {'n_output_chns': 10,
'with_bias': False,
'with_bn': True,
'w_regularizer': regularizers.l2_regularizer(0.5),
'b_regularizer': regularizers.l2_regularizer(0.5)}
self._test_fc_layer_output_shape(rank=2,
param_dict=input_param,
output_shape=(2, 10),
is_training=True)
def test_deconv_3d_bias_reg_shape(self):
input_param = {'n_output_chns': 10,
'kernel_size': 3,
'stride': [2, 2, 1],
'with_bias': True,
'w_regularizer': regularizers.l2_regularizer(0.5),
'b_regularizer': regularizers.l2_regularizer(0.5)}
self._test_deconv_output_shape(rank=3,
param_dict=input_param,
output_shape=(2, 32, 32, 16, 10))
def test_convlayer_2d_bias_reg_shape(self):
input_param = {'n_output_chns': 10,
'kernel_size': [3, 5],
'stride': [2, 1],
'with_bias': True,
'with_bn': False,
'w_regularizer': regularizers.l2_regularizer(0.5),
'b_regularizer': regularizers.l2_regularizer(0.5)}
self._test_conv_layer_output_shape(rank=2,
param_dict=input_param,
output_shape=(2, 8, 16, 10))
def test_convlayer_3d_bn_reg_shape(self):
input_param = {'n_output_chns': 10,
'kernel_size': [5, 1, 2],
'stride': 1,
'with_bias': False,
'with_bn': True,
'w_regularizer': regularizers.l2_regularizer(0.5),
'b_regularizer': regularizers.l2_regularizer(0.5)}
self._test_conv_layer_output_shape(rank=3,
param_dict=input_param,
output_shape=(2, 16, 16, 16, 10),
is_training=True)
def test_convlayer_2d_bn_reg_shape(self):
input_param = {'n_output_chns': 10,
'kernel_size': [3, 5],
'stride': [2, 1],
'with_bias': False,
'feature_normalization': 'batch',
'w_regularizer': regularizers.l2_regularizer(0.5),
'b_regularizer': regularizers.l2_regularizer(0.5)}
self._test_conv_layer_output_shape(rank=2,
param_dict=input_param,
output_shape=(2, 8, 16, 10),
is_training=True)
def test_fclayer_3d_bn_reg_shape(self):
input_param = {
'n_output_chns': 10,
'with_bias': False,
'feature_normalization': 'batch',
'w_regularizer': regularizers.l2_regularizer(0.5),
'b_regularizer': regularizers.l2_regularizer(0.5)
}
self._test_fc_layer_output_shape(rank=3,
param_dict=input_param,
output_shape=(2, 10),
is_training=True)
def item_scores(self):
# (N_USER_IDS, 1, K)
user = tf.expand_dims(tf.nn.embedding_lookup(self.user_embeddings,
self.score_user_ids),
1,
name='user_test')
# (1, N_ITEM, K)
item = tf.tile(tf.expand_dims(self.item_embeddings, 0),
[tf.shape(user)[0], 1, 1],
name='item_test')
feature = tf.tile(tf.expand_dims(self.feature_projection, 0),
[tf.shape(user)[0], 1, 1],
name='feature_test')
input = tf.concat([
tf.reshape(tf.tile(user, [1, tf.shape(item)[1], 1]),
[-1, self.embed_dim]),
tf.reshape(item, [-1, self.embed_dim]),
tf.reshape(feature, [-1, self.embed_dim])
],
1,
name='input_test')
with tf.variable_scope('dense'):
hidden_layer = tf.layers.dense(
inputs=tf.nn.l2_normalize(input, dim=1),
units=5 * self.embed_dim,
trainable=False,
kernel_regularizer=regularizers.l2_regularizer(100.0),
activation=tf.nn.tanh,
name='hidden_layer',
reuse=True)
hidden_layer1 = tf.layers.dense(
inputs=tf.nn.l2_normalize(hidden_layer, dim=1),
units=1 * self.embed_dim,
trainable=False,
kernel_regularizer=regularizers.l2_regularizer(100.0),
activation=tf.nn.relu,
name='hidden_layer1',
reuse=True)
attention_layer_score = self.embed_dim * tf.nn.softmax(
hidden_layer1, dim=-1)
attention_reshape = tf.reshape(
attention_layer_score, [-1, tf.shape(item)[1], self.embed_dim],
name='attention_test')
scores = -tf.reduce_sum(tf.squared_difference(
tf.multiply(attention_reshape, user),
tf.multiply(attention_reshape, item)),
2,
name="scores")
top_n = tf.nn.top_k(scores, 10 + self.max_train_count, name='top_n')
return top_n
def test_deconv_3d_bias_reg_shape(self):
input_param = {
'n_output_chns': 10,
'kernel_size': 3,
'stride': [2, 2, 1],
'with_bias': True,
'w_regularizer': regularizers.l2_regularizer(0.5),
'b_regularizer': regularizers.l2_regularizer(0.5)
}
self._test_deconv_output_shape(rank=3,
param_dict=input_param,
output_shape=(2, 32, 32, 16, 10))
def test_convlayer_2d_bias_reg_shape(self):
input_param = {
'n_output_chns': 10,
'kernel_size': [3, 5],
'stride': [2, 1],
'with_bias': True,
'with_bn': False,
'w_regularizer': regularizers.l2_regularizer(0.5),
'b_regularizer': regularizers.l2_regularizer(0.5)
}
self._test_conv_layer_output_shape(rank=2,
param_dict=input_param,
output_shape=(2, 8, 16, 10))
def test_convlayer_3d_relu_shape(self):
input_param = {'n_output_chns': 10,
'kernel_size': [5, 1, 2],
'stride': [1, 2, 2],
'with_bias': False,
'feature_normalization': 'batch',
'acti_func': 'relu',
'w_regularizer': regularizers.l2_regularizer(0.5),
'b_regularizer': regularizers.l2_regularizer(0.5)}
self._test_conv_layer_output_shape(rank=3,
param_dict=input_param,
output_shape=(2, 16, 8, 8, 10),
is_training=True)
def test_convlayer_3d_bn_reg_shape(self):
input_param = {
'n_output_chns': 10,
'kernel_size': [5, 1, 2],
'stride': 1,
'with_bias': False,
'with_bn': True,
'w_regularizer': regularizers.l2_regularizer(0.5),
'b_regularizer': regularizers.l2_regularizer(0.5)
}
self._test_conv_layer_output_shape(rank=3,
param_dict=input_param,
output_shape=(2, 16, 16, 16, 10),
is_training=True)
def discriminator_fn(code, code2, hidden_units):
kwargs = {
'kernel_regularizer': l2_regularizer(1e-4),
'bias_regularizer': l2_regularizer(1e-4)
}
h = code
for hidden_unit in hidden_units:
h = Dense(hidden_unit, activation=tf.nn.leaky_relu, **kwargs)(h)
h = Dense(1, name='dis_out', **kwargs)(h)
logits = h
return logits
def test_2d_reg_shape(self):
input_shape = (2, 32, 32, 1)
x = tf.ones(input_shape)
highres_layer = HighRes3DNet(
num_classes=5,
w_regularizer=regularizers.l2_regularizer(0.5),
b_regularizer=regularizers.l2_regularizer(0.5))
out = highres_layer(x, is_training=True)
# print(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
out = sess.run(out)
self.assertAllClose((2, 32, 32, 5), out.shape)
def test_2d_reg_shape(self):
input_shape = (2, 20, 20, 1)
x = tf.ones(input_shape)
holistic_net_instance = HolisticNet(
num_classes=3,
w_regularizer=regularizers.l2_regularizer(0.5),
b_regularizer=regularizers.l2_regularizer(0.5))
out = holistic_net_instance(x, is_training=False)
# print(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
with self.cached_session() as sess:
sess.run(tf.global_variables_initializer())
out = sess.run(out)
self.assertAllClose((2, 20, 20, 3), out.shape)
def test_2d_reg_shape(self):
input_shape = (2, 20, 20, 1)
x = tf.ones(input_shape)
holistic_net_instance = HolisticNet(
num_classes=3,
w_regularizer=regularizers.l2_regularizer(0.5),
b_regularizer=regularizers.l2_regularizer(0.5))
out = holistic_net_instance(x, is_training=False)
# print(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
out = sess.run(out)
self.assertAllClose((2, 20, 20, 3), out.shape)
def test_2d_reg_shape(self):
input_shape = (2, 57, 57, 1)
x = tf.ones(input_shape)
deepmedic_instance = DeepMedic(
num_classes=160,
w_regularizer=regularizers.l2_regularizer(0.5),
b_regularizer=regularizers.l2_regularizer(0.5))
out = deepmedic_instance(x, is_training=True)
# print(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
out = sess.run(out)
self.assertAllClose((2, 9, 9, 160), out.shape)
def test_2d_reg_shape(self):
input_shape = (2, 57, 57, 1)
x = tf.ones(input_shape)
deepmedic_instance = DeepMedic(
num_classes=160,
w_regularizer=regularizers.l2_regularizer(0.5),
b_regularizer=regularizers.l2_regularizer(0.5))
out = deepmedic_instance(x, is_training=True)
# print(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
out = sess.run(out)
self.assertAllClose((2, 9, 9, 160), out.shape)
def discriminator2(img,
training=True,
weight_decay=0.0001,
batch_norm_decay=0.997,
batch_norm_epsilon=1e-5,
batch_norm_scale=True):
batch_norm_params = {
'is_training': training,
'decay': batch_norm_decay,
'epsilon': batch_norm_epsilon,
'scale': batch_norm_scale,
'updates_collections': ops.GraphKeys.UPDATE_OPS,
}
disc = slim.conv2d(
img,
64, [4, 4], [2, 2],
weights_regularizer=regularizers.l2_regularizer(weight_decay),
weights_initializer=initializers.variance_scaling_initializer(),
activation_fn=lrelu,
scope="convolutionstart")
with arg_scope(
[slim.conv2d],
weights_regularizer=regularizers.l2_regularizer(weight_decay),
weights_initializer=initializers.variance_scaling_initializer(),
activation_fn=lrelu,
normalizer_fn=layers.batch_norm,
normalizer_params=batch_norm_params):
disc = slim.stack(disc,
slim.conv2d, [(128, [4, 4], [2, 2]),
(256, [4, 4], [2, 2]),
(512, [4, 4], [2, 2]),
(1024, [4, 4], [2, 2])],
scope="convolution")
disc = slim.conv2d(
disc,
1024, [1, 1],
weights_regularizer=regularizers.l2_regularizer(weight_decay),
weights_initializer=initializers.variance_scaling_initializer(),
activation_fn=lrelu,
scope="convolutionend")
disc = tf.reshape(disc, [batch_size, 4 * 4 * 1024])
disc = slim.fully_connected(disc, 1, activation_fn=None, scope="logits")
return disc
def resnet_arg_scope(is_training=True,
weight_decay=cfg.TRAIN.WEIGHT_DECAY,
batch_norm_decay=0.997,
batch_norm_epsilon=1e-5,
batch_norm_scale=True):
batch_norm_params = {
# NOTE 'is_training' here does not work because inside resnet it gets reset:
# https://github.com/tensorflow/models/blob/master/slim/nets/resnet_v1.py#L187
'is_training': False,
'decay': batch_norm_decay,
'epsilon': batch_norm_epsilon,
'scale': batch_norm_scale,
'trainable': cfg.RESNET.BN_TRAIN,
'updates_collections': ops.GraphKeys.UPDATE_OPS
}
with arg_scope(
[slim.conv2d],
weights_regularizer=regularizers.l2_regularizer(weight_decay),
weights_initializer=initializers.variance_scaling_initializer(),
trainable=is_training,
activation_fn=nn_ops.relu,
normalizer_fn=layers.batch_norm,
normalizer_params=batch_norm_params):
with arg_scope([layers.batch_norm], **batch_norm_params) as arg_sc:
return arg_sc
def test_deconvlayer_3d_bias_reg_shape(self):
input_param = {
'n_output_chns': 10,
'kernel_size': 3,
'stride': 1,
'with_bias': True,
'feature_normalization': None,
'w_regularizer': regularizers.l2_regularizer(0.5),
'b_regularizer': regularizers.l2_regularizer(0.5)
}
self._test_deconv_layer_output_shape(rank=3,
param_dict=input_param,
output_shape=(2, 16, 16, 16, 10))
self._test_deconv_layer_output_shape(rank=3,
param_dict=input_param,
output_shape=(2, 16, 16, 16, 10))
def build_graph(self):
self._construct_weights()
saver, logits, KL = self.forward_pass()
log_softmax_var = tf.nn.log_softmax(logits)
neg_ll = -tf.reduce_mean(
tf.reduce_sum(log_softmax_var * self.input_ph, axis=-1))
# apply regularization to weights
reg = l2_regularizer(self.lam)
reg_var = apply_regularization(reg, self.weights_q + self.weights_p)
# tensorflow l2 regularization multiply 0.5 to the l2 norm
# multiply 2 so that it is back in the same scale
neg_ELBO = neg_ll + self.anneal_ph * KL + 2 * reg_var
train_op = tf.train.AdamOptimizer(self.lr).minimize(neg_ELBO)
# add summary statistics
tf.summary.scalar('negative_multi_ll', neg_ll)
tf.summary.scalar('KL', KL)
tf.summary.scalar('neg_ELBO_train', neg_ELBO)
merged = tf.summary.merge_all()
return saver, logits, neg_ELBO, train_op, merged
def high_order_module(inputs, order, scope):
with tf.variable_scope(scope):
in_channels = inputs.shape[3]
inter_channels = in_channels // 8 * 2
attention = 0
with arg_scope([layers_lib.conv2d],
weights_regularizer=regularizers.l2_regularizer(0.0001),
weights_initializer=initializers.
variance_scaling_initializer(),
activation_fn=None,
biases_initializer=None):
for i in range(order):
out = 1.0
for j in range(i + 1):
out *= layers_lib.conv2d(
inputs,
inter_channels,
1,
scope='hoa_conv_stage_1_order_%d_%d' % (i + 1, j + 1))
out = tf.nn.relu(out)
out = layers_lib.conv2d(
out,
in_channels,
1,
scope='hoa_conv_stage_2_conv_order_%d' % (i + 1))
out = tf.nn.sigmoid(out)
attention += out
output = inputs * attention / order
return output
def __init__(self, mode, image, caption, is_training_inception,
is_first_time):
self.config = ModelConfig()
self.is_first_time = is_first_time
self.initializer = tf.random_uniform_initializer(
-self.config.initializer_scale, self.config.initializer_scale)
self.mode = mode
self.image = image
self.caption = caption
self.caption_len = None
self.target = None
self.global_step = tf.Variable(initial_value=0,
name="global_step",
trainable=False,
collections=[
tf.GraphKeys.GLOBAL_STEP,
tf.GraphKeys.GLOBAL_VARIABLES
])
self.build(is_training_inception)
assert mode in ["train", "eval", "inference"]
if mode == "train":
self.regularizer = l2_regularizer(0.00004)
else:
self.regularizer = None
def resnet_arg_scope(is_training=True,
weight_decay=cfg.TRAIN.WEIGHT_DECAY,
batch_norm_decay=0.997,
batch_norm_epsilon=1e-5,
batch_norm_scale=True):
batch_norm_params = {
# NOTE 'is_training' here does not work because inside resnet it gets reset:
# https://github.com/tensorflow/models/blob/master/slim/nets/resnet_v1.py#L187
'is_training': False,
'decay': batch_norm_decay,
'epsilon': batch_norm_epsilon,
'scale': batch_norm_scale,
'trainable': cfg.RESNET.BN_TRAIN,
'updates_collections': ops.GraphKeys.UPDATE_OPS,
'fused': True
}
with arg_scope(
[slim.conv2d, slim.separable_conv2d],
weights_regularizer=regularizers.l2_regularizer(weight_decay),
weights_initializer=initializers.variance_scaling_initializer(),
trainable=is_training,
activation_fn=nn_ops.relu,
# normalizer_fn=None,
# normalizer_params=None):
normalizer_fn=layers.batch_norm,
normalizer_params=batch_norm_params):
with arg_scope([layers.batch_norm], **batch_norm_params) as arg_sc:
return arg_sc
def resnet_arg_scope(is_training=True,
weight_decay=0.0001,
batch_norm_decay=0.997,
batch_norm_epsilon=1e-5,
batch_norm_scale=True):
batch_norm_params = {
'is_training': False,
'decay': batch_norm_decay,
'epsilon': batch_norm_epsilon,
'scale': batch_norm_scale,
'trainable': cfg.RESNET.BN_TRAIN,
'updates_collections': ops.GraphKeys.UPDATE_OPS
}
with arg_scope(
[slim.conv2d],
weights_regularizer=regularizers.l2_regularizer(weight_decay),
weights_initializer=initializers.variance_scaling_initializer(),
trainable=is_training,
activation_fn=nn_ops.relu,
normalizer_fn=layers.batch_norm,
normalizer_params=batch_norm_params):
with arg_scope([layers.batch_norm], **batch_norm_params):
# The following implies padding='SAME' for pool1, which makes feature
# alignment easier for dense prediction tasks. This is also used in
# https://github.com/facebook/fb.resnet.torch. However the accompanying
# code of 'Deep Residual Learning for Image Recognition' uses
# padding='VALID' for pool1. You can switch to that choice by setting
# tf.contrib.framework.arg_scope([tf.contrib.layers.max_pool2d], padding='VALID').
with arg_scope([layers.max_pool2d], padding='SAME') as arg_sc:
return arg_sc
def slim_net_original(image, keep_prob):
with arg_scope([layers.conv2d, layers.fully_connected], biases_initializer=tf.random_normal_initializer(stddev=0.1)):
# conv2d(inputs, num_outputs, kernel_size, stride=1, padding='SAME',
# activation_fn=nn.relu, normalizer_fn=None, normalizer_params=None,
# weights_initializer=initializers.xavier_initializer(), weights_regularizer=None,
# biases_initializer=init_ops.zeros_initializer, biases_regularizer=None, scope=None):
net = layers.conv2d(image, 32, [5, 5], scope='conv1', weights_regularizer=regularizers.l1_regularizer(0.5))
# max_pool(inputs, kernel_size, stride=2, padding='VALID', scope=None)
net = layers.max_pool2d(net, 2, scope='pool1')
net = layers.conv2d(net, 64, [5, 5], scope='conv2', weights_regularizer=regularizers.l2_regularizer(0.5))
summaries.summarize_tensor(net, tag='conv2')
net = layers.max_pool2d(net, 2, scope='pool2')
net = layers.flatten(net, scope='flatten1')
# fully_connected(inputs, num_outputs, activation_fn=nn.relu, normalizer_fn=None,
# normalizer_params=None, weights_initializer=initializers.xavier_initializer(),
# weights_regularizer=None, biases_initializer=init_ops.zeros_initializer,
# biases_regularizer=None, scope=None):
net = layers.fully_connected(net, 1024, scope='fc1')
# dropout(inputs, keep_prob=0.5, is_training=True, scope=None)
net = layers.dropout(net, keep_prob=keep_prob, scope='dropout1')
net = layers.fully_connected(net, 10, scope='fc2')
return net
def initialise_network(self):
w_regularizer = None
b_regularizer = None
reg_type = self.net_param.reg_type.lower()
decay = self.net_param.decay
if reg_type == 'l2' and decay > 0:
from tensorflow.contrib.layers.python.layers import regularizers
w_regularizer = regularizers.l2_regularizer(decay)
b_regularizer = regularizers.l2_regularizer(decay)
elif reg_type == 'l1' and decay > 0:
from tensorflow.contrib.layers.python.layers import regularizers
w_regularizer = regularizers.l1_regularizer(decay)
b_regularizer = regularizers.l1_regularizer(decay)
self.net = ApplicationNetFactory.create(self.net_param.name)(
w_regularizer=w_regularizer, b_regularizer=b_regularizer)
def resnet_arg_scope(bn_is_training,
bn_trainable,
trainable=True,
weight_decay=config.TRAIN.weight_decay_factor,
batch_norm_decay=0.99,
batch_norm_epsilon=1e-9,
batch_norm_scale=True,
data_format='NHWC'):
batch_norm_params = {
'is_training': bn_is_training,
'decay': batch_norm_decay,
'epsilon': batch_norm_epsilon,
'scale': batch_norm_scale,
'trainable': bn_trainable,
'updates_collections': ops.GraphKeys.UPDATE_OPS,
'fused': True
}
with arg_scope(
[slim.conv2d, slim.separable_conv2d],
weights_regularizer=regularizers.l2_regularizer(weight_decay),
weights_initializer=initializers.variance_scaling_initializer(),
trainable=trainable,
activation_fn=nn_ops.relu,
normalizer_fn=slim.batch_norm
if 'BN' in config.TRAIN.norm else GroupNorm,
normalizer_params=batch_norm_params
if 'BN' in config.TRAIN.norm else None,
data_format=data_format):
with arg_scope([layers.batch_norm, layers.max_pool2d],
data_format=data_format):
with arg_scope([layers.batch_norm], **batch_norm_params) as arg_sc:
return arg_sc
def prepare_inference(self):
self.prepare_data()
siamese = self.make_subnetwork(is_siamese = True)
pseudo = self.make_subnetwork(is_siamese = False)
fused_fts = tf.concat((siamese, pseudo), axis=1, name='fused_features')
with tf.variable_scope("fuse_layer") as scope: # @UnusedVariable
self.logits = tf.layers.dense(fused_fts, 2, None, use_bias=True, # @UndefinedVariable
kernel_initializer=tf.truncated_normal_initializer(stddev=self.stddev, dtype=tf.float32),
bias_initializer=tf.constant_initializer(0.01, dtype=tf.float32),
kernel_regularizer=l2_regularizer(self.wd),
bias_regularizer=l2_regularizer(self.wd),
name='fused_layer')
def resnet_arg_scope(is_training=True,
weight_decay=cfg.TRAIN.WEIGHT_DECAY,
batch_norm_decay=0.997,
batch_norm_epsilon=1e-5,
batch_norm_scale=True):
data_format = 'NCHW' if cfg.RESNET.USE_NCHW else 'NHWC'
batch_norm_params = {
# NOTE 'is_training' is set appropriately inside of the resnet if we pass it to it:
# https://github.com/tensorflow/models/blob/master/slim/nets/resnet_v1.py#L187
# 'is_training': False,
'decay': batch_norm_decay,
'epsilon': batch_norm_epsilon,
'scale': batch_norm_scale,
'trainable': True,
'data_format': data_format,
'fused': True
}
with arg_scope(
[slim.conv2d, slim.conv2d_transpose],
weights_regularizer=regularizers.l2_regularizer(weight_decay),
weights_initializer=initializers.variance_scaling_initializer(),
trainable=is_training,
activation_fn=nn_ops.relu,
normalizer_fn=layers.batch_norm,
data_format=data_format,
normalizer_params=batch_norm_params):
with arg_scope([slim.max_pool2d, resnet_utils.conv2d_same],
data_format=data_format):
with arg_scope([layers.batch_norm], **batch_norm_params) as arg_sc:
return arg_sc
def test_3d_reg_shape(self):
input_shape = (2, 32, 32, 32, 1)
x = tf.ones(input_shape)
# vnet_instance = VNet(num_classes=160)
vnet_instance = VNet(num_classes=160,
w_regularizer=regularizers.l2_regularizer(0.4),
b_regularizer=regularizers.l2_regularizer(0.4))
out = vnet_instance(x, is_training=True)
print(vnet_instance.num_trainable_params())
# print(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
out = sess.run(out)
self.assertAllClose((2, 32, 32, 32, 160), out.shape)
def resnet_arg_scope(bn_is_training,
bn_trainable,
trainable=True,
weight_decay=cfg.weight_decay,
batch_norm_decay=0.99,
batch_norm_epsilon=1e-9,
batch_norm_scale=True):
batch_norm_params = {
'is_training': bn_is_training,
'decay': batch_norm_decay,
'epsilon': batch_norm_epsilon,
'scale': batch_norm_scale,
'trainable': bn_trainable,
'updates_collections': ops.GraphKeys.UPDATE_OPS
}
with arg_scope(
[slim.conv2d],
weights_regularizer=regularizers.l2_regularizer(weight_decay),
weights_initializer=initializers.variance_scaling_initializer(),
trainable=trainable,
activation_fn=nn_ops.relu,
normalizer_fn=layers.batch_norm,
normalizer_params=batch_norm_params):
with arg_scope([layers.batch_norm], **batch_norm_params) as arg_sc:
return arg_sc
def deconv_layer(self, deconv_input, layers_to_skip, reuse):
feature_maps = self.feature_maps[:-1][::-1] + [
image_channels,
]
kernel_size = self.kernel_size[:-1][::-1] + [
3,
]
stride_size = self.stride_size[1:][::-1] + [
1,
]
assert len(kernel_size) == len(feature_maps) == len(
stride_size), "lens must be equal"
layers_to_skip_d = layers_to_skip[:-1][::-1]
net = deconv_input
with tf.variable_scope('deconv_autoencoder', reuse=reuse):
for i, (each_feat_map, each_kernel_size, each_stride) in enumerate(
zip(feature_maps, kernel_size, stride_size)):
activation = tf.nn.relu
if i == (len(stride_size) - 1):
# last layer !
activation = tf.nn.tanh
if i > 0:
# not first layer !
net = tf.concat([net, layers_to_skip_d[i - 1]], axis=3)
net = slim.conv2d_transpose(
net,
each_feat_map, [each_kernel_size, each_kernel_size],
stride=each_stride,
activation_fn=activation,
scope='deconv_' + str(i),
weights_initializer=trunc_normal(0.01),
weights_regularizer=regularizers.l2_regularizer(l2_val))
return net
def generator1(z, training=True, weight_decay=0.0001, batch_norm_decay=0.997,
batch_norm_epsilon=1e-5, batch_norm_scale=True):
batch_norm_params = {
'is_training': training,
'decay': batch_norm_decay,
'epsilon': batch_norm_epsilon,
'scale': batch_norm_scale,
'updates_collections': ops.GraphKeys.UPDATE_OPS,
}
c0 = tf.reshape(z, [batch_size, 1, 1, z_dim])
with arg_scope(
[slim.conv2d],
weights_regularizer=regularizers.l2_regularizer(weight_decay),
weights_initializer=initializers.variance_scaling_initializer(),
activation_fn=nn_ops.relu,
normalizer_fn=layers.batch_norm,
normalizer_params=batch_norm_params):
gen = tf.image.resize_nearest_neighbor(c0, [2,2])
gen = slim.conv2d(gen, 1024, [3,3], [1,1], scope="convolution1")
gen = tf.image.resize_nearest_neighbor(gen, [4,4])
gen = slim.conv2d(gen, 512, [3,3], [1,1], scope="convolution2")
gen = tf.image.resize_nearest_neighbor(gen, [8,8])
gen = slim.conv2d(gen, 256, [3,3], [1,1], scope="convolution3")
gen = tf.image.resize_nearest_neighbor(gen, [16,16])
gen = slim.conv2d(gen, 128, [3,3], [1,1], scope="convolution4")
gen = tf.image.resize_nearest_neighbor(gen, [32,32])
gen = slim.conv2d(gen, 64, [3,3], [1,1], scope="convolution5")
gen = tf.image.resize_nearest_neighbor(gen, [64,64])
# l = [(4096, [3,3], [2,2]), (2048, [3,3], [2,2]), (2048, [3,3], [2,2]),
# (2048, [3,3], [2,2]), (1024, [3,3], [2,2]), (512, [3,3], [2,2]), (256, [3,3], [2,2]), (128, [3,3], [1,1]), (3, [3,3], [1,1])]
gen = slim.conv2d(gen, 3, [3,3], [1,1], weights_regularizer=regularizers.l2_regularizer(weight_decay),
weights_initializer=initializers.variance_scaling_initializer(), activation_fn=tf.tanh, scope="convolutionend")
return gen
def test_apply_zero_regularization(self):
regularizer = regularizers.l2_regularizer(0.0)
array_weights_list = [[1.5], [2, 3, 4.2], [10, 42, 666.6]]
tensor_weights_list = [constant_op.constant(x) for x in array_weights_list]
with self.cached_session():
result = regularizers.apply_regularization(regularizer,
tensor_weights_list)
self.assertAllClose(0.0, result.eval())
def inception_2d_fields(img,
fields,
num_classes=30,
is_training=True,
dropout_keep_prob=0.6,
prediction_fn=layers_lib.softmax,
spatial_squeeze=True,
reuse=None,
scope='InceptionV1_Fields'
):
with arg_scope([layers.conv2d, layers_lib.fully_connected],
weights_initializer=tf.contrib.layers.xavier_initializer(),
biases_initializer=tf.constant_initializer(0.2),
weights_regularizer=regularizers.l2_regularizer(0.0002),
biases_regularizer=regularizers.l2_regularizer(0.0002)):
net, end_points = inception_2d.inception_v1_base(img, scope=scope, final_endpoint='Mixed_4b')
with variable_scope.variable_scope('Logits'):
net = layers_lib.avg_pool2d(net, [5, 5], stride=3, scope='AvgPool_0a_5x5')
net = layers.conv2d(inputs=net, num_outputs=128, kernel_size=1)
net = tf.reshape(net, [-1, 1, 1, 4 * 4 * 128])
net = array_ops.squeeze(net,[1,2],name='Squeeze4Fields')
net = tf.concat([net,fields],axis=1)
net = layers.fully_connected(inputs=net, num_outputs=1024)
net = layers_lib.dropout(net, dropout_keep_prob, scope='Dropout_0b')
logits = layers.fully_connected(inputs=net,
num_outputs=num_classes,
activation_fn=None,
weights_initializer=tf.contrib.layers.xavier_initializer(),
biases_initializer=tf.constant_initializer(0.0),
weights_regularizer=regularizers.l2_regularizer(0.0002),
biases_regularizer=regularizers.l2_regularizer(0.0002),
scope='InnerProduct')
# logits = layers.conv2d(
# net,
# num_classes, [1, 1],
# activation_fn=None,
# normalizer_fn=None,
# scope='Conv2d_0c_1x1')
if spatial_squeeze:
logits = array_ops.squeeze(logits, [1, 2], name='SpatialSqueeze')
end_points['Logits'] = logits
end_points['Predictions'] = prediction_fn(logits, scope='Predictions')
return logits, end_points
def stpnet_v4(inputs,
num_outputs=30,
dropout_keep_prob=0.8,
pooling_kernel_size=[[1, 2, 2], [1, 4, 4], [1, 8, 8]],
pooling_stride=[[1, 2, 2], [1, 4, 4], [1, 8, 8]],
scope='Inception3D'):
# Inception v1 based
with tf.variable_scope(scope, 'InceptionV1', [inputs]) as scope:
net, end_points = stpnet_base(inputs=inputs,
final_endpoint='Inception_5b')
with tf.variable_scope('Logits'):
pyramid_0 = tf.contrib.layers.avg_pool3d(
inputs=net,
kernel_size=pooling_kernel_size[0],
stride=pooling_stride[0],
scope='AvgPool3D_Pyramid_0')
pyramid_0_flattened = tf.contrib.layers.flatten(pyramid_0)
pyramid_1 = tf.contrib.layers.avg_pool3d(
inputs=net,
kernel_size=pooling_kernel_size[1],
stride=pooling_stride[1],
scope='AvgPool3D_Pyramid_1')
pyramid_1_flattened = tf.contrib.layers.flatten(pyramid_1)
pyramid_2 = tf.contrib.layers.avg_pool3d(
inputs=net,
kernel_size=pooling_kernel_size[2],
stride=pooling_stride[2],
scope='AvgPool3D_Pyramid_2')
pyramid_2_flattened = tf.contrib.layers.flatten(pyramid_2)
net = tf.concat([
pyramid_0_flattened, pyramid_1_flattened, pyramid_2_flattened
],
axis=1)
end_points['AvgPool3D'] = net
net = tf.layers.dropout(inputs=net, rate=dropout_keep_prob)
logits = tf.contrib.layers.fully_connected(
inputs=net,
num_outputs=num_outputs,
activation_fn=None,
weights_initializer=tf.contrib.layers.xavier_initializer(),
biases_initializer=tf.constant_initializer(0.0),
weights_regularizer=regularizers.l2_regularizer(0.0002),
biases_regularizer=regularizers.l2_regularizer(0.0002),
scope='InnerProduct')
end_points['Logits'] = logits
return logits, end_points
def initialise_network(self):
w_regularizer = None
b_regularizer = None
reg_type = self.net_param.reg_type.lower()
decay = self.net_param.decay
if reg_type == 'l2' and decay > 0:
from tensorflow.contrib.layers.python.layers import regularizers
w_regularizer = regularizers.l2_regularizer(decay)
b_regularizer = regularizers.l2_regularizer(decay)
elif reg_type == 'l1' and decay > 0:
from tensorflow.contrib.layers.python.layers import regularizers
w_regularizer = regularizers.l1_regularizer(decay)
b_regularizer = regularizers.l1_regularizer(decay)
self.net = ApplicationNetFactory.create(self.net_param.name)(
w_regularizer=w_regularizer,
b_regularizer=b_regularizer)
def test_3d_reg_shape(self):
input_shape = (2, 32, 32, 32, 1)
x = tf.ones(input_shape)
# vnet_instance = VNet(num_classes=160)
vnet_instance = VNet(
num_classes=160,
w_regularizer=regularizers.l2_regularizer(0.4),
b_regularizer=regularizers.l2_regularizer(0.4))
out = vnet_instance(x, is_training=True)
print(vnet_instance.num_trainable_params())
# print(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
out = sess.run(out)
self.assertAllClose((2, 32, 32, 32, 160), out.shape)
def alexnet_v2_arg_scope(weight_decay=0.0005):
with arg_scope(
[layers.conv2d, layers_lib.fully_connected],
activation_fn=nn_ops.relu,
biases_initializer=init_ops.constant_initializer(0.1),
weights_regularizer=regularizers.l2_regularizer(weight_decay)):
with arg_scope([layers.conv2d], padding='SAME'):
with arg_scope([layers_lib.max_pool2d], padding='VALID') as arg_sc:
return arg_sc
def alexnet_v2_arg_scope(weight_decay=0.0005):
with arg_scope(
[layers.conv2d, layers_lib.fully_connected],
activation_fn=nn_ops.relu,
biases_initializer=init_ops.constant_initializer(0.1),
weights_regularizer=regularizers.l2_regularizer(weight_decay)):
with arg_scope([layers.conv2d], padding='SAME'):
with arg_scope([layers_lib.max_pool2d], padding='VALID') as arg_sc:
return arg_sc
def train(config_file):
# 1, load configuration parameters
config = parse_config(config_file)
config_data = config['data']
config_net = config['network']
config_train = config['training']
random.seed(config_train.get('random_seed', 1))
assert (config_data['with_ground_truth'])
net_type = config_net['net_type']
net_name = config_net['net_name']
class_num = config_net['class_num']
batch_size = config_data.get('batch_size', 5)
# 2, construct graph
full_data_shape = [batch_size] + config_data['data_shape']
full_label_shape = [batch_size] + config_data['label_shape']
x = tf.placeholder(tf.float32, shape=full_data_shape)
w = tf.placeholder(tf.float32, shape=full_label_shape)
y = tf.placeholder(tf.int64, shape=full_label_shape)
w_regularizer = regularizers.l2_regularizer(config_train.get(
'decay', 1e-7))
b_regularizer = regularizers.l2_regularizer(config_train.get(
'decay', 1e-7))
net_class = NetFactory.create(net_type)
net = net_class(num_classes=class_num,
w_regularizer=w_regularizer,
b_regularizer=b_regularizer,
name=net_name)
net.set_params(config_net)
predicty = net(x, is_training=True)
proby = tf.nn.softmax(predicty)
loss_func = LossFunction(n_class=class_num)
loss = loss_func(predicty, y, weight_map=w)
print('size of predicty:', predicty)
# 3, initialize session and saver
lr = config_train.get('learning_rate', 1e-3)
opt_step = tf.train.AdamOptimizer(lr).minimize(loss)
tf.summary.FileWriter("./graphs/" + config_net['net_name'],
tf.get_default_graph()).close()
def test_3d_prelu_reg_shape(self):
x = self.get_3d_input()
prelu_layer = ActiLayer(func='prelu',
regularizer=regularizers.l2_regularizer(0.5),
name='regularized')
out_prelu = prelu_layer(x)
print(prelu_layer)
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
out = sess.run(out_prelu)
self.assertAllClose((2, 16, 16, 16, 8), out.shape)
def test_fclayer_3d_bn_reg_dropout_valid_shape(self):
input_param = {'n_output_chns': 10,
'with_bias': False,
'with_bn': True,
'w_regularizer': regularizers.l2_regularizer(0.5),
'acti_func': 'prelu', }
self._test_fc_layer_output_shape(rank=3,
param_dict=input_param,
output_shape=(2, 10),
is_training=True,
dropout_prob=0.4)
def test_convlayer_2d_bn_reg_prelu_shape(self):
input_param = {'n_output_chns': 10,
'kernel_size': 3,
'stride': 1,
'with_bias': False,
'with_bn': True,
'acti_func': 'prelu',
'w_regularizer': regularizers.l2_regularizer(0.5)}
self._test_conv_layer_output_shape(rank=2,
param_dict=input_param,
output_shape=(2, 16, 16, 10),
is_training=True)
def shape_test_reg(self, input_shape, expected_shape):
x = tf.ones(input_shape)
layer_param = {
'num_classes': 5,
'w_regularizer': regularizers.l2_regularizer(0.5),
'b_regularizer': regularizers.l2_regularizer(0.5)}
highres_layer = HighRes3DNet(**layer_param)
highres_layer_small = HighRes3DNetSmall(**layer_param)
highres_layer_large = HighRes3DNetLarge(**layer_param)
out = highres_layer(x, is_training=True)
out_small = highres_layer_small(x, is_training=True)
out_large = highres_layer_large(x, is_training=True)
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
out, out_large, out_small = sess.run([out, out_large, out_small])
self.assertAllClose(expected_shape, out.shape)
self.assertAllClose(expected_shape, out_large.shape)
self.assertAllClose(expected_shape, out_small.shape)
def test_2d_reg_shape(self):
input_shape = (2, 96, 96, 1)
x = tf.ones(input_shape)
unet_instance = UNet3D(num_classes=160,
w_regularizer=regularizers.l2_regularizer(0.4))
out = unet_instance(x, is_training=True)
print(unet_instance.num_trainable_params())
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
out = sess.run(out)
self.assertAllClose((2, 8, 8, 160), out.shape)
def resnet_arg_scope(is_training=True,
weight_decay=0.0001,
batch_norm_decay=0.997,
batch_norm_epsilon=1e-5,
batch_norm_scale=True):
"""Defines the default ResNet arg scope.
TODO(gpapan): The batch-normalization related default values above are
appropriate for use in conjunction with the reference ResNet models
released at https://github.com/KaimingHe/deep-residual-networks. When
training ResNets from scratch, they might need to be tuned.
Args:
is_training: Whether or not we are training the parameters in the batch
normalization layers of the model.
weight_decay: The weight decay to use for regularizing the model.
batch_norm_decay: The moving average decay when estimating layer activation
statistics in batch normalization.
batch_norm_epsilon: Small constant to prevent division by zero when
normalizing activations by their variance in batch normalization.
batch_norm_scale: If True, uses an explicit `gamma` multiplier to scale the
activations in the batch normalization layer.
Returns:
An `arg_scope` to use for the resnet models.
"""
batch_norm_params = {
'is_training': is_training,
'decay': batch_norm_decay,
'epsilon': batch_norm_epsilon,
'scale': batch_norm_scale,
'updates_collections': ops.GraphKeys.UPDATE_OPS,
}
with arg_scope(
[layers_lib.conv2d],
weights_regularizer=regularizers.l2_regularizer(weight_decay),
weights_initializer=initializers.variance_scaling_initializer(),
activation_fn=nn_ops.relu,
normalizer_fn=layers.batch_norm,
normalizer_params=batch_norm_params):
with arg_scope([layers.batch_norm], **batch_norm_params):
# The following implies padding='SAME' for pool1, which makes feature
# alignment easier for dense prediction tasks. This is also used in
# https://github.com/facebook/fb.resnet.torch. However the accompanying
# code of 'Deep Residual Learning for Image Recognition' uses
# padding='VALID' for pool1. You can switch to that choice by setting
# tf.contrib.framework.arg_scope([tf.contrib.layers.max_pool2d], padding='VALID').
with arg_scope([layers.max_pool2d], padding='SAME') as arg_sc:
return arg_sc
def test_convlayer_3d_bn_reg_dropout_valid_shape(self):
input_param = {'n_output_chns': 10,
'kernel_size': [5, 3, 2],
'stride': [2, 2, 3],
'with_bias': False,
'with_bn': True,
'w_regularizer': regularizers.l2_regularizer(0.5),
'acti_func': 'prelu',
'padding': 'VALID'}
self._test_conv_layer_output_shape(rank=3,
param_dict=input_param,
output_shape=(2, 6, 7, 5, 10),
is_training=True,
dropout_prob=0.4)
def initialise_network(self):
w_regularizer = None
b_regularizer = None
reg_type = self.net_param.reg_type.lower()
decay = self.net_param.decay
if reg_type == 'l2' and decay > 0:
from tensorflow.contrib.layers.python.layers import regularizers
w_regularizer = regularizers.l2_regularizer(decay)
b_regularizer = regularizers.l2_regularizer(decay)
elif reg_type == 'l1' and decay > 0:
from tensorflow.contrib.layers.python.layers import regularizers
w_regularizer = regularizers.l1_regularizer(decay)
b_regularizer = regularizers.l1_regularizer(decay)
self.net = ApplicationNetFactory.create(self.net_param.name)(
num_classes=self.classification_param.num_classes,
w_initializer=InitializerFactory.get_initializer(
name=self.net_param.weight_initializer),
b_initializer=InitializerFactory.get_initializer(
name=self.net_param.bias_initializer),
w_regularizer=w_regularizer,
b_regularizer=b_regularizer,
acti_func=self.net_param.activation_function)
def test_2d_reg_shape(self):
input_shape = (2, 32, 32, 4)
x = tf.ones(input_shape)
scalenet_layer = ScaleNet(num_classes=5,
w_regularizer=regularizers.l2_regularizer(
0.3))
out = scalenet_layer(x, is_training=True)
print(scalenet_layer.num_trainable_params())
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
out = sess.run(out)
self.assertAllClose((2, 32, 32, 5), out.shape)
def inception_v3_arg_scope(weight_decay=0.00004,
batch_norm_var_collection='moving_vars',
batch_norm_decay=0.9997,
batch_norm_epsilon=0.001,
updates_collections=ops.GraphKeys.UPDATE_OPS,
use_fused_batchnorm=True):
"""Defines the default InceptionV3 arg scope.
Args:
weight_decay: The weight decay to use for regularizing the model.
batch_norm_var_collection: The name of the collection for the batch norm
variables.
batch_norm_decay: Decay for batch norm moving average
batch_norm_epsilon: Small float added to variance to avoid division by zero
updates_collections: Collections for the update ops of the layer
use_fused_batchnorm: Enable fused batchnorm.
Returns:
An `arg_scope` to use for the inception v3 model.
"""
batch_norm_params = {
# Decay for the moving averages.
'decay': batch_norm_decay,
# epsilon to prevent 0s in variance.
'epsilon': batch_norm_epsilon,
# collection containing update_ops.
'updates_collections': updates_collections,
# Use fused batch norm if possible.
'fused': use_fused_batchnorm,
# collection containing the moving mean and moving variance.
'variables_collections': {
'beta': None,
'gamma': None,
'moving_mean': [batch_norm_var_collection],
'moving_variance': [batch_norm_var_collection],
}
}
# Set weight_decay for weights in Conv and FC layers.
with arg_scope(
[layers.conv2d, layers_lib.fully_connected],
weights_regularizer=regularizers.l2_regularizer(weight_decay)):
with arg_scope(
[layers.conv2d],
weights_initializer=initializers.variance_scaling_initializer(),
activation_fn=nn_ops.relu,
normalizer_fn=layers_lib.batch_norm,
normalizer_params=batch_norm_params) as sc:
return sc
def test_deconvlayer_2d_bn_reg_shape(self):
input_param = {'n_output_chns': 10,
'kernel_size': [3, 1],
'stride': [1, 3],
'with_bias': False,
'with_bn': True,
'w_regularizer': regularizers.l2_regularizer(0.5)}
self._test_deconv_layer_output_shape(rank=2,
param_dict=input_param,
output_shape=(2, 16, 48, 10),
is_training=True)
self._test_deconv_layer_output_shape(rank=2,
param_dict=input_param,
output_shape=(2, 16, 48, 10),
is_training=False)
def test_2d_reg_shape(self):
#input_shape = (2, 572, 572, 5)
input_shape = (2, 180, 180, 5)
x = tf.ones(input_shape)
unet_instance = UNet2D(num_classes=2,
w_regularizer=regularizers.l2_regularizer(0.4))
out = unet_instance(x, is_training=True)
print(unet_instance.num_trainable_params())
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
out = sess.run(out)
#self.assertAllClose((2, 388, 388, 2), out.shape)
self.assertAllClose((2, 4, 4, 2), out.shape)
def inception_v1_arg_scope(weight_decay=0.00004,
use_batch_norm=True,
batch_norm_var_collection='moving_vars'):
"""Defines the default InceptionV1 arg scope.
Note: Althougth the original paper didn't use batch_norm we found it useful.
Args:
weight_decay: The weight decay to use for regularizing the model.
use_batch_norm: "If `True`, batch_norm is applied after each convolution.
batch_norm_var_collection: The name of the collection for the batch norm
variables.
Returns:
An `arg_scope` to use for the inception v3 model.
"""
batch_norm_params = {
# Decay for the moving averages.
'decay': 0.9997,
# epsilon to prevent 0s in variance.
'epsilon': 0.001,
# collection containing update_ops.
'updates_collections': ops.GraphKeys.UPDATE_OPS,
# collection containing the moving mean and moving variance.
'variables_collections': {
'beta': None,
'gamma': None,
'moving_mean': [batch_norm_var_collection],
'moving_variance': [batch_norm_var_collection],
}
}
if use_batch_norm:
normalizer_fn = layers_lib.batch_norm
normalizer_params = batch_norm_params
else:
normalizer_fn = None
normalizer_params = {}
# Set weight_decay for weights in Conv and FC layers.
with arg_scope(
[layers.conv2d, layers_lib.fully_connected],
weights_regularizer=regularizers.l2_regularizer(weight_decay)):
with arg_scope(
[layers.conv2d],
weights_initializer=initializers.variance_scaling_initializer(),
activation_fn=nn_ops.relu,
normalizer_fn=normalizer_fn,
normalizer_params=normalizer_params) as sc:
return sc
