def fc_with_init(self, x, out_dim, name='fc'):
if not self._init_params:
output = utils._fc(x, self._hp.num_classes, name)
else:
fc_init_params = [
convert(self._init_params[self._init_params_index][0]), # W
convert(self._init_params[self._init_params_index + 1][0])
] # b
output = utils._fc(x, self._hp.num_classes, name, fc_init_params)
self._init_params_index += 2
return output
def inference(inputs, dp_mult, keep_prob, pre_define_vars, resnet_params,
train_params):
# through first conv/auto-encoding layer
# no relu and batch norm here as they are included in resnet
with tf.variable_scope('enc_layer') as scope:
inputs = fixed_padding(inputs, pre_define_vars['kernel1'].shape[0],
_DATA_FORMAT)
print_var('enc padding', inputs)
inputs = tf.nn.conv2d(
inputs,
pre_define_vars['kernel1'],
[1, train_params.enc_stride, train_params.enc_stride, 1],
padding='VALID')
inputs = inputs + dp_mult + pre_define_vars['biases1']
# h_conv1 = tf.nn.relu(inputs)
h_conv1 = inputs
print_var('enc output', inputs)
# resnet18 without first conv, return last hidden layer
inputs = resnet18_builder_mod(
inputs, keep_prob, True, _DATA_FORMAT, resnet_params.num_filters,
resnet_params.resnet_version, resnet_params.first_pool_size,
resnet_params.first_pool_stride, resnet_params.block_sizes,
resnet_params.bottleneck, resnet_params.block_fn,
resnet_params.block_strides, resnet_params.pre_activation,
train_params.num_classes, train_params.hk)
# the last fc layer
inputs = tf.clip_by_value(inputs, -1, 1)
inputs = _fc(inputs,
train_params.num_classes,
None,
reuse=tf.AUTO_REUSE,
name='fc2')
# inputs = _fc(inputs, train_params.num_classes, None, name='fc2')
return inputs, h_conv1
def _fc(self, x, out_dim, input_q=None, output_q=None, name="fc"):
b, in_dim = x.get_shape().as_list()
x = utils._fc(x, out_dim, input_q, output_q, name)
f = 2 * (in_dim + 1) * out_dim
w = (in_dim + 1) * out_dim
scope_name = tf.get_variable_scope().name + "/" + name
self._add_flops_weights(scope_name, f, w)
return x
def _fc(self, x, out_dim, bias=True, name="fc"):
b, in_dim = x.get_shape().as_list()
x = utils._fc(x, out_dim, bias, name)
f = 2 * (in_dim + 1) * out_dim if bias else 2 * in_dim * out_dim
w = (in_dim + 1) * out_dim if bias else in_dim * out_dim
scope_name = tf.get_variable_scope().name + "/" + name
self._add_flops_weights(scope_name, f, w)
print('%s: %s' % (name, str(x.get_shape().as_list())))
return x
def fc_split(self, x, in_splits, out_splits, name='unit'):
b, num_in = x.get_shape().as_list()
assert num_in == sum(in_splits)
with tf.variable_scope(name) as scope:
print('\tBuilding fc layer: %s with %d splits' % (scope.name, len(in_splits)))
outs = []
offset_in = 0
for i, (n_in, n_out) in enumerate(zip(in_splits, out_splits)):
sliced = tf.slice(x, [0, offset_in], [b, n_in])
self._flops += self._get_fc_flops(sliced, n_out)
sliced_fc = utils._fc(sliced, n_out, name="split_%d" % (i+1))
outs.append(sliced_fc)
offset_in += n_in
concat = tf.concat(1, outs)
return concat
def build_model(self):
print('Building model')
# Init. conv.
print('\tBuilding unit: conv1')
conv_1 = utils._conv(self._images, 3, 16, 1, name='conv_1')
conv_1_bn = utils._bn(conv_1,
self.is_train,
self._global_step,
name='conv_1_bn')
conv1_relu = utils._relu(conv_1_bn, name='conv1_relu')
# Residual Blocks
#_residual_mod(input,filter_size,kernel_num,has_side_conv,is_stride,is_train,global_step,name=basename)
with tf.variable_scope('conv2') as scope:
conv2_1 = utils._residual_mod(conv1_relu,
3,
64,
True,
False,
self.is_train,
self._global_step,
name='conv2_1')
conv2_2 = utils._residual_mod(conv2_1,
3,
64,
False,
False,
self.is_train,
self._global_step,
name='conv2_2')
with tf.variable_scope('conv3') as scope:
conv3 = utils._residual_mod(conv2_2,
3,
128,
False,
True,
self.is_train,
self._global_step,
name='conv3')
#_inception1(input,filter_size,kernel_num,is_train,global_step,name=basename):
with tf.variable_scope('inception') as scope:
inception = utils._inception1(conv3, [1, 3, 3, 3, 1],
[128, 64, 128, 64, 128],
self.is_train,
self._global_step,
name='inception')
with tf.variable_scope('conv4') as scope:
conv4 = utils._residual_mod(inception,
3,
256,
False,
True,
self.is_train,
self._global_step,
name='conv4')
conv4_ave_pool = utils._avg_pool(conv4,
'VALID',
name='conv4_ave_pool')
# Logit
with tf.variable_scope('logits') as scope:
print('\tBuilding unit: %s' % scope.name)
conv4_ave_pool_shape = conv4_ave_pool.get_shape().as_list()
dim_conv4_ave_pool = conv4_ave_pool_shape[
1] * conv4_ave_pool_shape[2] * conv4_ave_pool_shape[3]
x = tf.reshape(conv4_ave_pool,
[conv4_ave_pool_shape[0], dim_conv4_ave_pool])
#pdb.set_trace()
x = utils._fc(x, self._hp.num_classes)
print x.get_shape()
#pdb.set_trace()
self._logits = x
self.probs = tf.nn.softmax(x, name='probs')
self.preds = tf.to_int32(tf.argmax(self._logits, 1, name='preds'))
ones = tf.constant(np.ones([self._hp.batch_size]), dtype=tf.float32)
zeros = tf.constant(np.zeros([self._hp.batch_size]), dtype=tf.float32)
correct = tf.select(tf.equal(self.preds, self._labels), ones, zeros)
self.acc = tf.reduce_mean(correct, name='acc')
#tf.scalar_summary('accuracy', self.acc)
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(x, self._labels)
self.loss = tf.reduce_mean(loss, name='cross_entropy')
tf.tile(tf.constant([[1.0,0.0]]), [BATCH_SIZE,1])
])
with tf.name_scope("adv") as scope:
al = utils._conv(adv_inp, 5, 5, 10)
al = utils._half(al)
al = utils._conv(al, 3, 3, 15)
al = utils._half(al)
al = utils._conv(al, 3, 3, 25)
al = utils._half(al)
al = utils._conv(al, 3, 3, 25)
al = utils._half(al)
al = utils._conv(al, 3, 3, 35)
al = utils._half(al)
al = utils._conv(al, 3, 3, 35)
al = utils._fc(al, 2)
al = tf.nn.softmax(al)
def normclip(grads_and_vars):
#for g,v in grads_and_vars:
# print "G: " + str(g)
# print "V: " + str(v.name)
#return grads_and_vars
return [ (tf.clip_by_norm(g, 1.0),v) for g, v in grads_and_vars]
adv_entropy = -tf.reduce_sum(answers*tf.log(tf.clip_by_value(al, 1e-10, 1.0))) / BATCH_SIZE
tf.scalar_summary("adv_entropy", adv_entropy)
adv_opt = tf.train.AdagradOptimizer(3e-3)
adv_train_step = adv_opt.apply_gradients(normclip(adv_opt.compute_gradients(adv_entropy, var_list=[x for x in tf.trainable_variables() if "adv" in x.name])))
def build_model(self):
print('Building model')
# Init. conv.
print('\tBuilding unit: init_conv')
x = utils._conv(self._images, 3, 16, 1, name='init_conv')
# Residual Blocks
filters = [16, 16 * self._hp.k, 32 * self._hp.k, 64 * self._hp.k]
strides = [1, 2, 2]
for i in range(1, 4):
# First residual unit
with tf.variable_scope('unit_%d_0' % i) as scope:
print('\tBuilding residual unit: %s' % scope.name)
x = utils._bn(x, self.is_train, self._global_step, name='bn_1')
x = utils._relu(x, name='relu_1')
# Shortcut
if filters[i - 1] == filters[i]:
if strides[i - 1] == 1:
shortcut = tf.identity(x)
else:
shortcut = tf.nn.max_pool(
x, [1, strides[i - 1], strides[i - 1], 1],
[1, strides[i - 1], strides[i - 1], 1], 'VALID')
else:
shortcut = utils._conv(x,
1,
filters[i],
strides[i - 1],
name='shortcut')
# Residual
x = utils._conv(x,
3,
filters[i],
strides[i - 1],
name='conv_1')
x = utils._bn(x, self.is_train, self._global_step, name='bn_2')
x = utils._relu(x, name='relu_2')
x = utils._conv(x, 3, filters[i], 1, name='conv_2')
# Merge
x = x + shortcut
# Other residual units
for j in range(1, self._hp.num_residual_units):
with tf.variable_scope('unit_%d_%d' % (i, j)) as scope:
print('\tBuilding residual unit: %s' % scope.name)
# Shortcut
shortcut = x
# Residual
x = utils._bn(x,
self.is_train,
self._global_step,
name='bn_1')
x = utils._relu(x, name='relu_1')
x = utils._conv(x, 3, filters[i], 1, name='conv_1')
x = utils._bn(x,
self.is_train,
self._global_step,
name='bn_2')
x = utils._relu(x, name='relu_2')
x = utils._conv(x, 3, filters[i], 1, name='conv_2')
# Merge
x = x + shortcut
# Last unit
with tf.variable_scope('unit_last') as scope:
print('\tBuilding unit: %s' % scope.name)
x = utils._bn(x, self.is_train, self._global_step)
x = utils._relu(x)
x = tf.reduce_mean(x, [1, 2])
# Logit
with tf.variable_scope('logits') as scope:
print('\tBuilding unit: %s' % scope.name)
x_shape = x.get_shape().as_list()
x = tf.reshape(x, [-1, x_shape[1]])
x = utils._fc(x, self._hp.num_classes)
self._logits = x
# Probs & preds & acc
self.probs = tf.nn.softmax(x, name='probs')
self.preds = tf.to_int32(tf.argmax(self._logits, 1, name='preds'))
ones = tf.constant(np.ones([self._hp.batch_size]), dtype=tf.float32)
zeros = tf.constant(np.zeros([self._hp.batch_size]), dtype=tf.float32)
correct = tf.where(tf.equal(self.preds, self._labels), ones, zeros)
self.acc = tf.reduce_mean(correct, name='acc')
tf.summary.scalar('accuracy', self.acc)
# Loss & acc
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=x, labels=self._labels)
self.loss = tf.reduce_mean(loss, name='cross_entropy')
tf.summary.scalar('cross_entropy', self.loss)
def build_model(self):
print('Building model')
# Init. conv.
print('\tBuilding unit: conv1')
conv1 = utils._conv(self._images, 3, 64, 1, name='conv1')
conv1_bn = utils._bn(conv1,
self.is_train,
self._global_step,
name='conv1_bn')
conv1_relu = utils._relu(conv1_bn, name='conv1_relu')
# Residual Blocks
#_residual_mod(input,filter_size,kernel_num,has_side_conv,is_stride,is_train,global_step,name=basename)
#_basic_conv(x, filter_size, out_channel, strides,is_train,global_step,pad='SAME', name='conv'):
with tf.variable_scope('conv2') as scope:
conv2_1a = utils._basic_conv(conv1_relu,
3,
64,
1,
self.is_train,
self._global_step,
name='conv2_1a')
conv2_1a_2_3x3 = utils._conv(conv2_1a,
3,
64,
1,
name='conv2_1a_2_3x3')
conv2_1b_1x1 = utils._conv(conv1_relu,
1,
64,
1,
name='conv2_1b_1x1')
conv2_res2_1 = tf.add(conv2_1a_2_3x3,
conv2_1b_1x1,
name='conv2_res2_1')
conv2_2a_1_bn = utils._bn(conv2_res2_1,
self.is_train,
self._global_step,
name='conv2_2a_1_bn')
conv2_2a_1_relu = utils._relu(conv2_2a_1_bn,
name='conv2_2a_1_relu')
conv2_2a_1_3x3 = utils._basic_conv(conv2_2a_1_relu,
3,
64,
1,
self.is_train,
self._global_step,
name='conv2_2a_1_3x3')
conv2_2a_2_3x3 = utils._conv(conv2_2a_1_3x3,
3,
64,
1,
name='conv2_2a_2_3x3')
conv2_res2_2 = tf.add(conv2_2a_2_3x3,
conv2_res2_1,
name='conv2_res2_2')
conv2_res2_2_bn = utils._bn(conv2_res2_2,
self.is_train,
self._global_step,
name='conv2_res2_2_bn')
conv2_res2_2_relu = utils._relu(conv2_res2_2_bn,
name='conv2_res2_2_relu')
with tf.variable_scope('conv3') as scope:
conv3_1a_1_3x3 = utils._basic_conv(conv2_res2_2_relu,
3,
256,
2,
self.is_train,
self._global_step,
name='conv3_1a_1_3x3')
conv3_1a_2_3x3 = utils._conv(conv3_1a_1_3x3,
3,
256,
1,
name='conv3_1a_2_3x3')
conv3_1b_1x1 = utils._conv(conv2_res2_2_relu,
1,
256,
2,
name='conv3_1b_1x1')
conv3_res3_1 = tf.add(conv3_1a_2_3x3,
conv3_1b_1x1,
name='conv3_res3_1')
conv3_res3_2a_bn = utils._bn(conv3_res3_1,
self.is_train,
self._global_step,
name='conv3_res3_2a_bn')
conv3_res3_2a_relu = utils._relu(conv3_res3_2a_bn,
name='conv3_res3_2a_relu')
#_inception1(input,filter_size,kernel_num,is_train,global_step,name=basename):
with tf.variable_scope('inception') as scope:
inception = utils._inception1(conv3_res3_2a_relu, [1, 3, 3, 3, 1],
[256, 128, 256, 128, 256],
self.is_train,
self._global_step,
name='inception')
inception_add = tf.add(inception,
conv3_res3_1,
name='inception_add')
inception_bn = utils._bn(inception_add,
self.is_train,
self._global_step,
name='inception_bn')
inception_relu = utils._relu(inception_bn, name='inception_relu')
with tf.variable_scope('conv4') as scope:
conv4_1a_1_3x3 = utils._basic_conv(inception_relu,
3,
256,
2,
self.is_train,
self._global_step,
name='conv4_1a_1_3x3')
conv4_1a_2_3x3 = utils._conv(conv4_1a_1_3x3,
3,
256,
1,
name='conv4_1a_2_3x3')
conv4_1b_1x1 = utils._conv(inception_relu,
1,
256,
2,
name='conv4_1b_1x1')
conv4_res4_1 = tf.add(conv4_1a_2_3x3,
conv4_1b_1x1,
name='conv4_res4_1')
conv4_res4_2a_1_bn = utils._bn(conv4_res4_1,
self.is_train,
self._global_step,
name='conv4_res4_2a_1_bn')
conv4_res4_2a_1_relu = utils._relu(conv4_res4_2a_1_bn,
name='conv4_res4_2a_1_relu')
conv4_res4_2a_2_3x3 = utils._basic_conv(conv4_res4_2a_1_relu,
3,
256,
1,
self.is_train,
self._global_step,
name='conv4_res4_2a_2_3x3')
conv4_res4_2a_2_3x3_2 = utils._conv(conv4_res4_2a_2_3x3,
3,
256,
1,
name='conv4_res4_2a_2_3x3_2')
conv4_res4_2 = tf.add(conv4_res4_2a_2_3x3_2,
conv4_res4_1,
name='conv4_res4_2')
conv4_res4_bn = utils._bn(conv4_res4_2,
self.is_train,
self._global_step,
name='conv4_res4_bn')
conv4_res4_relu = utils._relu(conv4_res4_bn,
name='conv4_res4_relu')
conv4_ave_pool = utils._avg_pool(conv4_res4_relu,
'VALID',
name='conv4_ave_pool')
# Logit
with tf.variable_scope('logits') as scope:
print('\tBuilding unit: %s' % scope.name)
conv4_ave_pool_shape = conv4_ave_pool.get_shape().as_list()
dim_conv4_ave_pool = conv4_ave_pool_shape[
1] * conv4_ave_pool_shape[2] * conv4_ave_pool_shape[3]
x = tf.reshape(conv4_ave_pool,
[conv4_ave_pool_shape[0], dim_conv4_ave_pool])
#pdb.set_trace()
x = utils._fc(x, self._hp.num_classes)
print x.get_shape()
#pdb.set_trace()
self._logits = x
self.probs = tf.nn.softmax(x, name='probs')
self.preds = tf.to_int32(tf.argmax(self._logits, 1, name='preds'))
ones = tf.constant(np.ones([self._hp.batch_size]), dtype=tf.float32)
zeros = tf.constant(np.zeros([self._hp.batch_size]), dtype=tf.float32)
correct = tf.where(tf.equal(self.preds, self._labels), ones, zeros)
self.acc = tf.reduce_mean(correct, name='acc')
#tf.scalar_summary('accuracy', self.acc)
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=x, labels=self._labels)
self.loss = tf.reduce_mean(loss, name='cross_entropy')
