def fc_with_init(self, x, out_dim, name='fc'): if not self._init_params: output = utils._fc(x, self._hp.num_classes, name) else: fc_init_params = [ convert(self._init_params[self._init_params_index][0]), # W convert(self._init_params[self._init_params_index + 1][0]) ] # b output = utils._fc(x, self._hp.num_classes, name, fc_init_params) self._init_params_index += 2 return output
def inference(inputs, dp_mult, keep_prob, pre_define_vars, resnet_params, train_params): # through first conv/auto-encoding layer # no relu and batch norm here as they are included in resnet with tf.variable_scope('enc_layer') as scope: inputs = fixed_padding(inputs, pre_define_vars['kernel1'].shape[0], _DATA_FORMAT) print_var('enc padding', inputs) inputs = tf.nn.conv2d( inputs, pre_define_vars['kernel1'], [1, train_params.enc_stride, train_params.enc_stride, 1], padding='VALID') inputs = inputs + dp_mult + pre_define_vars['biases1'] # h_conv1 = tf.nn.relu(inputs) h_conv1 = inputs print_var('enc output', inputs) # resnet18 without first conv, return last hidden layer inputs = resnet18_builder_mod( inputs, keep_prob, True, _DATA_FORMAT, resnet_params.num_filters, resnet_params.resnet_version, resnet_params.first_pool_size, resnet_params.first_pool_stride, resnet_params.block_sizes, resnet_params.bottleneck, resnet_params.block_fn, resnet_params.block_strides, resnet_params.pre_activation, train_params.num_classes, train_params.hk) # the last fc layer inputs = tf.clip_by_value(inputs, -1, 1) inputs = _fc(inputs, train_params.num_classes, None, reuse=tf.AUTO_REUSE, name='fc2') # inputs = _fc(inputs, train_params.num_classes, None, name='fc2') return inputs, h_conv1
def _fc(self, x, out_dim, input_q=None, output_q=None, name="fc"): b, in_dim = x.get_shape().as_list() x = utils._fc(x, out_dim, input_q, output_q, name) f = 2 * (in_dim + 1) * out_dim w = (in_dim + 1) * out_dim scope_name = tf.get_variable_scope().name + "/" + name self._add_flops_weights(scope_name, f, w) return x
def _fc(self, x, out_dim, bias=True, name="fc"): b, in_dim = x.get_shape().as_list() x = utils._fc(x, out_dim, bias, name) f = 2 * (in_dim + 1) * out_dim if bias else 2 * in_dim * out_dim w = (in_dim + 1) * out_dim if bias else in_dim * out_dim scope_name = tf.get_variable_scope().name + "/" + name self._add_flops_weights(scope_name, f, w) print('%s: %s' % (name, str(x.get_shape().as_list()))) return x
def fc_split(self, x, in_splits, out_splits, name='unit'): b, num_in = x.get_shape().as_list() assert num_in == sum(in_splits) with tf.variable_scope(name) as scope: print('\tBuilding fc layer: %s with %d splits' % (scope.name, len(in_splits))) outs = [] offset_in = 0 for i, (n_in, n_out) in enumerate(zip(in_splits, out_splits)): sliced = tf.slice(x, [0, offset_in], [b, n_in]) self._flops += self._get_fc_flops(sliced, n_out) sliced_fc = utils._fc(sliced, n_out, name="split_%d" % (i+1)) outs.append(sliced_fc) offset_in += n_in concat = tf.concat(1, outs) return concat
def build_model(self): print('Building model') # Init. conv. print('\tBuilding unit: conv1') conv_1 = utils._conv(self._images, 3, 16, 1, name='conv_1') conv_1_bn = utils._bn(conv_1, self.is_train, self._global_step, name='conv_1_bn') conv1_relu = utils._relu(conv_1_bn, name='conv1_relu') # Residual Blocks #_residual_mod(input,filter_size,kernel_num,has_side_conv,is_stride,is_train,global_step,name=basename) with tf.variable_scope('conv2') as scope: conv2_1 = utils._residual_mod(conv1_relu, 3, 64, True, False, self.is_train, self._global_step, name='conv2_1') conv2_2 = utils._residual_mod(conv2_1, 3, 64, False, False, self.is_train, self._global_step, name='conv2_2') with tf.variable_scope('conv3') as scope: conv3 = utils._residual_mod(conv2_2, 3, 128, False, True, self.is_train, self._global_step, name='conv3') #_inception1(input,filter_size,kernel_num,is_train,global_step,name=basename): with tf.variable_scope('inception') as scope: inception = utils._inception1(conv3, [1, 3, 3, 3, 1], [128, 64, 128, 64, 128], self.is_train, self._global_step, name='inception') with tf.variable_scope('conv4') as scope: conv4 = utils._residual_mod(inception, 3, 256, False, True, self.is_train, self._global_step, name='conv4') conv4_ave_pool = utils._avg_pool(conv4, 'VALID', name='conv4_ave_pool') # Logit with tf.variable_scope('logits') as scope: print('\tBuilding unit: %s' % scope.name) conv4_ave_pool_shape = conv4_ave_pool.get_shape().as_list() dim_conv4_ave_pool = conv4_ave_pool_shape[ 1] * conv4_ave_pool_shape[2] * conv4_ave_pool_shape[3] x = tf.reshape(conv4_ave_pool, [conv4_ave_pool_shape[0], dim_conv4_ave_pool]) #pdb.set_trace() x = utils._fc(x, self._hp.num_classes) print x.get_shape() #pdb.set_trace() self._logits = x self.probs = tf.nn.softmax(x, name='probs') self.preds = tf.to_int32(tf.argmax(self._logits, 1, name='preds')) ones = tf.constant(np.ones([self._hp.batch_size]), dtype=tf.float32) zeros = tf.constant(np.zeros([self._hp.batch_size]), dtype=tf.float32) correct = tf.select(tf.equal(self.preds, self._labels), ones, zeros) self.acc = tf.reduce_mean(correct, name='acc') #tf.scalar_summary('accuracy', self.acc) loss = tf.nn.sparse_softmax_cross_entropy_with_logits(x, self._labels) self.loss = tf.reduce_mean(loss, name='cross_entropy')
tf.tile(tf.constant([[1.0,0.0]]), [BATCH_SIZE,1]) ]) with tf.name_scope("adv") as scope: al = utils._conv(adv_inp, 5, 5, 10) al = utils._half(al) al = utils._conv(al, 3, 3, 15) al = utils._half(al) al = utils._conv(al, 3, 3, 25) al = utils._half(al) al = utils._conv(al, 3, 3, 25) al = utils._half(al) al = utils._conv(al, 3, 3, 35) al = utils._half(al) al = utils._conv(al, 3, 3, 35) al = utils._fc(al, 2) al = tf.nn.softmax(al) def normclip(grads_and_vars): #for g,v in grads_and_vars: # print "G: " + str(g) # print "V: " + str(v.name) #return grads_and_vars return [ (tf.clip_by_norm(g, 1.0),v) for g, v in grads_and_vars] adv_entropy = -tf.reduce_sum(answers*tf.log(tf.clip_by_value(al, 1e-10, 1.0))) / BATCH_SIZE tf.scalar_summary("adv_entropy", adv_entropy) adv_opt = tf.train.AdagradOptimizer(3e-3) adv_train_step = adv_opt.apply_gradients(normclip(adv_opt.compute_gradients(adv_entropy, var_list=[x for x in tf.trainable_variables() if "adv" in x.name])))
def build_model(self): print('Building model') # Init. conv. print('\tBuilding unit: init_conv') x = utils._conv(self._images, 3, 16, 1, name='init_conv') # Residual Blocks filters = [16, 16 * self._hp.k, 32 * self._hp.k, 64 * self._hp.k] strides = [1, 2, 2] for i in range(1, 4): # First residual unit with tf.variable_scope('unit_%d_0' % i) as scope: print('\tBuilding residual unit: %s' % scope.name) x = utils._bn(x, self.is_train, self._global_step, name='bn_1') x = utils._relu(x, name='relu_1') # Shortcut if filters[i - 1] == filters[i]: if strides[i - 1] == 1: shortcut = tf.identity(x) else: shortcut = tf.nn.max_pool( x, [1, strides[i - 1], strides[i - 1], 1], [1, strides[i - 1], strides[i - 1], 1], 'VALID') else: shortcut = utils._conv(x, 1, filters[i], strides[i - 1], name='shortcut') # Residual x = utils._conv(x, 3, filters[i], strides[i - 1], name='conv_1') x = utils._bn(x, self.is_train, self._global_step, name='bn_2') x = utils._relu(x, name='relu_2') x = utils._conv(x, 3, filters[i], 1, name='conv_2') # Merge x = x + shortcut # Other residual units for j in range(1, self._hp.num_residual_units): with tf.variable_scope('unit_%d_%d' % (i, j)) as scope: print('\tBuilding residual unit: %s' % scope.name) # Shortcut shortcut = x # Residual x = utils._bn(x, self.is_train, self._global_step, name='bn_1') x = utils._relu(x, name='relu_1') x = utils._conv(x, 3, filters[i], 1, name='conv_1') x = utils._bn(x, self.is_train, self._global_step, name='bn_2') x = utils._relu(x, name='relu_2') x = utils._conv(x, 3, filters[i], 1, name='conv_2') # Merge x = x + shortcut # Last unit with tf.variable_scope('unit_last') as scope: print('\tBuilding unit: %s' % scope.name) x = utils._bn(x, self.is_train, self._global_step) x = utils._relu(x) x = tf.reduce_mean(x, [1, 2]) # Logit with tf.variable_scope('logits') as scope: print('\tBuilding unit: %s' % scope.name) x_shape = x.get_shape().as_list() x = tf.reshape(x, [-1, x_shape[1]]) x = utils._fc(x, self._hp.num_classes) self._logits = x # Probs & preds & acc self.probs = tf.nn.softmax(x, name='probs') self.preds = tf.to_int32(tf.argmax(self._logits, 1, name='preds')) ones = tf.constant(np.ones([self._hp.batch_size]), dtype=tf.float32) zeros = tf.constant(np.zeros([self._hp.batch_size]), dtype=tf.float32) correct = tf.where(tf.equal(self.preds, self._labels), ones, zeros) self.acc = tf.reduce_mean(correct, name='acc') tf.summary.scalar('accuracy', self.acc) # Loss & acc loss = tf.nn.sparse_softmax_cross_entropy_with_logits( logits=x, labels=self._labels) self.loss = tf.reduce_mean(loss, name='cross_entropy') tf.summary.scalar('cross_entropy', self.loss)
def build_model(self): print('Building model') # Init. conv. print('\tBuilding unit: conv1') conv1 = utils._conv(self._images, 3, 64, 1, name='conv1') conv1_bn = utils._bn(conv1, self.is_train, self._global_step, name='conv1_bn') conv1_relu = utils._relu(conv1_bn, name='conv1_relu') # Residual Blocks #_residual_mod(input,filter_size,kernel_num,has_side_conv,is_stride,is_train,global_step,name=basename) #_basic_conv(x, filter_size, out_channel, strides,is_train,global_step,pad='SAME', name='conv'): with tf.variable_scope('conv2') as scope: conv2_1a = utils._basic_conv(conv1_relu, 3, 64, 1, self.is_train, self._global_step, name='conv2_1a') conv2_1a_2_3x3 = utils._conv(conv2_1a, 3, 64, 1, name='conv2_1a_2_3x3') conv2_1b_1x1 = utils._conv(conv1_relu, 1, 64, 1, name='conv2_1b_1x1') conv2_res2_1 = tf.add(conv2_1a_2_3x3, conv2_1b_1x1, name='conv2_res2_1') conv2_2a_1_bn = utils._bn(conv2_res2_1, self.is_train, self._global_step, name='conv2_2a_1_bn') conv2_2a_1_relu = utils._relu(conv2_2a_1_bn, name='conv2_2a_1_relu') conv2_2a_1_3x3 = utils._basic_conv(conv2_2a_1_relu, 3, 64, 1, self.is_train, self._global_step, name='conv2_2a_1_3x3') conv2_2a_2_3x3 = utils._conv(conv2_2a_1_3x3, 3, 64, 1, name='conv2_2a_2_3x3') conv2_res2_2 = tf.add(conv2_2a_2_3x3, conv2_res2_1, name='conv2_res2_2') conv2_res2_2_bn = utils._bn(conv2_res2_2, self.is_train, self._global_step, name='conv2_res2_2_bn') conv2_res2_2_relu = utils._relu(conv2_res2_2_bn, name='conv2_res2_2_relu') with tf.variable_scope('conv3') as scope: conv3_1a_1_3x3 = utils._basic_conv(conv2_res2_2_relu, 3, 256, 2, self.is_train, self._global_step, name='conv3_1a_1_3x3') conv3_1a_2_3x3 = utils._conv(conv3_1a_1_3x3, 3, 256, 1, name='conv3_1a_2_3x3') conv3_1b_1x1 = utils._conv(conv2_res2_2_relu, 1, 256, 2, name='conv3_1b_1x1') conv3_res3_1 = tf.add(conv3_1a_2_3x3, conv3_1b_1x1, name='conv3_res3_1') conv3_res3_2a_bn = utils._bn(conv3_res3_1, self.is_train, self._global_step, name='conv3_res3_2a_bn') conv3_res3_2a_relu = utils._relu(conv3_res3_2a_bn, name='conv3_res3_2a_relu') #_inception1(input,filter_size,kernel_num,is_train,global_step,name=basename): with tf.variable_scope('inception') as scope: inception = utils._inception1(conv3_res3_2a_relu, [1, 3, 3, 3, 1], [256, 128, 256, 128, 256], self.is_train, self._global_step, name='inception') inception_add = tf.add(inception, conv3_res3_1, name='inception_add') inception_bn = utils._bn(inception_add, self.is_train, self._global_step, name='inception_bn') inception_relu = utils._relu(inception_bn, name='inception_relu') with tf.variable_scope('conv4') as scope: conv4_1a_1_3x3 = utils._basic_conv(inception_relu, 3, 256, 2, self.is_train, self._global_step, name='conv4_1a_1_3x3') conv4_1a_2_3x3 = utils._conv(conv4_1a_1_3x3, 3, 256, 1, name='conv4_1a_2_3x3') conv4_1b_1x1 = utils._conv(inception_relu, 1, 256, 2, name='conv4_1b_1x1') conv4_res4_1 = tf.add(conv4_1a_2_3x3, conv4_1b_1x1, name='conv4_res4_1') conv4_res4_2a_1_bn = utils._bn(conv4_res4_1, self.is_train, self._global_step, name='conv4_res4_2a_1_bn') conv4_res4_2a_1_relu = utils._relu(conv4_res4_2a_1_bn, name='conv4_res4_2a_1_relu') conv4_res4_2a_2_3x3 = utils._basic_conv(conv4_res4_2a_1_relu, 3, 256, 1, self.is_train, self._global_step, name='conv4_res4_2a_2_3x3') conv4_res4_2a_2_3x3_2 = utils._conv(conv4_res4_2a_2_3x3, 3, 256, 1, name='conv4_res4_2a_2_3x3_2') conv4_res4_2 = tf.add(conv4_res4_2a_2_3x3_2, conv4_res4_1, name='conv4_res4_2') conv4_res4_bn = utils._bn(conv4_res4_2, self.is_train, self._global_step, name='conv4_res4_bn') conv4_res4_relu = utils._relu(conv4_res4_bn, name='conv4_res4_relu') conv4_ave_pool = utils._avg_pool(conv4_res4_relu, 'VALID', name='conv4_ave_pool') # Logit with tf.variable_scope('logits') as scope: print('\tBuilding unit: %s' % scope.name) conv4_ave_pool_shape = conv4_ave_pool.get_shape().as_list() dim_conv4_ave_pool = conv4_ave_pool_shape[ 1] * conv4_ave_pool_shape[2] * conv4_ave_pool_shape[3] x = tf.reshape(conv4_ave_pool, [conv4_ave_pool_shape[0], dim_conv4_ave_pool]) #pdb.set_trace() x = utils._fc(x, self._hp.num_classes) print x.get_shape() #pdb.set_trace() self._logits = x self.probs = tf.nn.softmax(x, name='probs') self.preds = tf.to_int32(tf.argmax(self._logits, 1, name='preds')) ones = tf.constant(np.ones([self._hp.batch_size]), dtype=tf.float32) zeros = tf.constant(np.zeros([self._hp.batch_size]), dtype=tf.float32) correct = tf.where(tf.equal(self.preds, self._labels), ones, zeros) self.acc = tf.reduce_mean(correct, name='acc') #tf.scalar_summary('accuracy', self.acc) loss = tf.nn.sparse_softmax_cross_entropy_with_logits( logits=x, labels=self._labels) self.loss = tf.reduce_mean(loss, name='cross_entropy')