def create_model(self, images, num_classes, weight_decay=0.00004, scope='Flowers', reuse=None, is_training=True): """Creates a base part of the Model (no gradients, no loss, no summaries). Args: images: A tensor of size [batch_size, height, width, channels]. num_classes: The number of predicted classes. scope: Optional variable_scope. reuse: Whether or not the network or its variables should be reused. To be able to reuse 'scope' must be given. is_training: Whether is training or not. Returns: A named tuple OutputEndpoints. """ with tf.variable_scope(scope, [images], reuse=reuse): with slim.arg_scope(inception_v3.inception_v3_arg_scope(weight_decay=weight_decay)): logits, endpoints = inception_v3.inception_v3( inputs = images, num_classes=num_classes, is_training=is_training) return logits, endpoints
def network_fn(inputs): # return transformer_factory.transform(inputs, BATCH_PER_GPU, NUM_STN, (224, 224), NUM_CLASSES, FLAGS.weight_decay, True) end_points = {} # with slim.arg_scope([slim.batch_norm, slim.dropout], is_training=True): # with slim.arg_scope(inception_v3_arg_scope(weight_decay=FLAGS.weight_decay)): with slim.arg_scope([slim.batch_norm, slim.dropout], is_training=is_training): with slim.arg_scope(inception_v3_arg_scope(weight_decay=weight_decay)): with tf.variable_scope("loc") as scope: with tf.variable_scope("net") as scope2: # _, _end_points = inception_resnet_v2.inception_resnet_v2(inputs, num_classes=2, is_training=True, scope = scope2) loc_net, _ = inception_v2.inception_v2_base(inputs, scope=scope2) # loc_net = _end_points['Conv2d_7b_1x1'] loc_net = slim.conv2d(loc_net, 128, [1, 1], scope='Loc_1x1') default_kernel_size = [14, 14] # kernel_size = _reduced_kernel_size_for_small_input(loc_net, default_kernel_size) loc_net = slim.conv2d(loc_net, 128, loc_net.get_shape()[1:3], padding='VALID', activation_fn=tf.nn.tanh, scope='Loc_fc1') loc_net = slim.flatten(loc_net) iv = 4. initial = np.array([iv, 0, iv, 0] * NUM_STN, dtype=np.float32) b_fc_loc = tf.get_variable( "Loc_fc_b", shape=[4 * NUM_STN], initializer=init_ops.constant_initializer(initial), dtype=dtypes.float32) W_fc_loc = tf.get_variable( "Loc_fc_W", shape=[128, 4 * NUM_STN], initializer=init_ops.constant_initializer( np.zeros((128, 4 * NUM_STN))), dtype=dtypes.float32) theta = tf.nn.tanh(tf.matmul(loc_net, W_fc_loc) + b_fc_loc) _finals = [] for i in xrange(NUM_STN): scope_name = "stn%d" % i with tf.variable_scope(scope_name) as scope1: _theta = tf.slice(theta, [0, 4 * i], [-1, 4 * (i + 1)]) # loc_net = slim.conv2d(loc_net, 6, [1,1], activation_fn=tf.nn.tanh, scope='Loc_fc', biases_initializer = init_ops.constant_initializer([4.0,0.0,0.0,0.0,4.0,0.0]*128,dtype=dtypes.float32)) # loc_net = slim.conv2d(loc_net, 6, [1,1], activation_fn=tf.nn.tanh, scope='Loc_fc', biases_initializer = init_ops.constant_initializer([4.0],dtype=dtypes.float32)) # loc_net = slim.flatten(loc_net) stn_output_size = (STN_OUT_SIZE, STN_OUT_SIZE) x = transformer(inputs, _theta, stn_output_size) x.set_shape([ BATCH_PER_GPU, stn_output_size[0], stn_output_size[1], 3 ]) # x.set_shape(tf.shape(inputs)) # tf.reshape(x, tf.shape(inputs)) end_points['x'] = x # with tf.variable_scope("net") as scope2: # return inception_resnet_v2.inception_resnet_v2(x, num_classes=NUM_CLASSES, is_training=True, scope = scope2) with tf.variable_scope("net") as scope2: net, _ = inception_v2.inception_v2_base(x, scope=scope2) kernel_size = _reduced_kernel_size_for_small_input( net, [7, 7]) net = slim.avg_pool2d(net, kernel_size, padding='VALID', scope='AvgPool_1a') net = slim.dropout(net, keep_prob=0.7, scope='Dropout_1b') _finals.append(net) with tf.variable_scope('Logits'): net = tf.concat(axis=3, values=_finals) logits = slim.conv2d(net, NUM_CLASSES, [1, 1], activation_fn=None, normalizer_fn=None, scope='Conv2d_1c_1x1') logits = tf.squeeze(logits, [1, 2], name='SpatialSqueeze') predictions = slim.softmax(logits, scope='Predictions') end_points['Predictions'] = predictions logits_a = slim.conv2d(net, NUM_ATTRIBS, [1, 1], activation_fn=None, normalizer_fn=None, scope='Conv2d_1c_1x1_a') logits_a = tf.squeeze(logits_a, [1, 2], name='SpatialSqueeze_a') predictions_a = slim.sigmoid(logits_a, scope='Predictions_a') end_points['Predictions_a'] = predictions_a return logits, logits_a, end_points
def graph(x, y, i, x_max, x_min, accum_s, accum_g): eps = 2.0 * FLAGS.max_epsilon / 255.0 num_iter = FLAGS.num_iter alpha = eps / num_iter # momentum = FLAGS.momentum num_classes = 1001 beta_1 = FLAGS.beta_1 beta_2 = FLAGS.beta_2 x_nes = x with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_v3, end_points_v3 = inception_v3.inception_v3( x_nes, num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE) pred = tf.argmax(end_points_v3['Predictions'], 1) first_round = tf.cast(tf.equal(i, 0), tf.int64) y = first_round * pred + (1 - first_round) * y one_hot = tf.one_hot(y, num_classes) cross_entropy = tf.losses.softmax_cross_entropy(one_hot, logits_v3) grad = tf.gradients(cross_entropy, x)[0] x_nes_2 = 1 / 2 * x_nes with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_v3_2, end_points_v3 = inception_v3.inception_v3( x_nes_2, num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE) cross_entropy_2 = tf.losses.softmax_cross_entropy(one_hot, logits_v3_2) grad += tf.gradients(cross_entropy_2, x)[0] x_nes_4 = 1 / 4 * x_nes with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_v3_4, end_points_v3 = inception_v3.inception_v3( x_nes_4, num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE) cross_entropy_4 = tf.losses.softmax_cross_entropy(one_hot, logits_v3_4) grad += tf.gradients(cross_entropy_4, x)[0] x_nes_8 = 1 / 8 * x_nes with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_v3_8, end_points_v3 = inception_v3.inception_v3( x_nes_8, num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE) cross_entropy_8 = tf.losses.softmax_cross_entropy(one_hot, logits_v3_8) grad += tf.gradients(cross_entropy_8, x)[0] x_nes_16 = 1 / 16 * x_nes with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_v3_16, end_points_v3 = inception_v3.inception_v3( x_nes_16, num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE) cross_entropy_16 = tf.losses.softmax_cross_entropy(one_hot, logits_v3_16) grad += tf.gradients(cross_entropy_16, x)[0] grad_normed = grad / tf.reduce_mean(tf.abs(grad), [1, 2, 3], keep_dims=True) accum_g = grad_normed * (1 - beta_1) + accum_g * beta_1 accum_s = tf.multiply(grad, grad) * (1 - beta_2) + accum_s * beta_2 accum_g_hat = tf.divide(accum_g, (1 - tf.pow(beta_1, tf.cast(i + 1, tf.float32)))) accum_s_hat = tf.divide(accum_s, (1 - tf.pow(beta_2, tf.cast(i + 1, tf.float32)))) x = x + tf.multiply(tf.divide(alpha, tf.add(tf.sqrt(accum_s_hat), 1e-6)), tf.sign(accum_g_hat)) x = tf.clip_by_value(x, x_min, x_max) i = tf.add(i, 1) return x, y, i, x_max, x_min, accum_s, accum_g
''' def getimagedir(path): res = [] for root, dirs, files in os.walk(path): rootpath = os.path.join(os.path.abspath(path), root) for file in files: filepath = os.path.join(rootpath, file) res.append(filepath) return res ''' with tf.Graph().as_default(): with slim.arg_scope(inception_v3.inception_v3_arg_scope()): # set your image #imgPath = '/mnt/notebook/kihong/models/slim/images/dog.jpg' imgPath = 'images/dog2.jpg' testImage_string = tf.gfile.FastGFile(imgPath, 'rb').read() testImage = tf.image.decode_jpeg(testImage_string, channels=3) processed_image = inception_preprocessing.preprocess_image(testImage, image_size, image_size, is_training=False) processed_images = tf.expand_dims(processed_image, 0) logits, _ = inception_v3.inception_v3(processed_images, num_classes=257, is_training=False) probabilities = tf.nn.softmax(logits) init_fn = slim.assign_from_checkpoint_fn( os.path.join(checkpoints_dir, 'model.ckpt-500'), slim.get_model_variables('InceptionV3')) with tf.Session() as sess:
def graph(x, y, i, x_max, x_min, g1, g2, e): eps = 2.0 * FLAGS.max_epsilon / 255.0 num_iter = FLAGS.num_iter alpha = eps / num_iter num_classes = 1001 beta = 0.91 k1 = 0.99 k2 = 0.72 k3 = 1.44 with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_v3, end_points_v3 = inception_v3.inception_v3( input_diversity(x), num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE) with slim.arg_scope(inception_v4.inception_v4_arg_scope()): logits_v4, end_points_v4 = inception_v4.inception_v4( input_diversity(x), num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE) with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2( input_diversity(x), num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE) with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits_resnet, end_points_resnet = resnet_v2.resnet_v2_101( input_diversity(x), num_classes=num_classes, is_training=False, scope='resnet_v2_101', reuse=tf.AUTO_REUSE) logits = (logits_v3 + logits_v4 + logits_res_v2 + logits_resnet) / 4 auxlogits = (end_points_v3['AuxLogits'] + end_points_v4['AuxLogits'] + end_points_res_v2['AuxLogits']) / 3 cross_entropy = tf.losses.softmax_cross_entropy(y, logits, label_smoothing=0.0, weights=1.0) cross_entropy += tf.losses.softmax_cross_entropy(y, auxlogits, label_smoothing=0.0, weights=0.4) noise = tf.gradients(cross_entropy, x)[0] noise = tf.nn.depthwise_conv2d(noise, stack_kernel, strides=[1, 1, 1, 1], padding='SAME') noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True) g1 = beta * g1 + (1 - beta) * (noise - e) g2 = g2 + noise x = x + alpha * tf.sign(k1 * noise + k2 * g1 + k3 * g2) x = tf.clip_by_value(x, x_min, x_max) i = tf.add(i, 1) e = noise return x, y, i, x_max, x_min, g1, g2, e
def graph(x, y, i, x_max, x_min, grad, y_target, y_logits): eps = 2.0 * FLAGS.max_epsilon / 255.0 num_iter = FLAGS.num_iter alpha = eps / num_iter momentum = FLAGS.momentum num_classes = 1001 # should keep original x here for output with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_v3, end_points_v3 = inception_v3.inception_v3( input_diversity(x), num_classes=num_classes, is_training=False) with slim.arg_scope(inception_v4.inception_v4_arg_scope()): logits_v4, end_points_v4 = inception_v4.inception_v4( input_diversity(x), num_classes=num_classes, is_training=False) with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2( input_diversity(x), num_classes=num_classes, is_training=False) with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits_resnet, end_points_resnet = resnet_v2.resnet_v2_152( input_diversity(x), num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE) with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits_resnet_50, end_points_resnet_50 = resnet_v2.resnet_v2_50( input_diversity(x), num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE) with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits_resnet_101, end_points_resnet_101 = resnet_v2.resnet_v2_101( input_diversity(x), num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE) # Adv training models # with slim.arg_scope(inception_v3.inception_v3_arg_scope()): # logits_adv_v3, end_points_adv_v3 = inception_v3.inception_v3( # input_diversity(x), num_classes=num_classes, is_training=False, scope='AdvInceptionV3') # # with slim.arg_scope(inception_v3.inception_v3_arg_scope()): # logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3( # input_diversity(x), num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3') with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3( input_diversity(x), num_classes=num_classes, is_training=False, scope='Ens4AdvInceptionV3') with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2( input_diversity(x), num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2') logits = (logits_v4 + logits_res_v2 + logits_v3 + logits_resnet + logits_resnet_50 + logits_resnet_101 + logits_ens4_adv_v3 + logits_ensadv_res_v2) / 8 auxlogits = (end_points_v4['AuxLogits'] + end_points_res_v2['AuxLogits'] + end_points_v3['AuxLogits'] + end_points_ens4_adv_v3['AuxLogits'] + end_points_ensadv_res_v2['AuxLogits']) / 5 y_oh = tf.one_hot(y, num_classes) y_target_oh = tf.one_hot(y_target, num_classes) loss = -FLAGS.W_crs * tf.losses.softmax_cross_entropy( y_oh, logits, label_smoothing=0.0, weights=1.0) # loss = - Poincare_dis(tf.clip_by_value((y_oh-0.01), 0.0, 1.0), # logits / tf.reduce_sum(tf.abs(logits), [1], keep_dims=True) ) loss_ce = tf.losses.softmax_cross_entropy(y_target_oh, logits, label_smoothing=0.0, weights=1.0) loss_ce += tf.losses.softmax_cross_entropy(y_target_oh, auxlogits, label_smoothing=0.0, weights=0.9) loss_po = Poincare_dis( tf.clip_by_value((y_target_oh - 0.00001), 0.0, 1.0), logits / tf.reduce_sum(tf.abs(logits), [1], keep_dims=True)) loss_po += FLAGS.W_aux * Poincare_dis( tf.clip_by_value((y_target_oh - 0.00001), 0.0, 1.0), auxlogits / tf.reduce_sum(tf.abs(auxlogits), [1], keep_dims=True)) loss_cos = tf.clip_by_value( (Cos_dis(y_oh, logits) - Cos_dis(y_target_oh, logits) + 0.007), 0.0, 2.1) if FLAGS.loss == "ce": loss = loss_ce elif FLAGS.loss == "po": loss = loss_po elif FLAGS.loss == "trip_po": loss = loss_po + 0.01 * loss_cos # loss += cross_entropy # loss += -10*Cos_dis(y_target_oh, logits) noise = -tf.gradients(loss, x)[0] # TI- noise = tf.nn.depthwise_conv2d(noise, stack_kernel, strides=[1, 1, 1, 1], padding='SAME') # CE Cross-entry loss must add this term if FLAGS.loss == "ce": noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True) noise = momentum * grad + noise x = x + alpha * tf.sign(noise) x = tf.clip_by_value(x, x_min, x_max) i = tf.add(i, 1) return x, y, i, x_max, x_min, noise, y_target, logits
def graph(x, x_adv_v3, y, i, x_max, x_min, grad, pred): eps = 2.0 * FLAGS.max_epsilon / 255.0 num_iter = FLAGS.num_iter alpha = eps / num_iter momentum = FLAGS.momentum num_classes = 1001 with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_v3, end_points_v3 = inception_v3.inception_v3( x, num_classes=num_classes, is_training=False) with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_adv_v3, end_points_adv_v3 = inception_v3.inception_v3( x, num_classes=num_classes, is_training=False, scope='AdvInceptionV3') with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3( x, num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3') with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3( x, num_classes=num_classes, is_training=False, scope='Ens4AdvInceptionV3') with slim.arg_scope(inception_v4.inception_v4_arg_scope()): logits_v4, end_points_v4 = inception_v4.inception_v4( x, num_classes=num_classes, is_training=False) with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2( x, num_classes=num_classes, is_training=False) with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2( x, num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2') with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits_resnet, end_points_resnet = resnet_v2.resnet_v2_101( x, num_classes=num_classes, is_training=False) pred = tf.argmax( end_points_v3['Predictions'] + end_points_adv_v3['Predictions'] + end_points_ens3_adv_v3['Predictions'] + \ end_points_ens4_adv_v3['Predictions'] + end_points_v4['Predictions'] + \ end_points_res_v2['Predictions'] + end_points_ensadv_res_v2['Predictions'] + end_points_resnet['predictions'], 1) first_round = tf.cast(tf.equal(i, 0), tf.int64) y = first_round * pred + (1 - first_round) * y one_hot = tf.one_hot(y, num_classes) # logits = (logits_v3 + 0.25 * logits_adv_v3 + logits_ens3_adv_v3 + \ # logits_ens4_adv_v3 + logits_v4 + \ # logits_res_v2 + logits_ensadv_res_v2 + logits_resnet) / 7.25 logits = (logits_v3 + logits_ens3_adv_v3 + \ logits_ens4_adv_v3 + logits_v4 + \ logits_res_v2 + logits_ensadv_res_v2 + logits_resnet) / 7 auxlogits = (end_points_v3['AuxLogits'] + 0.25 * end_points_adv_v3['AuxLogits'] + end_points_ens3_adv_v3[ 'AuxLogits'] + \ end_points_ens4_adv_v3['AuxLogits'] + end_points_v4['AuxLogits'] + \ end_points_res_v2['AuxLogits'] + end_points_ensadv_res_v2['AuxLogits']) / 6.25 # auxlogits = (end_points_v3['AuxLogits'] + end_points_ens3_adv_v3[ # 'AuxLogits'] + \ # end_points_ens4_adv_v3['AuxLogits'] + end_points_v4['AuxLogits'] + \ # end_points_res_v2['AuxLogits'] + end_points_ensadv_res_v2['AuxLogits']) / 6 cross_entropy = tf.losses.softmax_cross_entropy(one_hot, logits, label_smoothing=0.0, weights=1.0) cross_entropy += tf.losses.softmax_cross_entropy(one_hot, auxlogits, label_smoothing=0.0, weights=0.4) noise = tf.gradients(cross_entropy, x)[0] noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True) noise = momentum * grad + noise x = x + alpha * tf.sign(noise) x = tf.clip_by_value(x, x_min, x_max) i = tf.add(i, 1) # with slim.arg_scope(inception_v3.inception_v3_arg_scope()): # logits_adv_v3_one, end_points_adv_v3_one = inception_v3.inception_v3( # x_adv_v3, num_classes=num_classes, is_training=False, reuse=True, scope='AdvInceptionV3') # pred_adv_v3 = tf.argmax(end_points_adv_v3_one['Predictions'], axis=1) return x, x_adv_v3, y, i, x_max, x_min, noise, pred
def main(_): start = time.clock() # tf.set_random_seed(1) # np.random.seed(1) # Images for inception classifier are normalized to be in [-1, 1] interval,3 # eps is a difference between pixels so it should be in [0, 2] interval. # Renormalizing epsilon from [0, 255] to [0, 2]. eps = 2.0 * FLAGS.max_epsilon / 255.0 batch_shape = [FLAGS.batch_size, FLAGS.image_height, FLAGS.image_width, 3] tf.logging.set_verbosity(tf.logging.INFO) x = tf.placeholder(dtype=tf.float32, shape=batch_shape) # , name='x_Placeholder' x_adv_v3 = tf.placeholder(dtype=tf.float32, shape=batch_shape) x_max = tf.clip_by_value(x + eps, -1.0, 1.0) x_min = tf.clip_by_value(x - eps, -1.0, 1.0) y = tf.zeros(shape=FLAGS.batch_size, dtype=tf.int64) i = tf.zeros(shape=1, dtype=tf.int8) grad = tf.zeros(shape=batch_shape) # pred = tf.zeros(shape=FLAGS.batch_size, dtype=tf.int64) # noise = tf.zeros(shape=[FLAGS.batch_size, 299, 299, 3],dtype=tf.float32) # cross_entropy = tf.zeros(shape=FLAGS.batch_size,dtype=tf.float32) with tf.Session() as sess: eps = 2.0 * FLAGS.max_epsilon / 255.0 num_iter = FLAGS.num_iter alpha = eps / num_iter momentum = FLAGS.momentum num_classes = 1001 # pred = tf.zeros(shape=FLAGS.batch_size, dtype=tf.int64) # noise = tf.zeros(shape=[FLAGS.batch_size, 299, 299, 3], dtype=tf.float32) # cross_entropy = tf.zeros(shape=FLAGS.batch_size, dtype=tf.float32) with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_v3, end_points_v3 = inception_v3.inception_v3( x, num_classes=num_classes, is_training=False) with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_adv_v3, end_points_adv_v3 = inception_v3.inception_v3( x, num_classes=num_classes, is_training=False, scope='AdvInceptionV3') with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3( x, num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3') with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3( x, num_classes=num_classes, is_training=False, scope='Ens4AdvInceptionV3') with slim.arg_scope(inception_v4.inception_v4_arg_scope()): logits_v4, end_points_v4 = inception_v4.inception_v4( x, num_classes=num_classes, is_training=False) with slim.arg_scope( inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2( x, num_classes=num_classes, is_training=False) with slim.arg_scope( inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2( x, num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2') with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits_resnet, end_points_resnet = resnet_v2.resnet_v2_101( x, num_classes=num_classes, is_training=False) pred_dict = end_points_v3['Predictions'] + end_points_adv_v3['Predictions'] + end_points_ens3_adv_v3['Predictions'] + \ end_points_ens4_adv_v3['Predictions'] + end_points_v4['Predictions'] + \ end_points_res_v2['Predictions'] + end_points_ensadv_res_v2['Predictions'] + end_points_resnet['predictions'] pred = tf.argmax(pred_dict, axis=1) logits_dict = [ logits_v3, logits_ens3_adv_v3, logits_ens4_adv_v3, logits_v4, logits_res_v2, logits_ensadv_res_v2, logits_resnet ] # for j in range(FLAGS.batch_size): # end_points_v3_Pred = end_points_v3['Predictions'][j][pred[j]] # # end_points_adv_v3_Pred = end_points_adv_v3['Predictions'] # end_points_ens3_adv_v3_Pred = end_points_ens3_adv_v3['Predictions'][j][pred[j]] # end_points_ens4_adv_v3_Pred = end_points_ens4_adv_v3['Predictions'][j][pred[j]] # end_points_v4_Pred = end_points_v4['Predictions'][j][pred[j]] # end_points_res_v2_Pred = end_points_res_v2['Predictions'][j][pred[j]] # end_points_ensadv_res_v2_Pred = end_points_ensadv_res_v2['Predictions'][j][pred[j]] # end_points_resnet_Pred = end_points_resnet['Predictions'][j][pred[j]] # # # ens_Pred_Value = [end_points_v3_Pred * -1, end_points_ens3_adv_v3_Pred * -1, # end_points_ens4_adv_v3_Pred * -1, # end_points_v4_Pred * -1, end_points_res_v2_Pred * -1, # end_points_ensadv_res_v2_Pred * -1, # end_points_resnet_Pred * -1] # TopKFit, TopKFitIndx = tf.nn.top_k(ens_Pred_Value, FLAGS.LOW_K) # # TopKFit = TopKFit * -1 # # logits_dictionary = tf.placeholder(dtype=tf.int32, shape=(7, FLAGS.batch_size, 1001)) # TopKFitIndex = tf.placeholder(dtype=tf.int32, shape=(FLAGS.batch_size)) # logits[j] = (logits_dictionary[TopKFitIndex[0]] * 5.0 + logits_dictionary[TopKFitIndex[1]] * 4.0 + logits_dictionary[TopKFitIndex[2]] * 3.0 + logits_dictionary[TopKFitIndex[3]] * 2.0 + logits_dictionary[TopKFitIndex[4]] * 1.0) / 15.0 #i_iteration = tf.placeholder(dtype=tf.int32) #first_round = tf.cast(tf.equal(i_iteration, 0), tf.int64) #y = first_round * pred y = tf.placeholder( dtype=tf.int32, shape=[ 2, ] ) # shape= FLAGS.batchsize !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! #y1 = first_round * pred + (1 - first_round) * y one_hot = tf.one_hot(y, num_classes) logits_gravity = tf.placeholder(dtype=tf.float32, shape=[FLAGS.batch_size, 7]) # logits = tf.zeros(shape=[FLAGS.batch_size, 1001], dtype=tf.float32) #for i in range(2): ########### 2 is the FLAGS.batch_size logits1 = (logits_dict[0][0] * logits_gravity[0][0] + logits_dict[1][0] * logits_gravity[0][1] + logits_dict[2][0] * logits_gravity[0][2] + logits_dict[3][0] * logits_gravity[0][3] + logits_dict[4][0] * logits_gravity[0][4] + logits_dict[5][0] * logits_gravity[0][5] + logits_dict[6][0] * logits_gravity[0][6]) / 28.0 logits2 = (logits_dict[0][1] * logits_gravity[1][0] + logits_dict[1][1] * logits_gravity[1][1] + logits_dict[2][1] * logits_gravity[1][2] + logits_dict[3][1] * logits_gravity[1][3] + logits_dict[4][1] * logits_gravity[1][4] + logits_dict[5][1] * logits_gravity[1][5] + logits_dict[6][1] * logits_gravity[1][6]) / 28.0 logits1 = tf.reshape(logits1, [1, 1001]) logits2 = tf.reshape(logits2, [1, 1001]) logits = tf.concat([logits1, logits2], 0) # logits = (logits_v3 + 0.25 * logits_adv_v3 + logits_ens3_adv_v3 + \ # logits_ens4_adv_v3 + logits_v4 + \ # logits_res_v2 + logits_ensadv_res_v2 + logits_resnet) / 7.25 # auxlogits = (end_points_v3['AuxLogits'] + 0.25 * end_points_adv_v3['AuxLogits'] + end_points_ens3_adv_v3['AuxLogits'] + \ # end_points_ens4_adv_v3['AuxLogits'] + end_points_v4['AuxLogits'] + \ # end_points_res_v2['AuxLogits'] + end_points_ensadv_res_v2['AuxLogits']) / 6.25 # cross_entropy = tf.zeros(shape=(FLAGS.batch_size)) #print(str(gravity)) cross_entropy = tf.losses.softmax_cross_entropy( one_hot, tf.clip_by_value(logits, 1e-8, tf.reduce_max(logits)), label_smoothing=0.0, weights=1.0) # cross_entropy += tf.losses.softmax_cross_entropy(one_hot, # auxlogits, # label_smoothing=0.0, # weights=0.4) #print(str(cross_entropy)) noise = tf.gradients(cross_entropy, x)[0] #print(str(tf.gradients(cross_entropy, x)) + '=========') # if(noise == None): # noise = tf.zeros(shape=[FLAGS.batch_size, 299, 299, 3]) # print('noise出了问题!') noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True) grad = momentum * grad + noise adv = x + alpha * tf.sign(grad) adv = tf.clip_by_value(adv, x_min, x_max) with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_adv_v3_one, end_points_adv_v3_one = inception_v3.inception_v3( x_adv_v3, num_classes=num_classes, is_training=False, reuse=True, scope='AdvInceptionV3') pred_adv_v3 = tf.argmax(end_points_adv_v3_one['Predictions'], axis=1) # x_adv, y, _, _, _, _, pred = tf.while_loop(stop, graph, [x, y, i, x_max, x_min, grad, pred]) # Run computation s1 = tf.train.Saver(slim.get_model_variables(scope='InceptionV3')) s2 = tf.train.Saver(slim.get_model_variables(scope='AdvInceptionV3')) s3 = tf.train.Saver( slim.get_model_variables(scope='Ens3AdvInceptionV3')) s4 = tf.train.Saver( slim.get_model_variables(scope='Ens4AdvInceptionV3')) s5 = tf.train.Saver(slim.get_model_variables(scope='InceptionV4')) s6 = tf.train.Saver( slim.get_model_variables(scope='InceptionResnetV2')) s7 = tf.train.Saver( slim.get_model_variables(scope='EnsAdvInceptionResnetV2')) s8 = tf.train.Saver(slim.get_model_variables(scope='resnet_v2')) s1.restore(sess, FLAGS.checkpoint_path_inception_v3) s2.restore(sess, FLAGS.checkpoint_path_adv_inception_v3) s3.restore(sess, FLAGS.checkpoint_path_ens3_adv_inception_v3) s4.restore(sess, FLAGS.checkpoint_path_ens4_adv_inception_v3) s5.restore(sess, FLAGS.checkpoint_path_inception_v4) s6.restore(sess, FLAGS.checkpoint_path_inception_resnet_v2) s7.restore(sess, FLAGS.checkpoint_path_ens_adv_inception_resnet_v2) s8.restore(sess, FLAGS.checkpoint_path_resnet) #sess.run(tf.global_variables_initializer()) ####### # test # for filenames, images in load_images(FLAGS.input_dir, batch_shape): # logits_v3_, Pred_dict = sess.run([logits_v3, pred_dict], feed_dict={x: images}) # label = np.argmax(Pred_dict, axis=1) # print('label::::::', label) # print('=================\n', logits_v3_) ####### sum = 0 loss_num = 0 l2_distance = 0 label_distance = 0 for filenames, images in load_images(FLAGS.input_dir, batch_shape): sum += len(filenames) images = images.astype(np.float32) images_flatten_initial = images.reshape((2, 268203)) # true_label = [] # for i in range(len(filenames)): # true_label.append(filenames[i].split('-')[2][:-4]) #print('image.astype::::::::', images) #adv_images = tf.zeros(shape=batch_shape) #adv_images = [] #prediction = 0!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! prediction = [] for i in range(FLAGS.num_iter): logits_proportion = [] if i == 0: prediction = sess.run(pred, feed_dict={x: images}) print('true_label::::::::', prediction) logits_dictionary, end_points_v3_run, end_points_ens3_adv_v3_run, \ end_points_ens4_adv_v3_run, end_points_v4_run, end_points_res_v2_run, end_points_ensadv_res_v2_run, end_points_resnet_run = \ sess.run([logits_dict,end_points_v3, end_points_ens3_adv_v3, end_points_ens4_adv_v3, end_points_v4, end_points_res_v2, end_points_ensadv_res_v2, end_points_resnet], feed_dict={x: images, y: prediction}) else: logits_dictionary, adv_pred, end_points_v3_run, end_points_ens3_adv_v3_run, \ end_points_ens4_adv_v3_run, end_points_v4_run, end_points_res_v2_run, end_points_ensadv_res_v2_run, end_points_resnet_run = \ sess.run([logits_dict, pred, end_points_v3, end_points_ens3_adv_v3, end_points_ens4_adv_v3, end_points_v4, end_points_res_v2, end_points_ensadv_res_v2, end_points_resnet], feed_dict={x: images, y: prediction}) # print('prediction::::::::::', prediction) # print('end_points_v3_run[Predictions]:::::::', end_points_v3_run['Predictions']) # if i == FLAGS.num_iter - 1: # print('adv_pred:::::::', adv_pred) # # for l in range(len(filenames)): # label_distance += abs(prediction[l] - adv_pred[l]) # # if int(prediction[l]) == adv_pred[l]: # The test tag is always 1 more than the real tag, so I subtract 1 here # loss_num += 1 # #print('sucess_num:::::', loss_num) for j in range(FLAGS.batch_size): end_points_v3_Pred = end_points_v3_run['Predictions'][j][ prediction[j]] # end_points_adv_v3_Pred = end_points_adv_v3['Predictions'] end_points_ens3_adv_v3_Pred = end_points_ens3_adv_v3_run[ 'Predictions'][j][prediction[j]] end_points_ens4_adv_v3_Pred = end_points_ens4_adv_v3_run[ 'Predictions'][j][prediction[j]] end_points_v4_Pred = end_points_v4_run['Predictions'][j][ prediction[j]] end_points_res_v2_Pred = end_points_res_v2_run[ 'Predictions'][j][prediction[j]] end_points_ensadv_res_v2_Pred = end_points_ensadv_res_v2_run[ 'Predictions'][j][prediction[j]] end_points_resnet_Pred = end_points_resnet_run[ 'predictions'][j][prediction[j]] ens_Pred_Value = [ end_points_v3_Pred, end_points_ens3_adv_v3_Pred, end_points_ens4_adv_v3_Pred, end_points_v4_Pred, end_points_res_v2_Pred, end_points_ensadv_res_v2_Pred, end_points_resnet_Pred ] TopKFitIndx = np.argsort(ens_Pred_Value) # logits_dictionary = tf.placeholder(dtype=tf.int32, shape=(7, FLAGS.batch_size, 1001)) # TopKFitIndex = tf.placeholder(dtype=tf.int32, shape=(FLAGS.batch_size)) a = [0.0] * 7 print('ens_Pred_Value:::::::::', ens_Pred_Value) # print('TopKFitIndx::::::', TopKFitIndx) # for m in range(7): # a[TopKFitIndx[m]] = 7 - m a[TopKFitIndx[0]] = 2 a[TopKFitIndx[1]] = 2 a[TopKFitIndx[2]] = 2 a[TopKFitIndx[3]] = 2 a[TopKFitIndx[4]] = 1 a[TopKFitIndx[5]] = 1 a[TopKFitIndx[6]] = 1 logits_proportion.append(a) One_hot, Logits_dict, Logits1, Logits2, Logits, images, Cross_Entropy, Noise, gradient = \ sess.run([one_hot, logits_dict, logits1, logits2, logits, adv, cross_entropy, noise, grad], feed_dict={x: images, logits_gravity: logits_proportion, y: prediction}) if i == FLAGS.num_iter - 1: images_flatten_adv = images.reshape((2, 268203)) l2_distance_list = np.linalg.norm( (images_flatten_initial - images_flatten_adv), axis=1, keepdims=True) for n in range(len(filenames)): l2_distance += l2_distance_list[n] # print('One_hot::::::::::::', One_hot.shape, One_hot) # print('Logits_dict::::::::::', Logits_dict) # print('Logits1::::::::::',Logits1.shape, Logits1) # print('Logits2::::::::::',Logits2.shape, Logits2) # # print('Logits:::::::::::', Logits) # print('Cross_Entropy:::::::::::::', Cross_Entropy) # print('Noise:::::::::::', Noise) # print('gradient::::::::::', gradient) prediction_adv_v3 = sess.run(pred_adv_v3, feed_dict={x_adv_v3: images}) print('prediction_adv_v3::::::', prediction_adv_v3) for l in range(len(filenames)): label_distance += abs(prediction[l] - prediction_adv_v3[l]) if int(prediction[l]) == prediction_adv_v3[ l]: # The test tag is always 1 more than the real tag, so I subtract 1 here loss_num += 1 save_images(images, filenames, FLAGS.output_dir) print('sum::::', sum) print('loss_num::::', loss_num) rate_wrong = loss_num / sum print('rate_wrong:::::::', rate_wrong) print('l2_distance:::::::', l2_distance) print('label_distance:::::::', label_distance) end = time.clock() print('run time:::::', end - start)
def graph(x, y, i, x_max, x_min, grad, amplification): one_hot = tf.one_hot(y, FLAGS.num_classes) eps = 2.0 * FLAGS.max_epsilon / 255.0 num_iter = FLAGS.num_iter alpha = eps / num_iter momentum = FLAGS.momentum # amplification factor alpha_beta = alpha * FLAGS.amplification_factor gamma = alpha_beta # DIM: https://arxiv.org/abs/1803.06978 # input_diversity(FLAG, x) with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_v3, end_points_v3 = inception_v3.inception_v3( x, num_classes=FLAGS.num_classes, is_training=False) auxlogits_v3 = end_points_v3['AuxLogits'] with slim.arg_scope(inception_v4.inception_v4_arg_scope()): logits_v4, end_points_v4 = inception_v4.inception_v4( x, num_classes=FLAGS.num_classes, is_training=False) auxlogits_v4 = end_points_v4['AuxLogits'] with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits_resnet, end_points_resnet = resnet_v2.resnet_v2_152( x, num_classes=FLAGS.num_classes, is_training=False) with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_Incres, end_points_IR = inception_resnet_v2.inception_resnet_v2( x, num_classes=FLAGS.num_classes, is_training=False) auxlogits_IR = end_points_IR['AuxLogits'] logits = (logits_v3 + logits_v4 + logits_resnet + logits_Incres) / 4.0 auxlogits = (auxlogits_v3 + auxlogits_v4 + auxlogits_IR) / 3.0 cross_entropy = tf.losses.softmax_cross_entropy(one_hot, logits, label_smoothing=0.0, weights=1.0) cross_entropy += tf.losses.softmax_cross_entropy(one_hot, auxlogits, label_smoothing=0.0, weights=1.0) noise = tf.gradients(cross_entropy, x)[0] # TI-FGSM: https://arxiv.org/pdf/1904.02884.pdf # noise = tf.nn.depthwise_conv2d(noise, T_kern, strides=[1, 1, 1, 1], padding='SAME') # MI-FGSM: https://arxiv.org/pdf/1710.06081.pdf # noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True) # noise = momentum * grad + noise # Project cut noise amplification += alpha_beta * tf.sign(noise) cut_noise = tf.clip_by_value(abs(amplification) - eps, 0.0, 10000.0) * tf.sign(amplification) projection = gamma * tf.sign(project_noise(cut_noise, P_kern, kern_size)) # Occasionally, when the adversarial examples are crafted for an ensemble of networks with residual block by combined methods, # you may neet to comment the following line to get better result. amplification += projection x = x + alpha_beta * tf.sign(noise) + projection x = tf.clip_by_value(x, x_min, x_max) i = tf.add(i, 1) return x, y, i, x_max, x_min, noise, amplification
def graph(x, target_class_input, i, x_max, x_min, grad): eps = 2.0 * FLAGS.max_epsilon / 255.0 alpha = eps / FLAGS.iterations momentum = FLAGS.momentum num_classes = 1001 x_div = input_diversity(x) #x_div = x with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_adv_v3, end_points_adv_v3 = inception_v3.inception_v3( x_div, num_classes=num_classes, is_training=False, scope='AdvInceptionV3') with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3( x_div, num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3') with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3( x_div, num_classes=num_classes, is_training=False, scope='Ens4AdvInceptionV3') with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2( x_div, num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2') with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_adv_res_v2, end_points_adv_res_v2 = inception_resnet_v2.inception_resnet_v2( x_div, num_classes=num_classes, is_training=False, scope='AdvInceptionResnetV2') one_hot_target_class = tf.one_hot(target_class_input, num_classes) logits = (logits_adv_v3 + logits_ens3_adv_v3 + logits_ens4_adv_v3 + logits_ensadv_res_v2 + logits_adv_res_v2) / 5 auxlogits = (end_points_adv_v3['AuxLogits'] + end_points_ens3_adv_v3['AuxLogits'] + end_points_ens4_adv_v3['AuxLogits'] + end_points_adv_res_v2['AuxLogits'] + end_points_ensadv_res_v2['AuxLogits']) / 5 cross_entropy = tf.losses.softmax_cross_entropy(one_hot_target_class, logits, label_smoothing=0.0, weights=1.0) cross_entropy += tf.losses.softmax_cross_entropy(one_hot_target_class, auxlogits, label_smoothing=0.0, weights=0.4) noise = tf.gradients(cross_entropy, x)[0] kernel = gkern(7, FLAGS.sig).astype(np.float32) stack_kernel = np.stack([kernel, kernel, kernel]).swapaxes(2, 0) stack_kernel = np.expand_dims(stack_kernel, 3) noise = tf.nn.depthwise_conv2d(noise, stack_kernel, strides=[1, 1, 1, 1], padding='SAME') noise = noise / tf.reshape(tf.contrib.keras.backend.std(tf.reshape(noise, [FLAGS.batch_size, -1]), axis=1), [FLAGS.batch_size, 1, 1, 1]) noise = momentum * grad + noise noise = noise / tf.reshape(tf.contrib.keras.backend.std(tf.reshape(noise, [FLAGS.batch_size, -1]), axis=1), [FLAGS.batch_size, 1, 1, 1]) x = x - alpha * tf.clip_by_value(tf.round(noise), -2, 2) x = tf.clip_by_value(x, x_min, x_max) i = tf.add(i, 1) return x, target_class_input, i, x_max, x_min, noise
}, 'inception_v1': { 'shape': [1, 224, 224, 3], 'scope': inception_v1.inception_v1_arg_scope(), 'net': inception_v1.inception_v1, 'num_classes': 1000 }, 'inception_v2': { 'shape': [1, 224, 224, 3], 'scope': inception_v2.inception_v2_arg_scope(), 'net': inception_v2.inception_v2, 'num_classes': 1000 }, 'inception_v3': { 'shape': [1, 299, 299, 3], 'scope': inception_v3.inception_v3_arg_scope(), 'net': inception_v3.inception_v3, 'num_classes': 1000 }, 'vgg_16': { 'shape': [1, 224, 224, 3], 'scope': vgg.vgg_arg_scope(), 'net': vgg.vgg_16, 'num_classes': 1000 }, 'vgg_19': { 'shape': [1, 224, 224, 3], 'scope': vgg.vgg_arg_scope(), 'net': vgg.vgg_19, 'num_classes': 1000 },
def graph(x, y, yi, i, x_max, x_min, grad, pred_w): #[x_input, adv_x, y, i, x_max, x_min, grad] eps = 2.0 * FLAGS.max_epsilon / 255.0 num_iter = FLAGS.num_iter alpha = eps / num_iter momentum = FLAGS.momentum num_classes = 1001 image = x with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_inc_v3, end_points_inc_v3 = inception_v3.inception_v3( image, num_classes=num_classes, is_training=False, scope='InceptionV3') with slim.arg_scope(inception_v4.inception_v4_arg_scope()): logits_inc_v4, end_points_inc_v4 = inception_v4.inception_v4( image, num_classes=num_classes, is_training=False, scope='InceptionV4') with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_inc_res_v2, end_points_inc_res_v2 = inception_resnet_v2.inception_resnet_v2( image, num_classes=num_classes, is_training=False, scope='InceptionResnetV2') with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2( image, num_classes=num_classes, is_training=False, scope='ens_adv_inception_resnet_v2') with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens3_inc_v3, end_points_ens3_inc_v3 = inception_v3.inception_v3( image, num_classes=num_classes, is_training=False, scope='ens3_adv_inception_v3') # with slim.arg_scope(inception_v3.inception_v3_arg_scope()): # logits_ens4_inc_v3, end_points_ens4_inc_v3 = inception_v3.inception_v3( # image, num_classes=num_classes, is_training=False, scope='ens4_adv_inception_v3') pred = tf.argmax(end_points_inc_v3['Predictions']+ \ end_points_inc_v4['Predictions']+end_points_inc_res_v2['Predictions']+ \ end_points_ensadv_res_v2['Predictions']+end_points_ens3_inc_v3['Predictions'], 1) first_round = tf.cast(tf.equal(i, 0), tf.int64) y = first_round * pred + (1 - first_round) * y one_hot = tf.one_hot(y, num_classes) logits = (logits_inc_v3*pred_w[0]+logits_inc_v4*pred_w[1]+logits_inc_res_v2*pred_w[2] + \ logits_ensadv_res_v2*pred_w[3]+logits_ens3_inc_v3*pred_w[4])/tf.reduce_sum(pred_w) auxlogits = (end_points_inc_v3['AuxLogits']*pred_w[0]+end_points_inc_v4['AuxLogits']*pred_w[1]+ \ end_points_inc_res_v2['AuxLogits']*pred_w[2]+end_points_ensadv_res_v2['AuxLogits']*pred_w[3]+\ end_points_ens3_inc_v3['AuxLogits']*pred_w[4])/(tf.reduce_sum(pred_w)) logits = logits + auxlogits * 0.4 cross_entropy = tf.losses.softmax_cross_entropy(one_hot, logits, label_smoothing=0.0, weights=1.0) cross_entropy += tf.losses.softmax_cross_entropy(one_hot, auxlogits, label_smoothing=0.0, weights=0.4) noise = tf.gradients(cross_entropy, x)[0] noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True) noise = momentum * grad + noise x = x + alpha * tf.sign(noise) x = tf.clip_by_value(x, x_min, x_max) i = tf.add(i, 1) inc3_pred = tf.concat([[ end_points_inc_v3['Predictions'][i, yi[i]] for i in range(yi.shape[0]) ]], axis=0) inc4_pred = tf.concat([[ end_points_inc_v4['Predictions'][i, yi[i]] for i in range(yi.shape[0]) ]], axis=0) res2_pred = tf.concat([[ end_points_inc_res_v2['Predictions'][i, yi[i]] for i in range(yi.shape[0]) ]], axis=0) ens_res2_pred = tf.concat([[ end_points_ensadv_res_v2['Predictions'][i, yi[i]] for i in range(yi.shape[0]) ]], axis=0) ens3_inc3_pred = tf.concat([[ end_points_ens3_inc_v3['Predictions'][i, yi[i]] for i in range(yi.shape[0]) ]], axis=0) # ens4_inc3_pred = tf.concat([[end_points_ens4_inc_v3['Predictions'][i,yi[i]] for i in range(yi.shape[0])]],axis=0) pred_list = tf.concat( [inc3_pred, inc4_pred, res2_pred, ens_res2_pred, ens3_inc3_pred], axis=0) return x, y, yi, i, x_max, x_min, noise, pred_list
def run(): # Create the log directory here. Must be done here otherwise import will activate this unneededly. if not os.path.exists(log_dir): os.mkdir(log_dir) # ======================= TRAINING PROCESS ========================= # Now we start to construct the graph and build our model with tf.Graph().as_default() as graph: tf.logging.set_verbosity(tf.logging.INFO) # Set the verbosity to INFO level # First create the dataset and load one batch dataset = get_split('train', dataset_dir, file_pattern=file_pattern) images, labels = load_batch(dataset, batch_size=batch_size) # Know the number steps to take before decaying the learning rate and batches per epoch num_batches_per_epoch = int(dataset.num_samples / batch_size) num_steps_per_epoch = num_batches_per_epoch # Because one step is one batch processed decay_steps = int(num_epochs_before_decay * num_steps_per_epoch) # Create the model inference with slim.arg_scope(inception_v3_arg_scope()): logits, end_points = inception_v3(images, num_classes=dataset.num_classes, is_training=True) # Define the scopes that you want to exclude for restoration exclude = ['InceptionV3/Logits', 'InceptionV3/AuxLogits'] variables_to_restore = slim.get_variables_to_restore(exclude=exclude) # Perform one-hot-encoding of the labels (Try one-hot-encoding within the load_batch function!) one_hot_labels = slim.one_hot_encoding(labels, dataset.num_classes) # Performs the equivalent to tf.nn.sparse_softmax_cross_entropy_with_logits but enhanced with checks loss = tf.losses.softmax_cross_entropy(onehot_labels=one_hot_labels, logits=logits) total_loss = tf.losses.get_total_loss(add_regularization_losses=True) # obtain the regularization losses as well # Create the global step for monitoring the learning_rate and training. global_step = get_or_create_global_step() # Define your exponentially decaying learning rate lr = tf.train.exponential_decay( learning_rate=initial_learning_rate, global_step=global_step, decay_steps=decay_steps, decay_rate=learning_rate_decay_factor, staircase=True) # Now we can define the optimizer that takes on the learning rate optimizer = tf.train.AdamOptimizer(learning_rate=lr) # Create the train_op. train_op = slim.learning.create_train_op(total_loss, optimizer) # State the metrics that you want to predict. We get a predictions that is not one_hot_encoded. predictions = tf.argmax(end_points['Predictions'], 1) probabilities = end_points['Predictions'] accuracy, accuracy_update = tf.contrib.metrics.streaming_accuracy(predictions, labels) metrics_op = tf.group(accuracy_update, probabilities) # Now finally create all the summaries you need to monitor and group them into one summary op. tf.summary.scalar('losses/Total_Loss', total_loss) tf.summary.scalar('accuracy', accuracy) tf.summary.scalar('learning_rate', lr) my_summary_op = tf.summary.merge_all() # Now we need to create a training step function that runs both the train_op, metrics_op and updates the global_step concurrently. def train_step(sess, train_op, global_step): ''' Simply runs a session for the three arguments provided and gives a logging on the time elapsed for each global step ''' # Check the time for each sess run start_time = time.time() total_loss, global_step_count, _ = sess.run([train_op, global_step, metrics_op]) time_elapsed = time.time() - start_time # Run the logging to print some results logging.info('global step %s: loss: %.4f (%.2f sec/step)', global_step_count, total_loss, time_elapsed) return total_loss, global_step_count # Now we create a saver function that actually restores the variables from a checkpoint file in a sess saver = tf.train.Saver(variables_to_restore) def restore_fn(sess): return saver.restore(sess, checkpoint_file) # Define your supervisor for running a managed session. Do not run the summary_op automatically or else it will consume too much memory sv = tf.train.Supervisor(logdir=log_dir, summary_op=None, init_fn=restore_fn) # Run the managed session with sv.managed_session() as sess: print (num_steps_per_epoch) print (num_epochs) for step in range(num_steps_per_epoch * num_epochs): # At the start of every epoch, show the vital information: if step % num_batches_per_epoch == 0: logging.info('Epoch %s/%s', step / num_batches_per_epoch + 1, num_epochs) learning_rate_value, accuracy_value = sess.run([lr, accuracy]) logging.info('Current Learning Rate: %s', learning_rate_value) logging.info('Current Streaming Accuracy: %s', accuracy_value) # optionally, print your logits and predictions for a sanity check that things are going fine. logits_value, probabilities_value, predictions_value, labels_value = sess.run( [logits, probabilities, predictions, labels]) print ('logits: \n', logits_value) print ('Probabilities: \n', probabilities_value) print ('predictions: \n', predictions_value) print ('Labels:\n:', labels_value) # Log the summaries every 10 step. if step % 10 == 0: train_writer = tf.summary.FileWriter('tmp/vis/depth_with_cp') loss, _ = train_step(sess, train_op, sv.global_step) summaries = sess.run(my_summary_op) sv.summary_computed(sess, summaries) train_writer.add_summary(summaries, step) train_writer.close() # If not, simply run the training step else: loss, _ = train_step(sess, train_op, sv.global_step) # We log the final training loss and accuracy logging.info('Final Loss: %s', loss) logging.info('Final Accuracy: %s', sess.run(accuracy)) # Once all the training has been done, save the log files and checkpoint model logging.info('Finished training! Saving model to disk now.') sv.saver.save(sess, sv.save_path, global_step=sv.global_step)
def graph(x, y, i, x_max, x_min, grad1, grad2, t): eps = 2 * tf.sqrt(299.0 * 299.0 * 3) * FLAGS.max_epsilon / 255.0 num_iter = FLAGS.num_iter alpha = eps / num_iter momentum = FLAGS.momentum num_classes = 1001 with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_v3, end_points_v3 = inception_v3.inception_v3( x, num_classes=num_classes, is_training=False) # with slim.arg_scope(inception_v3.inception_v3_arg_scope()): # logits_adv_v3, end_points_adv_v3 = inception_v3.inception_v3( # x, num_classes=num_classes, is_training=False, scope='AdvInceptionV3') with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3( x, num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3') with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3( x, num_classes=num_classes, is_training=False, scope='Ens4AdvInceptionV3') # with slim.arg_scope(inception_v4.inception_v4_arg_scope()): # logits_v4, end_points_v4 = inception_v4.inception_v4( # x, num_classes=num_classes, is_training=False) with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2( x, num_classes=num_classes, is_training=False) with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2( x, num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2') with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits_resnet, end_points_resnet = resnet_v2.resnet_v2_101( x, num_classes=num_classes, is_training=False) # pred = tf.argmax(end_points_v3['Predictions'] + end_points_adv_v3['Predictions'] + end_points_ens3_adv_v3['Predictions'] + \ # end_points_ens4_adv_v3['Predictions'] + end_points_v4['Predictions'] + \ # end_points_res_v2['Predictions'] + end_points_ensadv_res_v2['Predictions'] + end_points_resnet['predictions'], 1) pred = tf.argmax(end_points_v3['Predictions'] + end_points_ens3_adv_v3['Predictions'] + \ end_points_ens4_adv_v3['Predictions']+ \ end_points_res_v2['Predictions'] + end_points_ensadv_res_v2['Predictions'] + end_points_resnet['predictions'], 1) first_round = tf.cast(tf.equal(i, 0), tf.int64) y = first_round * pred + (1 - first_round) * y one_hot = tf.one_hot(y, num_classes) # logits = (logits_v3 + 0.25 * logits_adv_v3 + logits_ens3_adv_v3 + \ # logits_ens4_adv_v3 + logits_v4 + \ # logits_res_v2 + logits_ensadv_res_v2 + logits_resnet) / 7.25 # auxlogits = (end_points_v3['AuxLogits'] + 0.25 * end_points_adv_v3['AuxLogits'] + end_points_ens3_adv_v3['AuxLogits'] + \ # end_points_ens4_adv_v3['AuxLogits'] + end_points_v4['AuxLogits'] + \ # end_points_res_v2['AuxLogits'] + end_points_ensadv_res_v2['AuxLogits']) / 6.25 logits = (logits_v3 + logits_ens3_adv_v3 + \ logits_ens4_adv_v3 + \ logits_res_v2 + logits_ensadv_res_v2 + logits_resnet) / 6.0 auxlogits = (end_points_v3['AuxLogits'] + end_points_ens3_adv_v3['AuxLogits'] + \ end_points_ens4_adv_v3['AuxLogits'] + \ end_points_res_v2['AuxLogits'] + end_points_ensadv_res_v2['AuxLogits']) / 5.0 cross_entropy = tf.losses.softmax_cross_entropy(one_hot, logits, label_smoothing=0.0, weights=1.0) cross_entropy += tf.losses.softmax_cross_entropy(one_hot, auxlogits, label_smoothing=0.0, weights=0.4) noise = tf.gradients(cross_entropy, x)[0] b = tf.random_normal(shape=grad1.shape, mean=0, stddev=1.0) prob = 0.4 / (100 * ((i + 1) / FLAGS.num_iter)) b1 = tf.nn.dropout(b, keep_prob=prob) b2 = tf.sign(b1) b3 = 0.5 * tf.ones(shape=grad1.shape) + b2 b4 = tf.sign(b3) noise = (noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keepdims=True)) grad = momentum * grad2 + noise + (1.0 - (1.0 / FLAGS.num_iter) * i) * momentum * (grad2 - grad1) grad1 = grad2 grad2 = grad grad3 = grad * b4 noise_norm = tf.norm(grad3, axis=(1, 2), ord='fro', keepdims=True) x = x + alpha * (grad3 / noise_norm) x = tf.clip_by_value(x, x_min, x_max) i = tf.add(i, 1) return x, y, i, x_max, x_min, grad1, grad2, t
def run(): end_points = {} # Create the log directory here. Must be done here otherwise import will activate this unneededly. if not os.path.exists(log_dir): os.mkdir(log_dir) # ======================= TRAINING PROCESS ========================= # Now we start to construct the graph and build our model with tf.Graph().as_default() as graph: tf.logging.set_verbosity( tf.logging.INFO) # Set the verbosity to INFO level ######################################################## # Get RGB dataset and the Imagenet trained on RGB images ######################################################## # First create the dataset and load one batch dataset_rgb = get_split('train', dataset_dir_rgb, file_pattern=file_pattern) images_rgb, labels_rgb = load_batch(dataset_rgb, batch_size=batch_size) num_batches_per_epoch = int(dataset_rgb.num_samples / batch_size) num_steps_per_epoch = num_batches_per_epoch # Because one step is one batch processed decay_steps = int(num_epochs_before_decay * num_steps_per_epoch) with tf.variable_scope("net_rgb"): # Create the model inference with slim.arg_scope(inception_v3_arg_scope()): logits_rgb, end_points_rgb = inception_v3( images_rgb, num_classes=dataset_rgb.num_classes, is_training=True) net1_varlist = { v.name.lstrip("net_rgb/")[:-2]: v for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="net_rgb/") } ######################################################## # Get depth dataset and the Imagenet trained on depth images ######################################################## # First create the dataset and load one batch dataset_depth = get_split('train', dataset_dir_depth, file_pattern=file_pattern) images_depth, labels_depth = load_batch(dataset_depth, batch_size=batch_size) num_batches_per_epoch = int(dataset_rgb.num_samples / batch_size) num_steps_per_epoch = num_batches_per_epoch # Because one step is one batch processed decay_steps = int(num_epochs_before_decay * num_steps_per_epoch) # Create the model inference with tf.variable_scope("net_depth"): with slim.arg_scope(inception_v3_arg_scope()): logits_depth, end_points_depth = inception_v3( images_depth, num_classes=dataset_rgb.num_classes, is_training=True) net2_varlist = { v.name.lstrip("net_depth/")[:-2]: v for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="net_depth/") } ######################################################## # Combine the models with the concatenation operation # and add an FC layer on top ######################################################## # with tf.variable_scope("concat_dense"): W_master = tf.Variable(tf.random_uniform([10, 5], -0.01, 0.01), name="weights_concat") b_master = tf.Variable(tf.zeros([5]), name="bias_concat") h_master = tf.matmul(tf.concat( (logits_rgb, logits_depth), axis=1), W_master) + b_master variable_summaries(W_master, "concat") logits2 = tf.layers.dense(inputs=h_master, units=(num_classes * 2), name="dense_concat1") logits = tf.layers.dense(inputs=logits2, units=num_classes, name="dense_concat0") end_points['Logits'] = logits end_points['Predictions'] = slim.softmax(logits, scope='Predictions') one_hot_labels = slim.one_hot_encoding(labels_rgb, dataset_rgb.num_classes) loss = tf.losses.softmax_cross_entropy(onehot_labels=one_hot_labels, logits=logits) tf.summary.scalar('cross_entropy', loss) total_loss = tf.losses.get_total_loss(add_regularization_losses=True) global_step = get_or_create_global_step() lr = tf.train.exponential_decay(learning_rate=initial_learning_rate, global_step=global_step, decay_steps=decay_steps, decay_rate=learning_rate_decay_factor, staircase=True) optimizer = tf.train.AdamOptimizer(learning_rate=lr) train_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "concat_dense") train_op = slim.learning.create_train_op(total_loss, optimizer, variables_to_train=train_vars) # Metrics predictions = tf.argmax(end_points['Predictions'], 1) probabilities = end_points['Predictions'] accuracy, accuracy_update = tf.contrib.metrics.streaming_accuracy( predictions, labels_rgb) metrics_op = tf.group(accuracy_update, probabilities) tf.summary.scalar('losses/Total_Loss', total_loss) tf.summary.scalar('accuracy', accuracy) tf.summary.scalar('learning_rate', lr) my_summary_op = tf.summary.merge_all() def train_step(sess, train_op, global_step): ''' Simply runs a session for the three arguments provided and gives a logging on the time elapsed for each global step ''' # Check the time for each sess run start_time = time.time() total_loss, global_step_count, _ = sess.run( [train_op, global_step, metrics_op]) time_elapsed = time.time() - start_time # Run the logging to print some results logging.info('global step %s: loss: %.4f (%.2f sec/step)', global_step_count, total_loss, time_elapsed) return total_loss, global_step_count saver_rgb = tf.train.Saver(var_list=net1_varlist) saver_depth = tf.train.Saver(var_list=net2_varlist) sv = tf.train.Supervisor(logdir=log_dir, summary_op=None, init_fn=None) with sv.managed_session() as sess: print(num_steps_per_epoch) print(num_epochs) rgb_latest = tf.train.latest_checkpoint( os.path.join(os.getcwd(), 'tmp/log/rgb_no_cp')) depth_latest = tf.train.latest_checkpoint( os.path.join(os.getcwd(), 'tmp/log/depth_no_cp')) saver_rgb.restore(sess, rgb_latest) saver_depth.restore(sess, depth_latest) for step in range(num_steps_per_epoch * num_epochs): if step % num_batches_per_epoch == 0: logging.info('Epoch %s/%s', step / num_batches_per_epoch + 1, num_epochs) learning_rate_value, accuracy_value = sess.run( [lr, accuracy]) logging.info('Current Learning Rate: %s', learning_rate_value) logging.info('Current Streaming Accuracy: %s', accuracy_value) if step % 10 == 0: train_writer = tf.summary.FileWriter( 'tmp/vis/both_no_cp', sess.graph) loss, _ = train_step(sess, train_op, sv.global_step) summaries = sess.run(my_summary_op) sv.summary_computed(sess, summaries) train_writer.add_summary(summaries, step) train_writer.close() # If not, simply run the training step else: loss, _ = train_step(sess, train_op, sv.global_step) logging.info('Final Loss: %s', loss) logging.info('Final Accuracy: %s', sess.run(accuracy)) logging.info('Finished training! Saving model to disk now.') sv.saver.save(sess, sv.save_path, global_step=sv.global_step)
def Evaluator(x, y): num_classes = 1001 # should keep original x here for output with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_v3, end_points_v3 = inception_v3.inception_v3( x, num_classes=num_classes, is_training=False) with slim.arg_scope(inception_v4.inception_v4_arg_scope()): logits_v4, end_points_v4 = inception_v4.inception_v4( x, num_classes=num_classes, is_training=False) with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2( x, num_classes=num_classes, is_training=False, reuse=True) with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits_resnet, end_points_resnet = resnet_v2.resnet_v2_152( x, num_classes=num_classes, is_training=False) with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits_resnet_101, end_points_resnet_101 = resnet_v2.resnet_v2_101( x, num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE) with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits_resnet_50, end_points_resnet_50 = resnet_v2.resnet_v2_50( x, num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE) with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_adv_v3, end_points_adv_v3 = inception_v3.inception_v3( x, num_classes=num_classes, is_training=False, scope='AdvInceptionV3') with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3( x, num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3') with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3( x, num_classes=num_classes, is_training=False, scope='Ens4AdvInceptionV3') with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2( x, num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2') # with slim.arg_scope(inception_v3.inception_v3_arg_scope()): # logits_ens_v3, end_points_ens_v3 = inception_v3.inception_v3( # x, num_classes=num_classes, is_training=False) # acc_v3 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_v3['Predictions'], 1), y), tf.float32)) # acc_v4 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_v4['Predictions'], 1), y), tf.float32)) # acc_res_v2 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_res_v2['Predictions'], 1), y), tf.float32)) # acc_resnet = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_resnet['predictions'], 1), y), tf.float32)) # acc_resnet_50 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_resnet_50['predictions'], 1), y), tf.float32)) # acc_resnet_101 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_resnet_101['predictions'], 1), y), tf.float32)) # acc_ens_v3 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_ens_v3['predictions'], 1), y), tf.float32)) logits_esm = (logits_v3 + logits_v4 + logits_res_v2 + logits_resnet + logits_resnet_50 + logits_resnet_101) if FLAGS.att_model == "incep_v3": logits_esm = (logits_esm - logits_v3) / 5 elif FLAGS.att_model == "incep_v4": logits_esm = (logits_esm - logits_v4) / 5 elif FLAGS.att_model == "incep_res_v2": logits_esm = (logits_esm - logits_res_v2) / 5 elif FLAGS.att_model == "resnet_50": logits_esm = (logits_esm - logits_resnet_50) / 5 elif FLAGS.att_model == "resnet_101": logits_esm = (logits_esm - logits_resnet_101) / 5 elif FLAGS.att_model == "resnet_152": logits_esm = (logits_esm - logits_resnet) / 5 elif FLAGS.att_model == "ens3_adv_3": logits_esm = (logits_esm + logits_ens4_adv_v3 + logits_ensadv_res_v2) / 8 elif FLAGS.att_model == "ens4_adv_3": logits_esm = (logits_esm + logits_ens3_adv_v3 + logits_ensadv_res_v2) / 8 elif FLAGS.att_model == "ensadv_res_2": logits_esm = (logits_esm + logits_ens4_adv_v3 + logits_ens3_adv_v3) / 8 # top_k top_k = FLAGS.top_k acc_v3 = tf.reduce_sum( tf.cast(tf.nn.in_top_k(end_points_v3['Predictions'], y, k=top_k), tf.float32)) acc_v4 = tf.reduce_sum( tf.cast(tf.nn.in_top_k(end_points_v4['Predictions'], y, k=top_k), tf.float32)) acc_res_v2 = tf.reduce_sum( tf.cast(tf.nn.in_top_k(end_points_res_v2['Predictions'], y, k=top_k), tf.float32)) acc_resnet = tf.reduce_sum( tf.cast(tf.nn.in_top_k(end_points_resnet['predictions'], y, k=top_k), tf.float32)) acc_resnet_50 = tf.reduce_sum( tf.cast( tf.nn.in_top_k(end_points_resnet_50['predictions'], y, k=top_k), tf.float32)) acc_resnet_101 = tf.reduce_sum( tf.cast( tf.nn.in_top_k(end_points_resnet_101['predictions'], y, k=top_k), tf.float32)) acc_adv_v3 = tf.reduce_sum( tf.cast(tf.nn.in_top_k(end_points_adv_v3['Predictions'], y, k=top_k), tf.float32)) acc_ens3_adv_v3 = tf.reduce_sum( tf.cast( tf.nn.in_top_k(end_points_ens3_adv_v3['Predictions'], y, k=top_k), tf.float32)) acc_ens4_adv_v3 = tf.reduce_sum( tf.cast( tf.nn.in_top_k(end_points_ens4_adv_v3['Predictions'], y, k=top_k), tf.float32)) acc_ensadv_res_v2 = tf.reduce_sum( tf.cast( tf.nn.in_top_k(end_points_ensadv_res_v2['Predictions'], y, k=top_k), tf.float32)) acc_esm = tf.reduce_sum( tf.cast( tf.nn.in_top_k(slim.softmax(logits_esm, scope='predictions'), y, k=top_k), tf.float32)) # acc_ens_v3 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_ens_v3['predictions'], 1), y), tf.float32)) # pred1 = tf.argmax(end_points_v3['Predictions'], 1) # pred2 = tf.argmax(end_points_v4['Predictions'], 1) # pred3 = tf.argmax(end_points_res_v2['Predictions'], 1) # pred4 = tf.argmax(end_points_resnet['predictions'], 1) return acc_v3, acc_v4, acc_res_v2, acc_resnet, acc_resnet_50, acc_resnet_101, acc_adv_v3, acc_ens3_adv_v3, \ acc_ens4_adv_v3, acc_ensadv_res_v2, acc_esm, end_points_v3, end_points_v4, end_points_res_v2, end_points_resnet
def Evaluator(x, y): num_classes = 1001 # should keep original x here for output with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_v3, end_points_v3 = inception_v3.inception_v3( x, num_classes=num_classes, is_training=False) with slim.arg_scope(inception_v4.inception_v4_arg_scope()): logits_v4, end_points_v4 = inception_v4.inception_v4( x, num_classes=num_classes, is_training=False) with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2( x, num_classes=num_classes, is_training=False, reuse=True) with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits_resnet, end_points_resnet = resnet_v2.resnet_v2_152( x, num_classes=num_classes, is_training=False) with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits_resnet_101, end_points_resnet_101 = resnet_v2.resnet_v2_101( x, num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE) with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits_resnet_50, end_points_resnet_50 = resnet_v2.resnet_v2_50( x, num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE) # with slim.arg_scope(inception_v3.inception_v3_arg_scope()): # logits_adv_v3, end_points_adv_v3 = inception_v3.inception_v3( # x, num_classes=num_classes, is_training=False, scope='AdvInceptionV3') # # with slim.arg_scope(inception_v3.inception_v3_arg_scope()): # logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3( # x, num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3') # # with slim.arg_scope(inception_v3.inception_v3_arg_scope()): # logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3( # x, num_classes=num_classes, is_training=False, scope='Ens4AdvInceptionV3') # # with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()): # logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2( # x, num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2') # with slim.arg_scope(inception_v3.inception_v3_arg_scope()): # logits_ens_v3, end_points_ens_v3 = inception_v3.inception_v3( # x, num_classes=num_classes, is_training=False) # acc_v3 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_v3['Predictions'], 1), y), tf.float32)) # acc_v4 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_v4['Predictions'], 1), y), tf.float32)) # acc_res_v2 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_res_v2['Predictions'], 1), y), tf.float32)) # acc_resnet = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_resnet['predictions'], 1), y), tf.float32)) # acc_resnet_50 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_resnet_50['predictions'], 1), y), tf.float32)) # acc_resnet_101 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_resnet_101['predictions'], 1), y), tf.float32)) # acc_ens_v3 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_ens_v3['predictions'], 1), y), tf.float32)) # top_k top_k = FLAGS.top_k acc_v3 = tf.reduce_sum( tf.cast(tf.nn.in_top_k(end_points_v3['Predictions'], y, k=top_k), tf.float32)) acc_v4 = tf.reduce_sum( tf.cast(tf.nn.in_top_k(end_points_v4['Predictions'], y, k=top_k), tf.float32)) acc_res_v2 = tf.reduce_sum( tf.cast(tf.nn.in_top_k(end_points_res_v2['Predictions'], y, k=top_k), tf.float32)) acc_resnet = tf.reduce_sum( tf.cast(tf.nn.in_top_k(end_points_resnet['predictions'], y, k=top_k), tf.float32)) acc_resnet_50 = tf.reduce_sum( tf.cast( tf.nn.in_top_k(end_points_resnet_50['predictions'], y, k=top_k), tf.float32)) acc_resnet_101 = tf.reduce_sum( tf.cast( tf.nn.in_top_k(end_points_resnet_101['predictions'], y, k=top_k), tf.float32)) # acc_adv_v3 = tf.reduce_sum(tf.cast(tf.nn.in_top_k(end_points_adv_v3['Predictions'], y, k=top_k), tf.float32)) # acc_ens3_adv_v3 = tf.reduce_sum(tf.cast(tf.nn.in_top_k(end_points_ens3_adv_v3['Predictions'], y, k=top_k), tf.float32)) # acc_ens4_adv_v3 = tf.reduce_sum(tf.cast(tf.nn.in_top_k(end_points_ens4_adv_v3['Predictions'], y, k=top_k), tf.float32)) # acc_ensadv_res_v2 = tf.reduce_sum(tf.cast(tf.nn.in_top_k(end_points_ensadv_res_v2['Predictions'], y, k=top_k), tf.float32)) # acc_ens_v3 = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(end_points_ens_v3['predictions'], 1), y), tf.float32)) # pred1 = tf.argmax(end_points_v3['Predictions'], 1) # pred2 = tf.argmax(end_points_v4['Predictions'], 1) # pred3 = tf.argmax(end_points_res_v2['Predictions'], 1) # pred4 = tf.argmax(end_points_resnet['predictions'], 1) return acc_v3, acc_v4, acc_res_v2, acc_resnet, acc_resnet_50, acc_resnet_101, end_points_v3, end_points_v4, end_points_res_v2, end_points_resnet
def run(): if not os.path.exists(log_eval): os.mkdir(log_eval) with tf.Graph().as_default() as graph: tf.logging.set_verbosity(tf.logging.INFO) dataset = get_split('validation', dataset_dir) images, labels = load_batch(dataset, batch_size=batch_size, is_training=True) num_batches_per_epoch = dataset.num_samples / batch_size num_steps_per_epoch = num_batches_per_epoch with slim.arg_scope(inception_v3_arg_scope()): logits, end_points = inception_v3(images, num_classes=dataset.num_classes, is_training=True) variables_to_restore = slim.get_variables_to_restore() saver = tf.train.Saver(variables_to_restore) def restore_fn(sess): return saver.restore(sess, checkpoint_file) predictions = tf.argmax(end_points['Predictions'], 1) accuracy, accuracy_update = tf.contrib.metrics.streaming_accuracy( predictions, labels) acc, acc_op = tf.metrics.accuracy(labels=labels, predictions=predictions) metrics_op = tf.group(accuracy_update) global_step = get_or_create_global_step() global_step_op = tf.assign(global_step, global_step + 1) def eval_step(sess, metrics_op, global_step): ''' Simply takes in a session, runs the metrics op and some logging information. ''' start_time = time.time() _, global_step_count, accuracy_value = sess.run( [metrics_op, global_step_op, accuracy]) time_elapsed = time.time() - start_time logging.info( 'Global Step %s: Streaming Accuracy: %.4f (%.2f sec/step)', global_step_count, accuracy_value, time_elapsed) return accuracy_value tf.summary.scalar('Validation_Accuracy', accuracy) my_summary_op = tf.summary.merge_all() sv = tf.train.Supervisor(logdir=log_eval, summary_op=None, saver=None, init_fn=restore_fn) with sv.managed_session() as sess: num_steps_per_epoch = int(num_steps_per_epoch) for step in range(num_steps_per_epoch * num_epochs): sess.run(sv.global_step) if step % num_batches_per_epoch == 0: logging.info('Epoch: %s/%s', step / num_batches_per_epoch + 1, num_epochs) logging.info('Current Streaming Accuracy: %.4f', sess.run(accuracy)) if step % 10 == 0: eval_step(sess, metrics_op=metrics_op, global_step=sv.global_step) summaries = sess.run(my_summary_op) sv.summary_computed(sess, summaries) else: eval_step(sess, metrics_op=metrics_op, global_step=sv.global_step) logging.info('Final Streaming Accuracy: %.4f', sess.run(accuracy)) conf_m = confusion_matrix(labels, predictions) print(conf_m) for i in range(10): image, label, prediction = images[i], labels[i], predictions[i] prediction_name, label_name = dataset.labels_to_name[ prediction], dataset.labels_to_name[label] text = 'Prediction: %s \n Ground Truth: %s' % (prediction_name, label_name) print(text) logging.info( 'Model evaluation has completed! Visit TensorBoard for more information regarding your evaluation.' )
def graph(x, target_class_input, true_label_input, i, x_max, x_min, grad): eps = 2.0 * FLAGS.max_epsilon / 255.0 alpha = eps / FLAGS.iterations momentum = FLAGS.momentum num_classes = 1001 x_div = input_diversity(x) with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3( x_div, num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3') with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3( x_div, num_classes=num_classes, is_training=False, scope='Ens4AdvInceptionV3') with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2( x_div, num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2') with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_adv_res_v2, end_points_adv_res_v2 = inception_resnet_v2.inception_resnet_v2( x_div, num_classes=num_classes, is_training=False, scope='AdvInceptionResnetV2') with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_inc_v3, end_points_inc_v3 = inception_v3.inception_v3( x_div, num_classes=num_classes, is_training=False) with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_adv_v3, end_points_adv_v3 = inception_v3.inception_v3( x_div, num_classes=num_classes, is_training=False, scope='AdvInceptionV3') with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits_res_v2, end_points_res_v2 = resnet_v2.resnet_v2_50( x_div, num_classes=num_classes, is_training=False, scope='resnet_v2_50') with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits_res_v2_101, end_points_res_v2_101 = resnet_v2.resnet_v2_101( x_div, num_classes=num_classes, is_training=False, scope='resnet_v2_101') with TowerContext(tower_name='eval', is_training=False): logits_advresnext_101 = resnext.ResNeXtDenoiseAllModel().get_logits(preprocess_for_model(x_div)) one_hot_target_class = tf.one_hot(target_class_input, num_classes) one_hot_true_label = tf.one_hot(true_label_input, num_classes) one_hot_target_class_resnext = tf.one_hot(target_class_input-1, num_classes-1) one_hot_true_label_resnext = tf.one_hot(true_label_input-1, num_classes-1) logits = (logits_inc_v3 + logits_ens3_adv_v3 + logits_ens4_adv_v3 + logits_ensadv_res_v2 + logits_adv_res_v2 +logits_adv_v3+logits_res_v2+logits_res_v2_101) / 8 auxlogits = (end_points_inc_v3['AuxLogits'] + end_points_ens3_adv_v3['AuxLogits'] + end_points_ens4_adv_v3['AuxLogits'] + end_points_adv_res_v2['AuxLogits'] + end_points_ensadv_res_v2['AuxLogits'] +end_points_adv_v3['AuxLogits']+logits_res_v2+logits_res_v2_101) / 8 logits_resnext = logits_advresnext_101 cross_entropy = -tf.losses.softmax_cross_entropy(one_hot_true_label, logits, label_smoothing=0.0, weights=1.0) cross_entropy -= tf.losses.softmax_cross_entropy(one_hot_true_label, auxlogits, label_smoothing=0.0, weights=0.4) cross_entropy2 = tf.losses.softmax_cross_entropy(one_hot_true_label_resnext, logits_resnext, label_smoothing=0.0, weights=1.0) '''cross_entropy -= tf.losses.softmax_cross_entropy(one_hot_true_label, logits_ensadv_res_v2, label_smoothing=0.0, weights=1.0) cross_entropy -= tf.losses.softmax_cross_entropy(one_hot_true_label, end_points_ensadv_res_v2['AuxLogits'], label_smoothing=0.0, weights=0.4)''' rnd = i%2 cross_entropy = tf.cond(tf.greater(rnd,0),lambda:cross_entropy,lambda:cross_entropy2) noise = tf.gradients(cross_entropy, x)[0] kernel = gkern(7, FLAGS.sig).astype(np.float32) #kernel = gkern(15, 4).astype(np.float32) stack_kernel = np.stack([kernel, kernel, kernel]).swapaxes(2, 0) stack_kernel = np.expand_dims(stack_kernel, 3) noise = tf.nn.depthwise_conv2d(noise, stack_kernel, strides=[1, 1, 1, 1], padding='SAME') noise = noise / tf.reshape(tf.contrib.keras.backend.std(tf.reshape(noise, [FLAGS.batch_size, -1]), axis=1), [FLAGS.batch_size, 1, 1, 1]) noise = momentum * grad + noise noise = noise / tf.reshape(tf.contrib.keras.backend.std(tf.reshape(noise, [FLAGS.batch_size, -1]), axis=1), [FLAGS.batch_size, 1, 1, 1]) x = x - alpha * tf.clip_by_value(tf.round(noise), -2, 2) x = tf.clip_by_value(x, x_min, x_max) i = tf.add(i, 1) return x, target_class_input, true_label_input, i, x_max, x_min, noise
def main(_): start = time.clock() # Images for inception classifier are normalized to be in [-1, 1] interval, # eps is a difference between pixels so it should be in [0, 2] interval. # Renormalizing epsilon from [0, 255] to [0, 2]. eps = 2.0 * FLAGS.max_epsilon / 255.0 batch_shape = [FLAGS.batch_size, FLAGS.image_height, FLAGS.image_width, 3] tf.logging.set_verbosity(tf.logging.INFO) with tf.Graph().as_default(): # Prepare graph x_input = tf.placeholder(tf.float32, shape=batch_shape) x_adv_v3 = tf.placeholder(tf.float32, shape=batch_shape) x_max = tf.clip_by_value(x_input + eps, -1.0, 1.0) x_min = tf.clip_by_value(x_input - eps, -1.0, 1.0) y = tf.constant(np.zeros([FLAGS.batch_size]), tf.int64) i = tf.constant(0) grad = tf.zeros(shape=batch_shape) pred = tf.zeros(shape=[ 2, ], dtype=tf.int64) Prediction_adv_v3 = tf.zeros(shape=[ 2, ], dtype=tf.int64) x_adv, _, label, _, _, _, _, Pred = tf.while_loop( stop, graph, [x_input, x_adv_v3, y, i, x_max, x_min, grad, pred]) with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_adv_v3_one, end_points_adv_v3_one = inception_v3.inception_v3( x_adv_v3, num_classes=1001, is_training=False, reuse=True, scope='AdvInceptionV3') pred_adv_v3 = tf.argmax(end_points_adv_v3_one['Predictions'], axis=1) # Run computation s1 = tf.train.Saver(slim.get_model_variables(scope='InceptionV3')) s2 = tf.train.Saver(slim.get_model_variables(scope='AdvInceptionV3')) s3 = tf.train.Saver( slim.get_model_variables(scope='Ens3AdvInceptionV3')) s4 = tf.train.Saver( slim.get_model_variables(scope='Ens4AdvInceptionV3')) s5 = tf.train.Saver(slim.get_model_variables(scope='InceptionV4')) s6 = tf.train.Saver( slim.get_model_variables(scope='InceptionResnetV2')) s7 = tf.train.Saver( slim.get_model_variables(scope='EnsAdvInceptionResnetV2')) s8 = tf.train.Saver(slim.get_model_variables(scope='resnet_v2')) with tf.Session() as sess: s1.restore(sess, FLAGS.checkpoint_path_inception_v3) s2.restore(sess, FLAGS.checkpoint_path_adv_inception_v3) s3.restore(sess, FLAGS.checkpoint_path_ens3_adv_inception_v3) s4.restore(sess, FLAGS.checkpoint_path_ens4_adv_inception_v3) s5.restore(sess, FLAGS.checkpoint_path_inception_v4) s6.restore(sess, FLAGS.checkpoint_path_inception_resnet_v2) s7.restore(sess, FLAGS.checkpoint_path_ens_adv_inception_resnet_v2) s8.restore(sess, FLAGS.checkpoint_path_resnet) sum = 0 loss_num = 0 l2_distance = 0 label_distance = 0 for filenames, images in load_images(FLAGS.input_dir, batch_shape): images_flatten_initial = images.reshape((2, 268203)) sum += len(filenames) # for i in range(len(filenames)): # true_label.append(filenames[i].split('-')[2][:-4]) adv_images, true_label = sess.run([x_adv, label], feed_dict={x_input: images}) print('adv_images::::', type(adv_images[0][0][0][0]), adv_images.shape) Prediction = sess.run(pred_adv_v3, feed_dict={x_adv_v3: adv_images}) print('true_label::::::::', true_label) images_flatten_adv = adv_images.reshape((2, 268203)) l2_diatance_list = np.linalg.norm( (images_flatten_initial - images_flatten_adv), axis=1, keepdims=True) for n in range(len(filenames)): l2_distance += l2_diatance_list[n] for j in range(len(filenames)): label_distance += abs(true_label[j] - Prediction[j]) if int(true_label[j]) == Prediction[ j]: # The test tag is always 1 more than the real tag, so I subtract 1 here loss_num += 1 print('sucess_num:::::', loss_num) print('Prediction:::::::', Prediction) save_images(adv_images, filenames, FLAGS.output_dir) print('sum::::', sum) print('loss_num::::', loss_num) rate_wrong = loss_num / sum print('rate_wrong:::::::', rate_wrong) print('l2_distance:::::::', l2_distance) print('label_distance::::::', label_distance) end = time.clock() print('run time::::::::', end - start)
def train(): """Train classification for a number of steps.""" with tf.Graph().as_default(): # Returns and create (if necessary) the global step tensor. global_step = tf.train.get_or_create_global_step() # Get images and labels for datasets. # Force input pipeline to CPU:0 to avoid operations sometimes ending up on GPU and resulting in a slow down. with tf.device('/cpu:0'): # images:[batch size size channel], labels:[batch], filenames: nouseful images, labels, _ = tf_input.input(is_random=True, is_training=True) labels = labels - 1 # Build a Graph that computes the logits predictions from the inference model. # Attentation please, train mobilenet directly maybe very very difficult to convergence, # you should do retrain(transfer learning) with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits, _ = inception_v3.inception_v3(inputs=images, is_training=True, num_classes=NUM_CLASSES) # Or for simply # logits, _ = inception_v3.inception_v3(inputs=images, num_classes=NUM_CLASSES, is_training=True) # Calculate loss. loss = tf_inference.loss(logits, labels) # Build a Graph that trains the model with one batch of examples and # updates the model parameters. train_op = tf_inference.train(loss, global_step) class _LoggerHook(tf.train.SessionRunHook): """Logs loss and runtime.""" def begin(self): self._step = -1 self._start_time = time.time() def before_run(self, run_context): self._step += 1 return tf.train.SessionRunArgs(loss) # Asks for loss value. def after_run(self, run_context, run_values): if self._step % FLAGS.log_frequency == 0: current_time = time.time() duration = current_time - self._start_time self._start_time = current_time loss_value = run_values.results examples_per_sec = FLAGS.log_frequency * FLAGS.batch_size / duration sec_per_batch = float(duration / FLAGS.log_frequency) format_str = ( '%s: step %d, loss = %.2f (%.1f examples/sec; %.3f ' 'sec/batch)') print(format_str % (datetime.now(), self._step, loss_value, examples_per_sec, sec_per_batch)) with tf.train.MonitoredTrainingSession( checkpoint_dir=FLAGS.train_dir, hooks=[ tf.train.StopAtStepHook(last_step=FLAGS.max_steps), tf.train.NanTensorHook(loss), _LoggerHook() ], config=tf.ConfigProto(log_device_placement=FLAGS. log_device_placement)) as mon_sess: while not mon_sess.should_stop(): mon_sess.run(train_op)
def initialize(self): if not hasattr(self.algo, '_kwargs'): return if 'ablation_type' in self.algo._kwargs: self.ablation_type = self.algo._kwargs['ablation_type'] else: self.ablation_type = "None" #img = self.render('rgb_array') if 'imsize' in self.algo._kwargs: self.imsize = self.algo._kwargs['imsize'] self.name = self.algo._kwargs['name'] self.mode = self.algo._kwargs['mode'] if self.mode.startswith('inception'): self.layer = self.algo._kwargs['layer'] batch_size = 25 idims = self.imsize image = tf.placeholder(tf.uint8, (batch_size, ) + idims + (3, ), name='image') image_trans = tf.image.convert_image_dtype(image, dtype=tf.float32) image_trans = tf.subtract(image_trans, 0.5) image_trans = tf.multiply(image_trans, 2.0) with slim.arg_scope(inception_v3.inception_v3_arg_scope()): model = inception_v3.inception_v3(image_trans, num_classes=1001, is_training=False, dropout_keep_prob=1.0) variables_to_restore = slim.get_variables_to_restore() restorer = tf.train.Saver(variables_to_restore) config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) # sess.run(tf.global_variables_initializer()) restorer.restore(sess, "model/inception_v3.ckpt") bird = scipy.misc.imread('model/bird.jpg') bird = scipy.misc.imresize(bird, idims) logits = sess.run(model[0], {image: [bird] * batch_size}) print(np.argsort(logits[0])[-20:]) self.sess = sess self.image = image self.model = model if self.mode == 'inceptionsame': with open(self.algo._kwargs['experttheano'], 'rb') as pfile: expertpolicy = pickle.load(pfile) allfeats = [] for nroll in range(20): path = rollout(self.algo.env, expertpolicy, max_path_length=self.algo.max_path_length, animated=False) # import IPython # IPython.embed() imgs = [ img[0] for img in path['env_infos']['imgs'] if img is not None ] feat = self.sess.run(self.model[1][self.layer], {self.image: imgs}) allfeats.append(feat) # import IPython # IPython.embed() self.means = np.mean(allfeats, axis=0) self.std = np.std(allfeats, axis=0) else: data = np.load(self.algo._kwargs['meanfile']) self.means = data[self.layer] self.std = data[self.layer + 'std'] # print(efile, expertpolicy) elif self.mode.startswith('ours'): idim = self.imsize self.batch_size = 25 tfinput = tf.placeholder(tf.uint8, (3, self.batch_size) + idim + (3, ), name='x') image_trans = tf.image.convert_image_dtype(tfinput, dtype=tf.float32) image_trans = tf.subtract(image_trans, 0.5) image_trans = tf.multiply(image_trans, 2.0) self.image_trans = image_trans if self.mode == 'oursinception': with slim.arg_scope(inception_v3.inception_v3_arg_scope()): model = inception_v3.inception_v3(tf.reshape( tensor=image_trans, shape=(3 * self.batch_size, ) + idim + (3, )), num_classes=1001, is_training=False, dropout_keep_prob=1.0) autodc = arm_shaping.ContextAEInception2( strides=[1, 2, 1, 2], kernels=[3, 3, 3, 3], filters=[1024, 1024, 512, 512]) featlayer = model[1]['Mixed_7c'] featshape = featlayer.get_shape().as_list() featreshape = tf.reshape(featlayer, (3, self.batch_size, featshape[1], featshape[2], featshape[3])) with tf.variable_scope("contextmodel") as scope: autodc.build(featreshape) self.image_trans = featreshape else: if self.name == 'real' or self.name == 'sweep': autodc = arm_shaping.ContextAEReal() elif self.name == 'push' or self.name == 'reach' or self.name == 'strike' or self.name == 'throw': autodc = arm_shaping.ContextSkipNew() autodc.build(image_trans) config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) learning_rate = tf.placeholder(tf.float32, shape=[]) # sess.run(tf.global_variables_initializer()) saver = tf.train.Saver() saver.restore(sess, self.algo._kwargs['modelname']) if self.mode == 'oursinception': bird = scipy.misc.imread('model/bird.jpg') bird = scipy.misc.imresize(bird, idim) logits = sess.run(model[0], {tfinput: [[bird] * self.batch_size] * 3}) print(np.argsort(logits[0])[-20:]) self.nvp = self.algo._kwargs['nvp'] self.sess = sess self.image = tfinput self.model = autodc self.initialized = True
def run_training(path_db, pid, category, task_id, path_unknown, pretrained_dir, tensorflow_dir, path_save, num_epochs=1000, batch_size=32, finetune_last_layer=False, data_augmentation=True, mix_up=False, network_model='inception-v3', restore_all_parameters=False, initial_learning_rate=0.0002, learning_rate_decay_factor=0.7, num_epochs_before_decay=2): ##### start parameters for creating TFRecord files ##### #validation_size = 0.1 validation_size = 0.0 num_shards = 2 random_seed = 0 ##### end parameters for creating TFRecord files ##### dataset_dir = os.path.join(path_db, pid, category) log_dir = path_save tfrecord_filename = pid + '_' + category if _dataset_exists(dataset_dir=dataset_dir, _NUM_SHARDS=num_shards, output_filename=tfrecord_filename): print('Dataset files already exist. Overwrite them.') photo_filenames, class_names = _get_filenames_and_classes( dataset_dir, path_unknown) # dictionary for class name and class ID class_names_to_ids = dict(zip(class_names, range(len(class_names)))) # number of validation examples num_validation = int(validation_size * len(photo_filenames)) # divide to training and validation data random.seed(random_seed) random.shuffle(photo_filenames) training_filenames = photo_filenames[num_validation:] validation_filenames = photo_filenames[:num_validation] # find available GPU ID gpu_id = gpu_utils.pick_gpu_lowest_memory() # if log directory does not exist, create log directory and dataset if not os.path.exists(log_dir): os.makedirs(log_dir) print('found lowest memory gpu id : ' + str(gpu_id)) _convert_dataset(gpu_id, 'train', training_filenames, class_names_to_ids, dataset_dir=dataset_dir, tfrecord_filename=tfrecord_filename, _NUM_SHARDS=num_shards) _convert_dataset(gpu_id, 'validation', validation_filenames, class_names_to_ids, dataset_dir=dataset_dir, tfrecord_filename=tfrecord_filename, _NUM_SHARDS=num_shards) labels_to_class_names = dict(zip(range(len(class_names)), class_names)) write_label_file(labels_to_class_names, dataset_dir) print('finished creating dataset ' + tfrecord_filename) # start training output_label_filepath = os.path.join(dataset_dir, 'labels.txt') if network_model!='inception-v4' and network_model!='inception-v3' and network_model!='resnet-v2-50' and network_model!='resnet-v2-152' and \ network_model!='vgg-16' and network_model!='mobilenet-v1' and network_model!='nasnet-large' and network_model!='nasnet-mobile': print("invalid network model : " + network_model) sys.exit() # find pretrained model if os.path.exists(os.path.join(log_dir, 'model.ckpt')): checkpoint_file = os.path.join(log_dir, 'model.ckpt') else: if network_model == 'inception-v4': checkpoint_file = os.path.join( pretrained_dir, 'inception_resnet_v2_2016_08_30.ckpt') elif network_model == 'inception-v3': checkpoint_file = os.path.join(pretrained_dir, 'inception_v3.ckpt') elif network_model == 'resnet-v2-50': checkpoint_file = os.path.join(pretrained_dir, 'resnet_v2_50.ckpt') elif network_model == 'resnet-v2-152': checkpoint_file = os.path.join(pretrained_dir, 'resnet_v2_152.ckpt') elif network_model == 'vgg-16': checkpoint_file = os.path.join(pretrained_dir, 'vgg_16.ckpt') elif network_model == 'mobilenet-v1': checkpoint_file = os.path.join(pretrained_dir, 'mobilenet_v1_1.0_224.ckpt') elif network_model == 'nasnet-large': checkpoint_file = os.path.join(pretrained_dir, 'nasnet-a_large_04_10_2017', 'model.ckpt') elif network_model == 'nasnet-mobile': checkpoint_file = os.path.join(pretrained_dir, 'nasnet-a_mobile_04_10_2017', 'model.ckpt') else: print("invalid network model : " + network_model) sys.exit() # set image size if network_model == 'inception-v4' or network_model == 'inception-v3' or network_model == 'resnet-v2-50' or network_model == 'resnet-v2-152': image_size = 299 elif network_model == 'vgg-16' or network_model == 'mobilenet-v1' or network_model == 'nasnet-mobile': image_size = 224 elif network_model == 'nasnet-large': image_size = 331 else: print("invalid network model : " + network_model) sys.exit() # create the file pattern of TFRecord files file_pattern = tfrecord_filename + '_%s_*.tfrecord' file_pattern_for_counting = tfrecord_filename labels_to_name, label_list = load_labels(output_label_filepath) num_classes = len(label_list) # create a dataset discription items_to_descriptions = { 'image': 'A 3-channel RGB coloured image that is either ' + ','.join(label_list), 'label': 'A label that is as such -- ' + ','.join([ str(key) + ':' + labels_to_name[key] for key in labels_to_name.keys() ]) } # start training with tf.Graph().as_default() as graph: tf.logging.set_verbosity(tf.logging.INFO) # create dataset and load one batch dataset = get_split('train', dataset_dir, file_pattern, file_pattern_for_counting, labels_to_name, num_classes, items_to_descriptions) images, _, labels = load_batch(dataset, batch_size=batch_size, data_augmentation=data_augmentation, mix_up=mix_up, height=image_size, width=image_size) # number of steps to take before decaying the learning rate and batches per epoch num_batches_per_epoch = int(dataset.num_samples / batch_size) num_steps_per_epoch = num_batches_per_epoch # because one step is one batch processed decay_steps = int(num_epochs_before_decay * num_steps_per_epoch) # create model for inference finetune_vars = [] if network_model == 'inception-v4': with slim.arg_scope(inception_resnet_v2_arg_scope()): logits, end_points = inception_resnet_v2( images, num_classes=dataset.num_classes, is_training=True) finetune_vars = [ 'InceptionResnetV2/Logits', 'InceptionResnetV2/AuxLogits' ] elif network_model == 'inception-v3': with slim.arg_scope(inception_v3_arg_scope()): logits, end_points = inception_v3( images, num_classes=dataset.num_classes, is_training=True) finetune_vars = ['InceptionV3/Logits', 'InceptionV3/AuxLogits'] elif network_model == 'resnet-v2-50': with slim.arg_scope(resnet_arg_scope()): logits, end_points = resnet_v2_50( images, num_classes=dataset.num_classes, is_training=True) finetune_vars = ['resnet_v2_50/logits'] elif network_model == 'resnet-v2-152': with slim.arg_scope(resnet_arg_scope()): logits, end_points = resnet_v2_152( images, num_classes=dataset.num_classes, is_training=True) finetune_vars = ['resnet_v2_152/logits'] elif network_model == 'vgg-16': with slim.arg_scope(vgg_arg_scope()): logits, _ = vgg_16(images, num_classes=dataset.num_classes, is_training=True) finetune_vars = ['vgg_16/fc8'] elif network_model == 'mobilenet-v1': with slim.arg_scope(mobilenet_v1_arg_scope()): logits, end_points = mobilenet_v1( images, num_classes=dataset.num_classes, is_training=True) finetune_vars = ['MobilenetV1/Logits'] elif network_model == 'nasnet-large': with slim.arg_scope(nasnet.nasnet_large_arg_scope()): logits, end_points = nasnet.build_nasnet_large( images, dataset.num_classes) finetune_vars = [ 'final_layer', 'aux_11', 'cell_stem_0/comb_iter_0/left/global_step' ] elif network_model == 'nasnet-mobile': with slim.arg_scope(nasnet.nasnet_mobile_arg_scope()): logits, end_points = nasnet.build_nasnet_mobile( images, dataset.num_classes) finetune_vars = ['final_layer', 'aux_7'] else: print("Invalid network model : " + network_model) sys.exit() # define the scopes that you want to exclude for restoration exclude = [] if not restore_all_parameters: exclude = finetune_vars variables_to_restore = slim.get_variables_to_restore(exclude=exclude) if mix_up: labels.set_shape([batch_size, dataset.num_classes]) logits.set_shape([batch_size, dataset.num_classes]) loss = tf.losses.sigmoid_cross_entropy(labels, logits) else: # perform one-hot-encoding of the labels (Try one-hot-encoding within the load_batch function!) one_hot_labels = slim.one_hot_encoding(labels, dataset.num_classes) # performs the equivalent to tf.nn.sparse_softmax_cross_entropy_with_logits but enhanced with checks loss = tf.losses.softmax_cross_entropy( onehot_labels=one_hot_labels, logits=logits) total_loss = tf.losses.get_total_loss( ) #obtain the regularization losses as well # create the global step for monitoring the learning_rate and training. global_step = tf.train.get_or_create_global_step() # define your exponentially decaying learning rate lr = tf.train.exponential_decay(learning_rate=initial_learning_rate, global_step=global_step, decay_steps=decay_steps, decay_rate=learning_rate_decay_factor, staircase=True) # define optimizer optimizer = tf.train.AdamOptimizer(learning_rate=lr) # create train_op if finetune_last_layer: variables_to_train = get_variables_to_train_by_scopes( finetune_vars) print("finetune variables : " + str(variables_to_train)) train_op = slim.learning.create_train_op( total_loss, optimizer, variables_to_train=variables_to_train) else: train_op = slim.learning.create_train_op(total_loss, optimizer) # define prediction matrix if network_model=='inception-v4' or network_model=='inception-v3' or network_model=='mobilenet-v1' or \ network_model=='nasnet-large' or network_model=='nasnet-mobile': predictions = tf.argmax(end_points['Predictions'], 1) probabilities = end_points['Predictions'] elif network_model == 'resnet-v2-50' or network_model == 'resnet-v2-152': predictions = tf.argmax(end_points['predictions'], 1) probabilities = end_points['predictions'] elif network_model == 'vgg-16': predictions = tf.argmax(logits, 1) probabilities = tf.nn.softmax(logits) else: print("Invalid network model : " + network_model) sys.exit() if mix_up: argmax_labels = tf.argmax(labels, 1) accuracy, accuracy_update = tf.contrib.metrics.streaming_accuracy( predictions, argmax_labels) else: accuracy, accuracy_update = tf.contrib.metrics.streaming_accuracy( predictions, labels) metrics_op = tf.group(accuracy_update, probabilities) # create summaries tf.summary.scalar('losses/Total_Loss', total_loss) tf.summary.scalar('accuracy', accuracy) tf.summary.scalar('learning_rate', lr) my_summary_op = tf.summary.merge_all() # defube training step function that runs both the train_op, metrics_op and updates the global_step concurrently def train_step(sess, train_op, global_step): # check the time for each sess run start_time = time.time() total_loss, global_step_count, _ = sess.run( [train_op, global_step, metrics_op]) time_elapsed = time.time() - start_time # run the logging to print some results logging.info('global step %s: loss: %.4f (%.2f sec/step)', global_step_count, total_loss, time_elapsed) return total_loss, int(global_step_count) # create a saver function that actually restores the variables from a checkpoint file saver = tf.train.Saver(variables_to_restore) def restore_fn(sess): return saver.restore(sess, checkpoint_file) # define your supervisor for running a managed session sv = tf.train.Supervisor(logdir=log_dir, summary_op=None, init_fn=restore_fn) # run the managed session start_train_time = time.time() gpu_options = tf.ConfigProto( gpu_options=tf.GPUOptions(visible_device_list=str(gpu_id), per_process_gpu_memory_fraction=0.4)) with sv.prepare_or_wait_for_session(config=gpu_options) as sess: for step in range(num_steps_per_epoch * num_epochs): # check if training task is not canceled if not controller.check_train_task_alive( pid, category, task_id): print('Training task is canceled.') sv.stop() return False, "", "", output_label_filepath, global_step_count # at the start of every epoch, show the vital information: if step % num_batches_per_epoch == 0: logging.info('Epoch %s/%s', step / num_batches_per_epoch + 1, num_epochs) learning_rate_value, accuracy_value = sess.run( [lr, accuracy]) logging.info('Current Learning Rate: %s', learning_rate_value) logging.info('Current Streaming Accuracy: %s', accuracy_value) # optionally, print your logits and predictions for a sanity check that things are going fine. logits_value, probabilities_value, predictions_value, labels_value = sess.run( [logits, probabilities, predictions, labels]) print('logits: \n', logits_value) print('Probabilities: \n', probabilities_value) print('predictions: \n', predictions_value) print('Labels:\n:', labels_value) # log the summaries every 10 step. if step % 10 == 0: loss, global_step_count = train_step( sess, train_op, sv.global_step) summaries = sess.run(my_summary_op) sv.summary_computed(sess, summaries) # if not, simply run the training step else: loss, global_step_count = train_step( sess, train_op, sv.global_step) # if specific time passes, save model for evaluation time_elapsed_train = time.time() - start_train_time print('training time : ' + str(time_elapsed_train)) # log the final training loss and accuracy logging.info( 'Training Progress : %.2f %% ', 100.0 * step / float(num_steps_per_epoch * num_epochs)) logging.info('Final Loss: %s', loss) logging.info('Global Step: %s', global_step_count) logging.info('Final Accuracy: %s', sess.run(accuracy)) # after all the training has been done, save the log files and checkpoint model logging.info('Finished training! Saving model to disk now.') sv.saver.save(sess, sv.save_path, global_step=sv.global_step) # save graph definition file output_graph_filepath = os.path.join(log_dir, 'graph.pb') export_graph_command_exec = "./network/export_slim_graph.py" if not os.path.exists(export_graph_command_exec): print("fatal error, cannot find command : " + export_graph_command_exec) sys.exit() export_graph_command_env = os.environ.copy() export_graph_command_env["CUDA_VISIBLE_DEVICES"] = '' export_graph_command = [] export_graph_command.append(sys.executable) export_graph_command.append(export_graph_command_exec) export_graph_command.append(network_model) export_graph_command.append(str(dataset.num_classes)) export_graph_command.append(output_graph_filepath) print("start exec:" + " ".join(export_graph_command)) proc = subprocess.Popen(export_graph_command, env=export_graph_command_env) print("export graph process ID=" + str(proc.pid)) controller.upsert_train_child_process(task_id, proc.pid) proc.communicate() controller.delete_train_child_process(task_id, proc.pid) print("finish exec:" + " ".join(export_graph_command)) if not controller.check_train_task_alive(pid, category, task_id): print('Training task is canceled.') sv.stop() return False, "", "", output_label_filepath, global_step_count # save frozon graph, optimized graph, and quantized graph from graph definition and checkpoint latest_checkpoint_filepath = tf.train.latest_checkpoint(log_dir) # you can check output node name by tensorflow/tools/graph_transforms::summarize_graph # https://github.com/tensorflow/models/tree/master/research/slim#Export output_node_names = "" if network_model == 'inception-v4': output_node_names = "InceptionResnetV2/Logits/Predictions" elif network_model == 'inception-v3': output_node_names = "InceptionV3/AuxLogits/SpatialSqueeze,InceptionV3/Predictions/Reshape_1" elif network_model == 'resnet-v2-50': output_node_names = "resnet_v2_50/predictions/Reshape_1" elif network_model == 'resnet-v2-152': output_node_names = "resnet_v2_152/predictions/Reshape_1" elif network_model == 'vgg-16': output_node_names = "vgg_16/fc8/squeezed" elif network_model == 'mobilenet-v1': output_node_names = "MobilenetV1/Predictions/Reshape_1" elif network_model == 'nasnet-large' or network_model == 'nasnet-mobile': output_node_names = "final_layer/predictions" else: print("Invalid network model : " + network_model) sys.exit() output_frozen_graph_filepath = os.path.join( log_dir, 'frozen_graph.pb') freeze_graph_command_exec = os.path.join( tensorflow_dir, "bazel-bin/tensorflow/python/tools/freeze_graph") if not os.path.exists(freeze_graph_command_exec): print("fatal error, cannot find command : " + freeze_graph_command_exec) sys.exit() freeze_graph_command_env = os.environ.copy() freeze_graph_command_env["CUDA_VISIBLE_DEVICES"] = '' freeze_graph_command = [] freeze_graph_command.append(freeze_graph_command_exec) freeze_graph_command.append("--input_graph=" + output_graph_filepath) freeze_graph_command.append("--input_checkpoint=" + latest_checkpoint_filepath) freeze_graph_command.append("--input_binary=true") freeze_graph_command.append("--output_graph=" + output_frozen_graph_filepath) freeze_graph_command.append("--output_node_names=" + output_node_names) print("start exec:" + " ".join(freeze_graph_command)) proc = subprocess.Popen(freeze_graph_command, env=freeze_graph_command_env) print("freeze graph process ID=" + str(proc.pid)) controller.upsert_train_child_process(task_id, proc.pid) proc.communicate() controller.delete_train_child_process(task_id, proc.pid) print("finish exec:" + " ".join(freeze_graph_command)) if not controller.check_train_task_alive(pid, category, task_id): print('Training task is canceled.') sv.stop() return False, "", "", output_label_filepath, global_step_count output_optimized_graph_filepath = os.path.join( log_dir, 'optimized_graph.pb') optimize_graph_command_exec = os.path.join( tensorflow_dir, "bazel-bin/tensorflow/python/tools/optimize_for_inference") if not os.path.exists(optimize_graph_command_exec): print("fatal error, cannot find command : " + optimize_graph_command_exec) sys.exit() optimize_graph_command_env = os.environ.copy() optimize_graph_command_env["CUDA_VISIBLE_DEVICES"] = '' optimize_graph_command = [] optimize_graph_command.append(optimize_graph_command_exec) optimize_graph_command.append("--input=" + output_frozen_graph_filepath) optimize_graph_command.append("--output=" + output_optimized_graph_filepath) optimize_graph_command.append("--input_names=input") optimize_graph_command.append("--output_names=" + output_node_names) optimize_graph_command.append("--frozen_graph=true") print("start exec:" + " ".join(optimize_graph_command)) proc = subprocess.Popen(optimize_graph_command, env=optimize_graph_command_env) print("optimize graph process ID=" + str(proc.pid)) controller.upsert_train_child_process(task_id, proc.pid) proc.communicate() controller.delete_train_child_process(task_id, proc.pid) print("finish exec:" + " ".join(optimize_graph_command)) if not controller.check_train_task_alive(pid, category, task_id): print('Training task is canceled.') sv.stop() return False, "", "", output_label_filepath, global_step_count output_quantized_graph_filepath = os.path.join( log_dir, 'quantized_graph.pb') quantize_graph_command_exec = os.path.join( tensorflow_dir, "bazel-bin/tensorflow/tools/quantization/quantize_graph") if not os.path.exists(quantize_graph_command_exec): print("fatal error, cannot find command : " + quantize_graph_command_exec) sys.exit() quantize_graph_command_env = os.environ.copy() quantize_graph_command_env["CUDA_VISIBLE_DEVICES"] = '' quantize_graph_command = [] quantize_graph_command.append(quantize_graph_command_exec) quantize_graph_command.append("--input=" + output_optimized_graph_filepath) quantize_graph_command.append("--output=" + output_quantized_graph_filepath) quantize_graph_command.append("--input_node_names=input") quantize_graph_command.append("--output_node_names=" + output_node_names) quantize_graph_command.append("--mode=eightbit") print("start exec:" + " ".join(quantize_graph_command)) proc = subprocess.Popen(quantize_graph_command, env=quantize_graph_command_env) print("quantize graph process ID=" + str(proc.pid)) controller.upsert_train_child_process(task_id, proc.pid) proc.communicate() controller.delete_train_child_process(task_id, proc.pid) print("finish exec:" + " ".join(quantize_graph_command)) if not controller.check_train_task_alive(pid, category, task_id): print('Training task is canceled.') sv.stop() return False, "", "", output_label_filepath, global_step_count return True, output_optimized_graph_filepath, output_quantized_graph_filepath, output_label_filepath, global_step_count
def graph(x, mask, y, i, x_max, x_min, grad): eps = 2.0 * FLAGS.max_epsilon / 255.0 alpha = eps / 10 momentum = FLAGS.momentum num_classes = 1001 with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_v3, end_points_v3 = inception_v3.inception_v3( input_diversity(x), num_classes=num_classes, is_training=False) logits_v3_rotated, _ = inception_v3.inception_v3( rotate(x), num_classes=num_classes, is_training=False, reuse=True) with slim.arg_scope(inception_v4.inception_v4_arg_scope()): logits_v4, end_points_v4 = inception_v4.inception_v4( input_diversity(x), num_classes=num_classes, is_training=False) logits_v4_rotated, _ = inception_v4.inception_v4( rotate(x), num_classes=num_classes, is_training=False, reuse=True) with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_res_v2, end_points_res_v2 = inception_resnet_v2.inception_resnet_v2( input_diversity(x), num_classes=num_classes, is_training=False, reuse=True) logits_res_v2_rotated, _ = inception_resnet_v2.inception_resnet_v2( rotate(x), num_classes=num_classes, is_training=False, reuse=True) with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits_resnet, end_points_resnet = resnet_v2.resnet_v2_152( input_diversity(x), num_classes=num_classes, is_training=False) logits_resnet_rotated, _ = resnet_v2.resnet_v2_152( rotate(x), num_classes=num_classes, is_training=False, reuse=True) with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3( input_diversity(x), num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3') logits_ens3_adv_v3_rotated, _ = inception_v3.inception_v3( rotate(x), num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3', reuse=True) with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3( input_diversity(x), num_classes=num_classes, is_training=False, scope='Ens4AdvInceptionV3') logits_ens4_adv_v3_rotated, _ = inception_v3.inception_v3( rotate(x), num_classes=num_classes, is_training=False, scope='Ens4AdvInceptionV3', reuse=True) # with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()): # logits_ens_advres_v2, end_points_ens_advres_v2 = inception_resnet_v2.inception_resnet_v2( # input_diversity(x), num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2') # logits_ens_advres_v2_rotated, _ = inception_resnet_v2.inception_resnet_v2( # rotate(x), num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2', reuse=True) logits = (logits_v3 + logits_v4 + logits_res_v2 + logits_resnet + logits_v3_rotated + logits_v4_rotated + logits_res_v2_rotated + logits_resnet_rotated + logits_ens3_adv_v3 + logits_ens4_adv_v3 + logits_ens3_adv_v3_rotated + logits_ens4_adv_v3_rotated) / 12 auxlogits = ( end_points_v3['AuxLogits'] + end_points_v4['AuxLogits'] + end_points_res_v2['AuxLogits'] + end_points_ens3_adv_v3['AuxLogits'] + end_points_ens4_adv_v3['AuxLogits'] # + end_points_ens_advres_v2['AuxLogits'] ) / 5 cross_entropy = tf.losses.softmax_cross_entropy(y, logits, label_smoothing=0.0, weights=1.0) cross_entropy += tf.losses.softmax_cross_entropy(y, auxlogits, label_smoothing=0.0, weights=0.4) noise = tf.gradients(cross_entropy, x)[0] noise = tf.nn.depthwise_conv2d(noise, stack_kernel, strides=[1, 1, 1, 1], padding='SAME') noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True) noise = momentum * grad + noise noise = noise * mask[:, :, :, np.newaxis] x = x + alpha * tf.sign(noise) x = tf.clip_by_value(x, x_min, x_max) i = tf.add(i, 1) return x, mask, y, i, x_max, x_min, noise
def graph_large(x, target_class_input, i, x_max, x_min, grad): eps = 2.0 * FLAGS.max_epsilon / 255.0 alpha = eps / 12 momentum = FLAGS.momentum num_classes = 1001 with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_v3, end_points_v3 = inception_v3.inception_v3( x, num_classes=num_classes, is_training=False) with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_adv_v3, end_points_adv_v3 = inception_v3.inception_v3( x, num_classes=num_classes, is_training=False, scope='AdvInceptionV3') with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3( x, num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3') with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3( x, num_classes=num_classes, is_training=False, scope='Ens4AdvInceptionV3') with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2( x, num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2') one_hot_target_class = tf.one_hot(target_class_input, num_classes) logits = (4 * logits_v3 + logits_adv_v3 + logits_ens3_adv_v3 + logits_ens4_adv_v3 + 4 * logits_ensadv_res_v2) / 11 auxlogits = (4 * end_points_v3['AuxLogits'] + end_points_adv_v3['AuxLogits'] + end_points_ens3_adv_v3['AuxLogits'] + end_points_ens4_adv_v3['AuxLogits'] + 4 * end_points_ensadv_res_v2['AuxLogits']) / 11 cross_entropy = tf.losses.softmax_cross_entropy(one_hot_target_class, logits, label_smoothing=0.0, weights=1.0) cross_entropy += tf.losses.softmax_cross_entropy(one_hot_target_class, auxlogits, label_smoothing=0.0, weights=0.4) noise = tf.gradients(cross_entropy, x)[0] noise = noise / tf.reshape( tf.contrib.keras.backend.std(tf.reshape(noise, [FLAGS.batch_size, -1]), axis=1), [FLAGS.batch_size, 1, 1, 1]) noise = momentum * grad + noise noise = noise / tf.reshape( tf.contrib.keras.backend.std(tf.reshape(noise, [FLAGS.batch_size, -1]), axis=1), [FLAGS.batch_size, 1, 1, 1]) x = x - alpha * tf.clip_by_value(tf.round(noise), -2, 2) x = tf.clip_by_value(x, x_min, x_max) i = tf.add(i, 1) return x, target_class_input, i, x_max, x_min, noise
def graph(x, y, i, x_max, x_min, grad): eps = 2.0 * FLAGS.max_epsilon / 255.0 num_iter = FLAGS.num_iter alpha = eps / num_iter momentum = FLAGS.momentum num_classes = 1001 x_nes = x with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_v3, end_points_v3 = inception_v3.inception_v3( x_nes, num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE) pred = tf.argmax(end_points_v3['Predictions'], 1) first_round = tf.cast(tf.equal(i, 0), tf.int64) y = first_round * pred + (1 - first_round) * y one_hot = tf.one_hot(y, num_classes) cross_entropy = tf.losses.softmax_cross_entropy(one_hot, logits_v3) noise = tf.gradients(cross_entropy, x)[0] x_nes_2 = 1 / 2 * x_nes with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_v3_2, end_points_v3 = inception_v3.inception_v3( x_nes_2, num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE) cross_entropy_2 = tf.losses.softmax_cross_entropy(one_hot, logits_v3_2) noise += tf.gradients(cross_entropy_2, x)[0] x_nes_4 = 1 / 4 * x_nes with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_v3_4, end_points_v3 = inception_v3.inception_v3( x_nes_4, num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE) cross_entropy_4 = tf.losses.softmax_cross_entropy(one_hot, logits_v3_4) noise += tf.gradients(cross_entropy_4, x)[0] x_nes_8 = 1 / 8 * x_nes with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_v3_8, end_points_v3 = inception_v3.inception_v3( x_nes_8, num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE) cross_entropy_8 = tf.losses.softmax_cross_entropy(one_hot, logits_v3_8) noise += tf.gradients(cross_entropy_8, x)[0] x_nes_16 = 1 / 16 * x_nes with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_v3_16, end_points_v3 = inception_v3.inception_v3( x_nes_16, num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE) cross_entropy_16 = tf.losses.softmax_cross_entropy(one_hot, logits_v3_16) noise += tf.gradients(cross_entropy_16, x)[0] noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True) noise = momentum * grad + noise x = x + alpha * tf.sign(noise) x = tf.clip_by_value(x, x_min, x_max) i = tf.add(i, 1) return x, y, i, x_max, x_min, noise
def main(_): # Images for inception classifier are normalized to be in [-1, 1] interval, # eps is a difference between pixels so it should be in [0, 2] interval. # Renormalizing epsilon from [0, 255] to [0, 2]. eps = 2.0 * FLAGS.max_epsilon / 255.0 num_classes = 1001 batch_shape = [FLAGS.batch_size, FLAGS.image_height, FLAGS.image_width, 3] tf.logging.set_verbosity(tf.logging.INFO) with tf.Graph().as_default(): tf.set_random_seed(305) # Prepare graph x_input = tf.placeholder(tf.float32, shape = batch_shape) adv_img = tf.placeholder(tf.float32, shape = batch_shape) y = tf.placeholder(tf.int32, shape = batch_shape[0]) t_y = tf.placeholder(tf.int32, shape = batch_shape[0]) x_max = tf.clip_by_value(x_input + eps, -1.0, 1.0) x_min = tf.clip_by_value(x_input - eps, -1.0, 1.0) with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_v3, end_points_v3 = inception_v3.inception_v3( adv_img, num_classes=num_classes, is_training=False) pre_v3 = tf.argmax(logits_v3, 1) with slim.arg_scope(inception_v4.inception_v4_arg_scope()): logits_v4, end_points_v4 = inception_v4.inception_v4( adv_img, num_classes=num_classes, is_training=False) pre_v4 = tf.argmax(logits_v4, 1) with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits_resnet_152, end_points_resnet = resnet_v2.resnet_v2_152( adv_img, num_classes=num_classes, is_training=False) pre_resnet_152 = tf.argmax(logits_resnet_152, 1) with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits_resnet_101, end_points_resnet_101 = resnet_v2.resnet_v2_101( adv_img, num_classes=num_classes, is_training=False) pre_resnet_101 = tf.argmax(logits_resnet_101, 1) with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits_resnet_50, end_points_resnet_50 = resnet_v2.resnet_v2_50( adv_img, num_classes=num_classes, is_training=False) pre_resnet_50 = tf.argmax(logits_resnet_50, 1) with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_Incres, end_points_IR = inception_resnet_v2.inception_resnet_v2( adv_img, num_classes=num_classes, is_training=False) pre_Inc_res = tf.argmax(logits_Incres, 1) with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3( adv_img, num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3') pre_ens3_adv_v3 = tf.argmax(logits_ens3_adv_v3, 1) with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3( adv_img, num_classes=num_classes, is_training=False, scope='Ens4AdvInceptionV3') pre_ens4_adv_v3 = tf.argmax(logits_ens4_adv_v3, 1) with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2( adv_img, num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2') pre_ensadv_res_v2 = tf.argmax(logits_ensadv_res_v2, 1) pre_ensemble_logit = tf.argmax((logits_resnet_152 + logits_v4 + logits_v3 + + logits_resnet_101 + logits_resnet_50 +\ logits_Incres + logits_ens3_adv_v3 + logits_ens4_adv_v3), 1) mean_pert = 0.0 sum_v3, sum_v4, sum_res152, sum_res101, sum_res50, sum_Incres, sum_ensemble = 0, 0, 0, 0, 0, 0, 0 sum_ens3_adv_v3, sum_ens4_adv_v3, sum_ensadv_res_v2 = 0, 0, 0 i = tf.constant(0) grad = tf.zeros(shape=batch_shape) amplification = tf.zeros(shape=batch_shape) _, x_adv, _, _, _, _, _, _, _ = tf.while_loop(stop, graph, [x_input, adv_img, y, t_y, i, x_max, x_min, grad, amplification]) # Run computation s1 = tf.train.Saver(slim.get_model_variables(scope='InceptionV3')) s3 = tf.train.Saver(slim.get_model_variables(scope='Ens3AdvInceptionV3')) s4 = tf.train.Saver(slim.get_model_variables(scope='Ens4AdvInceptionV3')) s5 = tf.train.Saver(slim.get_model_variables(scope='InceptionV4')) s6 = tf.train.Saver(slim.get_model_variables(scope='InceptionResnetV2')) s7 = tf.train.Saver(slim.get_model_variables(scope='EnsAdvInceptionResnetV2')) s8 = tf.train.Saver(slim.get_model_variables(scope='resnet_v2_152')) s9 = tf.train.Saver(slim.get_model_variables(scope='resnet_v2_101')) s10 = tf.train.Saver(slim.get_model_variables(scope='resnet_v2_50')) with tf.Session() as sess: s1.restore(sess, model_checkpoint_map['inception_v3']) s3.restore(sess, model_checkpoint_map['ens3_adv_inception_v3']) s4.restore(sess, model_checkpoint_map['ens4_adv_inception_v3']) s5.restore(sess, model_checkpoint_map['inception_v4']) s6.restore(sess, model_checkpoint_map['inception_resnet_v2']) s7.restore(sess, model_checkpoint_map['ens_adv_inception_resnet_v2']) s8.restore(sess, model_checkpoint_map['resnet_v2_152']) s9.restore(sess, model_checkpoint_map['resnet_v2_101']) s10.restore(sess, model_checkpoint_map['resnet_v2_50']) import pandas as pd dev = pd.read_csv(FLAGS.input_csv) for idx in tqdm(range(0, 1000 // FLAGS.batch_size)): images, filenames, True_label, Target_label = load_images(FLAGS.input_dir, dev, idx * FLAGS.batch_size, batch_shape) my_adv_images = sess.run(x_adv, feed_dict={x_input: images, adv_img: images, y: True_label, t_y: Target_label}).astype(np.float32) mean_pert += abs(my_adv_images - images).mean() pre_v3_, pre_v4_, pre_resnet152_, pre_resnet101_, pre_resnet50_, pre_Inc_res_,\ pre_ens3_adv_v3_, pre_ens4_adv_v3_, pre_ensadv_res_v2_ ,pre_ensemble_ \ = sess.run([pre_v3, pre_v4, pre_resnet_152, pre_resnet_101, pre_resnet_50, pre_Inc_res, pre_ens3_adv_v3, pre_ens4_adv_v3, pre_ensadv_res_v2, pre_ensemble_logit], feed_dict = {adv_img: my_adv_images}) sum_v3 += (pre_v3_ == Target_label).sum() sum_v4 += (pre_v4_ == Target_label).sum() sum_res152 += (pre_resnet152_ == Target_label).sum() sum_res101 += (pre_resnet101_ == Target_label).sum() sum_res50 += (pre_resnet50_ == Target_label).sum() sum_Incres += (pre_Inc_res_ == Target_label).sum() sum_ens3_adv_v3 += (pre_ens3_adv_v3_ == Target_label).sum() sum_ens4_adv_v3 += (pre_ens4_adv_v3_ == Target_label).sum() sum_ensadv_res_v2 += (pre_ensadv_res_v2_ == Target_label).sum() sum_ensemble += (pre_ensemble_ == Target_label).sum() save_images(my_adv_images, filenames, FLAGS.output_dir) print('mean noise = ', (mean_pert / (1000.0 / FLAGS.batch_size)) * 255.0) print('sum_v3 = {}'.format(sum_v3)) print('sum_v4 = {}'.format(sum_v4)) print('sum_res2 = {}'.format(sum_res152)) print('sum_res1 = {}'.format(sum_res101)) print('sum_res1 = {}'.format(sum_res50)) print('sum_Incres_v2 = {}'.format(sum_Incres)) print('sum_ens3_adv_v3 = {}'.format(sum_ens3_adv_v3)) print('sum_ens4_adv_v3 = {}'.format(sum_ens4_adv_v3)) print('sum_ensadv_Incres_v2 = {}'.format(sum_ensadv_res_v2)) print('sum_ensmeble = {}'.format(sum_ensemble))
def main(_): batch_shape = [FLAGS.batch_size, FLAGS.image_height, FLAGS.image_width, 3] num_classes = 1001 itr = 30 tf.logging.set_verbosity(tf.logging.INFO) with tf.Graph().as_default(): # Prepare graph x_input = tf.placeholder(tf.float32, shape=batch_shape) img_resize_tensor = tf.placeholder(tf.int32, [2]) x_input_resize = tf.image.resize_images( x_input, img_resize_tensor, method=tf.image.ResizeMethod.NEAREST_NEIGHBOR) shape_tensor = tf.placeholder(tf.int32, [3]) padded_input = padding_layer_iyswim(x_input_resize, shape_tensor) # 330 is the last value to keep 8*8 output, 362 is the last value to keep 9*9 output, stride = 32 padded_input.set_shape( (FLAGS.batch_size, FLAGS.image_resize, FLAGS.image_resize, 3)) with slim.arg_scope( inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2( padded_input, num_classes=num_classes, is_training=False, create_aux_logits=True) with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_adv_v3, end_points_v3 = inception_v3.inception_v3( padded_input, num_classes=num_classes, is_training=False, scope='AdvInceptionV3') with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3( padded_input, num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3') with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens3_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3( padded_input, num_classes=num_classes, is_training=False, scope='Ens4AdvInceptionV3') with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits_resnet, end_points_resnet = resnet_v2.resnet_v2_101( padded_input, num_classes=num_classes, is_training=False) logits = (logits_ensadv_res_v2 + logits_adv_v3 + logits_ens3_adv_v3 + logits_ens3_adv_v3 + logits_resnet) / 5 Aux = (end_points_ensadv_res_v2['AuxLogits'] + end_points_v3['AuxLogits'] + end_points_ens3_adv_v3['AuxLogits'] + end_points_ens4_adv_v3['AuxLogits']) * 0.1 predicted_labels = tf.argmax((logits + Aux), 1) #predicted_labels = tf.argmax(end_points['Predictions'], 1) # Run computation #saver = tf.train.Saver(slim.get_model_variables()) #session_creator = tf.train.ChiefSessionCreator( # scaffold=tf.train.Scaffold(saver=saver), # checkpoint_filename_with_path=[FLAGS.checkpoint_path_ens3_adv_inception_v3,FLAGS.checkpoint_path_ens4_adv_inception_v3] # master=FLAGS.master) #with tf.train.MonitoredSession(session_creator=session_creator) as sess: s1 = tf.train.Saver(slim.get_model_variables(scope='AdvInceptionV3')) s2 = tf.train.Saver( slim.get_model_variables(scope='Ens3AdvInceptionV3')) s3 = tf.train.Saver( slim.get_model_variables(scope='Ens4AdvInceptionV3')) s4 = tf.train.Saver( slim.get_model_variables(scope='EnsAdvInceptionResnetV2')) s5 = tf.train.Saver(slim.get_model_variables(scope='resnet_v2')) with tf.Session() as sess: s1.restore(sess, FLAGS.checkpoint_path_adv_inception_v3) s2.restore(sess, FLAGS.checkpoint_path_ens3_adv_inception_v3) s3.restore(sess, FLAGS.checkpoint_path_ens4_adv_inception_v3) s4.restore(sess, FLAGS.checkpoint_path_ens_adv_inception_resnet_v2) s5.restore(sess, FLAGS.checkpoint_path_resnet) with tf.gfile.Open(FLAGS.output_file, 'w') as out_file: for filenames, images in load_images(FLAGS.input_dir, batch_shape): final_preds = np.zeros( [FLAGS.batch_size, num_classes, itr]) for j in range(itr): if np.random.randint(0, 2, size=1) == 1: images = images[:, :, ::-1, :] resize_shape_ = np.random.randint(310, 331) final_preds[..., j] = sess.run( [predicted_labels], feed_dict={ x_input: images, img_resize_tensor: [resize_shape_] * 2, shape_tensor: np.array([ random.randint( 0, FLAGS.image_resize - resize_shape_), random.randint( 0, FLAGS.image_resize - resize_shape_), FLAGS.image_resize ]) }) final_probs = np.sum(final_preds, axis=-1) labels = np.argmax(final_probs, 1) for filename, label in zip(filenames, labels): out_file.write('{0},{1}\n'.format(filename, label))
def graph(x, adv, y, t_y, i, x_max, x_min, grad, amplification): target_one_hot = tf.one_hot(t_y, 1001) true_one_hot = tf.one_hot(y, 1001) eps = 2.0 * FLAGS.max_epsilon / 255.0 num_iter = FLAGS.num_iter alpha = eps / num_iter alpha_beta = alpha * 10.0 gamma = alpha_beta * 0.8 momentum = FLAGS.momentum num_classes = 1001 # input_diversity(FLAGS, adv): for DI-FGSM with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_v3, end_points_v3 = inception_v3.inception_v3( input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, reuse=True) auxlogit_v3 = end_points_v3['AuxLogits'] with slim.arg_scope(inception_v4.inception_v4_arg_scope()): logits_v4, end_points_v4 = inception_v4.inception_v4( input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, reuse=True) auxlogit_v4 = end_points_v4['AuxLogits'] with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits_resnet_152, end_points_resnet = resnet_v2.resnet_v2_152( input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, reuse=True) with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits_resnet_101, end_points_resnet_101 = resnet_v2.resnet_v2_101( input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, reuse=True) with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits_resnet_50, end_points_resnet_50 = resnet_v2.resnet_v2_50( input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, reuse=True) with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_Incres, end_points_IR = inception_resnet_v2.inception_resnet_v2( input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, reuse=True) auxlogit_IR = end_points_IR['AuxLogits'] with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3( input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, scope='Ens3AdvInceptionV3',reuse=True) auxlogit_ens3 = end_points_ens3_adv_v3['AuxLogits'] with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3( input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, scope='Ens4AdvInceptionV3', reuse=True) auxlogit_ens4 = end_points_ens4_adv_v3['AuxLogits'] # with slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()): # logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2( # input_diversity(FLAGS, adv), num_classes=num_classes, is_training=False, scope='EnsAdvInceptionResnetV2', reuse=True) # auxlogit_resv2 = end_points_ensadv_res_v2['AuxLogits'] # FLAGS.temperature: for PI-FGSM++ logits = (logits_v4 + logits_resnet_152 + logits_v3 + logits_resnet_101 + logits_resnet_50 + \ logits_Incres + logits_ens3_adv_v3 + logits_ens4_adv_v3) / 8.0 / FLAGS.temperature auxlogits = (auxlogit_v4 + auxlogit_v3 + auxlogit_IR + auxlogit_ens3 + auxlogit_ens4) / 5.0 / FLAGS.temperature target_cross_entropy = tf.losses.softmax_cross_entropy(target_one_hot, logits, label_smoothing=0.0, weights=1.0) target_cross_entropy += tf.losses.softmax_cross_entropy(target_one_hot, auxlogits, label_smoothing=0.0, weights=1.0) noise = tf.gradients(target_cross_entropy, adv)[0] noise = tf.nn.depthwise_conv2d(noise, T_kern, strides=[1, 1, 1, 1], padding='SAME') # for MI-FGSM # noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True) # noise = momentum * grad + noise amplification += alpha_beta * n_staircase_sign(noise, num_of_K) cut_noise = tf.clip_by_value(abs(amplification) - eps, 0.0, 10000.0) * tf.sign(amplification) projection = gamma * n_staircase_sign(project_noise(cut_noise, P_kern, kern_size), num_of_K) adv = adv - alpha_beta * n_staircase_sign(noise, num_of_K) - projection # adv = adv - alpha * n_staircase_sign(noise, num_of_K) adv = tf.clip_by_value(adv, x_min, x_max) i = tf.add(i, 1) return x, adv, y, t_y, i, x_max, x_min, noise, amplification,
def Train(input_dir, output_dir): # some parameter batch_shape = [batch_size, FLAGS.img_size, FLAGS.img_size, 3] with tf.Graph().as_default(): # Prepare graph train_img = tf.placeholder( tf.float32, shape=[None, FLAGS.img_size, FLAGS.img_size, 3]) train_label = tf.placeholder(tf.float32, shape=[None, 2]) with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits, end_points = inception_v3.inception_v3(train_img, num_classes=2, is_training=True, scope=model_type) predict = tf.argmax(end_points['Predictions'], 1) logits = end_points['Logits'] cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(labels=train_label, logits=logits)) learning_rate = tf.placeholder(tf.float32, name='learning_rate') optimizer = tf.train.GradientDescentOptimizer( learning_rate=learning_rate) train_step = optimizer.minimize(cost) accuracy = tf.reduce_mean( tf.cast(tf.equal(predict, tf.argmax(train_label, 1)), tf.float32)) # Run computation with tf.Session() as sess: sess.run(tf.global_variables_initializer()) if FineTune: exclusion = [ '%s/Logits' % model_type, '%s/AuxLogits' % model_type ] #去除class类里的描述 except_logitis = slim.get_variables_to_restore( exclude=exclusion) #print(except_logitis) init_fn = slim.assign_from_checkpoint_fn( FLAGS.model, except_logitis, ignore_missing_vars=True) init_fn(sess) saver = tf.train.Saver( slim.get_model_variables(scope=model_type)) else: saver = tf.train.Saver( slim.get_model_variables(scope=model_type)) saver.restore(sess, FLAGS.model) for time in range(epoch_num): train_acc = [] count = 0 train_loss = [] val_acc = [] val_loss = [] epoch_learning_rate = init_learning_rate for raw_images, true_labels in load_images_with_true_label( input_dir, train=True): labels = one_hot(np.array(true_labels), 2) img = np.array((raw_images / 255.0) * 2.0 - 1.0) img = np.reshape( img, [batch_size, FLAGS.img_size, FLAGS.img_size, 3]) train_feed_dict = { train_img: img, train_label: labels, learning_rate: epoch_learning_rate } sess.run(train_step, feed_dict=train_feed_dict) batch_loss = cost.eval(feed_dict=train_feed_dict) batch_acc = accuracy.eval(feed_dict=train_feed_dict) train_acc.append(batch_acc) train_loss.append(batch_loss) count += 1 if count % 500 == 0: print('acc: ', np.mean(np.array(train_acc)), ' loss: ', np.mean(np.array(train_loss))) break saver.save(sess=sess, save_path='./train_model/%s_%s.ckpt' % (model_type, time)) for raw_images, true_labels in load_images_with_true_label( output_dir, train=False): labels = one_hot(np.array(true_labels), 2) img = np.array((raw_images / 255.0) * 2.0 - 1.0) img = np.reshape( img, [batch_size, FLAGS.img_size, FLAGS.img_size, 3]) train_feed_dict = { train_img: img, train_label: labels, learning_rate: epoch_learning_rate } batch_acc = sess.run([accuracy], feed_dict=train_feed_dict) val_acc.append(batch_acc) count += 1 print('val_acc: ', np.mean(np.array(val_acc)))
def main(_): # Images for inception classifier are normalized to be in [-1, 1] interval, # eps is a difference between pixels so it should be in [0, 2] interval. # Renormalizing epsilon from [0, 255] to [0, 2]. tf.logging.set_verbosity(tf.logging.INFO) full_start = timer() batch_shape = [FLAGS.batch_size, FLAGS.image_height, FLAGS.image_width, 3] all_images_taget_class, all_images_true_label = load_target_class( FLAGS.input_dir) if not os.path.exists(FLAGS.output_dir): os.mkdir(FLAGS.output_dir) with tf.Graph().as_default(): # Prepare graph x_input = tf.placeholder(tf.float32, shape=batch_shape) target_class_input = tf.placeholder(tf.int32, shape=[FLAGS.batch_size]) momentum = FLAGS.momentum eps = 2.0 * FLAGS.max_epsilon / 255.0 alpha = 0.2 num_classes = 1000 num_classes_a = 1001 # image = x_input image = input_diversity(x_input) # image = batch_dct2d(image) """ 224 input """ processed_imgs_res_v1_50 = preprocess_for_model(image, 'resnet_v1_50') with slim.arg_scope(resnet_v1.resnet_arg_scope()): logits_res_v1_50, end_points_res_v1_50 = resnet_v1.resnet_v1_50( processed_imgs_res_v1_50, num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE) processed_imgs_res_v1_101 = preprocess_for_model( image, 'resnet_v1_101') with slim.arg_scope(resnet_v1.resnet_arg_scope()): logits_res_v1_101, end_points_res_v1_101 = resnet_v1.resnet_v1_101( processed_imgs_res_v1_101, num_classes=num_classes, is_training=False, reuse=tf.AUTO_REUSE) processed_res_v1 = preprocess_for_model(image, 'resnet_v1_152') with slim.arg_scope(resnet_v1.resnet_arg_scope()): logits_res_v1_152, end_points_res_v1 = resnet_v1.resnet_v1_152( processed_res_v1, num_classes=num_classes, is_training=False, scope='resnet_v1_152', reuse=tf.AUTO_REUSE) processed_imgs_vgg_16 = preprocess_for_model(image, 'vgg_16') with slim.arg_scope(vgg.vgg_arg_scope()): logits_vgg_16, end_points_vgg_16 = vgg.vgg_16( processed_imgs_vgg_16, num_classes=num_classes, is_training=False, scope='vgg_16') processed_imgs_vgg_19 = preprocess_for_model(image, 'vgg_19') with slim.arg_scope(vgg.vgg_arg_scope()): logits_vgg_19, end_points_vgg_19 = vgg.vgg_19( processed_imgs_vgg_19, num_classes=num_classes, is_training=False, scope='vgg_19') logits_clean_a = (logits_res_v1_50 + logits_res_v1_101 + logits_res_v1_152 + logits_vgg_16 + logits_vgg_19) / 5.0 processed_imgs_inception_v1 = preprocess_for_model( image, 'inception_v1') with slim.arg_scope(inception_v1.inception_v1_arg_scope()): logits_inception_v1, end_points_inception_v1 = inception_v1.inception_v1( processed_imgs_inception_v1, num_classes=num_classes_a, is_training=False, reuse=tf.AUTO_REUSE) """ 299 input """ x_div = preprocess_for_model(image, 'inception_v3') with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_inc_v3, end_points_inc_v3 = inception_v3.inception_v3( x_div, num_classes=num_classes_a, is_training=False, scope='InceptionV3') with slim.arg_scope(inception_v4.inception_v4_arg_scope()): logits_inc_v4, end_points_inc_v4 = inception_v4.inception_v4( x_div, num_classes=num_classes_a, is_training=False, scope='InceptionV4') with slim.arg_scope( inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_inc_res_v2, end_points_inc_res_v2 = inception_resnet_v2.inception_resnet_v2( x_div, num_classes=num_classes_a, is_training=False, scope='InceptionResnetV2') logits_clean_b = (logits_inc_v3 + logits_inc_v4 + logits_inc_res_v2 + logits_inception_v1) / 4.0 """ add adv model """ with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_adv_v3, end_points_adv_v3 = inception_v3.inception_v3( x_div, num_classes=num_classes_a, is_training=False, scope='AdvInceptionV3') with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens3_adv_v3, end_points_ens3_adv_v3 = inception_v3.inception_v3( x_div, num_classes=num_classes_a, is_training=False, scope='Ens3AdvInceptionV3') with slim.arg_scope(inception_v3.inception_v3_arg_scope()): logits_ens4_adv_v3, end_points_ens4_adv_v3 = inception_v3.inception_v3( x_div, num_classes=num_classes_a, is_training=False, scope='Ens4AdvInceptionV3') with slim.arg_scope( inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_ensadv_res_v2, end_points_ensadv_res_v2 = inception_resnet_v2.inception_resnet_v2( x_div, num_classes=num_classes_a, is_training=False, scope='EnsAdvInceptionResnetV2') with slim.arg_scope( inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_adv_res_v2, end_points_adv_res_v2 = inception_resnet_v2.inception_resnet_v2( x_div, num_classes=num_classes_a, is_training=False, scope='AdvInceptionResnetV2') logits_ens_a = (logits_adv_v3 + logits_ens3_adv_v3 + logits_ens4_adv_v3 + logits_ensadv_res_v2 + logits_adv_res_v2) / 5.0 logits_ens_aux = (end_points_adv_v3['AuxLogits'] + end_points_ens3_adv_v3['AuxLogits'] + end_points_ens4_adv_v3['AuxLogits'] + end_points_adv_res_v2['AuxLogits'] + end_points_ensadv_res_v2['AuxLogits']) / 5.0 label_test = tf.argmax(logits_adv_v3, axis=1) """ ensemble model loss """ clean_logits = (logits_clean_a + logits_clean_b[:, 1:1001]) / 2.0 adv_logits = logits_ens_a[:, 1:1001] + logits_ens_aux[:, 1:1001] logits = (clean_logits + adv_logits) / 2.0 ens_labels = tf.argmax(logits, axis=1) one_hot = tf.one_hot(target_class_input, num_classes) loss_adv_v3 = tf.losses.softmax_cross_entropy(one_hot, logits_adv_v3[:, 1:1001], label_smoothing=0.0, weights=1.0) loss_ens3_adv_v3 = tf.losses.softmax_cross_entropy( one_hot, logits_ens3_adv_v3[:, 1:1001], label_smoothing=0.0, weights=1.0) loss_ens4_adv_v3 = tf.losses.softmax_cross_entropy( one_hot, logits_ens4_adv_v3[:, 1:1001], label_smoothing=0.0, weights=1.0) loss_ensadv_res_v2 = tf.losses.softmax_cross_entropy( one_hot, logits_ensadv_res_v2[:, 1:1001], label_smoothing=0.0, weights=1.0) loss_adv_res_v2 = tf.losses.softmax_cross_entropy( one_hot, logits_adv_res_v2[:, 1:1001], label_smoothing=0.0, weights=1.0) loss_res_v1_101 = tf.losses.softmax_cross_entropy(one_hot, logits_res_v1_101, label_smoothing=0.0, weights=1.0) loss_res_v1_50 = tf.losses.softmax_cross_entropy(one_hot, logits_res_v1_50, label_smoothing=0.0, weights=1.0) loss_vgg_16 = tf.losses.softmax_cross_entropy(one_hot, logits_vgg_16, label_smoothing=0.0, weights=1.0) loss_res_v1_152 = tf.losses.softmax_cross_entropy(one_hot, logits_res_v1_152, label_smoothing=0.0, weights=1.0) total_loss = tf.losses.softmax_cross_entropy(one_hot, logits, label_smoothing=0.0, weights=1.0) noise = tf.gradients(total_loss, x_input)[0] kernel = gkern(15, FLAGS.sig).astype(np.float32) stack_kernel = np.stack([kernel, kernel, kernel]).swapaxes(2, 0) stack_kernel = np.expand_dims(stack_kernel, 3) noise = tf.nn.depthwise_conv2d(noise, stack_kernel, strides=[1, 1, 1, 1], padding='SAME') # [batch, out_height, out_width, in_channels * channel_multiplier] noise = noise / tf.reshape( tf.contrib.keras.backend.std(tf.reshape(noise, [FLAGS.batch_size, -1]), axis=1), [FLAGS.batch_size, 1, 1, 1]) # noise = momentum * grad + noise noise = noise / tf.reshape( tf.contrib.keras.backend.std(tf.reshape(noise, [FLAGS.batch_size, -1]), axis=1), [FLAGS.batch_size, 1, 1, 1]) s1 = tf.train.Saver(slim.get_model_variables(scope='InceptionV1')) s2 = tf.train.Saver(slim.get_model_variables(scope='InceptionV3')) s3 = tf.train.Saver(slim.get_model_variables(scope='InceptionV4')) s4 = tf.train.Saver(slim.get_model_variables(scope='resnet_v1_50')) s5 = tf.train.Saver(slim.get_model_variables(scope='resnet_v1_101')) s6 = tf.train.Saver(slim.get_model_variables(scope='resnet_v1_152')) s7 = tf.train.Saver(slim.get_model_variables(scope='vgg_16')) s8 = tf.train.Saver(slim.get_model_variables(scope='vgg_19')) s9 = tf.train.Saver( slim.get_model_variables(scope='InceptionResnetV2')) s10 = tf.train.Saver( slim.get_model_variables(scope='AdvInceptionResnetV2')) s11 = tf.train.Saver( slim.get_model_variables(scope='Ens3AdvInceptionV3')) s12 = tf.train.Saver( slim.get_model_variables(scope='Ens4AdvInceptionV3')) s13 = tf.train.Saver( slim.get_model_variables(scope='EnsAdvInceptionResnetV2')) s14 = tf.train.Saver(slim.get_model_variables(scope='AdvInceptionV3')) print('Created Graph') with tf.Session() as sess: s1.restore(sess, FLAGS.checkpoint_path_inception_v1) s2.restore(sess, FLAGS.checkpoint_path_inception_v3) s3.restore(sess, FLAGS.checkpoint_path_inception_v4) s4.restore(sess, FLAGS.checkpoint_path_resnet_v1_50) s5.restore(sess, FLAGS.checkpoint_path_resnet_v1_101) s6.restore(sess, FLAGS.checkpoint_path_resnet_v1_152) s7.restore(sess, FLAGS.checkpoint_path_vgg_16) s8.restore(sess, FLAGS.checkpoint_path_vgg_19) s9.restore(sess, FLAGS.checkpoint_path_inception_resnet_v2) s10.restore(sess, FLAGS.checkpoint_path_adv_inception_resnet_v2) s11.restore(sess, FLAGS.checkpoint_path_ens3_adv_inception_v3) s12.restore(sess, FLAGS.checkpoint_path_ens4_adv_inception_v3) s13.restore(sess, FLAGS.checkpoint_path_ens_adv_inception_resnet_v2) s14.restore(sess, FLAGS.checkpoint_path_adv_inception_v3) print('Initialized Models') processed = 0.0 defense, tgt, untgt, final = 0.0, 0.0, 0.0, 0.0 idx = 0 for filenames, images in load_images(FLAGS.input_dir, batch_shape): target_class_for_batch = ( [all_images_taget_class[n[:-4]] for n in filenames] + [0] * (FLAGS.batch_size - len(filenames))) true_label_for_batch = ( [all_images_true_label[n[:-4]] for n in filenames] + [0] * (FLAGS.batch_size - len(filenames))) x_max = np.clip(images + eps, -1.0, 1.0) x_min = np.clip(images - eps, -1.0, 1.0) adv_img = np.copy(images) for i in range(FLAGS.iterations): # loss_set = sess.run([loss_adv_v3,loss_ens3_adv_v3,loss_ens4_adv_v3,loss_ensadv_res_v2, # loss_adv_res_v2,loss_res_v1_101,loss_res_v1_50,loss_vgg_16,loss_res_v1_152], # feed_dict={x_input: batch_NLM(adv_img), # target_class_input: target_class_for_batch}) # print ("loss:",loss_set) # label_ens_model = sess.run([a,b,c,d],feed_dict={x_input: adv_img,target_class_input: target_class_for_batch}) # print ("label_ens_model:",label_ens_model) # print (target_class_for_batch,true_label_for_batch) adv_img = batch_NLM(adv_img) if i % 5 == 0 else adv_img ens_loss, pred, grad, pred_adv_v3 = sess.run( [total_loss, ens_labels, noise, label_test], feed_dict={ x_input: adv_img, target_class_input: target_class_for_batch }) adv_img = adv_img - alpha * np.clip(np.round(grad), -2, 2) adv_img = np.clip(adv_img, x_min, x_max) print("{} \t total_loss {}".format(i, ens_loss)) print('prediction :', pred) print('target_label :', target_class_for_batch) print('true_label :', true_label_for_batch) # print ("{} \t total_loss {} predction {} \t target class {} \t true label {} \t ".format(i,ens_loss,pred,target_class_for_batch,true_label_for_batch)) # print ("model predction {} \t target class {} \t true label {} \t ".format(pred,target_class_for_batch,true_label_for_batch)) print( "final prediction {} \t target class {} \t true label {} \t " .format(pred, target_class_for_batch, true_label_for_batch)) processed += FLAGS.batch_size tgt += sum( np.equal(np.array(pred), np.array(target_class_for_batch))) defense += sum( np.equal(np.array(pred), np.array(true_label_for_batch))) untgt = processed - tgt - defense print("processed {} \t acc {} {} \t tgt {} {} \t untgt {} {} ". format(processed, defense, defense / processed, tgt, tgt / processed, untgt, untgt / processed)) full_end = timer() print("DONE: Processed {} images in {} sec".format( processed, full_end - full_start)) save_images(adv_img, filenames, FLAGS.output_dir) print("DONE: Processed {} images in {} sec".format( processed, full_end - full_start))