def predict(image, version='V3'):
tf.reset_default_graph()
# Process the image
raw_image, processed_image = process_image(image)
class_names = imagenet.create_readable_names_for_imagenet_labels()
# Create a placeholder for the images
X = tf.placeholder(tf.float32, [None, 299, 299, 3], name="X")
#inception_v3 function returns logits and end_points dictionary
#logits are output of the network before applying softmax activation
if version.upper() == 'V3':
model_ckpt_path = INCEPTION_V3_CKPT_PATH
with slim.arg_scope(inception.inception_v3_arg_scope()):
# Set the number of classes and is_training parameter
logits, end_points = inception.inception_v3(
X, num_classes=1001, is_training=False)
elif version.upper() == 'V4':
model_ckpt_path = INCEPTION_V4_CKPT_PATH
with slim.arg_scope(inception.inception_v3_arg_scope()):
# Set the number of classes and is_training parameter
# Logits
logits, end_points = inception.inception_v4(
X, num_classes=1001, is_training=False)
predictions = end_points.get('Predictions', 'No key named predictions')
saver = tf.train.Saver()
with tf.Session() as sess:
saver.restore(sess, model_ckpt_path)
prediction_values = predictions.eval({X: processed_image})
try:
# Add an index to predictions and then sort by probability
prediction_values = [
(i, prediction)
for i, prediction in enumerate(prediction_values[0, :])
]
prediction_values = sorted(
prediction_values, key=lambda x: x[1], reverse=True)
# Plot the image
# plot_color_image(raw_image)
plot_color_image(image)
print("Using Inception_{} CNN\nPrediction: Probability\n".format(
version))
# Display the image and predictions
for i in range(5):
predicted_class = class_names[prediction_values[i][0]]
probability = prediction_values[i][1]
print("{}: {:.2f}%".format(predicted_class, probability * 100))
# If the predictions do not come out right
except:
print(predictions)
def main(opt):
config = tf.ConfigProto(allow_soft_placement=True)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.1)
config.gpu_options.allow_growth = True
jpg_path = opt.image_path
images_lists = []
for subdir, dirs, files in os.walk(jpg_path):
for f in files:
f = f.strip()
images_lists.append(os.path.join(jpg_path, f))
att_dir = os.path.join(opt.out_dir, opt.att_dir)
fc_dir = os.path.join(opt.out_dir, opt.fc_dir)
if not tf.gfile.Exists(fc_dir):
tf.gfile.MakeDirs(fc_dir)
if not tf.gfile.Exists(att_dir):
tf.gfile.MakeDirs(att_dir)
checkpoints_dir = opt.model_path
slim = tf.contrib.slim
image_size = inception.inception_v3.default_image_size
tf_image = tf.placeholder(tf.string, None)
image = tf.image.decode_jpeg(tf_image, channels=3)
processed_image = inception_preprocessing.preprocess_image(
image, image_size, image_size, is_training=False)
processed_images = tf.expand_dims(processed_image, 0)
with slim.arg_scope(inception.inception_v3_arg_scope()):
tf_feats_att, tf_feats_fc = inception.inception_v3(processed_images,
num_classes=1001,
is_training=False)
init_fn = slim.assign_from_checkpoint_fn(
os.path.join(checkpoints_dir, 'inception_v3.ckpt'),
slim.get_model_variables('InceptionV3'))
with tf.Session(config=config) as sess:
init_fn(sess)
for idx, image_path in enumerate(images_lists):
print('{} {}'.format(idx, image_path))
image_name = os.path.basename(image_path)
image_id = get_image_id(image_name)
url = 'file://' + image_path
image_string = urllib.request.urlopen(url).read()
conv_feats, fc_feats = sess.run([tf_feats_att, tf_feats_fc],
feed_dict={tf_image: image_string})
conv_feats = np.squeeze(conv_feats)
fc_feats = np.squeeze(fc_feats)
np.save(os.path.join(fc_dir, str(image_id)), fc_feats)
np.savez_compressed(os.path.join(att_dir, str(image_id)),
feat=conv_feats)
def main(input_dir, output_dir):
eps = 2.0 * FLAGS.max_epsilon / 255.0
batch_shape = [FLAGS.batch_size, 299, 299, 3]
_check_or_create_dir(output_dir)
dev_dir = "./dev_data/dev_imgs.csv"
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
x_input = tf.placeholder(tf.float32, shape=batch_shape)
x_max = tf.clip_by_value(x_input + eps, -1.0, 1.0)
x_min = tf.clip_by_value(x_input - eps, -1.0, 1.0)
with slim.arg_scope(inception.inception_v3_arg_scope()):
logits, end_points = inception.inception_v3(x_input,
num_classes=1001,
is_training=False)
score = tf.nn.softmax(logits, name='pre')
pred_labels = tf.argmax(score, 1)
y = tf.one_hot(pred_labels, FLAGS.num_classes)
i = tf.constant(0)
grad = tf.zeros(shape=batch_shape)
grad_D = tf.zeros(shape=batch_shape)
x_adv, _, _, _, _, _, _ = tf.while_loop(
stop, graph_incv3, [x_input, y, i, x_max, x_min, grad, grad_D])
# Run computation
saver = tf.train.Saver(slim.get_model_variables(scope='InceptionV3'))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver.restore(sess, FLAGS.checkpoint_path)
for filenames, raw_images, true_labels in load_images(
dev_dir, input_dir, batch_shape):
adv_images = sess.run(x_adv, feed_dict={x_input: raw_images})
save_images(adv_images, filenames, output_dir)
def testNoBatchNormScaleByDefault(self):
height, width = 299, 299
num_classes = 1000
inputs = tf.placeholder(tf.float32, (1, height, width, 3))
with slim.arg_scope(inception.inception_v3_arg_scope()):
inception.inception_v3(inputs, num_classes, is_training=False)
self.assertEqual(tf.global_variables('.*/BatchNorm/gamma:0$'), [])
def testNoBatchNormScaleByDefault(self):
height, width = 299, 299
num_classes = 1000
inputs = tf.placeholder(tf.float32, (1, height, width, 3))
with slim.arg_scope(inception.inception_v3_arg_scope()):
inception.inception_v3(inputs, num_classes, is_training=False)
self.assertEqual(tf.global_variables('.*/BatchNorm/gamma:0$'), [])
def create(self, images, num_classes, is_training):
"""See baseclass."""
with slim.arg_scope(inception.inception_v3_arg_scope()):
_, endpoints = inception.inception_v3(images,
num_classes,
create_aux_logits=False,
is_training=is_training)
return endpoints
def inception_v3(inputs):
with slim.arg_scope(inception.inception_v3_arg_scope()):
logits, end_points = inception.inception_v3(
inputs,
num_classes=None,
is_training=False,
spatial_squeeze=True,
global_pool=True)
return logits, end_points, inception_v3_ckpt_path
def testModelHasExpectedNumberOfParameters(self):
batch_size = 5
height, width = 299, 299
inputs = tf.random.uniform((batch_size, height, width, 3))
with slim.arg_scope(inception.inception_v3_arg_scope()):
inception.inception_v3_base(inputs)
total_params, _ = slim.model_analyzer.analyze_vars(
slim.get_model_variables())
self.assertAlmostEqual(21802784, total_params)
def testModelHasExpectedNumberOfParameters(self):
batch_size = 5
height, width = 299, 299
inputs = tf.random_uniform((batch_size, height, width, 3))
with slim.arg_scope(inception.inception_v3_arg_scope()):
inception.inception_v3_base(inputs)
total_params, _ = slim.model_analyzer.analyze_vars(
slim.get_model_variables())
self.assertAlmostEqual(21802784, total_params)
def __call__(self, x_input):
"""Constructs model and return probabilities for given input."""
reuse = True if self.built else None
with slim.arg_scope(inception.inception_v3_arg_scope()):
_, end_points = inception.inception_v3(
x_input, num_classes=self.num_classes, is_training=False,
reuse=reuse)
self.built = True
output = end_points['Predictions']
# Strip off the extra reshape op at the output
probs = output.op.inputs[0]
return probs
def graph_incv3(x, y, i, x_max, x_min, grad, grad_D):
eps = 2.0 * FLAGS.max_epsilon / 255.0
kd = FLAGS.derivative
momentum = FLAGS.momentum
num_iter = FLAGS.num_iter
alpha = eps / FLAGS.num_iter
tf.get_variable_scope().reuse_variables()
x_nes = x + alpha * momentum * grad
x_b = x - alpha * grad_D
with slim.arg_scope(inception.inception_v3_arg_scope()):
logits_1, end_points_1 = inception.inception_v3(
x_b, num_classes=FLAGS.num_classes, is_training=False)
cross_entropy_1 = tf.losses.softmax_cross_entropy(y,
logits_1,
label_smoothing=0.0,
weights=1.0)
noise_1 = tf.gradients(cross_entropy_1, x_b)[0]
noise_1 = noise_1 / tf.reduce_mean(tf.abs(noise_1), [1, 2, 3],
keep_dims=True)
with slim.arg_scope(inception.inception_v3_arg_scope()):
logits, end_points = inception.inception_v3(
x_nes, num_classes=FLAGS.num_classes, is_training=False)
#logits = (end_points['Logits'])
cross_entropy = tf.losses.softmax_cross_entropy(y,
logits,
label_smoothing=0.0,
weights=1.0)
noise = tf.gradients(cross_entropy, x)[0]
noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True)
grad_D = grad_D + kd * (noise - noise_1)
noise_all = momentum * grad + noise - grad_D
x = x + alpha * tf.sign(noise_all)
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
return x, y, i, x_max, x_min, noise_all, grad_D
def testBatchNormScale(self):
height, width = 299, 299
num_classes = 1000
inputs = tf.placeholder(tf.float32, (1, height, width, 3))
with slim.arg_scope(
inception.inception_v3_arg_scope(batch_norm_scale=True)):
inception.inception_v3(inputs, num_classes, is_training=False)
gamma_names = set(
v.op.name for v in tf.global_variables('.*/BatchNorm/gamma:0$'))
self.assertGreater(len(gamma_names), 0)
for v in tf.global_variables('.*/BatchNorm/moving_mean:0$'):
self.assertIn(v.op.name[:-len('moving_mean')] + 'gamma', gamma_names)
def test_image_data():
slim = tf.contrib.slim
tf.reset_default_graph()
session = tf.Session()
names = imagenet.create_readable_names_for_imagenet_labels()
processed_images = tf.placeholder(tf.float32, shape=(None, 299, 299, 3))
with slim.arg_scope(inception.inception_v3_arg_scope()):
logits, _ = inception.inception_v3(processed_images,
num_classes=1001,
is_training=False)
probabilities = tf.nn.softmax(logits)
# Please correctly set the model path.
# Download the model at https://github.com/tensorflow/models/tree/master/research/slim
checkpoints_dir = 'model'
init_fn = slim.assign_from_checkpoint_fn(
os.path.join(checkpoints_dir, 'inception_v3.ckpt'),
slim.get_model_variables('InceptionV3'))
init_fn(session)
def predict_fn(images):
return session.run(probabilities, feed_dict={processed_images: images})
def f(x):
return x / 2 + 0.5
class_names = []
for item in names:
class_names.append(names[item])
images = transform_img_fn(['data/violin.JPEG'])
image = images[0]
explainer = xdeep_image.ImageExplainer(predict_fn, class_names)
explainer.explain('lime', image, top_labels=3)
explainer.show_explanation('lime', deprocess=f, positive_only=False)
explainer.explain('cle', image, top_labels=3)
explainer.show_explanation('cle', deprocess=f, positive_only=False)
explainer.explain('anchor', image)
explainer.show_explanation('anchor')
segments_slic = slic(image, n_segments=50, compactness=30, sigma=3)
explainer.initialize_shap(n_segment=50, segment=segments_slic)
explainer.explain('shap', image, nsamples=400)
explainer.show_explanation('shap', deprocess=f)
def build_network(batch_size, is_training):
# input
tf_raw_image_data = tf.placeholder(tf.string, shape=(batch_size, ))
tf_body_bbox = tf.placeholder(tf.int32, shape=(batch_size, 4))
tf_labels = tf.placeholder(tf.int32, shape=(batch_size, ))
# pre-processing pipeline
crops = []
for i in range(batch_size):
image = tf.image.decode_jpeg(tf_raw_image_data[i], channels=3)
body_crop = tf.image.crop_to_bounding_box(image, tf_body_bbox[i, 1],
tf_body_bbox[i, 0],
tf_body_bbox[i, 3],
tf_body_bbox[i, 2])
processed_crop = inception_preprocessing.preprocess_image(
body_crop, image_size, image_size, is_training=is_training)
crops.append(processed_crop)
processed_images = tf.stack(crops)
# training pipeline
with slim.arg_scope(inception.inception_v3_arg_scope()):
_, endpoints = inception.inception_v3(processed_images,
num_classes=num_identity,
is_training=is_training)
# load model parameters
init_fn = slim.assign_from_checkpoint_fn(
os.path.join(checkpoints_dir, checkpoint_name),
slim.get_model_variables(original_variable_namescope))
net_before_pool = tf.reshape(endpoints['Mixed_7c'], shape=(batch_size, -1))
net_before_pool_frozen = tf.stop_gradient(net_before_pool)
tf_features = slim.fully_connected(net_before_pool_frozen,
feature_length,
activation_fn=None)
tf_features_normalized = tf.nn.l2_normalize(tf_features, dim=1)
tf_loss = coco_loss_layer(tf_features_normalized, tf_labels, batch_size)
# optimizer
tf_lr = tf.placeholder(dtype=tf.float32, shape=(), name='learning_rate')
optimizer = tf.train.AdamOptimizer(learning_rate=0.001)
train = optimizer.minimize(tf_loss)
# summary
tf.summary.scalar('coco_loss', tf_loss)
summary_op = tf.summary.merge_all()
return (tf_raw_image_data, tf_body_bbox,
tf_labels), (init_fn, tf_loss, tf_lr, train,
summary_op), tf_features
def __init__(self, sess):
# placeholder
self.input_placeholder = tf.placeholder(tf.float32, shape=(None, 299, 299, 3))
# inception features extractor
device = tf.device('device:CPU:0')
if tf.test.is_gpu_available():
device = tf.device('device:GPU:0')
with device:
with tf.contrib.slim.arg_scope(inception_model.inception_v3_arg_scope()):
self.features_extractor, _ = inception_model.inception_v3(self.input_placeholder,
num_classes=0, is_training=False)
# init
init_fn = tf.contrib.slim.assign_from_checkpoint_fn(INCEPTION_MODEL_PATH,
tf.contrib.slim.get_model_variables("InceptionV3"))
init_fn(sess)
def __call__(self, x_input, batch_size=None, is_training=False):
"""Constructs model and return probabilities for given input."""
reuse = True if self.built else None
with slim.arg_scope(inception.inception_v3_arg_scope()):
with tf.variable_scope(self.name):
logits, end_points = inception.inception_v3(
x_input,
num_classes=self.num_classes,
is_training=is_training,
reuse=reuse)
preds = tf.argmax(logits, axis=1)
self.built = True
self.logits = logits
self.preds = preds
return logits
def load_pretrained_slim_model():
global session, names, probabilities, processed_images, slim_home_dir
names = imagenet.create_readable_names_for_imagenet_labels()
processed_images = tf.placeholder(tf.float32, shape=(None, 299, 299, 3))
with slim.arg_scope(inception.inception_v3_arg_scope()):
logits, _ = inception.inception_v3(processed_images,
num_classes=1001,
is_training=False)
probabilities = tf.nn.softmax(logits)
checkpoints_dir = os.path.join(slim_home_dir, 'pretrained')
session = tf.Session()
init_fn = slim.assign_from_checkpoint_fn(
os.path.join(checkpoints_dir, 'inception_v3.ckpt'),
slim.get_model_variables('InceptionV3'))
init_fn(session)
def __call__(self, x_input, batch_size=None, is_training=False):
"""Constructs model and return probabilities for given input."""
reuse = True if self.built else None
with slim.arg_scope(inception.inception_v3_arg_scope()):
with tf.variable_scope(self.ckpt):
logits, end_points = inception.inception_v3(
x_input, num_classes=self.num_classes, is_training=is_training,
reuse=reuse)
preds = tf.argmax(logits, axis=1)
self.built = True
self.logits = logits
self.preds = preds
#output = end_points['logits']
# Strip off the extra reshape op at the output
#probs = output.op.inputs[0]
return logits
def __init__(self,
cnn_model_path=None,
lr_model_path=None,
center_crop=True):
self.inception_dim = 299
self.arg_scope = inception.inception_v3_arg_scope()
self.endpoint = 'Mixed_7c'
self.moving_average_decay = 0.9999
self.center_crop = center_crop
self.label_class = LABEL_CLASS
self.graph = tf.Graph()
if cnn_model_path:
self.init_cnn_graph(cnn_model_path)
if lr_model_path:
self.init_lr_model(lr_model_path)
config_sess = tf.ConfigProto(allow_soft_placement=True)
config_sess.gpu_options.allow_growth = True
self.sess = tf.Session(config=config_sess, graph=self.graph)
self.init_fn(self.sess)
def inception_v3(inputs, is_training, opts):
with slim.arg_scope(inception.inception_v3_arg_scope(
weight_decay=opts.weight_decay,
use_batch_norm=opts.use_batch_norm,
batch_norm_decay=opts.batch_norm_decay,
batch_norm_epsilon=opts.batch_norm_epsilon,
activation_fn=tf.nn.relu)):
return inception.inception_v3(
inputs,
num_classes=opts.num_classes,
is_training=is_training,
dropout_keep_prob=opts.dropout_keep_prob,
min_depth=opts.min_depth,
depth_multiplier=opts.depth_multiplier,
prediction_fn=slim.softmax,
spatial_squeeze=opts.spatial_squeeze,
reuse=None,
create_aux_logits=opts.create_aux_logits,
global_pool=opts.global_pool)
def batch_results():
batch_size = 3 # number of samples to show
with tf.Graph().as_default():
tf.logging.set_verbosity(tf.logging.INFO)
images, images_raw, labels = load_batch(dataset,
height=image_size,
width=image_size)
# Create the model, use the default arg scope to configure the batch norm parameters.
with slim.arg_scope(inception.inception_v3_arg_scope()):
logits, _ = inception.inception_v3(images,
num_classes=dataset.num_classes,
is_training=False)
probabilities = tf.nn.softmax(logits)
checkpoint_path = tf.train.latest_checkpoint(train_dir)
init_fn = slim.assign_from_checkpoint_fn(
checkpoint_path, slim.get_variables_to_restore())
with tf.Session() as sess:
with slim.queues.QueueRunners(sess):
sess.run(tf.initialize_local_variables())
init_fn(sess)
np_probabilities, np_images_raw, np_labels = sess.run(
[probabilities, images_raw, labels])
for i in xrange(batch_size):
image = np_images_raw[i, :, :, :]
true_label = np_labels[i]
predicted_label = np.argmax(np_probabilities[i, :])
predicted_name = dataset.labels_to_names[predicted_label]
true_name = dataset.labels_to_names[true_label]
plt.figure()
plt.imshow(image.astype(np.uint8))
plt.title('Ground Truth: [%s], Prediction [%s]' %
(true_name, predicted_name))
plt.axis('off')
plt.show()
def url_results():
with tf.Graph().as_default():
url = 'http://www.vetprofessionals.com/catprofessional/images/home-cat.jpg'
image_string = urllib2.urlopen(url).read()
image = tf.image.decode_jpeg(image_string, channels=3)
processed_image = inception_preprocessing.preprocess_image(
image, image_size, image_size, is_training=False)
processed_images = tf.expand_dims(processed_image, 0)
# Create the model, use the default arg scope to configure the batch norm parameters.
with slim.arg_scope(inception.inception_v3_arg_scope()):
logits, _ = inception.inception_v3(processed_images,
num_classes=dataset.num_classes,
is_training=False)
probabilities = tf.nn.softmax(logits)
checkpoint_path = tf.train.latest_checkpoint(train_dir)
init_fn = slim.assign_from_checkpoint_fn(
checkpoint_path, slim.get_variables_to_restore())
with tf.Session() as sess:
init_fn(sess)
np_image, probabilities = sess.run([image, probabilities])
probabilities = probabilities[0, 0:]
sorted_inds = [
i[0]
for i in sorted(enumerate(-probabilities), key=lambda x: x[1])
]
plt.figure()
plt.imshow(np_image.astype(np.uint8))
plt.axis('off')
plt.show()
names = dataset.labels_to_names
for i in range(dataset.num_classes):
index = sorted_inds[i]
print('Probability %0.2f%% => [%s]' %
(probabilities[index], names[index + 1]))
def build_graph():
images_ph = tf.placeholder(tf.float32, shape=[None, 299, 299, 3])
labels_ph = tf.placeholder(tf.int32, shape=[
None,
])
with slim.arg_scope(inception.inception_v3_arg_scope()):
l, _ = inception.inception_v3(images_ph,
num_classes=2,
is_training=False,
reuse=False,
scope="crack/InceptionV3")
# Construct the scalar neuron tensor.
logits = tf.get_default_graph().get_tensor_by_name(
'crack/InceptionV3/Logits/SpatialSqueeze:0')
neuron_selector = tf.placeholder(tf.int32)
y = logits[0][neuron_selector]
# Construct tensor for predictions.
prediction = tf.argmax(logits, 1)
probs = tf.nn.softmax(l)
return images_ph, prediction, probs, y, neuron_selector
def __call__(self,
image_input,
training=False,
keep_prob=1.0,
endpoint_name='Mixed_6e'):
weight_decay = FLAGS.weight_decay
activation_fn = tf.nn.relu
end_points = {}
with slim.arg_scope(
inception.inception_v3_arg_scope(
weight_decay=FLAGS.weight_decay)):
with tf.variable_scope("", reuse=self.reuse):
with tf.variable_scope(None, 'InceptionV3',
[image_input]) as scope:
with slim.arg_scope([slim.batch_norm, slim.dropout],
is_training=training):
net, end_points = inception.inception_v3_base(
image_input, scope=scope)
feature_map = end_points[endpoint_name]
self.reuse = True
return feature_map
def nestedClassification(img_paths):
checkpoints_path = '/home/mehdi/Desktop/1.Projects/1.Image_Processing/1_.classification/inception_v3/all'
fileName = "../Project_Data/labels.txt"
names = []
crimefile = open(fileName, 'r')
for line in crimefile.readlines():
if line.strip():
names.append(line.strip().split(":")[-1])
BATCH_SIZE=FLAGS.Batch_Size
############################################################################
number_of_inputImage=len(img_paths)
if BATCH_SIZE>=number_of_inputImage:
batch_size=number_of_inputImage
else:
batch_size=BATCH_SIZE
NUM_CLASSES = len(names)
image_size=inception.inception_v3.default_image_size
repeat_count=2
checkpoints_path = tf.train.latest_checkpoint(checkpoints_path)
X = tf.placeholder(tf.float32, [batch_size, image_size, image_size, 3])
with slim.arg_scope(inception.inception_v3_arg_scope()):
model,_ = inception.inception_v3(X, num_classes = NUM_CLASSES, is_training = False)
probabilities = tf.nn.softmax(model)
init = slim.assign_from_checkpoint_fn(checkpoints_path,slim.get_model_variables('InceptionV3'))
def input_parser(img_path):
# # read the img from file
img_file = tf.read_file(img_path)
img_decoded = tf.image.decode_png(img_file,channels=3)
img_decoded=tf.image.convert_image_dtype(img_decoded,dtype=tf.float32)
processed_image = inception_preprocessing.preprocess_image(img_decoded, image_size, image_size,
is_training=False)
return processed_image
data=tf.data.Dataset.from_tensor_slices(img_paths)
print("===imaPath",img_paths)
data = data.map(input_parser, num_parallel_calls=1)
data= data.repeat(repeat_count) # Repeats dataset this # times
data = data.batch(batch_size)
# data= data.repeat(repeat_count) # Repeats dataset this # times
iterator=data.make_initializable_iterator()
next_batch = iterator.get_next()
test_init_op = iterator.make_initializer(data)
with tf.Session() as sess:
init(sess)
sess.run(test_init_op)
#
j = 0
k=0
epoch=int(number_of_inputImage/batch_size)
for i in range(epoch+1):
img_batch=(sess.run(next_batch))
probabilities1 = sess.run(probabilities, feed_dict={X: img_batch})
for prediction in range((probabilities1.shape)[0]): # batch_size
if j<number_of_inputImage:
print(img_paths[j])
j += 1
batchPredResult = probabilities1[prediction, 0:]
sorted_inds = [i[0] for i in sorted(enumerate(-batchPredResult), key=lambda x: x[1])]
for i in range(1):
index = sorted_inds[i]
print('Probability %0.4f => [%s]' % (batchPredResult[index], names[index]))
def train():
eps=2.0*float(FLAGS.max_epsilon)/256.0;
tf.logging.set_verbosity(tf.logging.INFO);
with tf.Graph().as_default():
# Design architecture
# input
x_data = tf.placeholder(tf.float32, [None, FLAGS.img_height, FLAGS.img_width,3], name="x_data")
y_label = tf.placeholder(tf.float32, [None, FLAGS.num_classes], name="y_label")
# generator
x_generated, g_params = build_generator(x_data,FLAGS);
x_generated = x_generated * eps;
x_generated = x_data + x_generated;
# discriminator(inception v3)
with slim.arg_scope(inception.inception_v3_arg_scope()):
_, end_points = inception.inception_v3(
x_generated, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['Predictions'], 1);
predicted_logits = end_points['Logits'];
disc_var_list=slim.get_model_variables();
# discriminator(resnet v2 50)
x_generated2=tf.image.resize_bilinear(x_generated,[224,224],align_corners=False);
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points2 = resnet_v2.resnet_v2_50(
x_generated2, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels2 = tf.argmax(end_points2['predictions'], 1);
predicted_logits2 = end_points2['predictions'];
disc_var_list2=slim.get_model_variables()[len(disc_var_list):];
# discriminator(resnet v2 152)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points3 = resnet_v2.resnet_v2_152(
x_generated2, num_classes=FLAGS.num_classes, is_training=False)
predicted_labels3 = tf.argmax(end_points3['predictions'], 1);
predicted_logits3 = end_points3['predictions'];
disc_var_list3=slim.get_model_variables()[(len(disc_var_list)+len(disc_var_list2)):];
# average
predicted_prob_avg = (end_points['Predictions']+end_points2['predictions']+end_points3['predictions'])/5.0;
predicted_labels_avg = tf.argmax((end_points['Predictions']+end_points2['predictions']+end_points3['predictions'])/5.0, 1);
# loss and optimizer
gen_acc=tf.reduce_mean(tf.cast(tf.equal(predicted_labels,tf.argmax(y_label,1)),tf.float32));
cross_entropy=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=predicted_logits, labels=y_label));
gen_acc2=tf.reduce_mean(tf.cast(tf.equal(predicted_labels2,tf.argmax(y_label,1)),tf.float32));
cross_entropy2=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=predicted_logits2, labels=y_label));
gen_acc3=tf.reduce_mean(tf.cast(tf.equal(predicted_labels3,tf.argmax(y_label,1)),tf.float32));
cross_entropy3=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=predicted_logits3, labels=y_label));
gen_acc_avg=tf.reduce_mean(tf.cast(tf.equal(predicted_labels_avg,tf.argmax(y_label,1)),tf.float32));
cross_entropy_avg=tf.reduce_mean(-tf.reduce_sum(y_label*tf.log(predicted_prob_avg),1));
infi_norm=tf.reduce_mean(tf.norm(tf.reshape(abs(x_data-x_generated),[-1,FLAGS.img_size]),ord=np.inf,axis=1));
g_loss=-1*cross_entropy_avg;
optimizer = tf.train.AdamOptimizer(0.0001)
g_trainer = optimizer.minimize(g_loss, var_list=g_params)
# get the data and label
img_list=np.sort(glob.glob(FLAGS.input_folder+"*.png"));
total_data=np.zeros((len(img_list),FLAGS.img_height,FLAGS.img_width,3),dtype=float);
for i in range(len(img_list)):
total_data[i]=imread(img_list[i],mode='RGB').astype(np.float) / 255.0;
total_data[i]=total_data[i]*2.0-1.0; # 0~1 -> -1~1
val_data=np.copy(total_data[0]);
f=open(FLAGS.label_folder+"true_label","r");
total_label2=np.array([i[:-1].split(",")[1] for i in f.readlines()],dtype=int);
total_label=np.zeros((len(total_data),FLAGS.num_classes),dtype=int);
for i in range(len(total_data)):
total_label[i,total_label2[i]]=1;
val_label=np.copy(total_label[0]);
# shuffle
total_idx=range(len(total_data)); np.random.shuffle(total_idx);
total_data=total_data[total_idx];total_label=total_label[total_idx];
# Run computation
saver = tf.train.Saver(disc_var_list);
saver2 = tf.train.Saver(disc_var_list2);
saver3 = tf.train.Saver(disc_var_list3);
saver_gen = tf.train.Saver(g_params);
# initialization
init = tf.global_variables_initializer();
with tf.Session() as sess:
sess.run(init)
saver.restore(sess,FLAGS.checkpoint_path+FLAGS.checkpoint_file_name);
saver2.restore(sess,FLAGS.checkpoint_path+FLAGS.checkpoint_file_name2);
saver3.restore(sess,FLAGS.checkpoint_path+FLAGS.checkpoint_file_name3);
# training
for i in range(FLAGS.max_epoch):
tr_infi=0;
tr_ce=0;
tr_gen_acc=0;
tr_ce2=0;
tr_gen_acc2=0;
tr_ce3=0;
tr_gen_acc3=0;
tr_ce_avg=0;
tr_gen_acc_avg=0;
for j in range(len(total_data) / FLAGS.batch_size):
batch_data=total_data[j*FLAGS.batch_size:(j+1)*FLAGS.batch_size];
batch_label=total_label[j*FLAGS.batch_size:(j+1)*FLAGS.batch_size];
acc_p_a,ce_p_a,acc_p3,ce_p3,acc_p2,ce_p2,acc_p,ce_p,infi_p,_=sess.run([gen_acc_avg,cross_entropy_avg,gen_acc3,cross_entropy3,gen_acc2,cross_entropy2,gen_acc,cross_entropy,infi_norm,g_trainer],feed_dict={x_data: batch_data, y_label: batch_label});
tr_infi+=infi_p;
tr_ce+=ce_p;
tr_gen_acc+=acc_p;
tr_ce2+=ce_p2;
tr_gen_acc2+=acc_p2;
tr_ce3+=ce_p3;
tr_gen_acc3+=acc_p3;
tr_ce_avg+=ce_p_a;
tr_gen_acc_avg+=acc_p_a;
print(str(i+1)+" Epoch InfiNorm:"+str(tr_infi/(j+1))+",CE: "+str(tr_ce/(j+1))+",Acc: "+str(tr_gen_acc/(j+1))+",CE2: "+str(tr_ce2/(j+1))+",Acc2: "+str(tr_gen_acc2/(j+1))+",CE3: "+str(tr_ce3/(j+1))+",Acc3: "+str(tr_gen_acc3/(j+1))+",Acc_avg: "+str(tr_gen_acc_avg/(j+1))+",CE_avg: "+str(tr_ce_avg/(j+1)));
total_idx=range(len(total_data)); np.random.shuffle(total_idx);
total_data=total_data[total_idx];total_label=total_label[total_idx];
saver_gen.save(sess,"my-models_iv3_rv250_rv2152_avg2/my-model_"+str(FLAGS.max_epsilon)+".ckpt");
def graph_incv3(x, y, i, x_max, x_min, grad, grad_D):
eps = 2.0 * FLAGS.max_epsilon / 255.0
kd = FLAGS.derivative
momentum = FLAGS.momentum
num_iter = FLAGS.num_iter
alpha = eps / FLAGS.num_iter
tf.get_variable_scope().reuse_variables()
batch_shape = [FLAGS.batch_size, 299, 299, 3]
x_nes = x + alpha * momentum * grad
x_b = x - alpha * grad_D
g_1 = tf.zeros(shape=batch_shape)
for j1 in range(5):
with slim.arg_scope(inception.inception_v3_arg_scope()):
logits_1, end_points_1 = inception.inception_v3(
input_diversity(x_b),
num_classes=FLAGS.num_classes,
is_training=False)
cross_entropy_1 = tf.losses.softmax_cross_entropy(y,
logits_1,
label_smoothing=0.0,
weights=1.0)
n_1 = tf.gradients(cross_entropy_1, x_b)[0]
g_1 = g_1 + n_1
x_lt = x_b / 2.0
noise_1 = g_1 / tf.constant(5.0)
noise_1 = tf.nn.depthwise_conv2d(noise_1,
stack_kernel,
strides=[1, 1, 1, 1],
padding='SAME')
noise_1 = noise_1 / tf.reduce_mean(tf.abs(noise_1), [1, 2, 3],
keep_dims=True)
g_2 = tf.zeros(shape=batch_shape)
for j2 in range(5):
with slim.arg_scope(inception.inception_v3_arg_scope()):
logits, end_points = inception.inception_v3(
input_diversity(x_nes),
num_classes=FLAGS.num_classes,
is_training=False)
#logits = (end_points['Logits'])
cross_entropy = tf.losses.softmax_cross_entropy(y,
logits,
label_smoothing=0.0,
weights=1.0)
n_2 = tf.gradients(cross_entropy, x)[0]
g_2 = g_2 + n_2
x_nes = x_nes / 2.0
noise = g_2 / tf.constant(5.0)
noise = tf.nn.depthwise_conv2d(noise,
stack_kernel,
strides=[1, 1, 1, 1],
padding='SAME')
noise = noise / tf.reduce_mean(tf.abs(noise), [1, 2, 3], keep_dims=True)
grad_D = grad_D + kd * (noise - noise_1)
noise_all = momentum * grad + noise - grad_D
x = x + alpha * tf.sign(noise_all)
x = tf.clip_by_value(x, x_min, x_max)
i = tf.add(i, 1)
return x, y, i, x_max, x_min, noise_all, grad_D
from nets import inception, resnet_v1
from preprocessing import vgg_preprocessing, inception_preprocessing
from tensorflow.contrib import slim
import h5py
df = pd.read_csv('sample_submission.csv')
synset = list(df.columns[1:])
inception_size = inception.inception_v3.default_image_size
resnet_size = resnet_v1.resnet_v1_152.default_image_size
model_dict = {
'InceptionV3': {
'model': inception.inception_v3,
'size': inception_size,
'scope': inception.inception_v3_arg_scope(),
'output': 'AvgPool_1a',
'numclasses': 1001,
'preprocessing': inception_preprocessing,
'ckpt_path': 'inception_v3.ckpt'
},
'resnet_v1_152': {
'model': resnet_v1.resnet_v1_152,
'size': resnet_size,
'scope': resnet_v1.resnet_arg_scope(),
'output': 'global_pool',
'numclasses': 1000,
'preprocessing': vgg_preprocessing,
'ckpt_path': 'resnet_v1_152.ckpt'
}
}
exit()
print('Start to read images!')
image_list = get_test_images(test_path)
processed_images = tf.placeholder(tf.float32,
shape=(None, image_size, image_size, 3))
if deep_lerning_architecture == "v1" or deep_lerning_architecture == "V1":
with slim.arg_scope(inception.inception_v1_arg_scope()):
logits, _ = inception.inception_v1(processed_images,
num_classes=nb_classes,
is_training=False)
else:
if deep_lerning_architecture == "v3" or deep_lerning_architecture == "V3":
with slim.arg_scope(inception.inception_v3_arg_scope()):
logits, _ = inception.inception_v3(processed_images,
num_classes=nb_classes,
is_training=False)
else:
if deep_lerning_architecture == "resv2" or deep_lerning_architecture == "inception_resnet2":
with slim.arg_scope(inception.inception_resnet_v2_arg_scope()):
logits, _ = inception.inception_resnet_v2(
processed_images,
num_classes=nb_classes,
is_training=False)
def predict_fn(images):
return session.run(probabilities, feed_dict={processed_images: images})
probabilities = tf.nn.softmax(logits)
def create(self, images, num_classes, is_training):
"""See baseclass."""
with slim.arg_scope(inception.inception_v3_arg_scope()):
_, endpoints = inception.inception_v3(
images, num_classes, create_aux_logits=False, is_training=is_training)
return endpoints
def main(_):
batch_shape = [FLAGS.batch_size, FLAGS.image_height, FLAGS.image_width, 3]
num_classes = 1001
# max_epsilon over checking
# get original images
origin_img_list=np.sort(glob.glob(FLAGS.origin_img_dir+"*.png"));
origin_imgs=np.zeros((len(origin_img_list),FLAGS.image_height,FLAGS.image_width,3),dtype=float);
for i in range(len(origin_img_list)):
origin_imgs[i]=imread(origin_img_list[i],mode='RGB').astype(np.float);
# get adv images
adv_img_list=np.sort(glob.glob(FLAGS.input_dir+"*.png"));
adv_imgs=np.zeros((len(adv_img_list),FLAGS.image_height,FLAGS.image_width,3),dtype=float);
for i in range(len(adv_img_list)):
adv_imgs[i]=imread(adv_img_list[i],mode='RGB').astype(np.float);
epsilon_list=np.linalg.norm(np.reshape(abs(origin_imgs-adv_imgs),[-1,FLAGS.image_height*FLAGS.image_width*3]),ord=np.inf,axis=1);
#print(epsilon_list);exit(1);
over_epsilon_list=np.zeros((len(origin_img_list),2),dtype=object);
cnt=0;
for i in range(len(origin_img_list)):
file_name=origin_img_list[i].split("/")[-1];
file_name=file_name.split(".")[0];
over_epsilon_list[i,0]=file_name;
if(epsilon_list[i]>FLAGS.max_epsilon):
over_epsilon_list[i,1]="1";
cnt+=1;
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
# Prepare graph
x_input = tf.placeholder(tf.float32, shape=batch_shape)
if(FLAGS.checkpoint_file_name=="inception_v3.ckpt"):
with slim.arg_scope(inception.inception_v3_arg_scope()):
_, end_points = inception.inception_v3(
x_input, num_classes=num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['Predictions'], 1)
elif(FLAGS.checkpoint_file_name=="inception_v4.ckpt"):
with slim.arg_scope(inception.inception_v4_arg_scope()):
_, end_points = inception.inception_v4(
x_input, num_classes=num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['Predictions'], 1)
elif(FLAGS.checkpoint_file_name=="inception_resnet_v2_2016_08_30.ckpt"):
with slim.arg_scope(inception.inception_resnet_v2_arg_scope()):
_, end_points = inception.inception_resnet_v2(
x_input, num_classes=num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['Predictions'], 1)
elif(FLAGS.checkpoint_file_name=="resnet_v2_101.ckpt"):
x_input2 = tf.image.resize_bilinear(x_input,[224,224],align_corners=False);
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = resnet_v2.resnet_v2_101(
x_input2, num_classes=num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['predictions'], 1)
elif(FLAGS.checkpoint_file_name=="resnet_v2_50.ckpt"):
x_input2 = tf.image.resize_bilinear(x_input,[224,224],align_corners=False);
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = resnet_v2.resnet_v2_50(
x_input2, num_classes=num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['predictions'], 1)
elif(FLAGS.checkpoint_file_name=="resnet_v2_152.ckpt"):
x_input2 = tf.image.resize_bilinear(x_input,[224,224],align_corners=False);
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = resnet_v2.resnet_v2_152(
x_input2, num_classes=num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['predictions'], 1)
elif(FLAGS.checkpoint_file_name=="inception_v1.ckpt"):
x_input2 = tf.image.resize_bilinear(x_input,[224,224],align_corners=False);
with slim.arg_scope(inception.inception_v1_arg_scope()):
_, end_points = inception.inception_v1(
x_input2, num_classes=num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['Predictions'], 1)
elif(FLAGS.checkpoint_file_name=="inception_v2.ckpt"):
x_input2 = tf.image.resize_bilinear(x_input,[224,224],align_corners=False);
with slim.arg_scope(inception.inception_v2_arg_scope()):
_, end_points = inception.inception_v2(
x_input2, num_classes=num_classes, is_training=False)
predicted_labels = tf.argmax(end_points['Predictions'], 1)
# Resnet v1 and vgg are not working now
elif(FLAGS.checkpoint_file_name=="vgg_16.ckpt"):
x_input_list=tf.unstack(x_input,FLAGS.batch_size,0);
for i in range(FLAGS.batch_size):
x_input_list[i]=vgg_preprocessing.preprocess_image(x_input_list[i],224,224);
x_input2=tf.stack(x_input_list,0);
with slim.arg_scope(vgg.vgg_arg_scope()):
_, end_points = vgg.vgg_16(
x_input2, num_classes=num_classes-1, is_training=False)
predicted_labels = tf.argmax(end_points['vgg_16/fc8'], 1)+1
elif(FLAGS.checkpoint_file_name=="vgg_19.ckpt"):
x_input_list=tf.unstack(x_input,FLAGS.batch_size,0);
for i in range(FLAGS.batch_size):
x_input_list[i]=vgg_preprocessing.preprocess_image(x_input_list[i],224,224);
x_input2=tf.stack(x_input_list,0);
with slim.arg_scope(vgg.vgg_arg_scope()):
_, end_points = vgg.vgg_19(
x_input2, num_classes=num_classes-1, is_training=False)
predicted_labels = tf.argmax(end_points['vgg_19/fc8'], 1)+1
elif(FLAGS.checkpoint_file_name=="resnet_v1_50.ckpt"):
x_input_list=tf.unstack(x_input,FLAGS.batch_size,0);
for i in range(FLAGS.batch_size):
x_input_list[i]=vgg_preprocessing.preprocess_image(x_input_list[i],224,224);
x_input2=tf.stack(x_input_list,0);
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = resnet_v1.resnet_v1_50(
x_input, num_classes=num_classes-1, is_training=False)
predicted_labels = tf.argmax(end_points['predictions'], 1)+1
elif(FLAGS.checkpoint_file_name=="resnet_v1_101.ckpt"):
x_input_list=tf.unstack(x_input,FLAGS.batch_size,0);
for i in range(FLAGS.batch_size):
x_input_list[i]=vgg_preprocessing.preprocess_image(x_input_list[i],224,224);
x_input2=tf.stack(x_input_list,0);
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = resnet_v1.resnet_v1_101(
x_input2, num_classes=num_classes-1, is_training=False)
predicted_labels = tf.argmax(end_points['predictions'], 1)+1
elif(FLAGS.checkpoint_file_name=="resnet_v1_152.ckpt"):
x_input_list=tf.unstack(x_input,FLAGS.batch_size,0);
for i in range(FLAGS.batch_size):
x_input_list[i]=vgg_preprocessing.preprocess_image(x_input_list[i],224,224);
x_input2=tf.stack(x_input_list,0);
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = resnet_v1.resnet_v1_152(
x_input2, num_classes=num_classes-1, is_training=False)
predicted_labels = tf.argmax(end_points['predictions'], 1)+1
# Run computation
saver = tf.train.Saver(slim.get_model_variables())
session_creator = tf.train.ChiefSessionCreator(
scaffold=tf.train.Scaffold(saver=saver),
checkpoint_filename_with_path=FLAGS.checkpoint_path+FLAGS.checkpoint_file_name,
master=FLAGS.master)
f=open(FLAGS.true_label,"r");
t_label_list=np.array([i[:-1].split(",") for i in f.readlines()]);
score=0;
with tf.train.MonitoredSession(session_creator=session_creator) as sess:
with tf.gfile.Open(FLAGS.output_file, 'w') as out_file:
for filenames, images in load_images(FLAGS.input_dir, batch_shape):
labels = sess.run(predicted_labels, feed_dict={x_input: images})
for filename, label in zip(filenames, labels):
f_name=filename.split(".")[0];
t_label=int(t_label_list[t_label_list[:,0]==f_name,1][0]);
if(t_label!=label):
if(over_epsilon_list[over_epsilon_list[:,0]==f_name,1]!="1"):
score+=1;
#out_file.write('{0},{1}\n'.format(filename, label))
print("Over max epsilon#: "+str(cnt));
print(str(FLAGS.max_epsilon)+" max epsilon Score: "+str(score));
def main(_):
batch_shape = [FLAGS.batch_size, FLAGS.image_height, FLAGS.image_width, 3]
num_classes = 1001
ensemble_type = FLAGS.ensemble_type
tf.logging.set_verbosity(tf.logging.INFO)
checkpoint_path_list = [
FLAGS.checkpoint_path_inception_v1, FLAGS.checkpoint_path_inception_v2,
FLAGS.checkpoint_path_inception_v3, FLAGS.checkpoint_path_inception_v4,
FLAGS.checkpoint_path_inception_resnet_v2,
FLAGS.checkpoint_path_resnet_v1_101,
FLAGS.checkpoint_path_resnet_v1_152,
FLAGS.checkpoint_path_resnet_v2_101,
FLAGS.checkpoint_path_resnet_v2_152, FLAGS.checkpoint_path_vgg_16,
FLAGS.checkpoint_path_vgg_19
]
normalization_method = [
'default', 'default', 'default', 'default', 'global', 'caffe_rgb',
'caffe_rgb', 'default', 'default', 'caffe_rgb', 'caffe_rgb'
]
pred_list = []
for idx, checkpoint_path in enumerate(checkpoint_path_list, 1):
with tf.Graph().as_default():
if int(FLAGS.test_idx) == 20 and idx in [3]:
continue
if int(FLAGS.test_idx) in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11
] and int(FLAGS.test_idx) != idx:
continue
# Prepare graph
if idx in [1, 2, 6, 7, 10, 11]:
_x_input = tf.placeholder(tf.float32, shape=batch_shape)
x_input = tf.image.resize_images(_x_input, [224, 224])
else:
_x_input = tf.placeholder(tf.float32, shape=batch_shape)
x_input = _x_input
x_input = image_normalize(x_input, normalization_method[idx - 1])
if idx == 1:
with slim.arg_scope(inception.inception_v1_arg_scope()):
_, end_points = inception.inception_v1(
x_input, num_classes=num_classes, is_training=False)
elif idx == 2:
with slim.arg_scope(inception.inception_v2_arg_scope()):
_, end_points = inception.inception_v2(
x_input, num_classes=num_classes, is_training=False)
elif idx == 3:
with slim.arg_scope(inception.inception_v3_arg_scope()):
_, end_points = inception.inception_v3(
x_input, num_classes=num_classes, is_training=False)
elif idx == 4:
with slim.arg_scope(inception.inception_v4_arg_scope()):
_, end_points = inception.inception_v4(
x_input, num_classes=num_classes, is_training=False)
elif idx == 5:
with slim.arg_scope(inception.inception_resnet_v2_arg_scope()):
_, end_points = inception.inception_resnet_v2(
x_input, num_classes=num_classes, is_training=False)
elif idx == 6:
with slim.arg_scope(resnet_v1.resnet_arg_scope()):
_, end_points = resnet_v1.resnet_v1_101(x_input,
num_classes=1000,
is_training=False)
elif idx == 7:
with slim.arg_scope(resnet_v1.resnet_arg_scope()):
_, end_points = resnet_v1.resnet_v1_152(x_input,
num_classes=1000,
is_training=False)
elif idx == 8:
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
_, end_points = resnet_v2.resnet_v2_101(
x_input, num_classes=num_classes, is_training=False)
elif idx == 9:
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
_, end_points = resnet_v2.resnet_v2_152(
x_input, num_classes=num_classes, is_training=False)
elif idx == 10:
with slim.arg_scope(vgg.vgg_arg_scope()):
_, end_points = vgg.vgg_16(x_input,
num_classes=1000,
is_training=False)
end_points['predictions'] = tf.nn.softmax(
end_points['vgg_16/fc8'])
elif idx == 11:
with slim.arg_scope(vgg.vgg_arg_scope()):
_, end_points = vgg.vgg_19(x_input,
num_classes=1000,
is_training=False)
end_points['predictions'] = tf.nn.softmax(
end_points['vgg_19/fc8'])
#end_points = tf.reduce_mean([end_points1['Predictions'], end_points2['Predictions'], end_points3['Predictions'], end_points4['Predictions']], axis=0)
#predicted_labels = tf.argmax(end_points, 1)
# Run computation
saver = tf.train.Saver(slim.get_model_variables())
session_creator = tf.train.ChiefSessionCreator(
scaffold=tf.train.Scaffold(saver=saver),
checkpoint_filename_with_path=checkpoint_path,
master=FLAGS.master)
pred_in = []
filenames_list = []
with tf.train.MonitoredSession(
session_creator=session_creator) as sess:
for filenames, images in load_images(FLAGS.input_dir,
batch_shape):
#if idx in [1,2,6,7,10,11]:
# # 16x299x299x3
# images = zoom(images, (1, 0.7491638795986622, 0.7491638795986622, 1), order=2)
filenames_list.extend(filenames)
end_points_dict = sess.run(end_points,
feed_dict={_x_input: images})
if idx in [6, 7, 10, 11]:
end_points_dict['predictions'] = \
np.concatenate([np.zeros([FLAGS.batch_size, 1]),
np.array(end_points_dict['predictions'].reshape(-1, 1000))],
axis=1)
try:
pred_in.extend(end_points_dict['Predictions'].reshape(
-1, num_classes))
except KeyError:
pred_in.extend(end_points_dict['predictions'].reshape(
-1, num_classes))
pred_list.append(pred_in)
if ensemble_type == 'mean':
pred = np.mean(pred_list, axis=0)
labels = np.argmax(
pred, axis=1
) # model_num X batch X class_num ==(np.mean)==> batch X class_num ==(np.argmax)==> batch
elif ensemble_type == 'vote':
pred = np.argmax(
pred_list, axis=2
) # model_num X batch X class_num ==(np.mean)==> batch X class_num ==(np.argmax)==> batch
labels = np.median(pred, axis=0)
with tf.gfile.Open(FLAGS.output_file, 'w') as out_file:
for filename, label in zip(filenames_list, labels):
out_file.write('{0},{1}\n'.format(filename, label))
def run(self, flist, labels):
kf = StratifiedKFold(n_splits=10, random_state=1, shuffle=True)
all_corr = pickle.load(
open('./correlations_file' + p_ROI + '.pkl', 'rb'))
np.random.shuffle(flist)
y_arr = np.array([utils.get_label(f, labels) for f in flist])
input_images = tf.placeholder(
shape=[None, self.central_size, self.central_size, 1],
dtype=tf.float32)
Y = tf.placeholder(tf.float32, shape=[None, 2])
MLM_Index = tf.placeholder(tf.int32, shape=[
None,
])
isTraining = tf.placeholder(tf.bool)
train_mode = tf.count_nonzero(isTraining)
if self.network_name == 'resnet_v2_50':
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
adv_data, var_noises = self.adv_program(
input_images, isTraining)
self.imagenet_logits, self.imagenet_prob = resnet_v2.resnet_v2_50(
adv_data, num_classes=1001, is_training=False)
self.imagenet_prob = self.imagenet_prob[:, 1:]
self.top_k_value, self.top_k_indices = tf.math.top_k(
self.imagenet_prob, 30)
self.freq_prob = mapping_func.freq_mapping(
self.imagenet_prob, MLM_Index)
#self.disturbed_prob = tf.matmul(self.imagenet_prob, mapping_func.label_mapping())
#self.multi_label_prob = mapping_func.multi_prob(self.disturbed_prob)
init_fn = slim.assign_from_checkpoint_fn(
os.path.join(self.pretrained_model_dir,
self.network_name + '.ckpt'),
slim.get_model_variables('resnet_v2_50'))
if self.network_name == 'inception_v3':
print("using inception")
with slim.arg_scope(inception.inception_v3_arg_scope()):
adv_data, var_noises = self.adv_program(
input_images, isTraining)
print('adversarial image shape:', adv_data.shape)
self.imagenet_logits, self.imagenet_prob, self.pre_logits = inception_v3.inception_v3(
adv_data, num_classes=1001, is_training=False)
self.imagenet_prob = self.imagenet_prob[:, 1:]
self.top_k_value, self.top_k_indices = tf.math.top_k(
self.imagenet_prob, 30)
self.freq_prob = mapping_func.freq_mapping(
self.imagenet_prob, MLM_Index)
#self.disturbed_prob = tf.matmul(self.imagenet_prob, self.label_mapping())
#self.multi_label_prob = self.multi_prob(self.disturbed_prob)
init_fn = slim.assign_from_checkpoint_fn(
os.path.join(self.pretrained_model_dir,
self.network_name + '.ckpt'),
slim.get_model_variables('InceptionV3'))
## Compute Gradient
y1 = tf.tile(Y, [self.q_batch + 1, 1])
#self.cross_entropy_loss = -tf.reduce_sum(y1 *tf.log(self.freq_prob + 1e-6), axis=1)
self.focal_loss = cal_func.multi_focal_loss(4, 0.8, y1, self.freq_prob)
estimate_grad = estimator.func(self.focal_loss, self.q_batch,
var_noises)
eGrad = tf.placeholder(tf.float32,
shape=[1, self.image_size, self.image_size, 3])
gradt = tf.gradients(self.focal_loss[0:self.batchsize], self.W)[0]
## Optimize
global_steps = tf.Variable(0, trainable=False)
starter_learning_rate = 0.01
end_learning_rate = 0.001
decay_steps = 2000
decay_rate = 0.96
learning_rate = tf.train.polynomial_decay(starter_learning_rate,
global_steps,
decay_steps,
end_learning_rate,
power=0.9)
gradvars = [(gradt, self.W)]
optim = tf.train.AdamOptimizer(0.01)
Step = optim.apply_gradients(gradvars, global_step=global_steps)
## Compute accuracy
correct_prediction = tf.equal(
tf.argmax(self.freq_prob[0:self.batchsize], 1), tf.argmax(Y, 1))
predict_probilities = tf.reduce_sum(self.freq_prob[0:self.batchsize] *
Y,
axis=-1)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
## Training
saver = tf.train.Saver()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
##Start training with mini batch size
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
init_opt = tf.global_variables_initializer()
## run cross-validation trainining with 10 split training set
for kk, (train_index, test_index) in enumerate(kf.split(flist, y_arr)):
sess.run(init_opt)
init_fn(sess)
train_samples = np.array(flist)[train_index]
test_samples = np.array(flist)[test_index]
train_data, train_label = utils.process_data(
all_corr, train_samples)
test_data, test_label = utils.process_data(all_corr, test_samples)
total_batch = int(train_data.shape[0] / self.batchsize)
epoch_starttime = time()
Asd = np.zeros((1000, 2))
NonAsd = np.zeros((1000, 2))
asd_cnt = 0
nonasd_cnt = 0
for i in range(train_data.shape[0]):
x_train_batch = np.expand_dims(train_data[i], axis=0)
y_train_batch = np.expand_dims(train_label[i], axis=0)
adv_image_batch, Top_k_indices, Top_k_values = sess.run(
[adv_data, self.top_k_indices, self.top_k_value],
feed_dict={
input_images: x_train_batch,
Y: y_train_batch,
isTraining: True
})
if y_train_batch[0][0] == 1:
for i in range(30):
NonAsd[Top_k_indices[0][i]][0] += Top_k_values[0][i]
NonAsd[Top_k_indices[0][i]][1] += 1
asd_cnt += 1
elif y_train_batch[0][1] == 1:
for i in range(30):
Asd[Top_k_indices[0][i]][0] += Top_k_values[0][i]
Asd[Top_k_indices[0][i]][1] += 1
nonasd_cnt += 1
mlm_index = mapping_func.freq_idx_search(NonAsd, Asd, self.mlm_num)
best_result = np.zeros(4)
for epoch in range(self.max_epoch):
epoch_starttime = time()
batch_loss = []
total_loss = []
#np.append(topK_array, Top_k_indices[0:self.batchsize], axis=0)
#shuffle#
s = np.arange(train_data.shape[0])
s = np.random.shuffle(s)
train_data = train_data[s][0]
train_label = train_label[s][0]
for batch in range(total_batch):
loss_list = []
est_Glist = []
x_train_batch = train_data[batch *
self.batchsize:(batch + 1) *
self.batchsize]
y_train_batch = train_label[batch *
self.batchsize:(batch + 1) *
self.batchsize]
for q_batch in range(int(self.num_rand_vec /
self.q_batch)):
adv_image_batch, Losses, est_Grad, tr_mode = sess.run(
[
adv_data, self.focal_loss, estimate_grad,
train_mode
],
feed_dict={
input_images: x_train_batch,
Y: y_train_batch,
MLM_Index: mlm_index,
isTraining: True
})
loss_list.append(np.mean(Losses, axis=0))
est_Glist.append(est_Grad)
avg_loss = np.mean(loss_list, axis=0, keepdims=False)
batch_loss.append(avg_loss)
avg_Glist = np.mean(est_Glist, axis=0, keepdims=True)
#weight, _ = sess.run([self.W, Step], feed_dict = {eGrad: avg_Glist, MLM_Index:mlm_index, isTraining:True})
weight, _ = sess.run(
[self.W, Step],
feed_dict={
input_images: x_train_batch,
Y: y_train_batch,
MLM_Index: mlm_index,
isTraining: True
})
## validate performance per 5 mini batch
if batch % 10 == 0:
valid_batch_acc, pred_probs, img_X_adv, tr_mode, LR, gp = sess.run([accuracy, predict_probilities, adv_data, train_mode, learning_rate, global_steps], \
feed_dict = {input_images: x_train_batch,Y: y_train_batch, MLM_Index:mlm_index, isTraining:False})
valid_acc = float(valid_batch_acc / self.batchsize)
#print('train mode: ', tr_mode)
print('epoch:{:03d}/{:03d}, batch: {:04d}/{}, loss: {:.4f}, valid_acc: {:.2f}'.format(epoch,\
self.max_epoch,batch,total_batch ,np.mean(batch_loss), valid_batch_acc))
total_loss.append(np.mean(batch_loss))
batch_loss = []
## save model per epoch
if (epoch + 1) % self.save_freq == 0:
saver.save(
sess,
os.path.join(self.train_dir + '_' + str(kk),
'model_{:06d}.ckpt'.format(epoch + 1)))
print('model_{:06d}.ckpt saved'.format(epoch + 1))
## End of Training
epoch_duration = time() - epoch_starttime
print("Training this epoch takes:",
"{:.2f}".format(epoch_duration))
print("Total average loss is:",
"{:.2f}".format(np.mean(total_loss)))
testing_start = time()
test_total_batch = int(test_data.shape[0] / self.batchsize)
Test_acc = 0.0
test_acc_sum = 0.0
test_sen_sum = 0.0
test_spe_sum = 0.0
for i in range(test_total_batch):
test_image_batch = test_data[i * self.batchsize:(i + 1) *
self.batchsize]
test_label_batch = test_label[i * self.batchsize:(i + 1) *
self.batchsize]
test_acc, test_batch_result, tr_mode = sess.run(
[
accuracy, self.freq_prob[0:self.batchsize],
train_mode
],
feed_dict={
input_images: test_image_batch,
Y: test_label_batch,
MLM_Index: mlm_index,
isTraining: False
})
test_batch_acc, test_batch_sen, test_batch_spe = cal_func.confusion(
np.argmax(test_batch_result, axis=1),
np.argmax(test_label_batch, axis=1))
label = np.argmax(test_label_batch[0], axis=0)
Test_acc += test_acc
test_acc_sum += test_batch_acc
test_sen_sum += test_batch_sen
test_spe_sum += test_batch_spe
final_acc = float(Test_acc / test_total_batch)
test_acc = float(test_acc_sum / test_total_batch)
test_sen = float(test_sen_sum / test_total_batch)
test_spe = float(test_spe_sum / test_total_batch)
testing_duration = time() - testing_start
if test_acc > best_result[1]:
best_result[:] = [epoch, test_acc, test_sen, test_spe]
elif test_acc == best_result[1]:
if test_sen > best_result[2]:
best_result[:] = [epoch, test_acc, test_sen, test_spe]
elif test_sen == best_result[2]:
if test_spe > best_result[3]:
best_result[:] = [
epoch, test_acc, test_sen, test_spe
]
#print('Test accuracy: {:.4f}'.format(final_acc))
#print('test accuracy: {:.4f} | test sensitivity: {:.4f} | test speciality: {:.4f}'.format(test_acc,test_sen, test_spe))
print(
'best accuracy: {:.4f} | best sensitivity: {:.4f} | best speciality: {:.4f}'
.format(best_result[1], best_result[2], best_result[3]))
with open('./check/ASD_withMLM_accuracy_focal.csv',
'a',
newline='') as csvfile:
writer = csv.writer(csvfile, delimiter=',')
writer.writerow([
kk, best_result[0], best_result[1], best_result[2],
best_result[3]
])
print(
"Testing finished takes:{:.2f} secs".format(testing_duration))
