def main(_):
assert FLAGS.output_dir, '--output_dir has to be provided'
if not tf.gfile.Exists(FLAGS.output_dir):
tf.gfile.MakeDirs(FLAGS.output_dir)
params = model_lib.default_hparams()
params.parse(FLAGS.hparams)
tf.logging.info('User provided hparams: %s', FLAGS.hparams)
tf.logging.info('All hyper parameters: %s', params)
batch_size = params.batch_size
graph = tf.Graph()
with graph.as_default():
with tf.device(
tf.train.replica_device_setter(ps_tasks=FLAGS.ps_tasks)):
# dataset
dataset, examples_per_epoch, num_classes, bounds = (
dataset_factory.get_dataset(FLAGS.dataset,
'train',
batch_size,
FLAGS.dataset_image_size,
is_training=True))
dataset_iterator = dataset.make_one_shot_iterator()
images, labels = dataset_iterator.get_next()
one_hot_labels = tf.one_hot(labels, num_classes)
# set up model
global_step = tf.train.get_or_create_global_step()
model_fn = model_lib.get_model(FLAGS.model_name, num_classes)
if params.train_adv_method == 'clean':
logits = model_fn(images, is_training=True)
adv_examples = None
else:
model_fn_eval_mode = lambda x: model_fn(x, is_training=False)
adv_examples = adversarial_attack.generate_adversarial_examples(
images, bounds, model_fn_eval_mode,
params.train_adv_method)
all_examples = tf.concat([images, adv_examples], axis=0)
logits = model_fn(all_examples, is_training=True)
one_hot_labels = tf.concat([one_hot_labels, one_hot_labels],
axis=0)
# update trainable variables if fine tuning is used
model_lib.filter_trainable_variables(
FLAGS.finetune_trainable_scopes)
# set up losses
total_loss = tf.losses.softmax_cross_entropy(
onehot_labels=one_hot_labels,
logits=logits,
label_smoothing=params.label_smoothing)
tf.summary.scalar('loss_xent', total_loss)
if params.train_lp_weight > 0:
images1, images2 = tf.split(logits, 2)
loss_lp = tf.losses.mean_squared_error(
images1, images2, weights=params.train_lp_weight)
tf.summary.scalar('loss_lp', loss_lp)
total_loss += loss_lp
if params.weight_decay > 0:
loss_wd = (params.weight_decay * tf.add_n(
[tf.nn.l2_loss(v) for v in tf.trainable_variables()]))
tf.summary.scalar('loss_wd', loss_wd)
total_loss += loss_wd
# Setup the moving averages:
if FLAGS.moving_average_decay and (FLAGS.moving_average_decay > 0):
with tf.name_scope('moving_average'):
moving_average_variables = tf.contrib.framework.get_model_variables(
)
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
else:
moving_average_variables = None
variable_averages = None
# set up optimizer and training op
learning_rate, steps_per_epoch = model_lib.get_lr_schedule(
params, examples_per_epoch, FLAGS.replicas_to_aggregate)
tf.Print(learning_rate, [learning_rate],
'THis issssssssssssssssss learning_rate:')
optimizer = model_lib.get_optimizer(params, learning_rate)
optimizer = tf.train.SyncReplicasOptimizer(
opt=optimizer,
replicas_to_aggregate=FLAGS.replicas_to_aggregate,
total_num_replicas=FLAGS.worker_replicas,
variable_averages=variable_averages,
variables_to_average=moving_average_variables)
train_op = tf.contrib.training.create_train_op(
total_loss,
optimizer,
update_ops=tf.get_collection(tf.GraphKeys.UPDATE_OPS))
tf.summary.image('images', images[0:FLAGS.num_summary_images])
if adv_examples is not None:
tf.summary.image('adv_images',
adv_examples[0:FLAGS.num_summary_images])
tf.summary.scalar('total_loss', total_loss)
tf.summary.scalar('learning_rate', learning_rate)
tf.summary.scalar('current_epoch',
tf.to_double(global_step) / steps_per_epoch)
# Training
is_chief = FLAGS.task == 0
scaffold = tf.train.Scaffold(
init_fn=_get_finetuning_init_fn(variable_averages))
hooks = [
tf.train.LoggingTensorHook(
{
'learning_rate': learning_rate,
'total_loss': total_loss,
'global_step': global_step
},
every_n_iter=20),
tf.train.NanTensorHook(total_loss),
]
chief_only_hooks = [
tf.train.SummarySaverHook(
save_steps=FLAGS.save_summaries_steps,
save_secs=FLAGS.save_summaries_secs,
output_dir=FLAGS.output_dir,
scaffold=scaffold),
tf.train.CheckpointSaverHook(FLAGS.output_dir,
save_steps=FLAGS.save_model_steps,
scaffold=scaffold),
]
if FLAGS.max_steps > 0:
hooks.append(
tf.train.StopAtStepHook(last_step=FLAGS.max_steps))
# hook for sync replica training
hooks.append(optimizer.make_session_run_hook(is_chief))
with tf.train.MonitoredTrainingSession(
#config=tf.ConfigProto(log_device_placement=True),
master=FLAGS.master,
is_chief=is_chief,
checkpoint_dir=FLAGS.output_dir,
scaffold=scaffold,
hooks=hooks,
chief_only_hooks=chief_only_hooks,
save_checkpoint_secs=None,
save_summaries_steps=None,
save_summaries_secs=None) as session:
while not session.should_stop():
session.run([train_op])
def main(_):
print(FLAGS.task_ids)
task_ids = FLAGS.task_ids.split(' ')
assert FLAGS.output_dir, '--output_dir has to be provided'
if not tf.gfile.Exists(FLAGS.output_dir):
tf.gfile.MakeDirs(FLAGS.output_dir)
params = model_lib.default_hparams()
params.parse(FLAGS.hparams)
tf.logging.info('User provided hparams: %s', FLAGS.hparams)
tf.logging.info('All hyper parameters: %s', params)
batch_size = params.batch_size
graph = tf.Graph()
with graph.as_default():
if FLAGS.job_name in ['ps', 'worker']:
pss = ["localhost:2210"]
workers = [
"localhost:221" + str(i + 1) for i in range(len(task_ids))
]
print(workers)
cluster = tf.train.ClusterSpec({"ps": pss, "worker": workers})
server = tf.train.Server(cluster,
job_name=FLAGS.job_name,
task_index=FLAGS.task)
if FLAGS.job_name == "ps":
server.join()
else:
if FLAGS.job_name == "worker":
device = tf.train.replica_device_setter(
worker_device="/job:worker/task:%d" % FLAGS.task,
ps_device="/job:ps/task:0",
cluster=cluster)
else:
device = '/device:GPU:0'
with tf.device(device):
img_bounds = (-1, 1) if 'finetune' in FLAGS.output_dir else (0,
1)
dataset, examples_per_epoch, num_classes, bounds = (
dataset_factory.get_dataset('tiny_imagenet',
'train',
batch_size,
64,
is_training=True,
bounds=img_bounds))
dataset_iterator = dataset.make_one_shot_iterator()
images, labels = dataset_iterator.get_next()
one_hot_labels = tf.one_hot(labels, num_classes)
# set up model
global_step = tf.train.get_or_create_global_step()
model_fn = model_lib.get_model(FLAGS.model_name, num_classes)
if FLAGS.add_noise:
images += tf.random_normal(images.shape,
mean=0,
stddev=16.0 / 255.0)
if params.train_adv_method == 'clean':
logits = model_fn(images, is_training=True)
adv_examples = None
correct = tf.equal(tf.argmax(logits, 1),
tf.argmax(one_hot_labels, 1))
train_acc = tf.reduce_mean(tf.cast(correct, tf.float32))
else:
model_fn_eval_mode = lambda x: model_fn(x,
is_training=False)
adv_examples = adversarial_attack.generate_adversarial_examples(
images, labels, bounds, model_fn_eval_mode,
params.train_adv_method)
all_examples = tf.concat([images, adv_examples], axis=0)
logits = model_fn(all_examples, is_training=True)
one_hot_labels = tf.concat(
[one_hot_labels, one_hot_labels], axis=0)
correct = tf.equal(
tf.argmax(logits[:batch_size], 1),
tf.argmax(one_hot_labels[:batch_size], 1))
train_acc = tf.reduce_mean(tf.cast(correct, tf.float32))
correct = tf.equal(
tf.argmax(logits[batch_size:], 1),
tf.argmax(one_hot_labels[batch_size:], 1))
adv_train_acc = tf.reduce_mean(tf.cast(
correct, tf.float32))
tf.summary.scalar('adv_train_acc', adv_train_acc)
tf.summary.scalar('train_acc', train_acc)
# update trainable variables if fine tuning is used
model_lib.filter_trainable_variables(
FLAGS.finetune_trainable_scopes)
# set up losses
# Note: batch_size is per-worker, not the global one
if 'Plain' in FLAGS.output_dir:
ce_labels, ce_logits = one_hot_labels[
0:batch_size], logits[0:batch_size]
elif '50% AT' in FLAGS.output_dir:
ce_labels, ce_logits = one_hot_labels[
0:2 * batch_size], logits[0:2 * batch_size]
elif '100% AT' in FLAGS.output_dir:
ce_labels, ce_logits = one_hot_labels[
batch_size:2 * batch_size], logits[batch_size:2 *
batch_size]
else:
ce_labels, ce_logits = one_hot_labels, logits
total_loss = tf.losses.softmax_cross_entropy(
onehot_labels=ce_labels,
logits=ce_logits,
label_smoothing=params.label_smoothing)
tf.summary.scalar('loss_xent', total_loss)
if params.train_lp_weight > 0:
images1, images2 = tf.split(logits, 2)
loss_lp = tf.losses.mean_squared_error(
images1, images2, weights=params.train_lp_weight)
tf.summary.scalar('loss_lp', loss_lp)
total_loss += loss_lp
if FLAGS.logit_squeezing > 0:
logit_squeezing_loss = tf.reduce_mean(
tf.norm(logits, ord='euclidean', axis=1))
total_loss += 0.05 * logit_squeezing_loss
tf.summary.scalar('logit_squeezing_loss',
logit_squeezing_loss)
if params.weight_decay > 0:
loss_wd = (params.weight_decay * tf.add_n(
[tf.nn.l2_loss(v) for v in tf.trainable_variables()]))
tf.summary.scalar('loss_wd', loss_wd)
total_loss += loss_wd
# Setup the moving averages:
if FLAGS.moving_average_decay and (FLAGS.moving_average_decay >
0):
with tf.name_scope('moving_average'):
moving_average_variables = tf.contrib.framework.get_model_variables(
)
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
else:
moving_average_variables = None
variable_averages = None
# set up optimizer and training op
learning_rate, steps_per_epoch = model_lib.get_lr_schedule(
params, examples_per_epoch, FLAGS.replicas_to_aggregate)
optimizer = model_lib.get_optimizer(params, learning_rate)
optimizer = tf.train.SyncReplicasOptimizer(
opt=optimizer,
replicas_to_aggregate=FLAGS.replicas_to_aggregate,
total_num_replicas=FLAGS.worker_replicas,
variable_averages=variable_averages,
variables_to_average=moving_average_variables)
train_op = tf.contrib.training.create_train_op(
total_loss,
optimizer,
update_ops=tf.get_collection(tf.GraphKeys.UPDATE_OPS))
tf.summary.image('images', images[0:FLAGS.num_summary_images])
if adv_examples is not None:
tf.summary.image('adv_images',
adv_examples[0:FLAGS.num_summary_images])
tf.summary.scalar('total_loss', total_loss)
tf.summary.scalar('learning_rate', learning_rate)
tf.summary.scalar('current_epoch',
tf.to_double(global_step) / steps_per_epoch)
# Training
is_chief = FLAGS.task == 0
scaffold = tf.train.Scaffold(
init_fn=_get_finetuning_init_fn(variable_averages))
hooks = [
tf.train.NanTensorHook(total_loss),
]
chief_only_hooks = [
tf.train.LoggingTensorHook(
{
'total_loss': total_loss,
'global_step': global_step
},
every_n_iter=10),
tf.train.SummarySaverHook(
save_steps=FLAGS.save_summaries_steps,
save_secs=FLAGS.save_summaries_secs,
output_dir=FLAGS.output_dir,
scaffold=scaffold),
tf.train.CheckpointSaverHook(
FLAGS.output_dir,
save_steps=FLAGS.save_model_steps,
scaffold=scaffold),
]
if FLAGS.max_steps > 0:
hooks.append(
tf.train.StopAtStepHook(last_step=FLAGS.max_steps))
# hook for sync replica training
hooks.append(optimizer.make_session_run_hook(is_chief))
if FLAGS.job_name == "worker":
master = server.target
else:
master = None
with tf.train.MonitoredTrainingSession(
master=master,
is_chief=is_chief,
checkpoint_dir=FLAGS.output_dir,
scaffold=scaffold,
hooks=hooks,
chief_only_hooks=chief_only_hooks,
save_checkpoint_secs=None,
save_summaries_steps=None,
save_summaries_secs=None,
config=tf.ConfigProto(
allow_soft_placement=True)) as session:
while not session.should_stop():
session.run([train_op])
def main(_):
if not FLAGS.train_dir and not FLAGS.checkpoint_path:
print(
'Either --train_dir or --checkpoint_path flags has to be provided.'
)
if FLAGS.train_dir and FLAGS.checkpoint_path:
print(
'Only one of --train_dir or --checkpoint_path should be provided.')
params = model_lib.default_hparams()
params.parse(FLAGS.hparams)
tf.logging.info('User provided hparams: %s', FLAGS.hparams)
tf.logging.info('All hyper parameters: %s', params)
batch_size = params.eval_batch_size
graph = tf.Graph()
with graph.as_default():
# dataset
dataset, num_examples, num_classes, bounds = dataset_factory.get_dataset(
FLAGS.dataset,
FLAGS.split_name,
batch_size,
FLAGS.dataset_image_size,
is_training=False)
dataset_iterator = dataset.make_one_shot_iterator()
images, labels = dataset_iterator.get_next()
if FLAGS.num_examples > 0:
num_examples = min(num_examples, FLAGS.num_examples)
# setup model
global_step = tf.train.get_or_create_global_step()
model_fn_two_args = model_lib.get_model(FLAGS.model_name, num_classes)
model_fn = lambda x: model_fn_two_args(x, is_training=False)
if not FLAGS.adv_method or FLAGS.adv_method == 'clean':
logits = model_fn(images)
else:
adv_examples = adversarial_attack.generate_adversarial_examples(
images, bounds, model_fn, FLAGS.adv_method)
logits = model_fn(adv_examples)
# update trainable variables if fine tuning is used
model_lib.filter_trainable_variables(FLAGS.trainable_scopes)
# Setup the moving averages
if FLAGS.moving_average_decay and (FLAGS.moving_average_decay > 0):
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
variables_to_restore = variable_averages.variables_to_restore(
tf.contrib.framework.get_model_variables())
variables_to_restore[global_step.op.name] = global_step
else:
variables_to_restore = tf.contrib.framework.get_variables_to_restore(
)
# Setup evaluation metric
with tf.name_scope('Eval'):
names_to_values, names_to_updates = (
tf.contrib.metrics.aggregate_metric_map({
'Accuracy':
tf.metrics.accuracy(labels, tf.argmax(logits, 1)),
'Top5':
tf.metrics.recall_at_k(tf.to_int64(labels), logits, 5)
}))
for name, value in names_to_values.iteritems():
tf.summary.scalar(name, value)
# Run evaluation
num_batches = int(num_examples / batch_size)
if FLAGS.train_dir:
output_dir = os.path.join(FLAGS.train_dir, FLAGS.eval_name)
if not tf.gfile.Exists(output_dir):
tf.gfile.MakeDirs(output_dir)
tf.contrib.training.evaluate_repeatedly(
FLAGS.train_dir,
master=FLAGS.master,
scaffold=tf.train.Scaffold(
saver=tf.train.Saver(variables_to_restore)),
eval_ops=names_to_updates.values(),
eval_interval_secs=FLAGS.eval_interval_secs,
hooks=[
tf.contrib.training.StopAfterNEvalsHook(num_batches),
tf.contrib.training.SummaryAtEndHook(output_dir),
tf.train.LoggingTensorHook(names_to_values, at_end=True),
],
max_number_of_evaluations=1 if FLAGS.eval_once else None)
else:
result = tf.contrib.training.evaluate_once(
FLAGS.checkpoint_path,
master=FLAGS.master,
scaffold=tf.train.Scaffold(
saver=tf.train.Saver(variables_to_restore)),
eval_ops=names_to_updates.values(),
final_ops=names_to_values,
hooks=[
tf.contrib.training.StopAfterNEvalsHook(num_batches),
tf.train.LoggingTensorHook(names_to_values, at_end=True),
])
if FLAGS.output_file:
with tf.gfile.Open(FLAGS.output_file, 'a') as f:
f.write(
'%s,%.3f,%.3f\n' %
(FLAGS.eval_name, result['Accuracy'], result['Top5']))
def main(_):
assert FLAGS.output_dir, '--output_dir has to be provided'
if not tf.gfile.Exists(FLAGS.output_dir):
tf.gfile.MakeDirs(FLAGS.output_dir)
params = model_lib.default_hparams()
params.parse(FLAGS.hparams)
tf.logging.info('User provided hparams: %s', FLAGS.hparams)
tf.logging.info('All hyper parameters: %s', params)
batch_size = params.batch_size
graph = tf.Graph()
with graph.as_default():
with tf.device(tf.train.replica_device_setter(ps_tasks=FLAGS.ps_tasks)):
# dataset
dataset, examples_per_epoch, num_classes, bounds = (
dataset_factory.get_dataset(
FLAGS.dataset,
'train',
batch_size,
FLAGS.dataset_image_size,
is_training=True))
dataset_iterator = dataset.make_one_shot_iterator()
images, labels = dataset_iterator.get_next()
one_hot_labels = tf.one_hot(labels, num_classes)
# set up model
global_step = tf.train.get_or_create_global_step()
model_fn = model_lib.get_model(FLAGS.model_name, num_classes)
if params.train_adv_method == 'clean':
logits = model_fn(images, is_training=True)
adv_examples = None
else:
model_fn_eval_mode = lambda x: model_fn(x, is_training=False)
adv_examples = adversarial_attack.generate_adversarial_examples(
images, bounds, model_fn_eval_mode, params.train_adv_method)
all_examples = tf.concat([images, adv_examples], axis=0)
logits = model_fn(all_examples, is_training=True)
one_hot_labels = tf.concat([one_hot_labels, one_hot_labels], axis=0)
# update trainable variables if fine tuning is used
model_lib.filter_trainable_variables(
FLAGS.finetune_trainable_scopes)
# set up losses
total_loss = tf.losses.softmax_cross_entropy(
onehot_labels=one_hot_labels,
logits=logits,
label_smoothing=params.label_smoothing)
tf.summary.scalar('loss_xent', total_loss)
if params.train_lp_weight > 0:
images1, images2 = tf.split(logits, 2)
loss_lp = tf.losses.mean_squared_error(
images1, images2, weights=params.train_lp_weight)
tf.summary.scalar('loss_lp', loss_lp)
total_loss += loss_lp
if params.weight_decay > 0:
loss_wd = (
params.weight_decay
* tf.add_n([tf.nn.l2_loss(v) for v in tf.trainable_variables()])
)
tf.summary.scalar('loss_wd', loss_wd)
total_loss += loss_wd
# Setup the moving averages:
if FLAGS.moving_average_decay and (FLAGS.moving_average_decay > 0):
with tf.name_scope('moving_average'):
moving_average_variables = tf.contrib.framework.get_model_variables()
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
else:
moving_average_variables = None
variable_averages = None
# set up optimizer and training op
learning_rate, steps_per_epoch = model_lib.get_lr_schedule(
params, examples_per_epoch, FLAGS.replicas_to_aggregate)
optimizer = model_lib.get_optimizer(params, learning_rate)
optimizer = tf.train.SyncReplicasOptimizer(
opt=optimizer,
replicas_to_aggregate=FLAGS.replicas_to_aggregate,
total_num_replicas=FLAGS.worker_replicas,
variable_averages=variable_averages,
variables_to_average=moving_average_variables)
train_op = tf.contrib.training.create_train_op(
total_loss, optimizer,
update_ops=tf.get_collection(tf.GraphKeys.UPDATE_OPS))
tf.summary.image('images', images[0:FLAGS.num_summary_images])
if adv_examples is not None:
tf.summary.image('adv_images', adv_examples[0:FLAGS.num_summary_images])
tf.summary.scalar('total_loss', total_loss)
tf.summary.scalar('learning_rate', learning_rate)
tf.summary.scalar('current_epoch',
tf.to_double(global_step) / steps_per_epoch)
# Training
is_chief = FLAGS.task == 0
scaffold = tf.train.Scaffold(
init_fn=_get_finetuning_init_fn(variable_averages))
hooks = [
tf.train.LoggingTensorHook({'total_loss': total_loss,
'global_step': global_step},
every_n_iter=1),
tf.train.NanTensorHook(total_loss),
]
chief_only_hooks = [
tf.train.SummarySaverHook(save_steps=FLAGS.save_summaries_steps,
save_secs=FLAGS.save_summaries_secs,
output_dir=FLAGS.output_dir,
scaffold=scaffold),
tf.train.CheckpointSaverHook(FLAGS.output_dir,
save_steps=FLAGS.save_model_steps,
scaffold=scaffold),
]
if FLAGS.max_steps > 0:
hooks.append(
tf.train.StopAtStepHook(last_step=FLAGS.max_steps))
# hook for sync replica training
hooks.append(optimizer.make_session_run_hook(is_chief))
with tf.train.MonitoredTrainingSession(
master=FLAGS.master,
is_chief=is_chief,
checkpoint_dir=FLAGS.output_dir,
scaffold=scaffold,
hooks=hooks,
chief_only_hooks=chief_only_hooks,
save_checkpoint_secs=None,
save_summaries_steps=None,
save_summaries_secs=None) as session:
while not session.should_stop():
session.run([train_op])
def main(_):
num_examples = FLAGS.num_examples
params = model_lib.default_hparams()
params.parse(FLAGS.hparams)
tf.logging.info('User provided hparams: %s', FLAGS.hparams)
tf.logging.info('All hyper parameters: %s', params)
graph = tf.Graph()
with graph.as_default():
img_bounds = (-1, 1) if 'Fine-tuned' in FLAGS.train_dir else (0, 1)
dataset, total_num_examples, num_classes, bounds = dataset_factory.get_dataset(
'tiny_imagenet',
'validation',
10000,
64,
is_training=False,
bounds=img_bounds)
dataset_iterator = dataset.make_one_shot_iterator()
images, labels = dataset_iterator.get_next()
images, labels = images[:num_examples], labels[:num_examples]
# setup model
global_step = tf.train.get_or_create_global_step()
model_fn_two_args = model_lib.get_model(FLAGS.model_name, num_classes)
model_fn = lambda x: model_fn_two_args(
x, is_training=False) # thin wrapper; args: images, is_training
# clean first
clean_logits = model_fn(images)
labels_for_ae = tf.identity(
labels
) # used only if 'rnd' in FLAGS.adv_method and FLAGS.n_restarts > 0
if FLAGS.adv_method == 'clean':
logits = clean_logits
else:
adv_examples = adversarial_attack.generate_adversarial_examples(
images,
labels_for_ae,
bounds,
model_fn,
FLAGS.adv_method,
n_restarts=FLAGS.n_restarts)
adv_logits = model_fn(adv_examples)
logits = adv_logits
correct = tf.equal(tf.argmax(logits, 1), tf.to_int64(labels))
# correct = tf.equal(tf.argmax(adv_logits, 1), tf.argmax(clean_logits, 1))
acc = tf.reduce_mean(tf.cast(correct, tf.float32))
# Setup the moving averages
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
variables_to_restore = variable_averages.variables_to_restore(
tf.contrib.framework.get_model_variables())
variables_to_restore[global_step.op.name] = global_step
saver = tf.train.Saver()
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.7)
config = tf.ConfigProto(gpu_options=gpu_options)
with tf.Session(graph=graph, config=config) as sess:
saver.restore(sess, get_latest_checkpoint(FLAGS.train_dir))
correct_vals = np.zeros([FLAGS.n_restarts, num_examples])
time_start = time.time()
# we select random classes only once, and fix them across multiple restarts
labels_for_rnd_multrest = np.random.random_integers(
0, num_classes - 1, size=[
num_examples,
])
for i_restart in range(FLAGS.n_restarts):
logits_val, correct_val, acc_val = sess.run(
[logits, correct, acc],
feed_dict={labels_for_ae: labels_for_rnd_multrest})
print('Accuracy: {:.2%}'.format(acc_val))
correct_vals[i_restart, :] = correct_val
print(
'[Elapsed {:.2f} min] avg_acc={:.2%} min_acc={:.2%} (n_restarts={})'
.format((time.time() - time_start) / 60, correct_vals.mean(),
correct_vals.min(axis=0).mean(), FLAGS.n_restarts))
def main(_):
if not FLAGS.train_dir and not FLAGS.checkpoint_path:
print('Either --train_dir or --checkpoint_path flags has to be provided.')
if FLAGS.train_dir and FLAGS.checkpoint_path:
print('Only one of --train_dir or --checkpoint_path should be provided.')
params = model_lib.default_hparams()
params.parse(FLAGS.hparams)
tf.logging.info('User provided hparams: %s', FLAGS.hparams)
tf.logging.info('All hyper parameters: %s', params)
batch_size = params.eval_batch_size
graph = tf.Graph()
with graph.as_default():
# dataset
dataset, num_examples, num_classes, bounds = dataset_factory.get_dataset(
FLAGS.dataset,
FLAGS.split_name,
batch_size,
FLAGS.dataset_image_size,
is_training=False)
dataset_iterator = dataset.make_one_shot_iterator()
images, labels = dataset_iterator.get_next()
if FLAGS.num_examples > 0:
num_examples = min(num_examples, FLAGS.num_examples)
# setup model
global_step = tf.train.get_or_create_global_step()
model_fn_two_args = model_lib.get_model(FLAGS.model_name, num_classes)
model_fn = lambda x: model_fn_two_args(x, is_training=False)
if not FLAGS.adv_method or FLAGS.adv_method == 'clean':
logits = model_fn(images)
else:
adv_examples = adversarial_attack.generate_adversarial_examples(
images, bounds, model_fn, FLAGS.adv_method)
logits = model_fn(adv_examples)
# update trainable variables if fine tuning is used
model_lib.filter_trainable_variables(FLAGS.trainable_scopes)
# Setup the moving averages
if FLAGS.moving_average_decay and (FLAGS.moving_average_decay > 0):
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
variables_to_restore = variable_averages.variables_to_restore(
tf.contrib.framework.get_model_variables())
variables_to_restore[global_step.op.name] = global_step
else:
variables_to_restore = tf.contrib.framework.get_variables_to_restore()
# Setup evaluation metric
with tf.name_scope('Eval'):
names_to_values, names_to_updates = (
tf.contrib.metrics.aggregate_metric_map({
'Accuracy': tf.metrics.accuracy(labels, tf.argmax(logits, 1)),
'Top5': tf.metrics.recall_at_k(tf.to_int64(labels), logits, 5)
}))
for name, value in names_to_values.iteritems():
tf.summary.scalar(name, value)
# Run evaluation
num_batches = int(num_examples / batch_size)
if FLAGS.train_dir:
output_dir = os.path.join(FLAGS.train_dir, FLAGS.eval_name)
if not tf.gfile.Exists(output_dir):
tf.gfile.MakeDirs(output_dir)
tf.contrib.training.evaluate_repeatedly(
FLAGS.train_dir,
master=FLAGS.master,
scaffold=tf.train.Scaffold(
saver=tf.train.Saver(variables_to_restore)),
eval_ops=names_to_updates.values(),
eval_interval_secs=FLAGS.eval_interval_secs,
hooks=[
tf.contrib.training.StopAfterNEvalsHook(num_batches),
tf.contrib.training.SummaryAtEndHook(output_dir),
tf.train.LoggingTensorHook(names_to_values, at_end=True),
],
max_number_of_evaluations=1 if FLAGS.eval_once else None)
else:
result = tf.contrib.training.evaluate_once(
FLAGS.checkpoint_path,
master=FLAGS.master,
scaffold=tf.train.Scaffold(
saver=tf.train.Saver(variables_to_restore)),
eval_ops=names_to_updates.values(),
final_ops=names_to_values,
hooks=[
tf.contrib.training.StopAfterNEvalsHook(num_batches),
tf.train.LoggingTensorHook(names_to_values, at_end=True),
])
if FLAGS.output_file:
with tf.gfile.Open(FLAGS.output_file, 'a') as f:
f.write('%s,%.3f,%.3f\n'
% (FLAGS.eval_name, result['Accuracy'], result['Top5']))
