def test_make_confidence_report_bundled():
"""
A very simple test that just makes sure make_confidence_report_bundled can run without crashing
"""
sess = tf.compat.v1.Session()
try:
nb_classes = 3
nb_features = 2
batch_size = 5
nb_test_examples = batch_size * 2
layer = Linear(num_hid=nb_classes)
model = MLP(layers=[layer], input_shape=(None, nb_features))
dataset = SimpleDataset(test_end=nb_test_examples, nb_classes=nb_classes)
model.dataset_factory = dataset.get_factory()
filepath = ".test_model.joblib"
with sess.as_default():
sess.run(tf.compat.v1.global_variables_initializer())
serial.save(filepath, model)
def recipe(sess, model, x, y, nb_classes, eps, clip_min,
clip_max, eps_iter, nb_iter,
report_path, eps_iter_small, batch_size):
"""
Mock recipe that just runs the Noise attack so the test runs fast
"""
attack_configs = [AttackConfig(Noise(model, sess), {'eps': eps})]
new_work_goal = {config: 1 for config in attack_configs}
goals = [Misclassify(new_work_goal=new_work_goal)]
bundle_attacks(sess, model, x, y, attack_configs, goals, report_path, attack_batch_size=batch_size,
eval_batch_size=batch_size)
make_confidence_report_bundled(filepath, test_end=nb_test_examples, recipe=recipe,
base_eps=.1, base_eps_iter=.01, batch_size=batch_size)
finally:
sess.close()
def main(argv=None):
"""
Make a confidence report and save it to disk.
"""
try:
_name_of_script, filepath = argv
except ValueError:
raise ValueError(argv)
print(filepath)
make_confidence_report_bundled(filepath=filepath,
test_start=FLAGS.test_start,
test_end=FLAGS.test_end,
which_set=FLAGS.which_set,
recipe=FLAGS.recipe,
report_path=FLAGS.report_path)
