def setUp(self): # Set master seed master_seed(1234) # Define input and output placeholders input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1]) output_ph = tf.placeholder(tf.int32, shape=[None, 10]) learning = tf.placeholder(tf.bool) # Define the tensorflow graph conv = tf.layers.conv2d(input_ph, 16, 5, activation=tf.nn.relu) conv = tf.layers.max_pooling2d(conv, 2, 2) fc = tf.contrib.layers.flatten(conv) # Logits layer logits = tf.layers.dense(fc, 10) # Train operator loss = tf.reduce_mean( tf.losses.softmax_cross_entropy(logits=logits, onehot_labels=output_ph)) optimizer = tf.train.AdamOptimizer(learning_rate=0.01) train = optimizer.minimize(loss) # Tensorflow session and initialization self.sess = tf.Session() self.sess.run(tf.global_variables_initializer()) # Create classifier and fit self.classifier = TFClassifier((0, 1), input_ph, logits, output_ph, train, loss, learning, self.sess) # Get MNIST (x_train, y_train), (x_test, y_test) = self.mnist self.classifier.fit(x_train, y_train, batch_size=100, nb_epochs=3)
def test_loss_gradient(self): # Get MNIST (_, _), (x_test, y_test), _, _ = load_mnist() x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST] # Test gradient tfc = TFClassifier(None, self._input_ph, self._logits, self._output_ph, None, self._loss, None, self._sess) grads = tfc.loss_gradient(x_test, y_test) self.assertTrue(np.array(grads.shape == (NB_TEST, 28, 28, 1)).all()) self.assertTrue(np.sum(grads) != 0)
def test_class_gradient(self): # Get MNIST (_, _), (x_test, y_test), _, _ = load_mnist() x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST] # Test gradient tfc = TFClassifier((0, 1), self._input_ph, self._logits, None, None, None, None, self._sess) grads = tfc.class_gradient(x_test) self.assertTrue( np.array(grads.shape == (NB_TEST, 10, 28, 28, 1)).all()) self.assertTrue(np.sum(grads) != 0) tf.reset_default_graph()
def test_failure_attack(self): """ Test the corner case when attack is failed. :return: """ # Build a TFClassifier # Define input and output placeholders input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1]) output_ph = tf.placeholder(tf.int32, shape=[None, 10]) # Define the tensorflow graph conv = tf.layers.conv2d(input_ph, 4, 5, activation=tf.nn.relu) conv = tf.layers.max_pooling2d(conv, 2, 2) fc = tf.contrib.layers.flatten(conv) # Logits layer logits = tf.layers.dense(fc, 10) # Train operator loss = tf.reduce_mean( tf.losses.softmax_cross_entropy(logits=logits, onehot_labels=output_ph)) optimizer = tf.train.AdamOptimizer(learning_rate=0.01) train = optimizer.minimize(loss) # Tensorflow session and initialization sess = tf.Session() sess.run(tf.global_variables_initializer()) # Get MNIST (x_train, y_train), (x_test, y_test) = self.mnist # Train the classifier tfc = TFClassifier((0, 1), input_ph, logits, output_ph, train, loss, None, sess) tfc.fit(x_train, y_train, batch_size=BATCH_SIZE, nb_epochs=10) # Failure attack cl2m = CarliniL2Method(classifier=tfc, targeted=True, max_iter=0, binary_search_steps=0, learning_rate=0, initial_const=1) params = {'y': random_targets(y_test, tfc.nb_classes)} x_test_adv = cl2m.generate(x_test, **params) self.assertTrue((x_test_adv <= 1.0001).all()) self.assertTrue((x_test_adv >= -0.0001).all()) np.testing.assert_almost_equal(x_test, x_test_adv, 3)
def set_up(args): if not os.path.isdir('output'): os.makedirs('output') # # Load the chestx dataset (x_train, y_train), (x_test, y_test), (mean_l2_train, mean_inf_train) = load_data( args.datapath, args.trainfile, args.testfile) if args.norm == '2': norm = 2 eps = mean_l2_train * args.eps elif args.norm == 'inf': norm = np.inf eps = mean_inf_train * args.eps # # Create the model sess, graph = create_model(args.weightspath, args.metaname, args.ckptname) # # Create the ART classifier input_tensor = graph.get_tensor_by_name("input_1:0") logit_tensor = graph.get_tensor_by_name("dense_3/MatMul:0") output_tensor = graph.get_tensor_by_name("dense_3/Softmax:0") label_tensor = graph.get_tensor_by_name("dense_3_target:0") loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits_v2(logits=logit_tensor, labels=label_tensor)) classifier = TFClassifier(input_ph=input_tensor, output=output_tensor, labels_ph=label_tensor, loss=loss, sess=sess) return (x_train, y_train), (x_test, y_test), (mean_l2_train, mean_inf_train), norm, eps, classifier
def get_madry_model(model_kwargs, wrapper_kwargs, weights_file=None): model = make_madry_model(**model_kwargs) input_ph = model.x_input labels_ph = model.y_input training_ph = tf.placeholder(tf.bool, shape=()) # Restore the checkpoint saver = tf.train.Saver() tf_sess = tf.Session() saved_model_dir = paths.DockerPaths().saved_model_dir filepath = os.path.join(saved_model_dir, weights_file) model_file = tf.train.latest_checkpoint(filepath) saver.restore(tf_sess, model_file) wrapped_model = TFClassifier( input_ph=input_ph, output=model.pre_softmax, labels_ph=labels_ph, loss=model.xent, learning=training_ph, sess=tf_sess, clip_values=(0, 255), **wrapper_kwargs ) return wrapped_model
def test_nb_classes(self): # Start to test tfc = TFClassifier((0, 1), self._input_ph, self._logits, self._output_ph, self._train, None, None, self._sess) self.assertTrue(tfc.nb_classes == 10) tf.reset_default_graph()
def _create_tfclassifier(): """ To create a simple TFClassifier for testing. :return: """ # Define input and output placeholders input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1]) output_ph = tf.placeholder(tf.int32, shape=[None, 10]) # Define the tensorflow graph conv = tf.layers.conv2d(input_ph, 4, 5, activation=tf.nn.relu) conv = tf.layers.max_pooling2d(conv, 2, 2) fc = tf.contrib.layers.flatten(conv) # Logits layer logits = tf.layers.dense(fc, 10) # Train operator loss = tf.reduce_mean( tf.losses.softmax_cross_entropy(logits=logits, onehot_labels=output_ph)) optimizer = tf.train.AdamOptimizer(learning_rate=0.01) train = optimizer.minimize(loss) # Tensorflow session and initialization sess = tf.Session() sess.run(tf.global_variables_initializer()) # Create the classifier tfc = TFClassifier((0, 1), input_ph, logits, output_ph, train, loss, None, sess) return tfc
def test_input_shape(self): # Start to test tfc = TFClassifier((0, 1), self._input_ph, self._logits, self._output_ph, self._train, None, None, self._sess) self.assertTrue(np.array(tfc.input_shape == (28, 28, 1)).all()) tf.reset_default_graph()
def get_art_model(model_kwargs, wrapper_kwargs, weights_file=None): model = Model(**model_kwargs) input_ph = model.x_input labels_ph = model.y_input training_ph = tf.placeholder(tf.bool, shape=()) tf_sess = tf.Session() tf_sess.run(tf.global_variables_initializer()) if weights_file: saver = tf.train.Saver() filepath = maybe_download_weights_from_s3(weights_file) tar = tarfile.open(filepath) tar.extractall(path=SAVED_MODEL_DIR) tar.close() model_file = tf.train.latest_checkpoint(SAVED_MODEL_DIR + '/free_checkpoint_tf') saver.restore(tf_sess, model_file) wrapped_model = TFClassifier(input_ph=input_ph, output=model.pre_softmax, labels_ph=labels_ph, loss=model.xent, learning=training_ph, sess=tf_sess, clip_values=(0, 255), **wrapper_kwargs) return wrapped_model
def _cnn_mnist_tf(input_shape): labels_tf = tf.placeholder(tf.float32, [None, 10]) inputs_tf = tf.placeholder(tf.float32, [None] + list(input_shape)) # Define the tensorflow graph conv = tf.layers.conv2d(inputs_tf, 4, 5, activation=tf.nn.relu) conv = tf.layers.max_pooling2d(conv, 2, 2) fc = tf.contrib.layers.flatten(conv) # Logits layer logits = tf.layers.dense(fc, 10) # Train operator loss = tf.reduce_mean( tf.losses.softmax_cross_entropy(logits=logits, onehot_labels=labels_tf)) optimizer = tf.train.AdamOptimizer(learning_rate=0.01) train_tf = optimizer.minimize(loss) sess = tf.Session() sess.run(tf.global_variables_initializer()) classifier = TFClassifier((0, 1), inputs_tf, logits, loss=loss, train=train_tf, output_ph=labels_tf, sess=sess) return classifier
def get_iris_classifier_tf(): """ Standard Tensorflow classifier for unit testing. The following hyper-parameters were used to obtain the weights and biases: - learning_rate: 0.01 - batch size: 5 - number of epochs: 200 - optimizer: tf.train.AdamOptimizer The model is trained of 70% of the dataset, and 30% of the training set is used as validation split. :return: The trained model for Iris dataset and the session. :rtype: `tuple(TFClassifier, tf.Session)` """ import tensorflow as tf from art.classifiers import TFClassifier # Define input and output placeholders input_ph = tf.placeholder(tf.float32, shape=[None, 4]) output_ph = tf.placeholder(tf.int32, shape=[None, 3]) # Define the tensorflow graph dense1 = tf.layers.dense( input_ph, 10, kernel_initializer=_tf_weights_loader('IRIS', 'W', 'DENSE1'), bias_initializer=_tf_weights_loader('IRIS', 'B', 'DENSE1')) dense2 = tf.layers.dense( dense1, 10, kernel_initializer=_tf_weights_loader('IRIS', 'W', 'DENSE2'), bias_initializer=_tf_weights_loader('IRIS', 'B', 'DENSE2')) logits = tf.layers.dense( dense2, 3, kernel_initializer=_tf_weights_loader('IRIS', 'W', 'DENSE3'), bias_initializer=_tf_weights_loader('IRIS', 'B', 'DENSE3')) # Train operator loss = tf.reduce_mean( tf.losses.softmax_cross_entropy(logits=logits, onehot_labels=output_ph)) # Tensorflow session and initialization sess = tf.Session() sess.run(tf.global_variables_initializer()) # Train the classifier tfc = TFClassifier(clip_values=(0, 1), input_ph=input_ph, logits=logits, output_ph=output_ph, train=None, loss=loss, learning=None, sess=sess, channel_index=1) return tfc, sess
def get_classifier_tf(): """ Standard Tensorflow classifier for unit testing. The following hyper-parameters were used to obtain the weights and biases: learning_rate: 0.01 batch size: 10 number of epochs: 2 optimizer: tf.train.AdamOptimizer :return: TFClassifier, tf.Session() """ import tensorflow as tf from art.classifiers import TFClassifier # Define input and output placeholders input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1]) output_ph = tf.placeholder(tf.int32, shape=[None, 10]) # Define the tensorflow graph conv = tf.layers.conv2d( input_ph, 1, 7, activation=tf.nn.relu, kernel_initializer=_tf_weights_loader('MNIST', 'W', 'CONV2D'), bias_initializer=_tf_weights_loader('MNIST', 'B', 'CONV2D')) conv = tf.layers.max_pooling2d(conv, 4, 4) flattened = tf.contrib.layers.flatten(conv) # Logits layer logits = tf.layers.dense( flattened, 10, kernel_initializer=_tf_weights_loader('MNIST', 'W', 'DENSE'), bias_initializer=_tf_weights_loader('MNIST', 'B', 'DENSE')) # Train operator loss = tf.reduce_mean( tf.losses.softmax_cross_entropy(logits=logits, onehot_labels=output_ph)) optimizer = tf.train.AdamOptimizer(learning_rate=0.01) train = optimizer.minimize(loss) # Tensorflow session and initialization sess = tf.Session() sess.run(tf.global_variables_initializer()) # Train the classifier tfc = TFClassifier(clip_values=(0, 1), input_ph=input_ph, logits=logits, output_ph=output_ph, train=train, loss=loss, learning=None, sess=sess) return tfc, sess
def test_fit_predict(self): # Get MNIST (x_train, y_train), (x_test, y_test), _, _ = load_mnist() x_train, y_train = x_train[:NB_TRAIN], y_train[:NB_TRAIN] x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST] # Test fit and predict tfc = TFClassifier(None, self._input_ph, self._logits, self._output_ph, self._train, self._loss, None, self._sess) tfc.fit(x_train, y_train, batch_size=100, nb_epochs=1) preds = tfc.predict(x_test) preds_class = np.argmax(preds, axis=1) trues_class = np.argmax(y_test, axis=1) acc = np.sum(preds_class == trues_class) / len(trues_class) print("\nAccuracy: %.2f%%" % (acc * 100)) self.assertGreater(acc, 0.1)
def test_layers(self): # Get MNIST (_, _), (x_test, y_test), _, _ = load_mnist() x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST] # Test and get layers tfc = TFClassifier((0, 1), self._input_ph, self._logits, self._output_ph, self._train, self._loss, None, self._sess) layer_names = tfc.layer_names print(layer_names) self.assertTrue(layer_names == [ 'conv2d/Relu:0', 'max_pooling2d/MaxPool:0', 'Flatten/flatten/Reshape:0', 'dense/BiasAdd:0' ]) for i, name in enumerate(layer_names): act_i = tfc.get_activations(x_test, i) act_name = tfc.get_activations(x_test, name) self.assertAlmostEqual(np.sum(act_name - act_i), 0) self.assertTrue( tfc.get_activations(x_test, 0).shape == (20, 24, 24, 16)) self.assertTrue( tfc.get_activations(x_test, 1).shape == (20, 12, 12, 16)) self.assertTrue(tfc.get_activations(x_test, 2).shape == (20, 2304)) self.assertTrue(tfc.get_activations(x_test, 3).shape == (20, 10)) tf.reset_default_graph()
def CIFAR(**args): # Get data for cifar x_train, y_train, x_test, y_test, x_val, y_val, _min, _max = load_train_eval_test('cifar') # Params params = cifar_param save_path = params.save_path # Load CNN if args['cnn_path'] is not None: if args['last_name'] is not None: sess_cnn, cnn = load_cnn(params.cnn, args['cnn_path'] + args['last_name'], load_last=False) else: sess_cnn, cnn = load_cnn(params.cnn, args['cnn_path'], load_last=True) classifier_cnn = KerasClassifier(model=cnn.model, clip_values=(_min, _max)) # Load VQ if args['vq_path'] is not None: path = args['vq_path'] vq_hard = load_vq(path, config, args['last_name']) classifer_e = TFClassifier(input_ph=vq_hard.input_x, output = vq_hard.pre_e, labels_ph= vq_hard.y_true,clip_values=(_min, _max), sess=vq_hard.sess, loss=vq_hard.loss_c2) vq_models = [vq_hard] # Black box test paths = ['save_val/attacks/CIFAR_pool3_BIM_copy_x_test_advs.npy', 'save_val/attacks/CIFAR_pool3_BIM_large_copy_x_test_advs.npy',\ 'save_val/attacks/CIFAR_pool3_FGSM_copy_x_test_advs.npy', 'save_val/attacks/CIFAR_pool3_FGSM_large_copy_x_test_advs.npy'] for index, black_flag in enumerate([args['BIM_black'], args['BIM_black_large'], args['FGSM_black'], args['FGSM_black_large']]): if black_flag: print(paths[index]) attack_file_path = paths[index] if args['cnn_path'] is not None: accs = cnn_black_box_test(attack_file_path, cnn, y_test, x_test) if args['vq_path'] is not None: vq_accs_q, vq_accs_e = vq_black_box_test(attack_file_path, [vq_hard], y_test, x_test) # White box attacks = [] if args['BIM_white']: attacks.append(BasicIterativeMethod) if args['FGSM_white']: attacks.append(FastGradientMethod) if args['CIM_white']: attacks.append(CarliniLInfMethod) if attacks != []: eps = eval(args['eps']) if args['cnn_path'] is not None: cnn_white_box_test(eps, attacks, [sess_cnn], [classifier_cnn], [cnn.model], ['cnn'], x_test, y_test) if args['vq_path'] is not None: vq_white_box_test(eps, attacks, [vq_hard.sess], [classifer_e], vq_models, ['VQ'], x_test, y_test )
def cw2(self, sess): from art.classifiers import TFClassifier tfc = TFClassifier(clip_values=(0, 1), input_ph=self.x_image, logits=self.pre_softmax, output_ph=self.y_input, train=None, loss=self.xent, learning=None, sess=sess) return tfc
def test_tfclassifier(self): """ First test with the TFClassifier. :return: """ # Build a TFClassifier # Define input and output placeholders input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1]) output_ph = tf.placeholder(tf.int32, shape=[None, 10]) # Define the tensorflow graph conv = tf.layers.conv2d(input_ph, 4, 5, activation=tf.nn.relu) conv = tf.layers.max_pooling2d(conv, 2, 2) fc = tf.contrib.layers.flatten(conv) # Logits layer logits = tf.layers.dense(fc, 10) # Train operator loss = tf.reduce_mean( tf.losses.softmax_cross_entropy(logits=logits, onehot_labels=output_ph)) optimizer = tf.train.AdamOptimizer(learning_rate=0.01) train = optimizer.minimize(loss) # Tensorflow session and initialization sess = tf.Session() sess.run(tf.global_variables_initializer()) # Get MNIST (x_train, y_train), (x_test, y_test) = self.mnist # Train the classifier tfc = TFClassifier((0, 1), input_ph, logits, output_ph, train, loss, None, sess) tfc.fit(x_train, y_train, batch_size=BATCH_SIZE, nb_epochs=2) # Attack # TODO Launch with all possible attacks attack_params = { "attacker": "newtonfool", "attacker_params": { "max_iter": 5 } } up = UniversalPerturbation(tfc) x_train_adv = up.generate(x_train, **attack_params) self.assertTrue((up.fooling_rate >= 0.2) or not up.converged) x_test_adv = x_test + up.v self.assertFalse((x_test == x_test_adv).all()) train_y_pred = np.argmax(tfc.predict(x_train_adv), axis=1) test_y_pred = np.argmax(tfc.predict(x_test_adv), axis=1) self.assertFalse((np.argmax(y_test, axis=1) == test_y_pred).all()) self.assertFalse((np.argmax(y_train, axis=1) == train_y_pred).all())
def get_art_model(model_kwargs, wrapper_kwargs, weights_file=None): input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1]) labels_ph = tf.placeholder(tf.int32, shape=[None, 10]) training_ph = tf.placeholder(tf.bool, shape=()) x = tf.layers.conv2d(input_ph, filters=4, kernel_size=(5, 5), activation=tf.nn.relu) x = tf.layers.max_pooling2d(x, 2, 2) x = tf.layers.conv2d(x, filters=10, kernel_size=(5, 5), activation=tf.nn.relu) x = tf.layers.max_pooling2d(x, 2, 2) x = tf.layers.flatten(x) x = tf.layers.dense(x, 100, activation=tf.nn.relu) logits = tf.layers.dense(x, 10) loss = tf.reduce_mean( tf.losses.softmax_cross_entropy(logits=logits, onehot_labels=labels_ph)) optimizer = tf.train.AdamOptimizer(learning_rate=0.01) train_op = optimizer.minimize(loss) sess = tf.Session() sess.run(tf.global_variables_initializer()) if weights_file: # Load Model using preferred save/restore method filepath = maybe_download_weights_from_s3(weights_file) tar = tarfile.open(filepath) tar.extractall(path=paths.runtime_paths().saved_model_dir) tar.close() # Restore variables... wrapped_model = TFClassifier(clip_values=(0.0, 1.0), input_ph=input_ph, output=logits, labels_ph=labels_ph, train=train_op, loss=loss, learning=training_ph, sess=sess, **wrapper_kwargs) return wrapped_model
def test_tfclassifier(self): """ First test with the TFClassifier. :return: """ # Build a TFClassifier # Define input and output placeholders self._input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1]) self._output_ph = tf.placeholder(tf.int32, shape=[None, 10]) # Define the tensorflow graph conv = tf.layers.conv2d(self._input_ph, 4, 5, activation=tf.nn.relu) conv = tf.layers.max_pooling2d(conv, 2, 2) fc = tf.contrib.layers.flatten(conv) # Logits layer self._logits = tf.layers.dense(fc, 10) # Train operator self._loss = tf.reduce_mean(tf.losses.softmax_cross_entropy(logits=self._logits, onehot_labels=self._output_ph)) optimizer = tf.train.AdamOptimizer(learning_rate=0.01) self._train = optimizer.minimize(self._loss) # Tensorflow session and initialization self._sess = tf.Session() self._sess.run(tf.global_variables_initializer()) # Get MNIST batch_size, nb_train, nb_test = 100, 1000, 10 (x_train, y_train), (x_test, y_test), _, _ = load_mnist() x_train, y_train = x_train[:nb_train], y_train[:nb_train] x_test, y_test = x_test[:nb_test], y_test[:nb_test] # Train the classifier tfc = TFClassifier((0, 1), self._input_ph, self._logits, self._output_ph, self._train, self._loss, None, self._sess) tfc.fit(x_train, y_train, batch_size=batch_size, nb_epochs=2) # Attack nf = NewtonFool(tfc) nf.set_params(max_iter=5) x_test_adv = nf.generate(x_test) self.assertFalse((x_test == x_test_adv).all()) y_pred = tfc.predict(x_test) y_pred_adv = tfc.predict(x_test_adv) y_pred_bool = y_pred.max(axis=1, keepdims=1) == y_pred y_pred_max = y_pred.max(axis=1) y_pred_adv_max = y_pred_adv[y_pred_bool] self.assertTrue((y_pred_max >= y_pred_adv_max).all())
def get_art_model(model_kwargs, wrapper_kwargs, weights_file=None): input_ph = tf.placeholder(tf.float32, shape=[None, 32, 32, 3]) labels_ph = tf.placeholder(tf.int32, shape=[None, 10]) training_ph = tf.placeholder(tf.bool, shape=()) # Conditional for handling training phase or inference phase output = tf.cond( training_ph, true_fn=lambda: _training_pass(input_ph), false_fn=lambda: _inference_pass(input_ph), ) loss = tf.reduce_mean( tf.losses.softmax_cross_entropy(logits=output, onehot_labels=labels_ph)) optimizer = tf.train.AdamOptimizer(learning_rate=0.003) train_op = optimizer.minimize(loss) sess = tf.Session() sess.run(tf.global_variables_initializer()) if weights_file: # Load Model using preferred save/restore method filepath = maybe_download_weights_from_s3(weights_file) tar = tarfile.open(filepath) tar.extractall(path=paths.runtime_paths().saved_model_dir) tar.close() # Restore variables... wrapped_model = TFClassifier(clip_values=(0.0, 1.0), input_ph=input_ph, output=output, labels_ph=labels_ph, train=train_op, loss=loss, learning=training_ph, sess=sess, **wrapper_kwargs) return wrapped_model
def __call__(self, session): inputs = tf.compat.v1.placeholder(tf.float32, shape=[ None, self._image_height, self._image_width, self._n_channels ]) target_ys = tf.compat.v1.placeholder(tf.float32, shape=[None, self._n_classes]) logits = self.calculate_logits(inputs) target_loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(labels=target_ys, logits=logits)) restorer = tf.compat.v1.train.Saver() restorer.restore(session, self._checkpoint_path) return TFClassifier(clip_values=(-1, 1), input_ph=inputs, output=logits, labels_ph=target_ys, sess=session, loss=target_loss)
class TestTFClassifier(unittest.TestCase): """ This class tests the functionalities of the Tensorflow-based classifier. """ @classmethod def setUpClass(cls): # Get MNIST (x_train, y_train), (x_test, y_test), _, _ = load_mnist() x_train, y_train = x_train[:NB_TRAIN], y_train[:NB_TRAIN] x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST] cls.mnist = (x_train, y_train), (x_test, y_test) def setUp(self): # Set master seed master_seed(1234) # Define input and output placeholders input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1]) output_ph = tf.placeholder(tf.int32, shape=[None, 10]) learning = tf.placeholder(tf.bool) # Define the tensorflow graph conv = tf.layers.conv2d(input_ph, 16, 5, activation=tf.nn.relu) conv = tf.layers.max_pooling2d(conv, 2, 2) fc = tf.contrib.layers.flatten(conv) # Logits layer logits = tf.layers.dense(fc, 10) # Train operator loss = tf.reduce_mean( tf.losses.softmax_cross_entropy(logits=logits, onehot_labels=output_ph)) optimizer = tf.train.AdamOptimizer(learning_rate=0.01) train = optimizer.minimize(loss) # Tensorflow session and initialization self.sess = tf.Session() self.sess.run(tf.global_variables_initializer()) # Create classifier and fit self.classifier = TFClassifier((0, 1), input_ph, logits, output_ph, train, loss, learning, self.sess) # Get MNIST (x_train, y_train), (x_test, y_test) = self.mnist self.classifier.fit(x_train, y_train, batch_size=100, nb_epochs=3) def tearDown(self): self.sess.close() def test_fit_predict(self): # Get MNIST (x_train, y_train), (x_test, y_test) = self.mnist # Test fit and predict preds = self.classifier.predict(x_test) preds_class = np.argmax(preds, axis=1) trues_class = np.argmax(y_test, axis=1) acc = np.sum(preds_class == trues_class) / len(trues_class) logger.info('Accuracy after fitting: %.2f%%', (acc * 100)) self.assertGreater(acc, 0.1) tf.reset_default_graph() def test_fit_generator(self): from art.data_generators import TFDataGenerator # Get MNIST (x_train, y_train), (x_test, y_test) = self.mnist # Create Tensorflow data generator x_tensor = tf.convert_to_tensor(x_train.reshape(10, 100, 28, 28, 1)) y_tensor = tf.convert_to_tensor(y_train.reshape(10, 100, 10)) dataset = tf.data.Dataset.from_tensor_slices((x_tensor, y_tensor)) iterator = dataset.make_initializable_iterator() data_gen = TFDataGenerator(sess=self.sess, iterator=iterator, iterator_type='initializable', iterator_arg={}, size=1000, batch_size=100) # Test fit and predict self.classifier.fit_generator(data_gen, nb_epochs=1) preds = self.classifier.predict(x_test) preds_class = np.argmax(preds, axis=1) trues_class = np.argmax(y_test, axis=1) acc = np.sum(preds_class == trues_class) / len(trues_class) logger.info('Accuracy after fitting: %.2f%%', (acc * 100)) self.assertGreater(acc, 0.1) tf.reset_default_graph() def test_fit_generator(self): from art.classifiers.keras import generator_fit from art.data_generators import KerasDataGenerator labels = np.argmax(self.mnist[1][1], axis=1) acc = np.sum( np.argmax(self.classifier.predict(self.mnist[1][0]), axis=1) == labels) / NB_TEST logger.info('Accuracy: %.2f%%', (acc * 100)) gen = generator_fit(self.mnist[0][0], self.mnist[0][1], batch_size=100) data_gen = KerasDataGenerator(generator=gen, size=NB_TRAIN, batch_size=100) self.classifier.fit_generator(generator=data_gen, nb_epochs=2) acc2 = np.sum( np.argmax(self.classifier.predict(self.mnist[1][0]), axis=1) == labels) / NB_TEST logger.info('Accuracy: %.2f%%', (acc2 * 100)) self.assertTrue(acc2 >= .8 * acc) def test_nb_classes(self): # Start to test self.assertTrue(self.classifier.nb_classes == 10) tf.reset_default_graph() def test_input_shape(self): # Start to test self.assertTrue( np.array(self.classifier.input_shape == (28, 28, 1)).all()) tf.reset_default_graph() def test_class_gradient(self): # Get MNIST (_, _), (x_test, _) = self.mnist # Test all gradients label = None grads = self.classifier.class_gradient(x_test) self.assertTrue( np.array(grads.shape == (NB_TEST, 10, 28, 28, 1)).all()) self.assertTrue(np.sum(grads) != 0) # Test 1 gradient label = 5 grads = self.classifier.class_gradient(x_test, label=5) self.assertTrue(np.array(grads.shape == (NB_TEST, 1, 28, 28, 1)).all()) self.assertTrue(np.sum(grads) != 0) # Test a set of gradients label = array label = np.random.randint(5, size=NB_TEST) grads = self.classifier.class_gradient(x_test, label=label) self.assertTrue(np.array(grads.shape == (NB_TEST, 1, 28, 28, 1)).all()) self.assertTrue(np.sum(grads) != 0) def test_loss_gradient(self): # Get MNIST (_, _), (x_test, y_test) = self.mnist # Test gradient grads = self.classifier.loss_gradient(x_test, y_test) self.assertTrue(np.array(grads.shape == (NB_TEST, 28, 28, 1)).all()) self.assertTrue(np.sum(grads) != 0) tf.reset_default_graph() def test_layers(self): # Get MNIST (_, _), (x_test, _) = self.mnist # Test and get layers layer_names = self.classifier.layer_names logger.debug(layer_names) self.assertTrue(layer_names == [ 'conv2d/Relu:0', 'max_pooling2d/MaxPool:0', 'Flatten/flatten/Reshape:0', 'dense/BiasAdd:0' ]) for i, name in enumerate(layer_names): act_i = self.classifier.get_activations(x_test, i) act_name = self.classifier.get_activations(x_test, name) self.assertAlmostEqual(np.sum(act_name - act_i), 0) self.assertTrue( self.classifier.get_activations(x_test, 0).shape == (20, 24, 24, 16)) self.assertTrue( self.classifier.get_activations(x_test, 1).shape == (20, 12, 12, 16)) self.assertTrue( self.classifier.get_activations(x_test, 2).shape == (20, 2304)) self.assertTrue( self.classifier.get_activations(x_test, 3).shape == (20, 10)) tf.reset_default_graph() def test_save(self): import os import re path = 'tmp' filename = 'model.ckpt' self.classifier.save(filename, path=path) self.assertTrue(os.path.isfile(os.path.join(path, filename + '.meta'))) self.assertTrue(os.path.isfile(os.path.join(path, filename + '.index'))) # Remove saved files for f in os.listdir(path): if re.search(filename, f): os.remove(os.path.join(path, f)) def test_set_learning(self): tfc = self.classifier self.assertTrue(tfc._feed_dict == {}) tfc.set_learning_phase(False) self.assertFalse(tfc._feed_dict[tfc._learning]) tfc.set_learning_phase(True) self.assertTrue(tfc._feed_dict[tfc._learning]) self.assertTrue(tfc.learning_phase)
# x_test = x_test[0:20, :] # y_test = y_test[0:20] data_path = config['data_path'] cifar = cifar10_input.CIFAR10Data(data_path) x_test = cifar.eval_data.xs[0:50, :] y_test = cifar.eval_data.ys[0:50] # print(x_test.shape) # print(min_pixel_value) # print(max_pixel_value) with tf.Session() as sess: saver.restore(sess, model_ckpt) classifier = TFClassifier(input_ph=input_ph, logits=logits, sess=sess, loss=loss, output_ph=labels_ph) predictions = classifier.predict(x_test) print(x_test[0]) # print(predictions) print(np.argmax(predictions, axis=1)) accuracy = np.sum( np.argmax(predictions, axis=1) == y_test) / len(y_test) print('Accuracy on benign test examples: {}%'.format(accuracy * 100)) # FGSM attack = FastGradientMethod(classifier=classifier, eps=epsilon,
x = tf.layers.max_pooling2d(x, 2, 2) x = tf.layers.conv2d(x, filters=10, kernel_size=5, activation=tf.nn.relu) x = tf.layers.max_pooling2d(x, 2, 2) x = tf.contrib.layers.flatten(x) x = tf.layers.dense(x, 100, activation=tf.nn.relu) logits = tf.layers.dense(x, 10) loss = tf.reduce_mean(tf.losses.softmax_cross_entropy(logits=logits, onehot_labels=labels_ph)) optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01) train = optimizer.minimize(loss) sess = tf.Session() sess.run(tf.global_variables_initializer()) # Step 3: Create the ART classifier classifier = TFClassifier(clip_values=(min_pixel_value, max_pixel_value), input_ph=input_ph, output=logits, labels_ph=labels_ph, train=train, loss=loss, learning=None, sess=sess) # Step 4: Train the ART classifier attack=art.attacks.CarliniLInfMethod(classifier) trainer = AdversarialTrainer(classifier,attack, ratio=1.0) trainer.fit(x_train, y_train, batch_size=128, nb_epochs=3) # Step 5: Evaluate the ART classifier on benign test examples ##hello predictions = classifier.predict(x_test) accuracy = np.sum(np.argmax(predictions, axis=1) == np.argmax(y_test, axis=1)) / len(y_test) print('Accuracy on benign test examples: {}%'.format(accuracy * 100)) # Step 6: Generate adversarial test examples attack = art.attacks.SaliencyMapMethod(classifier=classifier) x_test_adv = attack.generate(x=x_test)
def test_tfclassifier(self): """ First test with the TFClassifier. :return: """ # Build a TFClassifier # Define input and output placeholders input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1]) output_ph = tf.placeholder(tf.int32, shape=[None, 10]) # Define the tensorflow graph conv = tf.layers.conv2d(input_ph, 4, 5, activation=tf.nn.relu) conv = tf.layers.max_pooling2d(conv, 2, 2) fc = tf.contrib.layers.flatten(conv) # Logits layer logits = tf.layers.dense(fc, 10) # Train operator loss = tf.reduce_mean( tf.losses.softmax_cross_entropy(logits=logits, onehot_labels=output_ph)) optimizer = tf.train.AdamOptimizer(learning_rate=0.01) train = optimizer.minimize(loss) # Tensorflow session and initialization sess = tf.Session() sess.run(tf.global_variables_initializer()) # Get MNIST (x_train, y_train), (x_test, y_test) = self.mnist # Train the classifier tfc = TFClassifier((0, 1), input_ph, logits, output_ph, train, loss, None, sess) tfc.fit(x_train, y_train, batch_size=BATCH_SIZE, nb_epochs=10) # First attack clinfm = CarliniLInfMethod(classifier=tfc, targeted=True, max_iter=10, eps=0.5) params = {'y': random_targets(y_test, tfc.nb_classes)} x_test_adv = clinfm.generate(x_test, **params) self.assertFalse((x_test == x_test_adv).all()) self.assertTrue((x_test_adv <= 1.0001).all()) self.assertTrue((x_test_adv >= -0.0001).all()) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1) logger.debug('CW0 Target: %s', target) logger.debug('CW0 Actual: %s', y_pred_adv) logger.info('CW0 Success Rate: %.2f', (sum(target == y_pred_adv) / float(len(target)))) self.assertTrue((target == y_pred_adv).any()) # Second attack clinfm = CarliniLInfMethod(classifier=tfc, targeted=False, max_iter=10, eps=0.5) params = {'y': random_targets(y_test, tfc.nb_classes)} x_test_adv = clinfm.generate(x_test, **params) self.assertTrue((x_test_adv <= 1.0001).all()) self.assertTrue((x_test_adv >= -0.0001).all()) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1) logger.debug('CW0 Target: %s', target) logger.debug('CW0 Actual: %s', y_pred_adv) logger.info('CW0 Success Rate: %.2f', (sum(target != y_pred_adv) / float(len(target)))) self.assertTrue((target != y_pred_adv).any()) # Third attack clinfm = CarliniLInfMethod(classifier=tfc, targeted=False, max_iter=10, eps=0.5) params = {} x_test_adv = clinfm.generate(x_test, **params) self.assertFalse((x_test == x_test_adv).all()) self.assertTrue((x_test_adv <= 1.0001).all()) self.assertTrue((x_test_adv >= -0.0001).all()) y_pred = np.argmax(tfc.predict(x_test), axis=1) y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1) logger.debug('CW0 Target: %s', y_pred) logger.debug('CW0 Actual: %s', y_pred_adv) logger.info('CW0 Success Rate: %.2f', (sum(y_pred != y_pred_adv) / float(len(y_pred)))) self.assertTrue((y_pred != y_pred_adv).any()) # First attack without batching clinfmwob = CarliniLInfMethod(classifier=tfc, targeted=True, max_iter=10, eps=0.5, batch_size=1) params = {'y': random_targets(y_test, tfc.nb_classes)} x_test_adv = clinfmwob.generate(x_test, **params) self.assertFalse((x_test == x_test_adv).all()) self.assertTrue((x_test_adv <= 1.0001).all()) self.assertTrue((x_test_adv >= -0.0001).all()) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1) logger.debug('CW0 Target: %s', target) logger.debug('CW0 Actual: %s', y_pred_adv) logger.info('CW0 Success Rate: %.2f', (sum(target == y_pred_adv) / float(len(target)))) self.assertTrue((target == y_pred_adv).any()) # Second attack without batching clinfmwob = CarliniLInfMethod(classifier=tfc, targeted=False, max_iter=10, eps=0.5, batch_size=1) params = {'y': random_targets(y_test, tfc.nb_classes)} x_test_adv = clinfmwob.generate(x_test, **params) self.assertTrue((x_test_adv <= 1.0001).all()) self.assertTrue((x_test_adv >= -0.0001).all()) target = np.argmax(params['y'], axis=1) y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1) logger.debug('CW0 Target: %s', target) logger.debug('CW0 Actual: %s', y_pred_adv) logger.info('CW0 Success Rate: %.2f', (sum(target != y_pred_adv) / float(len(target)))) self.assertTrue((target != y_pred_adv).any()) # Third attack without batching clinfmwob = CarliniLInfMethod(classifier=tfc, targeted=False, max_iter=10, eps=0.5, batch_size=1) params = {} x_test_adv = clinfmwob.generate(x_test, **params) self.assertFalse((x_test == x_test_adv).all()) self.assertTrue((x_test_adv <= 1.0001).all()) self.assertTrue((x_test_adv >= -0.0001).all()) y_pred = np.argmax(tfc.predict(x_test), axis=1) y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1) logger.debug('CW0 Target: %s', y_pred) logger.debug('CW0 Actual: %s', y_pred_adv) logger.info('CW0 Success Rate: %.2f', (sum(y_pred != y_pred_adv) / float(len(y_pred)))) self.assertTrue((y_pred != y_pred_adv).any())
x = tf.layers.max_pooling2d(x, 2, 2) x = tf.layers.conv2d(x, filters=10, kernel_size=5, activation=tf.nn.relu) x = tf.layers.max_pooling2d(x, 2, 2) x = tf.layers.flatten(x) x = tf.layers.dense(x, 100, activation=tf.nn.relu) logits = tf.layers.dense(x, 10) loss = tf.reduce_mean(tf.losses.softmax_cross_entropy(logits=logits, onehot_labels=labels_ph)) optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01) train = optimizer.minimize(loss) sess = tf.Session() sess.run(tf.global_variables_initializer()) #print('success') # Step 3: Create the ART classifier classifier = TFClassifier(clip_values=(min_pixel_value, max_pixel_value), input_ph=input_ph, output=logits, labels_ph=labels_ph, train=train, loss=loss, learning=None, sess=sess) # Step 4: Train the ART classifier classifier.fit(x_train, y_train, batch_size=64, nb_epochs=3) print('successful') # Step 5: Evaluate the ART classifier on benign test examples predictions = classifier.predict(x_test) accuracy = np.sum(np.argmax(predictions, axis=1) == np.argmax(y_test, axis=1)) / len(y_test) print('Accuracy on benign test examples: {}%'.format(accuracy * 100)) # Step 6: Generate adversarial test examples attack = FastGradientMethod(classifier=classifier, eps=0.2) x_test_adv = attack.generate(x=x_test) # Step 7: Evaluate the ART classifier on adversarial test examples
def test_tfclassifier(self): """ First test with the TFClassifier. :return: """ # Build a TFClassifier # Define input and output placeholders input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1]) output_ph = tf.placeholder(tf.int32, shape=[None, 10]) # Define the tensorflow graph conv = tf.layers.conv2d(input_ph, 4, 5, activation=tf.nn.relu) conv = tf.layers.max_pooling2d(conv, 2, 2) flattened = tf.contrib.layers.flatten(conv) # Logits layer logits = tf.layers.dense(flattened, 10) # Train operator loss = tf.reduce_mean( tf.losses.softmax_cross_entropy(logits=logits, onehot_labels=output_ph)) optimizer = tf.train.AdamOptimizer(learning_rate=0.01) train = optimizer.minimize(loss) # Tensorflow session and initialization sess = tf.Session() sess.run(tf.global_variables_initializer()) # Get MNIST (x_train, y_train), (x_test, y_test) = self.mnist # Train the classifier tfc = TFClassifier((0, 1), input_ph, logits, output_ph, train, loss, None, sess) tfc.fit(x_train, y_train, batch_size=BATCH_SIZE, nb_epochs=2) # Attack attack_params = { "max_translation": 10.0, "num_translations": 3, "max_rotation": 30.0, "num_rotations": 3 } attack_st = SpatialTransformation(tfc) x_train_adv = attack_st.generate(x_train, **attack_params) self.assertTrue(abs(x_train_adv[0, 8, 13, 0] - 0.8066048) <= 0.01) # self.assertTrue(abs(attack_st.fooling_rate - 0.948) <= 0.01) self.assertTrue(attack_st.attack_trans_x == -3) self.assertTrue(attack_st.attack_trans_y == -3) self.assertTrue(attack_st.attack_rot == -30.0) x_test_adv = attack_st.generate(x_test) self.assertTrue(abs(x_test_adv[0, 14, 14, 0] - 0.6941315) <= 0.01) sess.close() tf.reset_default_graph()
def test_tfclassifier(self): """ First test with the TFClassifier. :return: """ # Build a TFClassifier # Define input and output placeholders input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1]) output_ph = tf.placeholder(tf.int32, shape=[None, 10]) # Define the tensorflow graph conv = tf.layers.conv2d(input_ph, 4, 5, activation=tf.nn.relu) conv = tf.layers.max_pooling2d(conv, 2, 2) fc = tf.contrib.layers.flatten(conv) # Logits layer logits = tf.layers.dense(fc, 10) # Train operator loss = tf.reduce_mean( tf.losses.softmax_cross_entropy(logits=logits, onehot_labels=output_ph)) optimizer = tf.train.AdamOptimizer(learning_rate=0.01) train = optimizer.minimize(loss) # Tensorflow session and initialization sess = tf.Session() sess.run(tf.global_variables_initializer()) # Get MNIST (x_train, y_train), (x_test, _) = self.mnist # Train the classifier tfc = TFClassifier((0, 1), input_ph, logits, output_ph, train, loss, None, sess) tfc.fit(x_train, y_train, batch_size=BATCH_SIZE, nb_epochs=2) # Attack # import time nf = NewtonFool(tfc, max_iter=5) # print("Test Tensorflow....") # starttime = time.clock() # x_test_adv = nf.generate(x_test, batch_size=1) # self.assertFalse((x_test == x_test_adv).all()) # endtime = time.clock() # print(1, endtime - starttime) # starttime = time.clock() # x_test_adv = nf.generate(x_test, batch_size=10) # endtime = time.clock() # print(10, endtime - starttime) # starttime = time.clock() x_test_adv = nf.generate(x_test, batch_size=100) # endtime = time.clock() # print(100, endtime - starttime) # # starttime = time.clock() # x_test_adv = nf.generate(x_test, batch_size=1000) # endtime = time.clock() # print(1000, endtime - starttime) self.assertFalse((x_test == x_test_adv).all()) y_pred = tfc.predict(x_test) y_pred_adv = tfc.predict(x_test_adv) y_pred_bool = y_pred.max(axis=1, keepdims=1) == y_pred y_pred_max = y_pred.max(axis=1) y_pred_adv_max = y_pred_adv[y_pred_bool] self.assertTrue((y_pred_max >= y_pred_adv_max).all())
class TestTFClassifier(unittest.TestCase): """ This class tests the functionalities of the Tensorflow-based classifier. """ @classmethod def setUpClass(cls): # Get MNIST (x_train, y_train), (x_test, y_test), _, _ = load_mnist() x_train, y_train = x_train[:NB_TRAIN], y_train[:NB_TRAIN] x_test, y_test = x_test[:NB_TEST], y_test[:NB_TEST] cls.mnist = (x_train, y_train), (x_test, y_test) def setUp(self): # Define input and output placeholders input_ph = tf.placeholder(tf.float32, shape=[None, 28, 28, 1]) output_ph = tf.placeholder(tf.int32, shape=[None, 10]) # Define the tensorflow graph conv = tf.layers.conv2d(input_ph, 16, 5, activation=tf.nn.relu) conv = tf.layers.max_pooling2d(conv, 2, 2) fc = tf.contrib.layers.flatten(conv) # Logits layer logits = tf.layers.dense(fc, 10) # Train operator loss = tf.reduce_mean( tf.losses.softmax_cross_entropy(logits=logits, onehot_labels=output_ph)) optimizer = tf.train.AdamOptimizer(learning_rate=0.01) train = optimizer.minimize(loss) # Tensorflow session and initialization self.sess = tf.Session() self.sess.run(tf.global_variables_initializer()) # Create classifier self.classifier = TFClassifier((0, 1), input_ph, logits, output_ph, train, loss, None, self.sess) def tearDown(self): self.sess.close() def test_fit_predict(self): # Get MNIST (x_train, y_train), (x_test, y_test) = self.mnist # Test fit and predict self.classifier.fit(x_train, y_train, batch_size=100, nb_epochs=1) preds = self.classifier.predict(x_test) preds_class = np.argmax(preds, axis=1) trues_class = np.argmax(y_test, axis=1) acc = np.sum(preds_class == trues_class) / len(trues_class) logger.info('Accuracy after fitting: %.2f%%', (acc * 100)) self.assertGreater(acc, 0.1) tf.reset_default_graph() def test_nb_classes(self): # Start to test self.assertTrue(self.classifier.nb_classes == 10) tf.reset_default_graph() def test_input_shape(self): # Start to test self.assertTrue( np.array(self.classifier.input_shape == (28, 28, 1)).all()) tf.reset_default_graph() def test_class_gradient(self): # Get MNIST (_, _), (x_test, y_test) = self.mnist # Test gradient grads = self.classifier.class_gradient(x_test) self.assertTrue( np.array(grads.shape == (NB_TEST, 10, 28, 28, 1)).all()) self.assertTrue(np.sum(grads) != 0) tf.reset_default_graph() def test_loss_gradient(self): # Get MNIST (_, _), (x_test, y_test) = self.mnist # Test gradient grads = self.classifier.loss_gradient(x_test, y_test) self.assertTrue(np.array(grads.shape == (NB_TEST, 28, 28, 1)).all()) self.assertTrue(np.sum(grads) != 0) tf.reset_default_graph() def test_layers(self): # Get MNIST (_, _), (x_test, y_test) = self.mnist # Test and get layers layer_names = self.classifier.layer_names logger.debug(layer_names) self.assertTrue(layer_names == [ 'conv2d/Relu:0', 'max_pooling2d/MaxPool:0', 'Flatten/flatten/Reshape:0', 'dense/BiasAdd:0' ]) for i, name in enumerate(layer_names): act_i = self.classifier.get_activations(x_test, i) act_name = self.classifier.get_activations(x_test, name) self.assertAlmostEqual(np.sum(act_name - act_i), 0) self.assertTrue( self.classifier.get_activations(x_test, 0).shape == (20, 24, 24, 16)) self.assertTrue( self.classifier.get_activations(x_test, 1).shape == (20, 12, 12, 16)) self.assertTrue( self.classifier.get_activations(x_test, 2).shape == (20, 2304)) self.assertTrue( self.classifier.get_activations(x_test, 3).shape == (20, 10)) tf.reset_default_graph()