def main(_): dataset = mnist_data.load_mnist() # print(dataset.train.images.shape) if tf.gfile.Exists(FLAGS.train_dir): tf.gfile.DeleteRecursively(FLAGS.train_dir) tf.gfile.MakeDirs(FLAGS.train_dir) lenet_train.train(dataset.train, dataset.validation)
def main(unused_args): assert FLAGS.job_name in ['ps', 'worker'], 'job_name must be ps or worker' # Extract all the hostnames for the ps and worker jobs to construct the # cluster spec. ps_hosts = FLAGS.ps_hosts.split(',') worker_hosts = FLAGS.worker_hosts.split(',') tf.logging.info('PS hosts are: %s' % ps_hosts) tf.logging.info('Worker hosts are: %s' % worker_hosts) cluster_spec = tf.train.ClusterSpec({ 'ps': ps_hosts, 'worker': worker_hosts }) server = tf.train.Server({ 'ps': ps_hosts, 'worker': worker_hosts }, job_name=FLAGS.job_name, task_index=FLAGS.task_id) if FLAGS.job_name == 'ps': # `ps` jobs wait for incoming connections from the workers. server.join() else: n_workers = len(worker_hosts) worker_id = int(FLAGS.task_id) dataset = mnist_data.load_mnist(worker_id=worker_id, n_workers=n_workers) # Only the chief checks for or creates train_dir. if FLAGS.task_id == 0: if not tf.gfile.Exists(FLAGS.train_dir): tf.gfile.MakeDirs(FLAGS.train_dir) distributed_train.train(server.target, dataset.train, cluster_spec)
from sklearn.metrics import accuracy_score from plotter import plot_decision_regions from sklearn.tree import DecisionTreeClassifier from sklearn.ensemble import AdaBoostClassifier import matplotlib.pyplot as plt import numpy as np # import the digit data from mnist_data import load_mnist X_train, y_train = load_mnist('mnist', kind='train') X_test, y_test = load_mnist('mnist', kind='t10k') tree = DecisionTreeClassifier(criterion='entropy', max_depth=16, random_state=0) ada = AdaBoostClassifier(base_estimator=tree, n_estimators=10, learning_rate=0.01, random_state=0) tree = tree.fit(X_train, y_train) y_train_pred = tree.predict(X_train) y_test_pred = tree.predict(X_test) tree_train = accuracy_score(y_train, y_train_pred) tree_test = accuracy_score(y_test, y_test_pred) print('Decision tree train/test accuracies %.3f/%.3f' % (tree_train, tree_test)) ada = ada.fit(X_train, y_train) y_train_pred = ada.predict(X_train) y_test_pred = ada.predict(X_test) ada_train = accuracy_score(y_train, y_train_pred) ada_test = accuracy_score(y_test, y_test_pred) print('Ada boost train/test accuracies %.3f/%.3f' % (ada_train, ada_test))
def main(_): dataset = mnist_data.load_mnist() # if tf.gfile.Exists(FLAGS.train_dir): # tf.gfile.DeleteRecursively(FLAGS.train_dir) # tf.gfile.MakeDirs(FLAGS.train_dir) evaluate(dataset.validation)
print('d_vars', [var.name for var in self.d_vars]) self.d_optim = tf.train.AdamOptimizer(learning_rate=0.0002, beta1=0.5).minimize(self.loss, var_list=self.d_vars) self.g_vars = [var for var in t_vars if 'Generator' in var.name] self.g_optim = tf.train.AdamOptimizer(learning_rate=0.0002, beta1=0.5).minimize(self.loss, var_list=self.g_vars) self.loss_sum = tf.summary.scalar('loss', self.loss) self.sum = tf.summary.merge([self.loss_sum]) # # hyper-parameter # batch_size = 200 dcgan = DCGAN(batch_size=batch_size, channel=1) # load MNIST data images, labels = mnist_data.load_mnist('./mnist') input_img = [] for i in range(60000): input_img.append(images[i]) input_img = np.array(input_img) image_util.save_images('output.png', input_img[0:64], [8,8]) # preprocess images input_img = input_img / 127.0 - 1.0 input_img = np.reshape(input_img, [60000, 28, 28, 1]) # init TF init = tf.global_variables_initializer() saver = tf.train.Saver() config = tf.ConfigProto() config.gpu_options.allow_growth = True
worker_hosts = 'localhost:1235' ps_hosts = ps_hosts.split(",") worker_hosts = worker_hosts.split(",") cluster_spec = tf.train.ClusterSpec({'ps': ps_hosts, 'worker': worker_hosts}) server = tf.train.Server({ 'ps': ps_hosts, 'worker': worker_hosts }, job_name=job, task_index=index) # --------------------------------------------- # Model setup # --------------------------------------------- dataset = mnist_data.load_mnist(worker_id=job, n_workers=1).train # Ops are assigned to worker by default. with tf.device( tf.train.replica_device_setter(worker_device='/job:worker/task:%s' % index, cluster=cluster_spec)): # Create a variable to count the number of train() calls. This equals the # number of updates applied to the variables. The PS holds the global step. global_step = tf.Variable(0, name="global_step", trainable=False) # Calculate the learning rate schedule. num_batches_per_epoch = (dataset.num_examples / 128) # Decay steps need to be divided by the number of replicas to aggregate.
#!/usr/bin/env python # coding: utf-8 from __future__ import print_function, division import numpy as np import tensorflow as tf import sklearn from mnist_data import load_mnist data_dir = './mnist' train_size = 60000 images, labels = load_mnist(data_dir) images = images / 127.0 - 1.0 images = np.reshape(images,[-1,28*28]) input_size = 28 * 28 output_size = 10 class MnistClassifier(sklearn.base.BaseEstimator): """ """ def __init__( self, \ batch_size = 128, hidden_units = 300, max_epoch = 100, activation = 'relu', input_dropout = 0.2, hidden_dropout = 0.5, learning_rate = 0.05,
layers = [784, 20, 10, 10] def init_network(): # 가중치와 편향값(weight and bias) 설정 network = {} network['W1'] = 0.01 * np.random.randn(layers[0], layers[1]) # 784행 20열 짜리 난수가 생김 network['W2'] = 0.01 * np.random.randn(layers[1], layers[2]) # 20,10 network['W3'] = 0.01 * np.random.randn(layers[2], layers[3]) # 10,10 network['b1'] = np.zeros(layers[1]) network['b2'] = np.zeros(layers[2]) network['b3'] = np.zeros(layers[3]) return network def predict(network, x): W1, W2, W3 = network['W1'], network['W2'], network['W3'] b1, b2, b3 = network['b1'], network['b2'], network['b3'] x1 = sigmoid(np.dot(x, W1) + b1) # 각각 변수에 담은 후에, dot 연산하고 bias 더해서 sigmoid 돌리면 다음 층으로 이동함 x2 = sigmoid(np.dot(x1, W2) + b2) x3 = np.dot(x2, W3) + b3 y = softmax(x3) return y def accuracy(network, x, t): y = predict(network, x) # 뉴럴넷 돌림 y = np.argmax(y, axis=1) # 그 결과를 디코딩해서 숫자만 뽑음 t = np.argmax(t, axis=1) accuracy = np.sum(y == t) / float(x.shape[0]) # y랑 t랑 같은 애들만 더해서 행의 개수(x.shape[0])로 나눔 return accuracy # 전체 행중에 둘이 같은게 몇개나 있는지 구함 (x_train, y_train), (x_test, y_test) = load_mnist() network = init_network() print(accuracy(network, x_train, y_train))
stage1_params = itertools.chain(encoder.parameters(), generator.parameters(), classifier.parameters()) stage2_params = discriminator.parameters() if sys.argv[1] == 'train': # =============================== TRAINING ==================================== stage1_solver = optim.Adam(stage1_params, lr=lr, betas=(0.5, 0.999)) stage2_solver = optim.Adam(stage2_params, lr=lr, betas=(0.5, 0.999)) best_VAE_loss = 1000. best_DIS_loss = 1000. best_epoch = -1 x_train, y_train, x_test, y_test = mnist_data.load_mnist(reshape=True, twoclass=None, binary=True, onehot=False) n_train_samples = x_train.shape[0] n_test_samples = x_test.shape[0] total_train_batch = n_train_samples // batch_size total_test_batch = n_test_samples // batch_size # =============================== Stage1 TRAINING ============================= if stage_flag[0]: if pretrain_epoch_num[0] >= 0: encoder.load_state_dict(torch.load('out/mnist/encoder_%d.pth' % (pretrain_epoch_num[0]))) generator.load_state_dict(torch.load('out/mnist/generator_%d.pth' % (pretrain_epoch_num[0]))) classifier.load_state_dict(torch.load('out/mnist/classifier_%d.pth' % (pretrain_epoch_num[0]))) for epoch in range(pretrain_epoch_num[0] + 1, epoch_num[0] + 1):
def train_MLP(args): classes = args.classes dim_z = args.dim_z batch_size = args.batch_size learning_rate = args.learning_rate epochs = args.epochs RESULT_DIR = args.results_path DATASET = 'mnist-MLP-dim_z{}'.format(dim_z) TEMP_DIR = './tmp' if not os.path.exists(RESULT_DIR): os.makedirs(RESULT_DIR) if not os.path.exists(TEMP_DIR): os.makedirs(TEMP_DIR) if not os.path.exists(os.path.join(RESULT_DIR, DATASET)): os.makedirs(os.path.join(RESULT_DIR, DATASET)) x_train, y_train, x_test, y_test = mnist_data.load_mnist(reshape=True, twoclass=None, binary=False, onehot=True) x = tf.placeholder(tf.float32, shape=[None, IMAGE_SIZE, IMAGE_SIZE, 1], name='input_img') y = tf.placeholder(tf.float32, shape=[None, classes], name='input_label') machine = mnist_model.MNIST_MLP(classes) loss, optim, metric, variables = machine.build(x, y, batch_size, learning_rate) n_train_samples = x_train.shape[0] n_test_samples = x_test.shape[0] total_train_batch = int(n_train_samples / batch_size) total_test_batch = int(n_test_samples / batch_size) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) saver = tf.train.Saver(write_version=tf.train.SaverDef.V1) for epoch in range(0, epochs + 1): # Random shuffling indexes = np.arange(0, n_train_samples) np.random.shuffle(indexes) x_train = x_train[indexes, ...] y_train = y_train[indexes, ...] for i in range(total_train_batch): offset = (i * batch_size) % n_train_samples batch_xs = x_train[offset:(offset + batch_size), ...] batch_ys = y_train[offset:(offset + batch_size), ...] feed_dict = {x: batch_xs, y: batch_ys} # update cla sess.run(optim['cla'], feed_dict=feed_dict) if i % 100 == 0: d_cla, acc = sess.run([loss['cla'], metric['acc']], feed_dict=feed_dict) msg = 'epoch:{}/{} step:{}/{} '.format( epoch, epochs, i, total_train_batch) msg += 'cla:{:.4}, acc{:.4}' \ .format(d_cla, acc) print(msg) # validate after each epoch val_d_cla, val_acc = 0.0, 0.0 for i in range(total_test_batch): offset = (i * batch_size) % n_train_samples batch_xs = x_test[offset:(offset + batch_size), ...] batch_ys = y_test[offset:(offset + batch_size), ...] feed_dict = {x: batch_xs, y: batch_ys} d_cla, acc = sess.run([loss['cla'], metric['acc']], feed_dict=feed_dict) val_d_cla += d_cla val_acc += acc msg = 'epoch:{}/{} '.format(epoch, epochs) msg += 'val_cla:{:.4}'.format(val_d_cla / total_test_batch) msg += 'val_acc:{:.4}'.format(val_acc / total_test_batch) print(msg) if epoch % 10 == 0: saver.save( sess, os.path.join(RESULT_DIR, DATASET, "{}_epoch_{}.data".format(DATASET, epoch))) saver.save( sess, os.path.join(RESULT_DIR, DATASET, "{}.data".format(DATASET)))
def train(args): classes = args.classes dim_z = args.dim_z epochs = args.epochs batch_size = args.batch_size learning_rate = args.learning_rate RESULT_DIR = args.results_path DATASET = 'mnist-dim_z{}'.format(dim_z) TEMP_DIR = './tmp' if not os.path.exists(RESULT_DIR): os.makedirs(RESULT_DIR) if not os.path.exists(TEMP_DIR): os.makedirs(TEMP_DIR) if not os.path.exists(os.path.join(RESULT_DIR, DATASET)): os.makedirs(os.path.join(RESULT_DIR, DATASET)) x_train, y_train, x_test, y_test = mnist_data.load_mnist(reshape=True, twoclass=None, binary=True, onehot=True) x = tf.placeholder(tf.float32, shape=[None, IMAGE_SIZE, IMAGE_SIZE, 1], name='input_img') y = tf.placeholder(tf.float32, shape=[None, classes], name='input_label') svae = mnist_model.SVAEMNIST(dim_z, classes) loss, optim, metric, variables = svae.build(x, y, batch_size, learning_rate) n_train_samples = x_train.shape[0] n_test_samples = x_test.shape[0] total_train_batch = int(n_train_samples / batch_size) total_test_batch = int(n_test_samples / batch_size) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) if True: # stage 1: train the supervised variational autoencoder saver = tf.train.Saver(write_version=tf.train.SaverDef.V1) # restore training # saver.restore(sess, os.path.join(RESULT_DIR, DATASET, "{}_epoch_80.data".format(DATASET))) for epoch in range(0, epochs + 1): # Random shuffling indexes = np.arange(0, n_train_samples) np.random.shuffle(indexes) x_train = x_train[indexes, ...] y_train = y_train[indexes, ...] for i in range(total_train_batch): # Compute the offset of the current minibatch in the data. offset = (i * batch_size) % n_train_samples batch_xs = x_train[offset:(offset + batch_size), ...] batch_ys = y_train[offset:(offset + batch_size), ...] feed_dict = {x: batch_xs, y: batch_ys} # update sae _, = sess.run([optim['SAE']], feed_dict=feed_dict) if i % 100 == 0: loss_tot, d_cla, d_KL, g_rec, acc = sess.run( [ loss['SAE'], loss['d_cla'], loss['d_KL'], loss['g_rec'], metric['acc'] ], feed_dict=feed_dict) msg = '[stage1] epoch:{}/{} step:{}/{} '.format( epoch, epochs, i, total_train_batch) msg += 'loss_tot:{:.4}, d_cla:{:.4}, d_KL:{:.4}, g_rec:{:.4}, acc{:.4}'.format( loss_tot, d_cla, d_KL, g_rec, acc) print(msg) # validate after each epoch batch_xs = x_test[0:100, ...] x_rec, x_gen = sess.run( [metric['x_hat'], metric['x_fake_hat']], feed_dict={x: batch_xs}) PAE = plot_utils.Plot_Adversarial_Example(TEMP_DIR) PAE.add_images(list(mnist_data.deprocess(batch_xs))) PAE.save_images(name="{}_ori.png".format(str(epoch).zfill(3))) PAE.clear() PAE.add_images(list(mnist_data.deprocess(x_rec))) PAE.save_images(name="{}_rec.png".format(str(epoch).zfill(3))) PAE.clear() PAE.add_images(list(mnist_data.deprocess(x_gen))) PAE.save_images(name="{}_gen.png".format(str(epoch).zfill(3))) val_loss_tot, val_d_cla, val_d_KL, val_g_rec, val_acc = 0.0, 0.0, 0.0, 0.0, 0.0 for i in range(total_test_batch): offset = (i * batch_size) % n_test_samples batch_xs = x_test[offset:(offset + batch_size), ...] batch_ys = y_test[offset:(offset + batch_size), ...] feed_dict = {x: batch_xs, y: batch_ys} loss_tot, d_cla, d_KL, g_rec, acc = sess.run( [ loss['SAE'], loss['d_cla'], loss['d_KL'], loss['g_rec'], metric['acc'] ], feed_dict=feed_dict) val_loss_tot += loss_tot val_d_cla += d_cla val_d_KL += d_KL val_g_rec += g_rec val_acc += acc msg = 'epoch:{}/{} '.format(epoch, epochs) msg += 'val_loss_tot:{:.4}, val_d_cla:{:.4}, val_d_KL:{:.4}, val_g_rec:{:.4}'.format( val_loss_tot / total_test_batch, val_d_cla / total_test_batch, val_d_KL / total_test_batch, val_g_rec / total_test_batch) msg += 'val_acc:{:.4}'.format(val_acc / total_test_batch) print(msg) if epoch % 10 == 0: saver.save( sess, os.path.join(RESULT_DIR, DATASET, "{}_epoch_{}.data".format(DATASET, epoch))) saver.save( sess, os.path.join(RESULT_DIR, DATASET, "{}_stage1.data".format(DATASET))) if True: # stage2: train a discriminator to estimate the manifold in the gaussian latent space saver_loader = tf.train.Saver(write_version=tf.train.SaverDef.V1, var_list=variables['enc'] + variables['gen'] + variables['cla']) saver_loader.restore( sess, os.path.join(RESULT_DIR, DATASET, '{}_stage1.data'.format(DATASET))) saver_writer = tf.train.Saver(write_version=tf.train.SaverDef.V1, var_list=variables['dis']) for epoch in range(0, epochs + 1): # Random shuffling indexes = np.arange(0, n_train_samples) np.random.shuffle(indexes) x_train = x_train[indexes, ...] y_train = y_train[indexes, ...] for i in range(total_train_batch): # Compute the offset of the current minibatch in the data. offset = (i * batch_size) % n_train_samples batch_xs = x_train[offset:(offset + batch_size), ...] batch_ys = y_train[offset:(offset + batch_size), ...] feed_dict = {x: batch_xs, y: batch_ys} # update sae _ = sess.run(optim['DIS'], feed_dict=feed_dict) if i % 100 == 0: loss_dis, acc_dis_true, acc_dis_fake = sess.run( [ loss['dis'], metric['acc_dis_true'], metric['acc_dis_fake'] ], feed_dict=feed_dict) msg = '[stage2] epoch:{}/{} step:{}/{} '.format( epoch, epochs, i, total_train_batch) msg += 'loss_dis:{:.4}, acc_dis_true:{:.4}, acc_dis_fake:{:.4}'.format( loss_dis, acc_dis_true, acc_dis_fake) print(msg) if epoch % 10 == 0: saver_writer.save( sess, os.path.join( RESULT_DIR, DATASET, "{}_stage2_epoch_{}.data".format(DATASET, epoch))) saver_writer.save( sess, os.path.join(RESULT_DIR, DATASET, "{}_stage2.data".format(DATASET)))
import numpy as np from mnist_data import load_mnist from matplotlib.pylab import plt (x_train, y_train), (x_test, y_test) = load_mnist( ) # mnist_data.py에서 load_mnist 함수를 불러와서 실행하고, 그 결과값을 튜플에 담음 for i in range(10): img = x_train[i] label = np.argmax( y_train[i]) # 파라미터 안에서 제일 큰 값을 return하는게 argmax임 / 원핫 인코딩 반대로 하는거임 print(label, end=', ') img = img.reshape(28, 28) plt.subplot(1, 10, i + 1) plt.imshow(img) print() plt.show()
from sklearn.neighbors import KNeighborsClassifier from plotter import plot_decision_regions from sklearn.preprocessing import StandardScaler from sklearn.metrics import accuracy_score import numpy as np import matplotlib.pyplot as plt # import the digit data from mnist_data import load_mnist X_train, y_train = load_mnist('mnist', kind='train', abbrv=True) X_test, y_test = load_mnist('mnist', kind='t10k', abbrv=True) sc = StandardScaler() sc.fit(X_train) X_train_std = sc.transform(X_train) X_test_std = sc.transform(X_test) neighbors = 2 knn = KNeighborsClassifier(n_neighbors=neighbors, p=2, metric='minkowski') knn.fit(X_train_std, y_train) y_pred_train = knn.predict(X_train_std) y_pred_test = knn.predict(X_test_std) train_accuracy = accuracy_score(y_train, y_pred_train) test_accuracy = accuracy_score(y_test, y_pred_test) print('KNN k=%d: train/test accuracy: %.3f/%.3f' % (neighbors, train_accuracy, test_accuracy))
onehot = tf.one_hot(label, output_size, axis=1) with tf.name_scope("cross_entropy_loss"): loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=output, labels=onehot)) trainer = tf.train.AdamOptimizer(learning_rate=0.001) optimize = trainer.minimize(loss, var_list=[ weights1, biases1, weights2, biases2, weights3, biases3, weights4, biases4 ]) correct = tf.equal(tf.argmax(pred, axis=1), label) accuracy = tf.reduce_mean(tf.cast(correct, tf.float32)) images, labels = load_mnist('./mnist') t_images, t_labels = load_mnist_t10k('./mnist') images = images / 127.0 - 1.0 t_images = t_images / 127.0 - 1.0 config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as session: init = tf.global_variables_initializer() session.run(init) writer = tf.summary.FileWriter('./graphs', session.graph) # training AE first ae_batch_count = ae_train_size // ae_batch_size for ae_epoch in range(ae_max_epoch):
def main(unused_argv=None): dataset = mnist_data.load_mnist() if tf.gfile.Exists(FLAGS.eval_dir): tf.gfile.DeleteRecursively(FLAGS.eval_dir) tf.gfile.MakeDirs(FLAGS.eval_dir) nn_eval.evaluate(dataset.validation)
def train(max_iter=24000): shape_x = (1, 28, 28) n_h = args.n_units n_y = args.n_class # Load MNIST Dataset from mnist_data import load_mnist, data_iterator_mnist images, labels = load_mnist(train=True) rng = np.random.RandomState(706) inds = rng.permutation(len(images)) def feed_labeled(i): j = inds[i] return images[j], labels[j] def feed_unlabeled(i): j = inds[i] return images[j], labels[j] di_l = I.data_iterator_simple( feed_labeled, args.n_labeled, args.batchsize_l, shuffle=True, rng=rng, with_file_cache=False, ) di_u = I.data_iterator_simple( feed_unlabeled, args.n_train, args.batchsize_u, shuffle=True, rng=rng, with_file_cache=False, ) di_v = data_iterator_mnist(args.batchsize_v, train=False) # Create networks # feed-forward-net building function def forward(x, test=False): return I.mlp_net(x, n_h, n_y, test) # Net for learning labeled data xl = nn.Variable((args.batchsize_l,) + shape_x, need_grad=False) yl = forward(xl, test=False) tl = nn.Variable((args.batchsize_l, 1), need_grad=False) loss_l = F.mean(F.softmax_cross_entropy(yl, tl)) # Net for learning unlabeled data xu = nn.Variable((args.batchsize_u,) + shape_x, need_grad=False) yu = forward(xu, test=False) y1 = yu.get_unlinked_variable() y1.need_grad = False noise = nn.Variable((args.batchsize_u,) + shape_x, need_grad=True) r = noise / (F.sum(noise ** 2, [1, 2, 3], keepdims=True)) ** 0.5 r.persistent = True y2 = forward(xu + args.xi_for_vat * r, test=False) y3 = forward(xu + args.eps_for_vat * r, test=False) loss_k = F.mean(I.distance(y1, y2)) loss_u = F.mean(I.distance(y1, y3)) # Net for evaluating validation data xv = nn.Variable((args.batchsize_v,) + shape_x, need_grad=False) hv = forward(xv, test=True) tv = nn.Variable((args.batchsize_v, 1), need_grad=False) err = F.mean(F.top_n_error(hv, tv, n=1)) # Create solver solver = S.Adam(args.learning_rate) solver.set_parameters(nn.get_parameters()) # Monitor training and validation stats. path = cache_dir(os.path.join(I.name, "monitor")) monitor = M.Monitor(path) monitor_verr = M.MonitorSeries("val_error", monitor, interval=240) monitor_time = M.MonitorTimeElapsed("time", monitor, interval=240) # Training Loop. for i in range(max_iter): # Validation Test if i % args.val_interval == 0: valid_error = I.calc_validation_error(di_v, xv, tv, err, args.val_iter) monitor_verr.add(i, valid_error) # forward, backward and update xl.d, tl.d = di_l.next() xl.d = xl.d / 255 solver.zero_grad() loss_l.forward(clear_no_need_grad=True) loss_l.backward(clear_buffer=True) solver.weight_decay(args.weight_decay) solver.update() # Calculate y without noise, only once. xu.d, _ = di_u.next() xu.d = xu.d / 255 yu.forward(clear_buffer=True) # Do power method iteration noise.d = np.random.normal(size=xu.shape).astype(np.float32) for k in range(args.n_iter_for_power_method): r.grad.zero() loss_k.forward(clear_no_need_grad=True) loss_k.backward(clear_buffer=True) noise.data.copy_from(r.grad) # forward, backward and update solver.zero_grad() loss_u.forward(clear_no_need_grad=True) loss_u.backward(clear_buffer=True) solver.weight_decay(args.weight_decay) solver.update() if i % args.iter_per_epoch == 0: solver.set_learning_rate(solver.learning_rate() * args.learning_rate_decay) monitor_time.add(i) # Evaluate the final model by the error rate with validation dataset valid_error = I.calc_validation_error(di_v, xv, tv, err, args.val_iter) monitor_verr.add(i, valid_error) monitor_time.add(i) return path
from sklearn.metrics import accuracy_score from plotter import plot_decision_regions from sklearn.tree import DecisionTreeClassifier from sklearn.ensemble import AdaBoostClassifier import matplotlib.pyplot as plt import numpy as np # import the digit data from mnist_data import load_mnist X_train, y_train = load_mnist('mnist', kind='train') X_test, y_test = load_mnist('mnist', kind='t10k') tree = DecisionTreeClassifier(criterion='entropy', max_depth=16, random_state=0) ada = AdaBoostClassifier(base_estimator=tree, n_estimators=10, learning_rate=0.01, random_state=0) tree = tree.fit(X_train, y_train) y_train_pred = tree.predict(X_train) y_test_pred = tree.predict(X_test) tree_train = accuracy_score(y_train, y_train_pred) tree_test = accuracy_score(y_test, y_test_pred) print('Decision tree train/test accuracies %.3f/%.3f' % (tree_train, tree_test)) ada = ada.fit(X_train, y_train)
def transition(args): test_img_index = args.test_index # modify this to test on different images target_img_label = args.target_label # modify this to transition the current image to different label classes = args.classes dim_z = args.dim_z batch_size = args.batch_size learning_rate = args.learning_rate RESULT_DIR = args.results_path DATASET = 'mnist-dim_z{}'.format(dim_z) TEMP_DIR = './tmp' if not os.path.exists(RESULT_DIR): os.makedirs(RESULT_DIR) if not os.path.exists(TEMP_DIR): os.makedirs(TEMP_DIR) if not os.path.exists(os.path.join(RESULT_DIR, DATASET)): os.makedirs(os.path.join(RESULT_DIR, DATASET)) # load dataset x_train, y_train, x_test, y_test = mnist_data.load_mnist(reshape=True, twoclass=None, binary=True, onehot=True) test_img = x_test[test_img_index, ...] test_label = np.argmax(y_test[test_img_index, ...]).squeeze() # target_img_label = (test_label + 1) if test_label != 9 else 0 print('target label is ', target_img_label) # choose test image and its target label target_label = np.zeros(shape=[1, classes]) target_label[0, target_img_label] = 1 # build the testing network x = tf.placeholder(tf.float32, shape=[None, IMAGE_SIZE, IMAGE_SIZE, 1], name='input_img') y = tf.placeholder(tf.float32, shape=[None, classes], name='input_label') z = tf.get_variable('z_var', shape=[1, dim_z], dtype=tf.float32) svae = mnist_model.SVAEMNIST(dim_z, classes) loss, optim, metric, variables = svae.build(x, y, batch_size, learning_rate) logits = svae.classify(z) x_hat = svae.decode(z) dis_logits = svae.discriminate(z) l_dis = tf.squeeze( tf.nn.sigmoid_cross_entropy_with_logits( logits=dis_logits, labels=tf.ones_like(dis_logits))) l_cla = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y)) l_tot = l_cla + 0.01 * l_dis + tf.norm(z) * 0.0005 # using adam optimizer to manipulate the latent space opt = tf.train.AdamOptimizer(5e-3).minimize(l_tot, var_list=[z]) assign_op = tf.assign(z, metric['latent']) with tf.Session() as sess: svae_saver = tf.train.Saver(write_version=tf.train.SaverDef.V1, var_list=variables['enc'] + variables['gen'] + variables['cla']) mc_gan_saver = tf.train.Saver(write_version=tf.train.SaverDef.V1, var_list=variables['dis']) sess.run(tf.global_variables_initializer()) # load the parameters of svae svae_saver.restore( sess, os.path.join(RESULT_DIR, DATASET, '{}_stage1.data'.format(DATASET))) # load the parameters of discrminator on manifold mc_gan_saver.restore( sess, os.path.join(RESULT_DIR, DATASET, '{}_stage2.data'.format(DATASET))) # initialize z_var _ = sess.run(assign_op, feed_dict={x: np.expand_dims(test_img, 0)}) # plot utils PAE = plot_utils.Plot_Adversarial_Example(os.path.join( RESULT_DIR, 'images/'), n_img_x=30) PAE.add_image(mnist_data.deprocess(test_img).squeeze()) loss_cla_buf = [] dis_buf = [] verbose_period = 10 iterations = 20000 for i in range(iterations + 1): if i % verbose_period == 0: x_rec, loss_cla, dis = sess.run([x_hat, l_cla, l_dis], feed_dict={y: target_label}) PAE.add_image(mnist_data.deprocess(x_rec).squeeze()) loss_cla_buf.append(np.log(loss_cla + 1e-5)) dis_buf.append(np.log(dis + 1e-5)) _, loss_tot, loss_cla, dis, val_z = sess.run( [opt, l_tot, l_cla, l_dis, z], feed_dict={y: target_label}) if i % verbose_period == 0: msg = 'step {}/{}, loss: {:.6}, loss_cla:{:.6}, dis: {:.6}, max_z: {:.4}'\ .format(i, iterations, loss_tot, loss_cla, dis, np.max(np.abs(val_z))) print(msg) IA = plot_utils.Image_Animation(os.path.join(RESULT_DIR, 'images/'), PAE.img_list) IA.save_animation('[{}]{}_to_{}.mp4'.format(test_img_index, test_label, target_img_label)) PAE.save_images('[{}]{}_to_{}.rec.jpg'.format(test_img_index, test_label, target_img_label)) plt.plot(np.arange(0, verbose_period * len(loss_cla_buf), verbose_period), loss_cla_buf, c='r', label='oracle classifier loss') plt.plot(np.arange(0, verbose_period * len(dis_buf), verbose_period), dis_buf, c='b', label='discriminator loss') plt.xlabel('steps') plt.ylabel('log loss') plt.legend(loc='upper right') plt.savefig( os.path.join( RESULT_DIR, 'images/', '[{}]loss_{}_to_{}.jpg'.format(test_img_index, test_label, target_img_label))) print('transition outputs to {}'.format( os.path.join(RESULT_DIR, 'images/')))
from sklearn.svm import SVC from sklearn.preprocessing import StandardScaler from sklearn.metrics import accuracy_score from plotter import plot_decision_regions import matplotlib.pyplot as plt import numpy as np from mnist_data import load_mnist X_train, y_train = load_mnist('mnist', kind='train', abbrv=True) X_test, y_test = load_mnist('mnist', kind='t10k', abbrv=True) sc = StandardScaler() sc.fit(X_train) X_train_std = sc.transform(X_train) X_test_std = sc.transform(X_test) svm = SVC(kernel='poly', C=2, random_state=0, gamma=0.5) svm.fit(X_train_std, y_train) y_pred = svm.predict(X_test_std) print('Misclassified samples: %d' % (y_test != y_pred).sum()) y_pred_train = svm.predict(X_train_std) y_pred_test = svm.predict(X_test_std) train_accuracy = accuracy_score(y_train, y_pred_train) test_accuracy = accuracy_score(y_test, y_pred_test) print('SVM Kernel train/test accuracy: %.3f/%.3f' % (train_accuracy, test_accuracy))
def attack(args): # attack mlp on mnist test_img_index = args.test_index # modify this to test on different images target_img_label = args.target_label # modify this to attack the current image to different label classes = args.classes dim_z = args.dim_z batch_size = args.batch_size learning_rate = args.learning_rate RESULT_DIR = args.results_path DATASET = 'mnist-dim_z{}'.format(dim_z) MLP_DIR = 'mnist-MLP-dim_z{}'.format(dim_z) TEMP_DIR = './tmp' if not os.path.exists(RESULT_DIR): os.makedirs(RESULT_DIR) if not os.path.exists(TEMP_DIR): os.makedirs(TEMP_DIR) if not os.path.exists(os.path.join(RESULT_DIR, DATASET)): os.makedirs(os.path.join(RESULT_DIR, DATASET)) # load mnist data x_train, y_train, x_test, y_test = mnist_data.load_mnist(reshape=True, twoclass=None, binary=True, onehot=True) test_label = np.argmax(y_test[test_img_index, ...]).squeeze() # target_img_label = (test_label + 1) if test_label < 9 else 0 test_img = x_test[test_img_index, ...] true_label = np.zeros(shape=[1, classes]) true_label[0, test_label] = 1 target_label = np.zeros(shape=[1, classes]) target_label[0, target_img_label] = 1 # build the testing network x = tf.placeholder(tf.float32, shape=[None, IMAGE_SIZE, IMAGE_SIZE, 1], name='input_img') y = tf.placeholder(tf.float32, shape=[None, classes], name='input_label') y_true = tf.placeholder(tf.float32, shape=[None, classes], name='true_label') k_t = tf.placeholder(tf.float32, shape=(), name='k_t') val_kt = 0 z = tf.get_variable('z_var', shape=[1, dim_z], dtype=tf.float32) svae = mnist_model.SVAEMNIST(dim_z, classes) loss, optim, metric, variables = svae.build(x, y, batch_size, learning_rate) logits = svae.classify(z) x_hat = svae.decode(z) # build the MLP on mnist MLP = mnist_model.MNIST_MLP(classes) MLP_loss, MLP_optim, MLP_metric, MLP_variables = MLP.build( x_hat, y_true, batch_size, learning_rate) dis_logits = svae.discriminate(z) l_dis = tf.squeeze( tf.nn.sigmoid_cross_entropy_with_logits( logits=dis_logits, labels=tf.ones_like(dis_logits))) l_cla = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y)) l_tot = l_cla + 0.01 * l_dis + tf.norm(z) * 0.0001 + MLP_loss['cla'] * k_t opt = tf.train.AdamOptimizer(5e-3).minimize(l_tot, var_list=[z]) assign_op = tf.assign(z, metric['latent']) with tf.Session() as sess: svae_saver = tf.train.Saver(write_version=tf.train.SaverDef.V1, var_list=variables['enc'] + variables['gen'] + variables['cla']) mc_gan_saver = tf.train.Saver(write_version=tf.train.SaverDef.V1, var_list=variables['dis']) MLP_saver = tf.train.Saver(write_version=tf.train.SaverDef.V1, var_list=MLP_variables['cla']) sess.run(tf.global_variables_initializer()) svae_saver.restore( sess, os.path.join(RESULT_DIR, DATASET, '{}_stage1.data'.format(DATASET))) mc_gan_saver.restore( sess, os.path.join(RESULT_DIR, DATASET, '{}_stage2.data'.format(DATASET))) MLP_saver.restore( sess, os.path.join(RESULT_DIR, MLP_DIR, '{}.data'.format(MLP_DIR))) # initialize z_var _ = sess.run(assign_op, feed_dict={x: np.expand_dims(test_img, 0)}) # plot utils PAE = plot_utils.Plot_Adversarial_Example(os.path.join( RESULT_DIR, 'images/'), n_img_x=30) PAE.add_image(mnist_data.deprocess(test_img).squeeze()) verbose_period = 10 loss_cla_buf = [] dis_buf = [] weak_loss_buf = [] iteration = 20000 for i in range(iteration + 1): feed_dict = {y: target_label, y_true: true_label, k_t: val_kt} if i % verbose_period == 0: x_rec, loss_cla, dis, weak_prob, loss_weak = sess.run( [x_hat, l_cla, l_dis, MLP_metric['prob'], MLP_loss['cla']], feed_dict=feed_dict) PAE.add_image(mnist_data.deprocess(x_rec).squeeze()) loss_cla_buf.append(np.log(loss_cla + 1e-5)) dis_buf.append(np.log(dis + 1e-5)) weak_loss_buf.append(np.log(loss_weak + 1e-5)) _, loss_tot, loss_cla, val_dis, val_z, val_loss_weak, weak_prob = sess.run( [ opt, l_tot, l_cla, l_dis, z, MLP_loss['cla'], MLP_metric['prob'] ], feed_dict=feed_dict) val_kt += 1e-4 * (val_loss_weak * 1e-3 - loss_cla + np.maximum(val_loss_weak - 0.01, 0)) val_kt = np.clip(val_kt, 0.0, 0.001) if i % verbose_period == 0: msg = 'step {}/{}, loss: {:.4}, loss_cla:{:.6}, val_dis: {:.6}, max_z: {:.4}, loss_MLP: {:.3}, MLP_pred/prob: {}/{:.4}, kt:{:.4}' \ .format(i, iteration, loss_tot, loss_cla, val_dis, np.max(np.abs(val_z)), val_loss_weak, np.argmax(weak_prob), np.max(weak_prob), val_kt) print(msg) IA = plot_utils.Image_Animation(os.path.join(RESULT_DIR, 'images/'), PAE.img_list) IA.save_animation('[{}]attack_{}_to_{}.mp4'.format( test_img_index, test_label, target_img_label)) PAE.save_images('[{}]attack_rec_{}_to_{}.jpg'.format( test_img_index, test_label, target_img_label)) plt.figure(figsize=(10, 5)) plt.ylim(-15, 5) plt.plot(np.arange(0, len(loss_cla_buf) * verbose_period, verbose_period), loss_cla_buf, c='r', label="oracle $f_2$ loss@" + "$y'$={}".format(target_img_label)) plt.plot(np.arange(0, len(dis_buf) * verbose_period, verbose_period), dis_buf, c='b', label='discriminator $f_{dis}$ loss') plt.plot(np.arange(0, len(weak_loss_buf) * verbose_period, verbose_period), weak_loss_buf, c='g', label="MLP $f_1$ loss @" + "$y$={}".format(test_label)) # plt.plot(np.arange(0, len(loss_tot_buf)*verbose_period, verbose_period), loss_tot_buf, c='black', label='total loss $J_{SA}$') plt.xlabel('iteration steps') plt.ylabel('log loss') plt.legend(loc='upper right') plt.savefig( os.path.join( RESULT_DIR, 'images/', '[{}]attack_loss_{}_to_{}.jpg'.format(test_img_index, test_label, target_img_label))) # output the final image prob, x_img = sess.run([MLP_metric['prob'], x_hat]) x_img_out = mnist_data.deprocess(x_img).squeeze() final_path = os.path.join(RESULT_DIR, 'images/', '[{}]final_{}_{:.3}.jpg').format( test_img_index, prob.argmax(), np.max(prob)) print('output final adversarial example to:', final_path) plot_utils.imsave(final_path, x_img_out)
def main(): """ Main script. Steps: * Get and set context. * Load Dataset * Initialize DataIterator. * Create Networks * Net for Labeled Data * Net for Unlabeled Data * Net for Test Data * Create Solver. * Training Loop. * Test * Training * by Labeled Data * Calculate Supervised Loss * by Unlabeled Data * Calculate Virtual Adversarial Noise * Calculate Unsupervised Loss """ args = get_args() # Get context. from nnabla.ext_utils import get_extension_context logger.info("Running in %s" % args.context) ctx = get_extension_context(args.context, device_id=args.device_id, type_config=args.type_config) nn.set_default_context(ctx) shape_x = (1, 28, 28) n_h = args.n_units n_y = args.n_class # Load MNIST Dataset from mnist_data import load_mnist, data_iterator_mnist images, labels = load_mnist(train=True) rng = np.random.RandomState(706) inds = rng.permutation(len(images)) def feed_labeled(i): j = inds[i] return images[j], labels[j] def feed_unlabeled(i): j = inds[i] return images[j], labels[j] di_l = data_iterator_simple(feed_labeled, args.n_labeled, args.batchsize_l, shuffle=True, rng=rng, with_file_cache=False) di_u = data_iterator_simple(feed_unlabeled, args.n_train, args.batchsize_u, shuffle=True, rng=rng, with_file_cache=False) di_v = data_iterator_mnist(args.batchsize_v, train=False) # Create networks # feed-forward-net building function def forward(x, test=False): return mlp_net(x, n_h, n_y, test) # Net for learning labeled data xl = nn.Variable((args.batchsize_l, ) + shape_x, need_grad=False) yl = forward(xl, test=False) tl = nn.Variable((args.batchsize_l, 1), need_grad=False) loss_l = F.mean(F.softmax_cross_entropy(yl, tl)) # Net for learning unlabeled data xu = nn.Variable((args.batchsize_u, ) + shape_x, need_grad=False) yu = forward(xu, test=False) y1 = yu.get_unlinked_variable() y1.need_grad = False noise = nn.Variable((args.batchsize_u, ) + shape_x, need_grad=True) r = noise / (F.sum(noise**2, [1, 2, 3], keepdims=True))**0.5 r.persistent = True y2 = forward(xu + args.xi_for_vat * r, test=False) y3 = forward(xu + args.eps_for_vat * r, test=False) loss_k = F.mean(distance(y1, y2)) loss_u = F.mean(distance(y1, y3)) # Net for evaluating validation data xv = nn.Variable((args.batchsize_v, ) + shape_x, need_grad=False) hv = forward(xv, test=True) tv = nn.Variable((args.batchsize_v, 1), need_grad=False) err = F.mean(F.top_n_error(hv, tv, n=1)) # Create solver solver = S.Adam(args.learning_rate) solver.set_parameters(nn.get_parameters()) # Monitor training and validation stats. import nnabla.monitor as M monitor = M.Monitor(args.model_save_path) monitor_verr = M.MonitorSeries("Test error", monitor, interval=240) monitor_time = M.MonitorTimeElapsed("Elapsed time", monitor, interval=240) # Training Loop. t0 = time.time() for i in range(args.max_iter): # Validation Test if i % args.val_interval == 0: valid_error = calc_validation_error(di_v, xv, tv, err, args.val_iter) monitor_verr.add(i, valid_error) ################################# ## Training by Labeled Data ##### ################################# # forward, backward and update xl.d, tl.d = di_l.next() xl.d = xl.d / 255 solver.zero_grad() loss_l.forward(clear_no_need_grad=True) loss_l.backward(clear_buffer=True) solver.weight_decay(args.weight_decay) solver.update() ################################# ## Training by Unlabeled Data ### ################################# # Calculate y without noise, only once. xu.d, _ = di_u.next() xu.d = xu.d / 255 yu.forward(clear_buffer=True) ##### Calculate Adversarial Noise ##### # Do power method iteration noise.d = np.random.normal(size=xu.shape).astype(np.float32) for k in range(args.n_iter_for_power_method): r.grad.zero() loss_k.forward(clear_no_need_grad=True) loss_k.backward(clear_buffer=True) noise.data.copy_from(r.grad) ##### Calculate loss for unlabeled data ##### # forward, backward and update solver.zero_grad() loss_u.forward(clear_no_need_grad=True) loss_u.backward(clear_buffer=True) solver.weight_decay(args.weight_decay) solver.update() ##### Learning rate update ##### if i % args.iter_per_epoch == 0: solver.set_learning_rate(solver.learning_rate() * args.learning_rate_decay) monitor_time.add(i) # Evaluate the final model by the error rate with validation dataset valid_error = calc_validation_error(di_v, xv, tv, err, args.val_iter) monitor_verr.add(i, valid_error) monitor_time.add(i) # Save the model. parameter_file = os.path.join(args.model_save_path, 'params_%06d.h5' % args.max_iter) nn.save_parameters(parameter_file)
'''this is to verify how the data works''' import mnist_data if __name__ == "__main__": dataset = mnist_data.load_mnist(reshape=True) dataset.train.test_print #print(dataset.train.num_examples, dataset.train.data_point_shape) #print(dataset.validation.num_examples, dataset.validation.data_point_shape)