def main(_):
dataset = mnist_data.load_mnist()
# print(dataset.train.images.shape)
if tf.gfile.Exists(FLAGS.train_dir):
tf.gfile.DeleteRecursively(FLAGS.train_dir)
tf.gfile.MakeDirs(FLAGS.train_dir)
lenet_train.train(dataset.train, dataset.validation)
def main(unused_args):
assert FLAGS.job_name in ['ps', 'worker'], 'job_name must be ps or worker'
# Extract all the hostnames for the ps and worker jobs to construct the
# cluster spec.
ps_hosts = FLAGS.ps_hosts.split(',')
worker_hosts = FLAGS.worker_hosts.split(',')
tf.logging.info('PS hosts are: %s' % ps_hosts)
tf.logging.info('Worker hosts are: %s' % worker_hosts)
cluster_spec = tf.train.ClusterSpec({
'ps': ps_hosts,
'worker': worker_hosts
})
server = tf.train.Server({
'ps': ps_hosts,
'worker': worker_hosts
},
job_name=FLAGS.job_name,
task_index=FLAGS.task_id)
if FLAGS.job_name == 'ps':
# `ps` jobs wait for incoming connections from the workers.
server.join()
else:
n_workers = len(worker_hosts)
worker_id = int(FLAGS.task_id)
dataset = mnist_data.load_mnist(worker_id=worker_id,
n_workers=n_workers)
# Only the chief checks for or creates train_dir.
if FLAGS.task_id == 0:
if not tf.gfile.Exists(FLAGS.train_dir):
tf.gfile.MakeDirs(FLAGS.train_dir)
distributed_train.train(server.target, dataset.train, cluster_spec)
from sklearn.metrics import accuracy_score
from plotter import plot_decision_regions
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import AdaBoostClassifier
import matplotlib.pyplot as plt
import numpy as np
# import the digit data
from mnist_data import load_mnist
X_train, y_train = load_mnist('mnist', kind='train')
X_test, y_test = load_mnist('mnist', kind='t10k')
tree = DecisionTreeClassifier(criterion='entropy', max_depth=16, random_state=0)
ada = AdaBoostClassifier(base_estimator=tree, n_estimators=10, learning_rate=0.01, random_state=0)
tree = tree.fit(X_train, y_train)
y_train_pred = tree.predict(X_train)
y_test_pred = tree.predict(X_test)
tree_train = accuracy_score(y_train, y_train_pred)
tree_test = accuracy_score(y_test, y_test_pred)
print('Decision tree train/test accuracies %.3f/%.3f' % (tree_train, tree_test))
ada = ada.fit(X_train, y_train)
y_train_pred = ada.predict(X_train)
y_test_pred = ada.predict(X_test)
ada_train = accuracy_score(y_train, y_train_pred)
ada_test = accuracy_score(y_test, y_test_pred)
print('Ada boost train/test accuracies %.3f/%.3f' % (ada_train, ada_test))
def main(_): dataset = mnist_data.load_mnist() # if tf.gfile.Exists(FLAGS.train_dir): # tf.gfile.DeleteRecursively(FLAGS.train_dir) # tf.gfile.MakeDirs(FLAGS.train_dir) evaluate(dataset.validation)
print('d_vars', [var.name for var in self.d_vars])
self.d_optim = tf.train.AdamOptimizer(learning_rate=0.0002, beta1=0.5).minimize(self.loss, var_list=self.d_vars)
self.g_vars = [var for var in t_vars if 'Generator' in var.name]
self.g_optim = tf.train.AdamOptimizer(learning_rate=0.0002, beta1=0.5).minimize(self.loss, var_list=self.g_vars)
self.loss_sum = tf.summary.scalar('loss', self.loss)
self.sum = tf.summary.merge([self.loss_sum])
#
# hyper-parameter
#
batch_size = 200
dcgan = DCGAN(batch_size=batch_size, channel=1)
# load MNIST data
images, labels = mnist_data.load_mnist('./mnist')
input_img = []
for i in range(60000):
input_img.append(images[i])
input_img = np.array(input_img)
image_util.save_images('output.png', input_img[0:64], [8,8])
# preprocess images
input_img = input_img / 127.0 - 1.0
input_img = np.reshape(input_img, [60000, 28, 28, 1])
# init TF
init = tf.global_variables_initializer()
saver = tf.train.Saver()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
worker_hosts = 'localhost:1235'
ps_hosts = ps_hosts.split(",")
worker_hosts = worker_hosts.split(",")
cluster_spec = tf.train.ClusterSpec({'ps': ps_hosts, 'worker': worker_hosts})
server = tf.train.Server({
'ps': ps_hosts,
'worker': worker_hosts
},
job_name=job,
task_index=index)
# ---------------------------------------------
# Model setup
# ---------------------------------------------
dataset = mnist_data.load_mnist(worker_id=job, n_workers=1).train
# Ops are assigned to worker by default.
with tf.device(
tf.train.replica_device_setter(worker_device='/job:worker/task:%s' %
index,
cluster=cluster_spec)):
# Create a variable to count the number of train() calls. This equals the
# number of updates applied to the variables. The PS holds the global step.
global_step = tf.Variable(0, name="global_step", trainable=False)
# Calculate the learning rate schedule.
num_batches_per_epoch = (dataset.num_examples / 128)
# Decay steps need to be divided by the number of replicas to aggregate.
#!/usr/bin/env python
# coding: utf-8
from __future__ import print_function, division
import numpy as np
import tensorflow as tf
import sklearn
from mnist_data import load_mnist
data_dir = './mnist'
train_size = 60000
images, labels = load_mnist(data_dir)
images = images / 127.0 - 1.0
images = np.reshape(images,[-1,28*28])
input_size = 28 * 28
output_size = 10
class MnistClassifier(sklearn.base.BaseEstimator):
"""
"""
def __init__( self, \
batch_size = 128,
hidden_units = 300,
max_epoch = 100,
activation = 'relu',
input_dropout = 0.2,
hidden_dropout = 0.5,
learning_rate = 0.05,
layers = [784, 20, 10, 10]
def init_network(): # 가중치와 편향값(weight and bias) 설정
network = {}
network['W1'] = 0.01 * np.random.randn(layers[0], layers[1]) # 784행 20열 짜리 난수가 생김
network['W2'] = 0.01 * np.random.randn(layers[1], layers[2]) # 20,10
network['W3'] = 0.01 * np.random.randn(layers[2], layers[3]) # 10,10
network['b1'] = np.zeros(layers[1])
network['b2'] = np.zeros(layers[2])
network['b3'] = np.zeros(layers[3])
return network
def predict(network, x):
W1, W2, W3 = network['W1'], network['W2'], network['W3']
b1, b2, b3 = network['b1'], network['b2'], network['b3']
x1 = sigmoid(np.dot(x, W1) + b1) # 각각 변수에 담은 후에, dot 연산하고 bias 더해서 sigmoid 돌리면 다음 층으로 이동함
x2 = sigmoid(np.dot(x1, W2) + b2)
x3 = np.dot(x2, W3) + b3
y = softmax(x3)
return y
def accuracy(network, x, t):
y = predict(network, x) # 뉴럴넷 돌림
y = np.argmax(y, axis=1) # 그 결과를 디코딩해서 숫자만 뽑음
t = np.argmax(t, axis=1)
accuracy = np.sum(y == t) / float(x.shape[0]) # y랑 t랑 같은 애들만 더해서 행의 개수(x.shape[0])로 나눔
return accuracy # 전체 행중에 둘이 같은게 몇개나 있는지 구함
(x_train, y_train), (x_test, y_test) = load_mnist()
network = init_network()
print(accuracy(network, x_train, y_train))
stage1_params = itertools.chain(encoder.parameters(), generator.parameters(), classifier.parameters())
stage2_params = discriminator.parameters()
if sys.argv[1] == 'train':
# =============================== TRAINING ====================================
stage1_solver = optim.Adam(stage1_params, lr=lr, betas=(0.5, 0.999))
stage2_solver = optim.Adam(stage2_params, lr=lr, betas=(0.5, 0.999))
best_VAE_loss = 1000.
best_DIS_loss = 1000.
best_epoch = -1
x_train, y_train, x_test, y_test = mnist_data.load_mnist(reshape=True, twoclass=None, binary=True, onehot=False)
n_train_samples = x_train.shape[0]
n_test_samples = x_test.shape[0]
total_train_batch = n_train_samples // batch_size
total_test_batch = n_test_samples // batch_size
# =============================== Stage1 TRAINING =============================
if stage_flag[0]:
if pretrain_epoch_num[0] >= 0:
encoder.load_state_dict(torch.load('out/mnist/encoder_%d.pth' % (pretrain_epoch_num[0])))
generator.load_state_dict(torch.load('out/mnist/generator_%d.pth' % (pretrain_epoch_num[0])))
classifier.load_state_dict(torch.load('out/mnist/classifier_%d.pth' % (pretrain_epoch_num[0])))
for epoch in range(pretrain_epoch_num[0] + 1, epoch_num[0] + 1):
def train_MLP(args):
classes = args.classes
dim_z = args.dim_z
batch_size = args.batch_size
learning_rate = args.learning_rate
epochs = args.epochs
RESULT_DIR = args.results_path
DATASET = 'mnist-MLP-dim_z{}'.format(dim_z)
TEMP_DIR = './tmp'
if not os.path.exists(RESULT_DIR):
os.makedirs(RESULT_DIR)
if not os.path.exists(TEMP_DIR):
os.makedirs(TEMP_DIR)
if not os.path.exists(os.path.join(RESULT_DIR, DATASET)):
os.makedirs(os.path.join(RESULT_DIR, DATASET))
x_train, y_train, x_test, y_test = mnist_data.load_mnist(reshape=True,
twoclass=None,
binary=False,
onehot=True)
x = tf.placeholder(tf.float32,
shape=[None, IMAGE_SIZE, IMAGE_SIZE, 1],
name='input_img')
y = tf.placeholder(tf.float32, shape=[None, classes], name='input_label')
machine = mnist_model.MNIST_MLP(classes)
loss, optim, metric, variables = machine.build(x, y, batch_size,
learning_rate)
n_train_samples = x_train.shape[0]
n_test_samples = x_test.shape[0]
total_train_batch = int(n_train_samples / batch_size)
total_test_batch = int(n_test_samples / batch_size)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(write_version=tf.train.SaverDef.V1)
for epoch in range(0, epochs + 1):
# Random shuffling
indexes = np.arange(0, n_train_samples)
np.random.shuffle(indexes)
x_train = x_train[indexes, ...]
y_train = y_train[indexes, ...]
for i in range(total_train_batch):
offset = (i * batch_size) % n_train_samples
batch_xs = x_train[offset:(offset + batch_size), ...]
batch_ys = y_train[offset:(offset + batch_size), ...]
feed_dict = {x: batch_xs, y: batch_ys}
# update cla
sess.run(optim['cla'], feed_dict=feed_dict)
if i % 100 == 0:
d_cla, acc = sess.run([loss['cla'], metric['acc']],
feed_dict=feed_dict)
msg = 'epoch:{}/{} step:{}/{} '.format(
epoch, epochs, i, total_train_batch)
msg += 'cla:{:.4}, acc{:.4}' \
.format(d_cla, acc)
print(msg)
# validate after each epoch
val_d_cla, val_acc = 0.0, 0.0
for i in range(total_test_batch):
offset = (i * batch_size) % n_train_samples
batch_xs = x_test[offset:(offset + batch_size), ...]
batch_ys = y_test[offset:(offset + batch_size), ...]
feed_dict = {x: batch_xs, y: batch_ys}
d_cla, acc = sess.run([loss['cla'], metric['acc']],
feed_dict=feed_dict)
val_d_cla += d_cla
val_acc += acc
msg = 'epoch:{}/{} '.format(epoch, epochs)
msg += 'val_cla:{:.4}'.format(val_d_cla / total_test_batch)
msg += 'val_acc:{:.4}'.format(val_acc / total_test_batch)
print(msg)
if epoch % 10 == 0:
saver.save(
sess,
os.path.join(RESULT_DIR, DATASET,
"{}_epoch_{}.data".format(DATASET, epoch)))
saver.save(
sess, os.path.join(RESULT_DIR, DATASET, "{}.data".format(DATASET)))
def train(args):
classes = args.classes
dim_z = args.dim_z
epochs = args.epochs
batch_size = args.batch_size
learning_rate = args.learning_rate
RESULT_DIR = args.results_path
DATASET = 'mnist-dim_z{}'.format(dim_z)
TEMP_DIR = './tmp'
if not os.path.exists(RESULT_DIR):
os.makedirs(RESULT_DIR)
if not os.path.exists(TEMP_DIR):
os.makedirs(TEMP_DIR)
if not os.path.exists(os.path.join(RESULT_DIR, DATASET)):
os.makedirs(os.path.join(RESULT_DIR, DATASET))
x_train, y_train, x_test, y_test = mnist_data.load_mnist(reshape=True,
twoclass=None,
binary=True,
onehot=True)
x = tf.placeholder(tf.float32,
shape=[None, IMAGE_SIZE, IMAGE_SIZE, 1],
name='input_img')
y = tf.placeholder(tf.float32, shape=[None, classes], name='input_label')
svae = mnist_model.SVAEMNIST(dim_z, classes)
loss, optim, metric, variables = svae.build(x, y, batch_size,
learning_rate)
n_train_samples = x_train.shape[0]
n_test_samples = x_test.shape[0]
total_train_batch = int(n_train_samples / batch_size)
total_test_batch = int(n_test_samples / batch_size)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
if True: # stage 1: train the supervised variational autoencoder
saver = tf.train.Saver(write_version=tf.train.SaverDef.V1)
# restore training
# saver.restore(sess, os.path.join(RESULT_DIR, DATASET, "{}_epoch_80.data".format(DATASET)))
for epoch in range(0, epochs + 1):
# Random shuffling
indexes = np.arange(0, n_train_samples)
np.random.shuffle(indexes)
x_train = x_train[indexes, ...]
y_train = y_train[indexes, ...]
for i in range(total_train_batch):
# Compute the offset of the current minibatch in the data.
offset = (i * batch_size) % n_train_samples
batch_xs = x_train[offset:(offset + batch_size), ...]
batch_ys = y_train[offset:(offset + batch_size), ...]
feed_dict = {x: batch_xs, y: batch_ys}
# update sae
_, = sess.run([optim['SAE']], feed_dict=feed_dict)
if i % 100 == 0:
loss_tot, d_cla, d_KL, g_rec, acc = sess.run(
[
loss['SAE'], loss['d_cla'], loss['d_KL'],
loss['g_rec'], metric['acc']
],
feed_dict=feed_dict)
msg = '[stage1] epoch:{}/{} step:{}/{} '.format(
epoch, epochs, i, total_train_batch)
msg += 'loss_tot:{:.4}, d_cla:{:.4}, d_KL:{:.4}, g_rec:{:.4}, acc{:.4}'.format(
loss_tot, d_cla, d_KL, g_rec, acc)
print(msg)
# validate after each epoch
batch_xs = x_test[0:100, ...]
x_rec, x_gen = sess.run(
[metric['x_hat'], metric['x_fake_hat']],
feed_dict={x: batch_xs})
PAE = plot_utils.Plot_Adversarial_Example(TEMP_DIR)
PAE.add_images(list(mnist_data.deprocess(batch_xs)))
PAE.save_images(name="{}_ori.png".format(str(epoch).zfill(3)))
PAE.clear()
PAE.add_images(list(mnist_data.deprocess(x_rec)))
PAE.save_images(name="{}_rec.png".format(str(epoch).zfill(3)))
PAE.clear()
PAE.add_images(list(mnist_data.deprocess(x_gen)))
PAE.save_images(name="{}_gen.png".format(str(epoch).zfill(3)))
val_loss_tot, val_d_cla, val_d_KL, val_g_rec, val_acc = 0.0, 0.0, 0.0, 0.0, 0.0
for i in range(total_test_batch):
offset = (i * batch_size) % n_test_samples
batch_xs = x_test[offset:(offset + batch_size), ...]
batch_ys = y_test[offset:(offset + batch_size), ...]
feed_dict = {x: batch_xs, y: batch_ys}
loss_tot, d_cla, d_KL, g_rec, acc = sess.run(
[
loss['SAE'], loss['d_cla'], loss['d_KL'],
loss['g_rec'], metric['acc']
],
feed_dict=feed_dict)
val_loss_tot += loss_tot
val_d_cla += d_cla
val_d_KL += d_KL
val_g_rec += g_rec
val_acc += acc
msg = 'epoch:{}/{} '.format(epoch, epochs)
msg += 'val_loss_tot:{:.4}, val_d_cla:{:.4}, val_d_KL:{:.4}, val_g_rec:{:.4}'.format(
val_loss_tot / total_test_batch,
val_d_cla / total_test_batch, val_d_KL / total_test_batch,
val_g_rec / total_test_batch)
msg += 'val_acc:{:.4}'.format(val_acc / total_test_batch)
print(msg)
if epoch % 10 == 0:
saver.save(
sess,
os.path.join(RESULT_DIR, DATASET,
"{}_epoch_{}.data".format(DATASET,
epoch)))
saver.save(
sess,
os.path.join(RESULT_DIR, DATASET,
"{}_stage1.data".format(DATASET)))
if True: # stage2: train a discriminator to estimate the manifold in the gaussian latent space
saver_loader = tf.train.Saver(write_version=tf.train.SaverDef.V1,
var_list=variables['enc'] +
variables['gen'] + variables['cla'])
saver_loader.restore(
sess,
os.path.join(RESULT_DIR, DATASET,
'{}_stage1.data'.format(DATASET)))
saver_writer = tf.train.Saver(write_version=tf.train.SaverDef.V1,
var_list=variables['dis'])
for epoch in range(0, epochs + 1):
# Random shuffling
indexes = np.arange(0, n_train_samples)
np.random.shuffle(indexes)
x_train = x_train[indexes, ...]
y_train = y_train[indexes, ...]
for i in range(total_train_batch):
# Compute the offset of the current minibatch in the data.
offset = (i * batch_size) % n_train_samples
batch_xs = x_train[offset:(offset + batch_size), ...]
batch_ys = y_train[offset:(offset + batch_size), ...]
feed_dict = {x: batch_xs, y: batch_ys}
# update sae
_ = sess.run(optim['DIS'], feed_dict=feed_dict)
if i % 100 == 0:
loss_dis, acc_dis_true, acc_dis_fake = sess.run(
[
loss['dis'], metric['acc_dis_true'],
metric['acc_dis_fake']
],
feed_dict=feed_dict)
msg = '[stage2] epoch:{}/{} step:{}/{} '.format(
epoch, epochs, i, total_train_batch)
msg += 'loss_dis:{:.4}, acc_dis_true:{:.4}, acc_dis_fake:{:.4}'.format(
loss_dis, acc_dis_true, acc_dis_fake)
print(msg)
if epoch % 10 == 0:
saver_writer.save(
sess,
os.path.join(
RESULT_DIR, DATASET,
"{}_stage2_epoch_{}.data".format(DATASET, epoch)))
saver_writer.save(
sess,
os.path.join(RESULT_DIR, DATASET,
"{}_stage2.data".format(DATASET)))
import numpy as np
from mnist_data import load_mnist
from matplotlib.pylab import plt
(x_train, y_train), (x_test, y_test) = load_mnist(
) # mnist_data.py에서 load_mnist 함수를 불러와서 실행하고, 그 결과값을 튜플에 담음
for i in range(10):
img = x_train[i]
label = np.argmax(
y_train[i]) # 파라미터 안에서 제일 큰 값을 return하는게 argmax임 / 원핫 인코딩 반대로 하는거임
print(label, end=', ')
img = img.reshape(28, 28)
plt.subplot(1, 10, i + 1)
plt.imshow(img)
print()
plt.show()
from sklearn.neighbors import KNeighborsClassifier
from plotter import plot_decision_regions
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score
import numpy as np
import matplotlib.pyplot as plt
# import the digit data
from mnist_data import load_mnist
X_train, y_train = load_mnist('mnist', kind='train', abbrv=True)
X_test, y_test = load_mnist('mnist', kind='t10k', abbrv=True)
sc = StandardScaler()
sc.fit(X_train)
X_train_std = sc.transform(X_train)
X_test_std = sc.transform(X_test)
neighbors = 2
knn = KNeighborsClassifier(n_neighbors=neighbors, p=2, metric='minkowski')
knn.fit(X_train_std, y_train)
y_pred_train = knn.predict(X_train_std)
y_pred_test = knn.predict(X_test_std)
train_accuracy = accuracy_score(y_train, y_pred_train)
test_accuracy = accuracy_score(y_test, y_pred_test)
print('KNN k=%d: train/test accuracy: %.3f/%.3f' %
(neighbors, train_accuracy, test_accuracy))
onehot = tf.one_hot(label, output_size, axis=1)
with tf.name_scope("cross_entropy_loss"):
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=output, labels=onehot))
trainer = tf.train.AdamOptimizer(learning_rate=0.001)
optimize = trainer.minimize(loss,
var_list=[
weights1, biases1, weights2, biases2, weights3,
biases3, weights4, biases4
])
correct = tf.equal(tf.argmax(pred, axis=1), label)
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
images, labels = load_mnist('./mnist')
t_images, t_labels = load_mnist_t10k('./mnist')
images = images / 127.0 - 1.0
t_images = t_images / 127.0 - 1.0
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as session:
init = tf.global_variables_initializer()
session.run(init)
writer = tf.summary.FileWriter('./graphs', session.graph)
# training AE first
ae_batch_count = ae_train_size // ae_batch_size
for ae_epoch in range(ae_max_epoch):
def main(unused_argv=None):
dataset = mnist_data.load_mnist()
if tf.gfile.Exists(FLAGS.eval_dir):
tf.gfile.DeleteRecursively(FLAGS.eval_dir)
tf.gfile.MakeDirs(FLAGS.eval_dir)
nn_eval.evaluate(dataset.validation)
def train(max_iter=24000):
shape_x = (1, 28, 28)
n_h = args.n_units
n_y = args.n_class
# Load MNIST Dataset
from mnist_data import load_mnist, data_iterator_mnist
images, labels = load_mnist(train=True)
rng = np.random.RandomState(706)
inds = rng.permutation(len(images))
def feed_labeled(i):
j = inds[i]
return images[j], labels[j]
def feed_unlabeled(i):
j = inds[i]
return images[j], labels[j]
di_l = I.data_iterator_simple(
feed_labeled,
args.n_labeled,
args.batchsize_l,
shuffle=True,
rng=rng,
with_file_cache=False,
)
di_u = I.data_iterator_simple(
feed_unlabeled,
args.n_train,
args.batchsize_u,
shuffle=True,
rng=rng,
with_file_cache=False,
)
di_v = data_iterator_mnist(args.batchsize_v, train=False)
# Create networks
# feed-forward-net building function
def forward(x, test=False):
return I.mlp_net(x, n_h, n_y, test)
# Net for learning labeled data
xl = nn.Variable((args.batchsize_l,) + shape_x, need_grad=False)
yl = forward(xl, test=False)
tl = nn.Variable((args.batchsize_l, 1), need_grad=False)
loss_l = F.mean(F.softmax_cross_entropy(yl, tl))
# Net for learning unlabeled data
xu = nn.Variable((args.batchsize_u,) + shape_x, need_grad=False)
yu = forward(xu, test=False)
y1 = yu.get_unlinked_variable()
y1.need_grad = False
noise = nn.Variable((args.batchsize_u,) + shape_x, need_grad=True)
r = noise / (F.sum(noise ** 2, [1, 2, 3], keepdims=True)) ** 0.5
r.persistent = True
y2 = forward(xu + args.xi_for_vat * r, test=False)
y3 = forward(xu + args.eps_for_vat * r, test=False)
loss_k = F.mean(I.distance(y1, y2))
loss_u = F.mean(I.distance(y1, y3))
# Net for evaluating validation data
xv = nn.Variable((args.batchsize_v,) + shape_x, need_grad=False)
hv = forward(xv, test=True)
tv = nn.Variable((args.batchsize_v, 1), need_grad=False)
err = F.mean(F.top_n_error(hv, tv, n=1))
# Create solver
solver = S.Adam(args.learning_rate)
solver.set_parameters(nn.get_parameters())
# Monitor training and validation stats.
path = cache_dir(os.path.join(I.name, "monitor"))
monitor = M.Monitor(path)
monitor_verr = M.MonitorSeries("val_error", monitor, interval=240)
monitor_time = M.MonitorTimeElapsed("time", monitor, interval=240)
# Training Loop.
for i in range(max_iter):
# Validation Test
if i % args.val_interval == 0:
valid_error = I.calc_validation_error(di_v, xv, tv, err, args.val_iter)
monitor_verr.add(i, valid_error)
# forward, backward and update
xl.d, tl.d = di_l.next()
xl.d = xl.d / 255
solver.zero_grad()
loss_l.forward(clear_no_need_grad=True)
loss_l.backward(clear_buffer=True)
solver.weight_decay(args.weight_decay)
solver.update()
# Calculate y without noise, only once.
xu.d, _ = di_u.next()
xu.d = xu.d / 255
yu.forward(clear_buffer=True)
# Do power method iteration
noise.d = np.random.normal(size=xu.shape).astype(np.float32)
for k in range(args.n_iter_for_power_method):
r.grad.zero()
loss_k.forward(clear_no_need_grad=True)
loss_k.backward(clear_buffer=True)
noise.data.copy_from(r.grad)
# forward, backward and update
solver.zero_grad()
loss_u.forward(clear_no_need_grad=True)
loss_u.backward(clear_buffer=True)
solver.weight_decay(args.weight_decay)
solver.update()
if i % args.iter_per_epoch == 0:
solver.set_learning_rate(solver.learning_rate() * args.learning_rate_decay)
monitor_time.add(i)
# Evaluate the final model by the error rate with validation dataset
valid_error = I.calc_validation_error(di_v, xv, tv, err, args.val_iter)
monitor_verr.add(i, valid_error)
monitor_time.add(i)
return path
from sklearn.metrics import accuracy_score
from plotter import plot_decision_regions
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import AdaBoostClassifier
import matplotlib.pyplot as plt
import numpy as np
# import the digit data
from mnist_data import load_mnist
X_train, y_train = load_mnist('mnist', kind='train')
X_test, y_test = load_mnist('mnist', kind='t10k')
tree = DecisionTreeClassifier(criterion='entropy',
max_depth=16,
random_state=0)
ada = AdaBoostClassifier(base_estimator=tree,
n_estimators=10,
learning_rate=0.01,
random_state=0)
tree = tree.fit(X_train, y_train)
y_train_pred = tree.predict(X_train)
y_test_pred = tree.predict(X_test)
tree_train = accuracy_score(y_train, y_train_pred)
tree_test = accuracy_score(y_test, y_test_pred)
print('Decision tree train/test accuracies %.3f/%.3f' %
(tree_train, tree_test))
ada = ada.fit(X_train, y_train)
def transition(args):
test_img_index = args.test_index # modify this to test on different images
target_img_label = args.target_label # modify this to transition the current image to different label
classes = args.classes
dim_z = args.dim_z
batch_size = args.batch_size
learning_rate = args.learning_rate
RESULT_DIR = args.results_path
DATASET = 'mnist-dim_z{}'.format(dim_z)
TEMP_DIR = './tmp'
if not os.path.exists(RESULT_DIR):
os.makedirs(RESULT_DIR)
if not os.path.exists(TEMP_DIR):
os.makedirs(TEMP_DIR)
if not os.path.exists(os.path.join(RESULT_DIR, DATASET)):
os.makedirs(os.path.join(RESULT_DIR, DATASET))
# load dataset
x_train, y_train, x_test, y_test = mnist_data.load_mnist(reshape=True,
twoclass=None,
binary=True,
onehot=True)
test_img = x_test[test_img_index, ...]
test_label = np.argmax(y_test[test_img_index, ...]).squeeze()
# target_img_label = (test_label + 1) if test_label != 9 else 0
print('target label is ', target_img_label)
# choose test image and its target label
target_label = np.zeros(shape=[1, classes])
target_label[0, target_img_label] = 1
# build the testing network
x = tf.placeholder(tf.float32,
shape=[None, IMAGE_SIZE, IMAGE_SIZE, 1],
name='input_img')
y = tf.placeholder(tf.float32, shape=[None, classes], name='input_label')
z = tf.get_variable('z_var', shape=[1, dim_z], dtype=tf.float32)
svae = mnist_model.SVAEMNIST(dim_z, classes)
loss, optim, metric, variables = svae.build(x, y, batch_size,
learning_rate)
logits = svae.classify(z)
x_hat = svae.decode(z)
dis_logits = svae.discriminate(z)
l_dis = tf.squeeze(
tf.nn.sigmoid_cross_entropy_with_logits(
logits=dis_logits, labels=tf.ones_like(dis_logits)))
l_cla = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y))
l_tot = l_cla + 0.01 * l_dis + tf.norm(z) * 0.0005
# using adam optimizer to manipulate the latent space
opt = tf.train.AdamOptimizer(5e-3).minimize(l_tot, var_list=[z])
assign_op = tf.assign(z, metric['latent'])
with tf.Session() as sess:
svae_saver = tf.train.Saver(write_version=tf.train.SaverDef.V1,
var_list=variables['enc'] +
variables['gen'] + variables['cla'])
mc_gan_saver = tf.train.Saver(write_version=tf.train.SaverDef.V1,
var_list=variables['dis'])
sess.run(tf.global_variables_initializer())
# load the parameters of svae
svae_saver.restore(
sess,
os.path.join(RESULT_DIR, DATASET,
'{}_stage1.data'.format(DATASET)))
# load the parameters of discrminator on manifold
mc_gan_saver.restore(
sess,
os.path.join(RESULT_DIR, DATASET,
'{}_stage2.data'.format(DATASET)))
# initialize z_var
_ = sess.run(assign_op, feed_dict={x: np.expand_dims(test_img, 0)})
# plot utils
PAE = plot_utils.Plot_Adversarial_Example(os.path.join(
RESULT_DIR, 'images/'),
n_img_x=30)
PAE.add_image(mnist_data.deprocess(test_img).squeeze())
loss_cla_buf = []
dis_buf = []
verbose_period = 10
iterations = 20000
for i in range(iterations + 1):
if i % verbose_period == 0:
x_rec, loss_cla, dis = sess.run([x_hat, l_cla, l_dis],
feed_dict={y: target_label})
PAE.add_image(mnist_data.deprocess(x_rec).squeeze())
loss_cla_buf.append(np.log(loss_cla + 1e-5))
dis_buf.append(np.log(dis + 1e-5))
_, loss_tot, loss_cla, dis, val_z = sess.run(
[opt, l_tot, l_cla, l_dis, z], feed_dict={y: target_label})
if i % verbose_period == 0:
msg = 'step {}/{}, loss: {:.6}, loss_cla:{:.6}, dis: {:.6}, max_z: {:.4}'\
.format(i, iterations, loss_tot, loss_cla, dis, np.max(np.abs(val_z)))
print(msg)
IA = plot_utils.Image_Animation(os.path.join(RESULT_DIR, 'images/'),
PAE.img_list)
IA.save_animation('[{}]{}_to_{}.mp4'.format(test_img_index, test_label,
target_img_label))
PAE.save_images('[{}]{}_to_{}.rec.jpg'.format(test_img_index,
test_label,
target_img_label))
plt.plot(np.arange(0, verbose_period * len(loss_cla_buf),
verbose_period),
loss_cla_buf,
c='r',
label='oracle classifier loss')
plt.plot(np.arange(0, verbose_period * len(dis_buf), verbose_period),
dis_buf,
c='b',
label='discriminator loss')
plt.xlabel('steps')
plt.ylabel('log loss')
plt.legend(loc='upper right')
plt.savefig(
os.path.join(
RESULT_DIR, 'images/',
'[{}]loss_{}_to_{}.jpg'.format(test_img_index, test_label,
target_img_label)))
print('transition outputs to {}'.format(
os.path.join(RESULT_DIR, 'images/')))
from sklearn.svm import SVC
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score
from plotter import plot_decision_regions
import matplotlib.pyplot as plt
import numpy as np
from mnist_data import load_mnist
X_train, y_train = load_mnist('mnist', kind='train', abbrv=True)
X_test, y_test = load_mnist('mnist', kind='t10k', abbrv=True)
sc = StandardScaler()
sc.fit(X_train)
X_train_std = sc.transform(X_train)
X_test_std = sc.transform(X_test)
svm = SVC(kernel='poly', C=2, random_state=0, gamma=0.5)
svm.fit(X_train_std, y_train)
y_pred = svm.predict(X_test_std)
print('Misclassified samples: %d' % (y_test != y_pred).sum())
y_pred_train = svm.predict(X_train_std)
y_pred_test = svm.predict(X_test_std)
train_accuracy = accuracy_score(y_train, y_pred_train)
test_accuracy = accuracy_score(y_test, y_pred_test)
print('SVM Kernel train/test accuracy: %.3f/%.3f' % (train_accuracy, test_accuracy))
def attack(args): # attack mlp on mnist
test_img_index = args.test_index # modify this to test on different images
target_img_label = args.target_label # modify this to attack the current image to different label
classes = args.classes
dim_z = args.dim_z
batch_size = args.batch_size
learning_rate = args.learning_rate
RESULT_DIR = args.results_path
DATASET = 'mnist-dim_z{}'.format(dim_z)
MLP_DIR = 'mnist-MLP-dim_z{}'.format(dim_z)
TEMP_DIR = './tmp'
if not os.path.exists(RESULT_DIR):
os.makedirs(RESULT_DIR)
if not os.path.exists(TEMP_DIR):
os.makedirs(TEMP_DIR)
if not os.path.exists(os.path.join(RESULT_DIR, DATASET)):
os.makedirs(os.path.join(RESULT_DIR, DATASET))
# load mnist data
x_train, y_train, x_test, y_test = mnist_data.load_mnist(reshape=True,
twoclass=None,
binary=True,
onehot=True)
test_label = np.argmax(y_test[test_img_index, ...]).squeeze()
# target_img_label = (test_label + 1) if test_label < 9 else 0
test_img = x_test[test_img_index, ...]
true_label = np.zeros(shape=[1, classes])
true_label[0, test_label] = 1
target_label = np.zeros(shape=[1, classes])
target_label[0, target_img_label] = 1
# build the testing network
x = tf.placeholder(tf.float32,
shape=[None, IMAGE_SIZE, IMAGE_SIZE, 1],
name='input_img')
y = tf.placeholder(tf.float32, shape=[None, classes], name='input_label')
y_true = tf.placeholder(tf.float32,
shape=[None, classes],
name='true_label')
k_t = tf.placeholder(tf.float32, shape=(), name='k_t')
val_kt = 0
z = tf.get_variable('z_var', shape=[1, dim_z], dtype=tf.float32)
svae = mnist_model.SVAEMNIST(dim_z, classes)
loss, optim, metric, variables = svae.build(x, y, batch_size,
learning_rate)
logits = svae.classify(z)
x_hat = svae.decode(z)
# build the MLP on mnist
MLP = mnist_model.MNIST_MLP(classes)
MLP_loss, MLP_optim, MLP_metric, MLP_variables = MLP.build(
x_hat, y_true, batch_size, learning_rate)
dis_logits = svae.discriminate(z)
l_dis = tf.squeeze(
tf.nn.sigmoid_cross_entropy_with_logits(
logits=dis_logits, labels=tf.ones_like(dis_logits)))
l_cla = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y))
l_tot = l_cla + 0.01 * l_dis + tf.norm(z) * 0.0001 + MLP_loss['cla'] * k_t
opt = tf.train.AdamOptimizer(5e-3).minimize(l_tot, var_list=[z])
assign_op = tf.assign(z, metric['latent'])
with tf.Session() as sess:
svae_saver = tf.train.Saver(write_version=tf.train.SaverDef.V1,
var_list=variables['enc'] +
variables['gen'] + variables['cla'])
mc_gan_saver = tf.train.Saver(write_version=tf.train.SaverDef.V1,
var_list=variables['dis'])
MLP_saver = tf.train.Saver(write_version=tf.train.SaverDef.V1,
var_list=MLP_variables['cla'])
sess.run(tf.global_variables_initializer())
svae_saver.restore(
sess,
os.path.join(RESULT_DIR, DATASET,
'{}_stage1.data'.format(DATASET)))
mc_gan_saver.restore(
sess,
os.path.join(RESULT_DIR, DATASET,
'{}_stage2.data'.format(DATASET)))
MLP_saver.restore(
sess, os.path.join(RESULT_DIR, MLP_DIR, '{}.data'.format(MLP_DIR)))
# initialize z_var
_ = sess.run(assign_op, feed_dict={x: np.expand_dims(test_img, 0)})
# plot utils
PAE = plot_utils.Plot_Adversarial_Example(os.path.join(
RESULT_DIR, 'images/'),
n_img_x=30)
PAE.add_image(mnist_data.deprocess(test_img).squeeze())
verbose_period = 10
loss_cla_buf = []
dis_buf = []
weak_loss_buf = []
iteration = 20000
for i in range(iteration + 1):
feed_dict = {y: target_label, y_true: true_label, k_t: val_kt}
if i % verbose_period == 0:
x_rec, loss_cla, dis, weak_prob, loss_weak = sess.run(
[x_hat, l_cla, l_dis, MLP_metric['prob'], MLP_loss['cla']],
feed_dict=feed_dict)
PAE.add_image(mnist_data.deprocess(x_rec).squeeze())
loss_cla_buf.append(np.log(loss_cla + 1e-5))
dis_buf.append(np.log(dis + 1e-5))
weak_loss_buf.append(np.log(loss_weak + 1e-5))
_, loss_tot, loss_cla, val_dis, val_z, val_loss_weak, weak_prob = sess.run(
[
opt, l_tot, l_cla, l_dis, z, MLP_loss['cla'],
MLP_metric['prob']
],
feed_dict=feed_dict)
val_kt += 1e-4 * (val_loss_weak * 1e-3 - loss_cla +
np.maximum(val_loss_weak - 0.01, 0))
val_kt = np.clip(val_kt, 0.0, 0.001)
if i % verbose_period == 0:
msg = 'step {}/{}, loss: {:.4}, loss_cla:{:.6}, val_dis: {:.6}, max_z: {:.4}, loss_MLP: {:.3}, MLP_pred/prob: {}/{:.4}, kt:{:.4}' \
.format(i, iteration, loss_tot, loss_cla, val_dis, np.max(np.abs(val_z)), val_loss_weak, np.argmax(weak_prob), np.max(weak_prob), val_kt)
print(msg)
IA = plot_utils.Image_Animation(os.path.join(RESULT_DIR, 'images/'),
PAE.img_list)
IA.save_animation('[{}]attack_{}_to_{}.mp4'.format(
test_img_index, test_label, target_img_label))
PAE.save_images('[{}]attack_rec_{}_to_{}.jpg'.format(
test_img_index, test_label, target_img_label))
plt.figure(figsize=(10, 5))
plt.ylim(-15, 5)
plt.plot(np.arange(0,
len(loss_cla_buf) * verbose_period, verbose_period),
loss_cla_buf,
c='r',
label="oracle $f_2$ loss@" +
"$y'$={}".format(target_img_label))
plt.plot(np.arange(0,
len(dis_buf) * verbose_period, verbose_period),
dis_buf,
c='b',
label='discriminator $f_{dis}$ loss')
plt.plot(np.arange(0,
len(weak_loss_buf) * verbose_period,
verbose_period),
weak_loss_buf,
c='g',
label="MLP $f_1$ loss @" + "$y$={}".format(test_label))
# plt.plot(np.arange(0, len(loss_tot_buf)*verbose_period, verbose_period), loss_tot_buf, c='black', label='total loss $J_{SA}$')
plt.xlabel('iteration steps')
plt.ylabel('log loss')
plt.legend(loc='upper right')
plt.savefig(
os.path.join(
RESULT_DIR, 'images/',
'[{}]attack_loss_{}_to_{}.jpg'.format(test_img_index,
test_label,
target_img_label)))
# output the final image
prob, x_img = sess.run([MLP_metric['prob'], x_hat])
x_img_out = mnist_data.deprocess(x_img).squeeze()
final_path = os.path.join(RESULT_DIR, 'images/',
'[{}]final_{}_{:.3}.jpg').format(
test_img_index, prob.argmax(),
np.max(prob))
print('output final adversarial example to:', final_path)
plot_utils.imsave(final_path, x_img_out)
def main():
"""
Main script.
Steps:
* Get and set context.
* Load Dataset
* Initialize DataIterator.
* Create Networks
* Net for Labeled Data
* Net for Unlabeled Data
* Net for Test Data
* Create Solver.
* Training Loop.
* Test
* Training
* by Labeled Data
* Calculate Supervised Loss
* by Unlabeled Data
* Calculate Virtual Adversarial Noise
* Calculate Unsupervised Loss
"""
args = get_args()
# Get context.
from nnabla.ext_utils import get_extension_context
logger.info("Running in %s" % args.context)
ctx = get_extension_context(args.context,
device_id=args.device_id,
type_config=args.type_config)
nn.set_default_context(ctx)
shape_x = (1, 28, 28)
n_h = args.n_units
n_y = args.n_class
# Load MNIST Dataset
from mnist_data import load_mnist, data_iterator_mnist
images, labels = load_mnist(train=True)
rng = np.random.RandomState(706)
inds = rng.permutation(len(images))
def feed_labeled(i):
j = inds[i]
return images[j], labels[j]
def feed_unlabeled(i):
j = inds[i]
return images[j], labels[j]
di_l = data_iterator_simple(feed_labeled,
args.n_labeled,
args.batchsize_l,
shuffle=True,
rng=rng,
with_file_cache=False)
di_u = data_iterator_simple(feed_unlabeled,
args.n_train,
args.batchsize_u,
shuffle=True,
rng=rng,
with_file_cache=False)
di_v = data_iterator_mnist(args.batchsize_v, train=False)
# Create networks
# feed-forward-net building function
def forward(x, test=False):
return mlp_net(x, n_h, n_y, test)
# Net for learning labeled data
xl = nn.Variable((args.batchsize_l, ) + shape_x, need_grad=False)
yl = forward(xl, test=False)
tl = nn.Variable((args.batchsize_l, 1), need_grad=False)
loss_l = F.mean(F.softmax_cross_entropy(yl, tl))
# Net for learning unlabeled data
xu = nn.Variable((args.batchsize_u, ) + shape_x, need_grad=False)
yu = forward(xu, test=False)
y1 = yu.get_unlinked_variable()
y1.need_grad = False
noise = nn.Variable((args.batchsize_u, ) + shape_x, need_grad=True)
r = noise / (F.sum(noise**2, [1, 2, 3], keepdims=True))**0.5
r.persistent = True
y2 = forward(xu + args.xi_for_vat * r, test=False)
y3 = forward(xu + args.eps_for_vat * r, test=False)
loss_k = F.mean(distance(y1, y2))
loss_u = F.mean(distance(y1, y3))
# Net for evaluating validation data
xv = nn.Variable((args.batchsize_v, ) + shape_x, need_grad=False)
hv = forward(xv, test=True)
tv = nn.Variable((args.batchsize_v, 1), need_grad=False)
err = F.mean(F.top_n_error(hv, tv, n=1))
# Create solver
solver = S.Adam(args.learning_rate)
solver.set_parameters(nn.get_parameters())
# Monitor training and validation stats.
import nnabla.monitor as M
monitor = M.Monitor(args.model_save_path)
monitor_verr = M.MonitorSeries("Test error", monitor, interval=240)
monitor_time = M.MonitorTimeElapsed("Elapsed time", monitor, interval=240)
# Training Loop.
t0 = time.time()
for i in range(args.max_iter):
# Validation Test
if i % args.val_interval == 0:
valid_error = calc_validation_error(di_v, xv, tv, err,
args.val_iter)
monitor_verr.add(i, valid_error)
#################################
## Training by Labeled Data #####
#################################
# forward, backward and update
xl.d, tl.d = di_l.next()
xl.d = xl.d / 255
solver.zero_grad()
loss_l.forward(clear_no_need_grad=True)
loss_l.backward(clear_buffer=True)
solver.weight_decay(args.weight_decay)
solver.update()
#################################
## Training by Unlabeled Data ###
#################################
# Calculate y without noise, only once.
xu.d, _ = di_u.next()
xu.d = xu.d / 255
yu.forward(clear_buffer=True)
##### Calculate Adversarial Noise #####
# Do power method iteration
noise.d = np.random.normal(size=xu.shape).astype(np.float32)
for k in range(args.n_iter_for_power_method):
r.grad.zero()
loss_k.forward(clear_no_need_grad=True)
loss_k.backward(clear_buffer=True)
noise.data.copy_from(r.grad)
##### Calculate loss for unlabeled data #####
# forward, backward and update
solver.zero_grad()
loss_u.forward(clear_no_need_grad=True)
loss_u.backward(clear_buffer=True)
solver.weight_decay(args.weight_decay)
solver.update()
##### Learning rate update #####
if i % args.iter_per_epoch == 0:
solver.set_learning_rate(solver.learning_rate() *
args.learning_rate_decay)
monitor_time.add(i)
# Evaluate the final model by the error rate with validation dataset
valid_error = calc_validation_error(di_v, xv, tv, err, args.val_iter)
monitor_verr.add(i, valid_error)
monitor_time.add(i)
# Save the model.
parameter_file = os.path.join(args.model_save_path,
'params_%06d.h5' % args.max_iter)
nn.save_parameters(parameter_file)
'''this is to verify how the data works'''
import mnist_data
if __name__ == "__main__":
dataset = mnist_data.load_mnist(reshape=True)
dataset.train.test_print
#print(dataset.train.num_examples, dataset.train.data_point_shape)
#print(dataset.validation.num_examples, dataset.validation.data_point_shape)
