def problem_29(args, n):
argp = _argparse().parse_args(args[1:])
train_set, valid_set, test_set = utils.load_MNIST(current_dir + "../mnist.pkl.gz")
data = np.asarray(np.vstack((train_set[0], valid_set[0], test_set[0])), dtype=np.float64)
y = np.hstack((train_set[1], valid_set[1], test_set[1]))
data = data[:n]
y = y[:n]
X_2d = np.zeros((n, 2))
print "loaded data"
k = 0
for result in bh_tsne(data, no_dims=argp.no_dims, perplexity=argp.perplexity, theta=argp.theta, randseed=argp.randseed,
verbose=argp.verbose, initial_dims=argp.initial_dims):
X_2d[k,:] = result
k+=1
plt.figure(figsize=(120, 120))
plt.axis('off')
X_2d *= 100
c = ['b', 'g', 'r', 'y','#12efff','#eee111','#123456','#abc222','#000999','#32efff']
for i in xrange(10):
plt.scatter(X_2d[(y == i) , 0], X_2d[(y == i), 1], label=str(i), marker='$'+str(i)+'$', s=18)
plt.savefig("figure5_"+str(n)+".svg", format="svg")
def evaluate_lenet(learning_rate=0.1, n_epochs=200,
nkerns=[20, 50], batch_size=500):
rng = np.random.RandomState(12345)
print("Loading datasets...")
train_set, valid_set, test_set = utils.load_MNIST()
train_set_X, train_set_y = utils.shared_dataset(train_set)
valid_set_X, valid_set_y = utils.shared_dataset(valid_set)
test_set_X, test_set_y = utils.shared_dataset(test_set)
# we cut data to batches so that we can efficiently load them to
# GPU (if needed)
n_train_batches = train_set_X.get_value(borrow=True).shape[0]
n_valid_batches = valid_set_X.get_value(borrow=True).shape[0]
n_test_batches = test_set_X.get_value(borrow=True).shape[0]
n_train_batches //= batch_size
n_valid_batches //= batch_size
n_test_batches //= batch_size
index = T.lscalar() # index to batches
X = T.matrix("X")
y = T.ivector("y")
print("Building the model...")
# now we construct a 4-layer CNN
# our inputs are 28*28 images with only one feature map, so we
# reshape it to (batch_size, 1, 28, 28)
layer0_input = X.reshape((batch_size, 1, 28, 28))
# layer0: convolution+max-pooling layer
layer0 = layers.ConvPoolLayer(
rng=rng,
input=layer0_input,
input_shape=(batch_size, 1, 28, 28),
filter_shape=(nkerns[0], 1, 5, 5),
poolsize=(2, 2)
)
# layer1: convolution+max-pooling layer
layer1 = layers.ConvPoolLayer(
rng=rng,
input=layer0.output,
input_shape=layer0.output_shape,
filter_shape=(nkerns[1], nkerns[0], 5, 5),
poolsize=(2, 2)
)
# layer2: fully-connected hidden layer
layer2 = layers.MLPLayer(
rng=rng,
input=layer1.output.flatten(2),
n_in=np.prod(layer1.output_shape[1:]),
n_out=layer1.output_shape[0],
activation=T.tanh
)
# layer3: logistic regression
layer3 = layers.LogisticRegression(input=layer2.output,
n_in=batch_size, n_out=10)
cost = layer3.negative_log_likelihood(y)
# construct functions to compute errors on test/validation sets
valid_error = theano.function(
[index],
layer3.errors(y),
givens={
X: valid_set_X[index*batch_size:(index+1)*batch_size],
y: valid_set_y[index*batch_size:(index+1)*batch_size]
}
)
test_error = theano.function(
[index],
layer3.errors(y),
givens={
X: test_set_X[index*batch_size:(index+1)*batch_size],
y: test_set_y[index*batch_size:(index+1)*batch_size]
}
)
# a list of all parameters in this model
params = layer0.params + layer1.params + layer2.params + layer3.params
grads = T.grad(cost, params)
# parameter update rule in stochastic gradient descent
updates = [(param_i, param_i - learning_rate * grad_i)
for param_i, grad_i in zip(params, grads)]
train_model = theano.function(
[index],
cost,
updates=updates,
givens={
X: train_set_X[index*batch_size:(index+1)*batch_size],
y: train_set_y[index*batch_size:(index+1)*batch_size]
}
)
predict_model = theano.function([X], layer3.output)
print("Training...")
# we use the early-stopping strategy
patience = 10000
patience_increase = 2
improvement_threshold = 0.995
validation_frequency = min(n_train_batches, patience // 2)
best_validation_score = 0.
best_iter = 0
test_score = 0.
start_time = timeit.default_timer()
epoch = 0
done = False
while (epoch < n_epochs) and (not done):
epoch += 1
for minibatch_index in range(n_train_batches):
iter = (epoch-1) * n_train_batches + minibatch_index
if iter % 100 == 0:
print("iter =", iter)
train_model(minibatch_index)
if (iter+1) % validation_frequency == 0:
valid_errors = [valid_error(i)
for i in range(n_valid_batches)]
score = 1 - np.mean(valid_errors)
print('epoch {}, minibatch {}/{}, validation accuracy {}'
.format(epoch, minibatch_index + 1,
n_train_batches, score))
if score > best_validation_score:
best_validation_score = score
best_iter = iter
# increase patience if improvement is large enough
if (1-score) < \
(1-best_validation_score) * improvement_threshold:
patience = max(patience, iter * patience_increase)
# test it on test set
test_errors = [test_error(i)
for i in range(n_test_batches)]
test_score = 1 - np.mean(test_errors)
print(' test score:', test_score)
# store best model to file
with open('tmp/best_cnn.pkl', 'wb') as f:
pickle.dump((predict_model, batch_size), f)
if patience <= iter:
done = True
break # break the batches loop
end_time = timeit.default_timer()
print('Finished training. Total time:',
(end_time - start_time) / 60, 'min')
print('Best validation score of', best_validation_score,
'obtained at iter', best_iter)
print('Precision: ', test_score)
continue
X_ = X_train[(y_train == i) + (y_train == j)]
y_ = y_train[(y_train == i) + (y_train == j)]
X_transformed = train_model(X_)
print "pca " , i, " and " , j
plots[i, j].scatter(X_transformed[:, 0], X_transformed[:, 1], c=y_)
plots[i, j].set_xticks(())
plots[i, j].set_yticks(())
plots[j, i].scatter(X_transformed[:, 0], X_transformed[:, 1], c=y_)
plots[j, i].set_xticks(())
plots[j, i].set_yticks(())
if i == 0:
plots[i, j].set_title(j)
plots[j, i].set_ylabel(j)
plt.tight_layout()
plt.savefig(name)
if __name__ == '__main__':
train_set, valid_set, test_set = utils.load_MNIST(current_dir + "../mnist.pkl.gz")
data = np.asarray(np.vstack((train_set[0], valid_set[0], test_set[0])), dtype=np.float64)
y = np.hstack((train_set[1], valid_set[1], test_set[1]))
print 'loaded!'
train(data, y, "scatterplotMNIST.png")
trainx, trainy, testx, testy = utils.load_cifar(current_dir + "../cifar-10-batches-py/")
trainx = np.asarray(np.vstack((trainx, testx)), dtype=np.float64)
trainy = np.hstack((trainy, testy))
print "loaded!"
train(trainx, trainy, "scatterplotCIFAR.png")
act_threshold = args['act_threshold'] if args['act_threshold'] else 0
top_k = args['k_neurons'] if args['k_neurons'] else 3
k_sect = args['k_sections'] if args['k_sections'] else 1000
selected_class = args[
'class'] if not args['class'] == None else -1 #ALL CLASSES
repeat = args['repeat'] if args['repeat'] else 1
logfile_name = args['logfile'] if args['logfile'] else 'result.log'
quantization_granularity = args['quantize'] if args['quantize'] else 3
adv_type = args['advtype'] if args['advtype'] else 'fgsm'
logfile = open(logfile_name, 'a')
####################
# 0) Load data
if dataset == 'mnist':
X_train, Y_train, X_test, Y_test = load_MNIST(channel_first=False)
img_rows, img_cols = 28, 28
else:
X_train, Y_train, X_test, Y_test = load_CIFAR()
img_rows, img_cols = 32, 32
if not selected_class == -1:
X_train, Y_train = filter_val_set(
selected_class, X_train,
Y_train) #Get training input for selected_class
X_test, Y_test = filter_val_set(
selected_class, X_test,
Y_test) #Get testing input for selected_class
####################
# 1) Setup the model
selected_class = args['class'] if not args['class'] == None else 0
step_size = args['step_size'] if not args['step_size'] == None else 1
distance = args['distance'] if not args['distance'] == None else 0.1
approach = args['approach'] if not args['approach'] == None else 'random'
susp_num = args[
'suspicious_num'] if not args['suspicious_num'] == None else 1
repeat = args['repeat'] if not args['repeat'] == None else 1
seed = args['seed'] if not args['seed'] == None else random.randint(0, 10)
star = args['star'] if not args['star'] == None else 3
logfile_name = args[
'logfile'] if not args['logfile'] == None else 'result.log'
####################
# 0) Load MNIST or CIFAR10 data
if dataset == 'mnist':
X_train, Y_train, X_test, Y_test = load_MNIST(one_hot=True)
else:
X_train, Y_train, X_test, Y_test = load_CIFAR()
X_train, X_val, Y_train, Y_val = train_test_split(X_train,
Y_train,
test_size=1 / 6.0,
random_state=seed)
logfile = open(logfile_name, 'a')
####################
# 1) Load the pretrained network.
try:
model = load_model(path.join(model_path, model_name))
except:
# Add hidden layers.
model.add(Dense(300, activation='relu'))
model.add(Dense(100, activation='relu'))
# Add output layer
model.add(Dense(10, activation='softmax'))
# Compile the model
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# Print information about the model
print(model.summary())
X_train, Y_train, X_test, Y_test = load_MNIST()
X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train,
test_size=1/6.0,
random_state=seed)
# Fit the model
model.fit(X_train, Y_train, batch_size=32, epochs=10, verbose=1)
# Save the model
model_json = model.to_json()
with open('mnist_test_model.json', "w") as json_file:
json_file.write(model_json)
model.save_weights("mnist_test_model.h5")
def main():
""" This represents the main function
"""
X, Y = utils.load_MNIST()
# Converting inputs to binary variables, which are Bernoulli random variables (not necessary, but improves performance)
X = (X > 0.5).astype(np.float32)
vae = VariationalAutoencoder(784, [200, 100])
vae.fit(X.copy(
)) # Added copy as fit shuffles the data and we wish to preserve the order
# To test the network, we plot the reconstruction
# plot reconstruction
done = False
while not done:
i = np.random.choice(len(X))
x = X[i]
im = vae.posterior_predictive_sample([x]).reshape(28, 28)
plt.subplot(1, 2, 1)
plt.imshow(x.reshape(28, 28), cmap='gray')
plt.title("Original")
plt.subplot(1, 2, 2)
plt.imshow(im, cmap='gray')
plt.title("Sampled")
plt.show()
ans = input("Generate another?")
if ans and ans[0] in ('n' or 'N'):
done = True
# plot output from random samples in latent space
done = False
while not done:
im, probs = vae.prior_predictive_sample_with_probability()
im = im.reshape(28, 28)
probs = probs.reshape(28, 28)
plt.subplot(1, 2, 1)
plt.imshow(im, cmap='gray')
plt.title("Prior predictive sample")
plt.subplot(1, 2, 2)
plt.imshow(probs, cmap='gray')
plt.title("Prior predictive probs")
plt.show()
ans = input("Generate another?")
if ans and ans[0] in ('n' or 'N'):
done = True
# Finally, we want to add some functionality for visualizing the latent space
# We have a latent space of dimensionality 2 centered at 0
# These images are a visual representation of what the latent space has learned to encode
Z = vae.transform(X)
plt.scatter(Z[:, 0], Z[:, 1], c=Y, s=10)
plt.show()
# We create a 20x20 grid in range [-3,3] centered on the origin.
number_images_per_side = 20
x_values = np.linspace(-3, 3, number_images_per_side)
y_values = np.linspace(-3, 3, number_images_per_side)
image = np.empty(
(28 * number_images_per_side, 28 *
number_images_per_side)) # Final image formed of 28x28 MNIST images
# We do a loop to collect all the Z data points from the grid.
# This is done in order to call the predict function once.
Z_points = []
for i, x in enumerate(x_values):
for j, y in enumerate(y_values):
z_point = [x, y]
Z_points.append(z_point)
X_reconstructed = vae.prior_predictive_sample_with_input(Z_points)
# Finally, we place each reconstructed image in its correponding spot
k = 0
for i, x in enumerate(x_values):
for j, y in enumerate(y_values):
x_reconstructed = X_reconstructed[k]
k += 1
x_reconstructed = x_reconstructed.reshape(28, 28)
image[(number_images_per_side - i - 1) *
28:(number_images_per_side - i) * 28,
j * 28:(j + 1) * 28] = x_reconstructed
plt.imshow(image, cmap='gray')
plt.show()
def train():
train_data, train_label, test_data, test_label = load_MNIST()
lr = 1e-4
Epoch = 1
Batchsize_test = 10
Batchsize_train = 600
Iteration = len(train_data) // Batchsize_train
train_data, train_label, test_data, test_label = load_MNIST()
cnn = CNN()
optimizer = optim.Adam(cnn.parameters(),
lr=lr) # optimize all cnn parameters
loss_func = nn.CrossEntropyLoss() # the target label is not one-hotted
TrainLOSS = []
TestLOSS = []
for epoch in range(Epoch):
for j in range(Iteration):
x_train, x_label = random_draw(train_data, train_label,
Batchsize_train)
x_train1 = torch.from_numpy(x_train).to(torch.float32)
x_train11 = x_train1.reshape(-1, 1, 28, 28)
x_label1 = torch.from_numpy(x_label).to(torch.float32)
x_label11 = torch.nonzero(x_label1).squeeze()
x_label11 = torch.unbind(x_label11, 1)
x_label111 = x_label11[1]
output = cnn(x_train11)
loss = loss_func(output, x_label111)
optimizer.zero_grad()
loss.backward()
optimizer.step()
TrainLOSS.append(loss.item())
x_test, x_testlabel = random_draw(test_data, test_label,
Batchsize_test)
x_test1 = torch.from_numpy(x_test).to(torch.float32)
x_test11 = x_test1.reshape(-1, 1, 28, 28)
x_testlabel1 = torch.from_numpy(x_testlabel).to(torch.float32)
x_testlabel11 = torch.nonzero(x_testlabel1).squeeze()
x_testlabel11 = torch.unbind(x_testlabel11, 1)
x_testlabel111 = x_testlabel11[1]
output2 = cnn(x_test11)
loss2 = loss_func(output2, x_testlabel111)
TestLOSS.append(loss2.item())
print("epoch = %d, loss = %.4f, test loss = %.4f" %
(epoch, loss, loss2))
trainLoss = np.array(TrainLOSS)
testLoss = np.array(TestLOSS)
from matplotlib import pyplot as plt
plt.plot(trainLoss, label="Training")
plt.plot(testLoss, label="Test")
plt.legend()
plt.show()
x_test, x_testlabel = random_draw(test_data, test_label, Batchsize_test)
x_test1 = torch.from_numpy(x_test).to(torch.float32)
x_test11 = x_test1.reshape(-1, 1, 28, 28)
x_testlabel1 = torch.from_numpy(x_testlabel).to(torch.float32)
x_testlabel11 = torch.nonzero(x_testlabel1).squeeze()
x_testlabel11 = torch.unbind(x_testlabel11, 1)
x_testlabel111 = x_testlabel11[1]
output2 = cnn(x_test11)
pred_y = torch.max(output2, 1)[1].data.numpy().squeeze()
print(pred_y, 'predicted number')
print(x_testlabel111.numpy(), 'real number')
loss2 = loss_func(output2, x_testlabel111)
# import pdb
# pdb.set_trace()
print('After Training.\nTest loss = %.4f' % loss2)