def plot_bottom_half(data, labels, theta, size, save_image):
result_arr = []
theta_b = theta[:, 392:]
c, d = theta_b.shape
pi = 0.1
for i in range(0, size):
top_half = data[i, :392]
bottom_half = data[i, 392:]
P_cx = np.exp(log_bernoulli_prod_top(data[i], theta, pi))
img_arr = []
for n in range(392, 784):
temp_sum = 0
for class_j in range(0, c):
prob = [1 - theta[class_j][n], theta[class_j][n]]
choice = [0, 1]
temp_sample = np.random.choice(choice, 1, p=prob)
temp_sum += temp_sample * P_cx[class_j]
img_arr.append(temp_sum)
bottom_half = np.asarray(img_arr).reshape(392, )
result = np.where(np.concatenate((top_half, bottom_half)) > 0.5, 1, 0)
result_arr.append(result)
if save_image:
save_images(np.asarray(result_arr), "2_f.jpg")
return np.asarray(result_arr)
def print_perf(params, iter, gradient):
if iter % 30 == 0:
save_images(sigmoid(params),
'results/4/thetas.png',
vmin=0.0,
vmax=1.0)
print(batched_loss(params, iter))
def auto_gd(train_images, train_labels, save_image):
weights = np.zeros((784, 10))
lr = 0.8
for i in range(0, 100):
weights -= lr * softmax_grad(weights, train_images, train_labels)
if save_image:
save_images(weights.T, "3_c.jpg")
return weights.T
def print_perf(combined_params, iter, grad):
if iter % 10 == 0:
gen_params, rec_params = combined_params
bound = np.mean(objective(combined_params, iter))
print("{:15}|{:20}".format(iter // num_batches, bound))
fake_data = generate_from_prior(gen_params, 20, latent_dim, seed)
save_images(fake_data, 'vae_samples.png', vmin=0, vmax=1)
def print_perf(combined_params, iter, grad):
if iter % 10 == 0:
gen_params, rec_params = combined_params
bound = np.mean(objective(combined_params, iter))
print("{:15}|{:20}".format(iter//num_batches, bound))
fake_data = generate_from_prior(gen_params, 20, latent_dim, seed)
save_images(fake_data, 'vae_samples.png', vmin=0, vmax=1)
def compute_means(train_images, train_labels):
np.where(train_images > 0.5, 1, 0)
means = []
for i in range(0, 10):
i_digits = get_digits_by_label(train_images, train_labels, i)
means.append(np.mean(i_digits, axis=0))
save_images(np.array(means), "1_c.jpg")
return np.array(means)
def print_perf(gen_params, dsc_params, iter, gen_gradient, dsc_gradient):
if iter % 10 == 0:
ability = np.mean(objective(gen_params, dsc_params, iter))
fake_data = generate_from_noise(gen_params, 20, noise_dim, seed)
real_data = train_images[batch_indices(iter)]
probs_fake = np.mean(sigmoid(neural_net_predict(dsc_params, fake_data)))
probs_real = np.mean(sigmoid(neural_net_predict(dsc_params, real_data)))
print("{:15}|{:20}|{:20}|{:20}".format(iter//num_batches, ability, probs_fake, probs_real))
save_images(fake_data, 'gan_samples.png', vmin=0, vmax=1)
def print_perf(combined_params, iter, grad):
if iter % 10 == 0:
gen_params, rec_params = combined_params
bound = np.mean(objective(combined_params, iter))
message = "{:15}|{:20}|".format(iter//num_batches, bound)
if iter % 100 == 0:
test_bound = -vae_lower_bound(gen_params, rec_params, test_images, seed) / data_dim
message += "{:20}".format(test_bound)
print(message)
fake_data = generate_from_prior(gen_params, 20, latent_dim, seed)
save_images(fake_data, 'vae_samples.png', vmin=0, vmax=1)
def print_perf(combined_params, iter, grad):
if iter % 10 == 0:
gen_params, rec_params = combined_params
bound = np.mean(objective(combined_params, iter))
message = "{:15}|{:20}|".format(iter//num_batches, bound)
if iter % 100 == 0:
test_bound = -vae_lower_bound(gen_params, rec_params, test_images, seed) / data_dim
message += "{:20}".format(test_bound)
print(message)
fake_data = generate_from_prior(gen_params, 20, latent_dim, seed)
save_images(fake_data, 'vae_samples.png', vmin=0, vmax=1)
def generate_image(train_images, train_labels, save_image):
np.where(train_images > 0.5, 1, 0)
generated_data = []
means = []
for i in range(0, 10):
i_digits = get_digits_by_label(train_images, train_labels, i)
means.append(np.mean(i_digits, axis=0))
temp_generated_data = np.random.rand(784) - means[i]
generated_data.append(np.where(temp_generated_data > 0, 0, 1))
if save_image:
save_images(np.array(generated_data), "2_c.jpg")
return np.array(means)
def print_perf(gen_params, dsc_params, iter, gen_gradient, dsc_gradient):
if iter % 10 == 0:
ability = np.mean(objective(gen_params, dsc_params, iter))
fake_data = generate_from_noise(gen_params, 20, noise_dim, seed)
real_data = train_images[batch_indices(iter)]
probs_fake = np.mean(
sigmoid(neural_net_predict(dsc_params, fake_data)))
probs_real = np.mean(
sigmoid(neural_net_predict(dsc_params, real_data)))
print("{:15}|{:20}|{:20}|{:20}".format(iter // num_batches,
ability, probs_fake,
probs_real))
save_images(fake_data, 'gan_samples.png', vmin=0, vmax=1)
def training_loop(train_images, train_labels, save_image):
weights = np.zeros((784, 10))
for i in range (0, ITERATION):
prob=np.matmul(train_images,weights)
prob_sum = np.array([scipy.misc.logsumexp(prob,axis=1)]).T
softmax_w = -np.exp(prob-prob_sum)
grad = np.matmul(softmax_w.T,train_images).T + np.matmul(train_images.T,train_labels)
grad -= np.divide(weights, np.power(SIGMA,2))
weights+=lr*grad
if save_image:
save_images(weights.T, "1_c.jpg")
return weights.T
def training(epochs=1, batch_size=32):
#Loading Data
x_train = load_images()
batches = x_train.shape[0] / batch_size
# Creating GAN
generator = create_generator()
discriminator = create_discriminator()
gan = create_gan(generator, discriminator)
# Adversarial Labels
y_valid = np.ones(batch_size)*0.9
y_fake = np.zeros(batch_size)
discriminator_loss, generator_loss = [], []
for epoch in range(1, epochs+1):
print('-'*15, 'Epoch', epoch, '-'*15)
g_loss = 0; d_loss = 0
for _ in tqdm(range(int(batches))):
# Random Noise and Images Set
noise = generate_noise(batch_size)
image_batch = x_train[np.random.randint(0, x_train.shape[0], size=batch_size)]
# Generate Fake Images
generated_images = generator.predict(noise)
# Train Discriminator (Fake and Real)
discriminator.trainable = True
d_valid_loss = discriminator.train_on_batch(image_batch, y_valid)
d_fake_loss = discriminator.train_on_batch(generated_images, y_fake)
d_loss += (d_fake_loss + d_valid_loss)/2
# Train Generator
noise = generate_noise(batch_size)
discriminator.trainable = False
g_loss += gan.train_on_batch(noise, y_valid)
discriminator_loss.append(d_loss/batches)
generator_loss.append(g_loss/batches)
if epoch % PLOT_FRECUENCY == 0:
plot_images(epoch, generator)
plot_loss(epoch, generator_loss, discriminator_loss)
save_images(generator)
def print_perf(var_params, iter, gradient):
mean_params, logstd_params = var_params
print(".", end='')
if iter % 30 == 0:
save_images(mean_params, 'a3plotmean.png')
save_images(logstd_params, 'a3plotsgd.png')
sample = sample_diag_gaussian(mean_params, logstd_params, num_samples=1, rs=npr.RandomState(iter))
save_images(sample[0, :, :], 'a3plotsample.png')
## uncomment for Question 2f)
# plot_posterior_contours(mean_params,logstd_params)
print(iter)
print(objective(var_params,iter))
self.x_input: images,
self.y_input: labels
})
return adv_images[:batch_size]
def restore(self):
network.restore(self.sess, FLAGS.attack_networks[0])
RHP_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope='RHP')
RHP_variables_saver = tf.train.Saver(RHP_variables)
ckpt_filename = tf.train.latest_checkpoint(FLAGS.RHP_savepath)
RHP_variables_saver.restore(sess, ckpt_filename)
if __name__ == '__main__':
sess = tf.Session()
model = Attacker(sess)
df = PNGDataFlow(FLAGS.img_dir,
FLAGS.test_list_filename,
FLAGS.ground_truth_file,
result_dir=None,
img_num=FLAGS.img_num)
df = BatchData(df, FLAGS.batch_size, remainder=True)
df.reset_state()
total_batch = int((df.ds.img_num - 1) / FLAGS.batch_size) + 1
for batch_index, (x_batch, y_batch, name_batch) in tqdm(enumerate(df),
total=total_batch):
advs = model.perturb(x_batch, y_batch)
save_images(advs, name_batch, FLAGS.result_dir)
cost_grad = elementwise_grad(cost_function)
#update weights
w = w + lr*cost_grad(w)
print(i)
return w
#run A2 Q3C code
new_images = train_images
new_labels = train_labels
grad_images=grad_labels = new_images #temporary just to create a gobal var for use with autograd
current_c = 0 #temporary just to create a gobal var for use with autograd
weights = logistic_gradient_desc(1000, 0.01) #5000 iterations with a common learning rate of 0.01
save_images(np.transpose(weights),'Q1a')
#A2 Q3d code
def avg_pred_log(w,images):
log_pc_x = 0
for i in range(0,images.shape[0]):
current_log_pc_x = np.dot(np.transpose(w),images[i,:]) - logsumexp(np.dot(np.transpose(w),images[i,:]))
log_pc_x = log_pc_x + current_log_pc_x
return np.sum(log_pc_x)/float(images.shape[0])
def predict_regression(images, w):
predictions = np.zeros((images.shape[0],w.shape[1])) #N by 10
def prediction(X, weight, Y):
log_prob = log_softmax(X, weight)
avg = avg_log_likelihood(((log_prob)), Y)
pred = np.argmax(np.exp(log_prob), axis=1)
return pred, avg
if __name__ == "__main__":
train_images = binarize_data(train_images)
test_images = binarize_data(test_images)
testY = np.argmax(test_labels, axis=1)
trainY = np.argmax(train_labels, axis=1)
weight, loss = optimization(0.1, train_images[:300], train_labels[:300])
data.save_images(weight.T, "unreg_Q3_weight")
data.plot_images(weight.T, plt.figure().add_subplot(111))
plt.show()
print("Train Set Result:")
class_label = np.argmax(train_labels[:300], axis=1)
pred, avg = prediction(train_images[:300], weight, class_label)
print("avg prediction Error is {}".format(np.mean(pred != class_label)))
print("The predictive avg log-likelihood is {}".format(avg))
print("\n")
print("Test Set Result:")
class_label = np.argmax(test_labels, axis=1)
pred, avg = prediction(test_images, weight, class_label)
print("avg prediction Error is {}".format(np.mean(pred != class_label)))
def train(train_images, test_images, param_dump='opt-params.pkl', seed=0):
"""
Optimize gradients of weights over batches of data with elbo estimate.
"""
# Model hyper-parameters
latent_dim = 2
data_dim = 784 # How many pixels in each image (28x28).
gen_layer_sizes = [latent_dim, 500, data_dim] # decoder has 500 hidden
rec_layer_sizes = [data_dim, 500, latent_dim * 2] # encoder has 500 hidden
# Training parameters
param_scale = 0.01
batch_size = 200
num_epochs = 100 # train for 100 epochs
learning_rate = 0.001
key = random.PRNGKey(seed)
key, enc_k, dec_k = random.split(key, 3)
init_gen_params = init_net_params(param_scale, gen_layer_sizes,
dec_k) # encoder
init_rec_params = init_net_params(param_scale, rec_layer_sizes,
enc_k) # decoder
combined_init_params = dict(dec=init_gen_params, enc=init_rec_params)
num_batches = int(np.ceil(len(train_images) / batch_size))
def batch_indices(iter):
idx = iter % num_batches
return slice(idx * batch_size, (idx + 1) * batch_size)
objective_grad = jit(value_and_grad(
batch_loss, argnums=1)) # differentiate w.r.t params
opt_init, opt_update, opt_get_params = adam(step_size=learning_rate)
opt_state = opt_init(combined_init_params)
it = 0
for epoch in tqdm(range(num_epochs)):
for batch in tqdm(range(num_batches)):
batch_x = train_images[batch_indices(batch)]
params = opt_get_params(opt_state)
key, *subkeys = random.split(key, batch_size + 1)
subkeys = np.stack(subkeys, axis=0)
print("subkeys shape: ", subkeys.shape)
loss_, grad_ = objective_grad(batch_x, params, subkeys)
opt_state = opt_update(it, grad_, opt_state)
if it % 100 == 0: # save samples during training
gen_params, rec_params = params['dec'], params['enc']
fake_data = generate_from_prior(gen_params, 20, latent_dim,
key)
save_images(fake_data, 'vae_samples.png', vmin=0, vmax=1)
if it == 0 or (it + 1) % 100 == 0:
test_size = test_images.shape[0]
print("test size: ", test_images.shape, train_images.shape)
key, *subkeys = random.split(key, test_size + 1)
subkeys = np.stack(subkeys, axis=0)
# print performance
loss_t = batch_loss(test_images, params, subkeys)
message = f"Epoch: {epoch} \t Batch: {batch} \t Loss: {loss_:.3f} \t Test Loss: {loss_t:.3f}"
tqdm.write(message)
it += 1
# pickle to save trained weights
params = opt_get_params(opt_state)
with open(param_dump, 'wb') as file:
pickle.dump(params, file, protocol=pickle.HIGHEST_PROTOCOL)
for i in range(0, theta_map.shape[0]):
img_digit_locs = train_labels[:,
i] #get all images locations that have a digit i
current_data = np.transpose(train_images)
Nd = np.dot(
current_data, img_digit_locs
) #multiply data with current digit locs to get data only for current digit (true digit)
N = np.sum(img_digit_locs) #get total points for current digit
current_theta = np.divide((Nd + 1), (N + 2))
theta_map[i, :] = current_theta #save the theta for this digit
return theta_map
#run Q1C code and save image
theta_map = find_theta_MAP(train_images, train_labels)
save_images(theta_map, 'Q1C')
#Q1e code
def find_log_likelihood(images, theta_map, pi_c):
#find using formula generated in q1d
likelihoods = np.zeros((images.shape[0], 10))
#find the likelihood for each digit/datapoint
for digit in range(0, 10): #loop through each digit
for i in range(0, images.shape[0]):
current_data = images[i, :] #get current data point
current_theta = theta_map[digit] #get theta for current digit
likelihoods[i, digit] = np.dot(
current_data, np.log(current_theta)) + np.dot(
(1 - current_data), np.log(1 - current_theta)) #q1d
adv_images = sess.run(self.x_adv,
feed_dict={
self.x_input: images,
self.y_input: labels
})
return adv_images
def restore(self):
for network_name in FLAGS.attack_networks:
network.restore(self.sess, network_name)
if __name__ == '__main__':
sess = tf.Session()
model = Attacker(sess)
df = PNGDataFlow(FLAGS.img_dir,
FLAGS.test_list_filename,
FLAGS.ground_truth_file,
img_num=FLAGS.img_num)
df = BatchData(df, FLAGS.batch_size)
# df = PrefetchDataZMQ(df)
df.reset_state()
total_batch = df.ds.img_num / FLAGS.batch_size
for batch_index, (x_batch, y_batch, name_batch) in tqdm(enumerate(df),
total=total_batch):
advs = model.perturb(x_batch, y_batch)
save_images(advs, name_batch,
FLAGS.result_dir) # TODO: optimize this line
train_images = np.round(train_images)
if n_examples is not None:
train_images = train_images[:n_examples]
train_labels = train_labels[0:n_examples]
if 1 in parts:
print('PART 1')
# c
theta = np.ndarray(shape=(10, 784))
for c in range(10):
x_c = train_images[train_labels[:, c] == 1]
n = x_c.shape[0]
theta[c] = (x_c.sum(axis=0) + 1) / (n + 2)
f, ax = plt.subplots()
save_images(theta, 'results/1/thetas.png', vmin=0.0, vmax=1.0)
# e
def c_given_x(x):
p = np.ndarray(shape=(x.shape[0], 10))
for c in range(10):
p[:, c] = np.log(theta[c]**x * (1 - theta[c])**(1 - x)).sum(axis=1)
p = p - logsumexp(p, axis=1, keepdims=True)
p = np.exp(p)
return p
p_train = c_given_x(train_images)
p_test = c_given_x(test_images)
avg_logp_train = average_logp(p_train, train_labels)
avg_logp_test = average_logp(p_test, test_labels)
avg_acc_train = average_accuracy(p_train, train_labels)
if args.fullaccuracy:
for i in range(0, NUM_ITERATIONS):
print("Iteration {} log-likelihood {}".format(i, liklog[i]))
train_accuracy, test_accuracy = accuracies(param_log[i])
print("Iteration {} train_accuracy: {}".format(i, train_accuracy))
print("Iteration {} test_accuracy: {}".format(i, test_accuracy))
else:
train_accuracy, test_accuracy = accuracies(optimized_params)
print("train_accuracy: {}".format(train_accuracy))
print("test_accuracy: {}".format(test_accuracy))
#return estimated convergence point (first log-likelihood in the log within <margin> of the average of the last <finalsamples> log-liklihoods)
finalsamples = 5
margin = 10
rejectmargin = 50
goal = np.average(liklog[-finalsamples:])
convergenceindex = np.argmax(
liklog >
(goal - margin)) #argmax returns first index of a True in this case
if (np.abs(goal - liklog[-1]) > rejectmargin):
print("log-likelihood has not yet converged")
else:
print("log-likelihood converges by iteration " +
str(convergenceindex))
# Plot weights
optimized_c_l_r, optimized_w = unpack_params(optimized_params)
print(softmax(optimized_c_l_r))
weights = optimized_w.reshape(C, 28, 28)
save_images(weights, "weights.jpg")
def print_perf(params, iter, gradient):
if iter % 30 == 0:
save_images(sigmoid(params), 'q4plot.png')
print(batched_loss(params, iter))
def training(epochs=1, batch_size=32):
#Loading Data
#x_train = load_imgs("./data256/*png")
# Rescale -1 to 1
#x_train = x_train / 127.5 - 1.
train_datagen = ImageDataGenerator()
train_generator = train_datagen.flow_from_directory('imgs/',
batch_size=batch_size,
shuffle=True)
batches = 12385. / batch_size
# Creating GAN
generator = create_generator()
discriminator = create_discriminator()
gan = create_gan(generator, discriminator)
# Adversarial Labels
y_valid = np.ones(batch_size) * 0.9
y_fake = np.zeros(batch_size)
discriminator_loss, generator_loss = [], []
for epoch in range(1, epochs + 1):
print('-' * 15, 'Epoch', epoch, '-' * 15)
g_loss = 0
d_loss = 0
for _ in tqdm(range(int(batches - 1))):
# Random Noise and Images Set
noise = generate_noise(batch_size)
image_batch, _ = train_generator.next()
image_batch = image_batch / 127.5 - 1.
if image_batch.shape[0] == batch_size:
# Generate Fake Images
generated_images = generator.predict(noise)
# Train Discriminator (Fake and Real)
discriminator.trainable = True
d_valid_loss = discriminator.train_on_batch(
image_batch, y_valid)
d_fake_loss = discriminator.train_on_batch(
generated_images, y_fake)
d_loss += (d_fake_loss + d_valid_loss) / 2
# Train Generator
noise = generate_noise(batch_size)
discriminator.trainable = False
g_loss += gan.train_on_batch(noise, y_valid)
train_generator.on_epoch_end()
discriminator_loss.append(d_loss / batches)
generator_loss.append(g_loss / batches)
if epoch % PLOT_FRECUENCY == 0:
plot_images(epoch, generator)
plot_loss(epoch, generator_loss, discriminator_loss)
save_path = "./models"
if not os.path.exists(save_path):
os.makedirs(save_path)
discriminator.save(save_path + "/discrim_%s.h5" % epoch)
generator.save(save_path + "/generat_%s.h5" % epoch)
gan.save(save_path + "/gan_%s.h5" % epoch)
save_images(generator)
results = []
for c in range(theta.shape[0]):
cur_theta = theta[c, 0:half_image.shape[0]]
results.append(
np.prod(cur_theta**half_image * (1 - cur_theta)**(1 - half_image)))
return np.array(results)
# The optimizers provided by autograd can optimize lists, tuples, or dicts of parameters.
# You may use these optimizers for Q4, but implement your own gradient descent optimizer for Q3!
optimized_params = adam(objective_grad,
init_params,
step_size=0.2,
num_iters=10000,
callback=print_perf)
num_images = 20
result = np.zeros((20, 784))
images_from_train = train_images[0:20, :]
result[:, 0:392] = images_from_train[:, 0:392]
for im in range(num_images):
top_probs = top_prob_list(sigmoid(optimized_params),
images_from_train[im, 0:392])
top_prob_sum = np.sum(top_probs)
for i in range(392, 784):
numerator = 0.0
for c in range(K):
numerator += sigmoid(optimized_params[c, i]) * top_probs[c]
result[im, i] = numerator / top_prob_sum
save_images(result, "q4d.jpg")
x_top_temp = np.full(theta_top_temp.shape, x_top_temp)
first = theta_top_temp**x_top_temp * (1 - theta_top_temp)**(1 -
x_top_temp)
first = np.prod(first, axis=1)
first = np.full(theta_bottom_temp.T.shape, first).T
x_bottom_temp = np.full(theta_bottom_temp.shape, x_bottom_temp)
second = theta_bottom_temp
second = np.sum(first * second, axis=0)
first = np.sum(first, axis=0)
result[i, int(theta.shape[1] / 2):] = second / first
result[i, :int(theta.shape[1] / 2)] = x[:int(theta.shape[1] / 2)]
return result
if __name__ == '__main__':
N_data, train_images, train_labels, test_images, test_labels = load_mnist()
optimized_params = adam(objective_grad,
init_params,
step_size=0.2,
num_iters=10000,
callback=print_perf)
optimized_params = sigmoid(optimized_params)
save_images(optimized_params, '4_c.jpg')
picked_images = train_images[
np.random.permutation(train_images.shape[0])[:20], :]
images = plot_bottom_half(picked_images, optimized_params)
save_images(images, '4_d.jpg')
for s in diff_sigma:
weight,loss = optimization(0.1,train_images[:300],train_labels[:300],s)
z = log_softmax(test_images, weight)
likelihood = log_likelihood(test_labels, z, weight, s) # compute test likelihood for sigma
# update the highest likelihood sigma
if (likelihood) > highest_likeli:
highest_likeli = (likelihood)
best_weight = weight
best_sig = s
print("highest likelihood sigma is {}".format(best_sig))
data.save_images(best_weight.T,"Q3_weight")
data.plot_images(best_weight.T, plt.figure().add_subplot(111))
#plt.close()
print("Train Set Result:")
class_label = np.argmax(train_labels[:300], axis=1)
pred, avg = prediction(train_images[:300], best_weight, class_label)
print("avg prediction train accuracy is {}".format(np.mean(pred == class_label)))
print("The predictive avg log-likelihood is {}".format(avg))
print("\n")
print("Test Set Result:")
