def main(device, train_data_loader, test_data_loader):
# select which autoencoder to train
# available autoencoders [MLP, CNN]
is_mlp = False
if constants.RUN_MLP_AUTOEN:
is_mlp = True
autoencoder = MLPAutoencoder()
train_output_dir = constants.MLP_AUTO_EN_TRAIN_OUT_DIR
test_output_dir = constants.MLP_AUTO_EN_TEST_OUT_DIR
model_path = constants.MLP_AUTO_EN_MODEL_PATH
else:
autoencoder = ImprvdCnnAutoencoder()
if constants.RUN_CNN_AUTOEN:
autoencoder = CnnAutoencoder()
train_output_dir = constants.CNN_AUTO_EN_TRAIN_OUT_DIR
test_output_dir = constants.CNN_AUTO_EN_TEST_OUT_DIR
model_path = constants.CNN_AUTO_EN_MODEL_PATH
# Migrate all operations on GPU if available else on CPU
autoencoder.to(device)
# Get the loss (MSE)
criterian = autoencoder.criterian
optimizer = optim.Adam(autoencoder.parameters(), constants.LR)
train_loss = 0
print("Training Initiated")
for epoch in range(1, constants.NUM_EPOCHS + 1):
for data in train_data_loader:
images, _ = data
optimizer.zero_grad()
images = images.to(device)
if is_mlp:
images = images.view(-1, 28 * 28)
output = autoencoder(images)
loss = criterian(output, images.float())
loss.backward()
optimizer.step()
train_loss += loss.item() * images.size(0)
train_loss = train_loss / len(train_data_loader)
if epoch % constants.PRINT_EVERY == 0:
print(
f"epoch {epoch}/{constants.NUM_EPOCHS} loss is :{train_loss}")
save_images(images[0], train_output_dir, epoch)
save_images(output[0], train_output_dir, epoch, 'out')
if not os.path.exists(model_path):
os.makedirs(model_path)
print(f"saving checkpoint for epoch: {epoch}")
torch.save(autoencoder.state_dict(),
os.path.join(model_path, f"epoch{epoch}.pth"))
if epoch == constants.SAVE_CHECKPOINT:
break
# activating evaluation mode so better to switch off the autograd
autoencoder.eval()
with torch.no_grad():
print("Performing inference on test set")
for data in test_data_loader:
images, _ = data
images = images.to(device)
if is_mlp:
images = images.view(-1, 28 * 28)
output = autoencoder(images)
for i in range(len(images)):
save_images(images[i], test_output_dir, i)
save_images(output[i], test_output_dir, i, image_name='out')
print("Process completed!")
real_images = (np.reshape(real_images,[BATCH_SIZE,28,28,1]) - 0.5) * 2.0
real_images = np.lib.pad(real_images, ((0,0),(2,2),(2,2),(0,0)),'constant', constant_values=(-1, -1))
feed_dict = {noise_vector:noise_samples, image:real_images}
loss_d, _ = sess.run([discriminator_loss, updateD], feed_dict = feed_dict)
feed_dict = {noise_vector:noise_samples}
loss_g, _ = sess.run([generator_loss, updateG], feed_dict = feed_dict)
loss_g, _ = sess.run([generator_loss, updateG], feed_dict = feed_dict)
logger.log_scalar(tag='Generator Loss',value=float(loss_g),step=epoch)
logger.log_scalar(tag='Discriminator Loss',value=float(loss_d),step=epoch)
if (epoch%10000) == 0:
print("LossG: {} and LossD: {}".format(float(loss_g), float(loss_d)))
test_noise = helper.sample_noise(BATCH_SIZE, NOISE_SIZE)
images = sess.run(generated_image, feed_dict = {noise_vector:test_noise})
ImagesPath = os.path.join(os.getcwd(), LOG_FILE, 'ImagesPath')
if not os.path.exists(ImagesPath):
os.mkdir(ImagesPath)
helper.save_images(np.reshape(images[0:36],[36,32,32]), [6,6], ImagesPath+'/fig'+str(epoch)+'.png')
ModelPath = os.path.join(os.getcwd(), LOG_FILE, 'ModelPath')
if not os.path.exists(ModelPath):
os.mkdir(ModelPath)
saver.save(sess,ModelPath+'/model'+str(epoch)+'.ckpt')
helper.log_string('######'+str(epoch)+'######', LOG_FILE)
helper.log_string('Generator Loss: '+str(loss_g), LOG_FILE)
helper.log_string('Discriminator Loss: '+str(loss_d), LOG_FILE)
def main():
out_dir = './out'
helper.check_and_create_dir(out_dir)
batch_size = 100
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets('../MNIST_data', one_hot=True)
train_step_per_epoch = mnist.train.num_examples / batch_size
height, width, channel = 28, 28, 1
with tf.Graph().as_default():
session_conf = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
session_conf.gpu_options.allow_growth = True
sess = tf.Session(config=session_conf)
with sess.as_default(), tf.device('/gpu:0'):
pixel_rnn = PixelRNN(sess, 28, 28, 1, 64, 1., 'pixel_rnn')
sess.run(tf.global_variables_initializer())
for epoch in range(100000):
total_train_costs = []
for idx in range(int(train_step_per_epoch)):
x_mb, _ = mnist.train.next_batch(batch_size)
x_mb = x_mb.reshape([batch_size, height, width, channel])
x_mb = (np.random.uniform(size=x_mb.shape) <
x_mb).astype('float32')
_, cost, outputs = sess.run(
[pixel_rnn.optim, pixel_rnn.loss, pixel_rnn.output],
feed_dict={pixel_rnn.inputs: x_mb})
total_train_costs.append(cost)
test_mb, _ = mnist.test.next_batch(batch_size)
test_mb = test_mb.reshape(
[batch_size, height, width, channel])
test_mb = (np.random.uniform(size=test_mb.shape) <
test_mb).astype('float32')
test_cost = sess.run(pixel_rnn.loss,
feed_dict={pixel_rnn.inputs: test_mb})
print(
"Epoch: %d, iteration:%d, train loss: %f, test loss: %f"
% (epoch, idx, cost, test_cost))
if idx % 100 == 0:
avg_train_cost = np.mean(total_train_costs)
#print("Epoch: %d, iteration:%d, train l: %f" % (epoch, idx, cost))
#print("Epoch: %d, iteration:%d, train l: %f" % (epoch, idx, avg_train_cost))
samples = pixel_rnn.generate()
helper.save_images(samples,
height,
width,
10,
10,
dir=out_dir,
prefix='test_')
avg_train_cost = np.mean(total_train_costs)
samples = pixel_rnn.generate()
helper.save_images(samples,
height,
width,
10,
10,
dir=out_dir,
prefix='test_')
print("Epoch: %d, iteration:%d, train l: %f" %
(epoch, idx, avg_train_cost))
def main(argv):
filename = argv.filename
img_name = argv.image
full = argv.fullcoeff
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
os.environ['OMP_NUM_THREADS'] = '8'
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
cwd = os.getcwd()
config = tf.compat.v1.ConfigProto()
# Model directory name
directory = 'baselineModel'
model_dir = cwd + '/' + directory
load_chk_pt = tf.train.latest_checkpoint(model_dir)
epsilons = np.asarray([0, 2.5, 5, 7.5, 10, 12.5, 15, 17.5, 20])
#epsilons = np.asarray([0, 2.5])
perturb_sizes = list(np.linspace(0, 30000, num=9))
epsilons_perturb = zip(epsilons, perturb_sizes)
size_array = []
size_tarray = []
for epsilon, perturb in epsilons_perturb:
tst_org, tst_inp = rd.get_validation_data(num_from=20, num_img=1, acc=4, full=full)
tst_org_full, tst_inp_full = rd.get_validation_data(num_from=20, num_img=1, acc=4, full=True)
mu = np.mean(tst_org, axis=(-1, -2), keepdims=True)
st = np.std(tst_org, axis=(-1, -2), keepdims=True)
tst_inp = (tst_inp - mu) / st
tst_inp = np.clip(tst_inp, -6, 6)
tst_org_adv = np.copy(tst_org)
tst_inp_orig = np.copy(tst_inp)
mu_full = np.mean(tst_org_full, axis=(-1, -2), keepdims=True)
st_full = np.std(tst_org_full, axis=(-1, -2), keepdims=True)
tst_inp_full = (tst_inp_full - mu_full) / st_full
tst_inp_full = np.clip(tst_inp_full, -6, 6)
# Load model and predict
tst_org_adv = tst_org_adv[..., np.newaxis]
tst_inp = tst_inp[..., np.newaxis]
tst_inp_full = tst_inp_full[..., np.newaxis]
tst_rec = np.zeros(tst_inp.shape, dtype=np.float32)
tst_inp_adv = np.copy(tst_inp)
tst_intr_adv = np.zeros(tst_inp_adv.shape, dtype=np.float32)
tst_rec_adv = np.zeros(tst_inp_adv.shape, dtype=np.float32)
# Initialize graph
tf.compat.v1.reset_default_graph()
with tf.compat.v1.Session(config=config) as sess:
new_saver = tf.compat.v1.train.import_meta_graph(model_dir + '/modelTst.meta')
new_saver.restore(sess, load_chk_pt)
graph = tf.compat.v1.get_default_graph()
predT = graph.get_tensor_by_name('predTst:0')
senseT = graph.get_tensor_by_name('sense:0')
# Run reconstruction for original reconstruction
tst_rec[0] = sess.run(predT, feed_dict={senseT: tst_inp[[0]]})
# Get gradient of loss with respect to input
origT = tf.convert_to_tensor(tst_org_adv)
loss = tf.norm(origT - predT, ord='euclidean')
g = tf.gradients(loss, senseT)
grad = sess.run(g, feed_dict={senseT: tst_inp[[0]]})
# Create adversarial example and run reconstruction
tst_intr_adv = epsilon * np.copy(grad)[0]
if epsilon != 0:
print('tst_inp, tst_inp_full', np.sum(np.abs(tst_inp_adv)), np.sum(np.abs(tst_inp_full)))
tst_intr_adv *= perturb / (np.sum(np.abs(tst_intr_adv)))
fraction = np.sum(np.abs(tst_inp_full)) / np.sum(np.abs(tst_intr_adv))
#fraction = np.linalg.norm(tst_inp_adv) / np.linalg.norm(tst_intr_adv)
#fraction = np.linalg.norm(tst_inp_full) / np.linalg.norm(tst_intr_adv)
else:
fraction = 0
# Fraction of original image is perturbation
print('Fraction', fraction)
tst_inp_adv += np.copy(tst_intr_adv)
tst_rec_adv[0] = sess.run(predT, feed_dict={senseT: tst_inp_adv[[0]]})
# Corrupt the input Fourier coefficients with Gaussian noise
mean, std = mu[0][0][0], st
gauss_noise = np.random.normal(mean, std, tst_inp_orig.shape)
gauss_noise *= (np.linalg.norm(tst_intr_adv)) / np.linalg.norm(gauss_noise)
img_shape = np.shape(tst_inp_orig)
tst_inp_gauss = np.copy(tst_inp_orig) + np.copy(gauss_noise)
tst_inp_gauss = tst_inp_gauss[..., np.newaxis]
tst_rec_gauss = np.zeros(tst_inp_gauss.shape, dtype=np.float32)
# Run reconstruction for Gaussian case
tst_rec_gauss[0] = sess.run(predT, feed_dict={senseT: tst_inp_gauss[[0]]})
# Transform images into original shape
tst_inp = tst_inp[..., 0]
tst_inp_full = tst_inp_full[..., 0]
tst_rec = tst_rec[..., 0]
tst_inp_gauss = tst_inp_gauss[..., 0]
tst_rec_gauss = tst_rec_gauss[..., 0]
tst_inp_adv = tst_inp_adv[..., 0]
tst_intr_adv = tst_intr_adv[..., 0]
tst_rec_adv = tst_rec_adv[..., 0]
tst_inp, tst_rec = tst_inp * st + mu, tst_rec * st + mu
tst_inp_gauss, tst_rec_gauss = tst_inp_gauss * st + mu, tst_rec_gauss * st + mu
tst_inp_adv, tst_rec_adv = tst_inp_adv * st + mu, tst_rec_adv * st + mu
tst_inp_full = tst_inp_full * st_full + mu_full
# Add noise to FC to see how it reacts
coeff_inp = sf.fft2c(tst_inp)
coeff_inp_full = sf.fft2c(tst_inp_full)
first_img = sf.sos(sf.crop(sf.ifft2c(coeff_inp), (320, 320)))
first_img = first_img[np.newaxis, ...]
shp = list(coeff_inp.shape)
# Scale should be 320, so dividing by it in the Fourier space will yield correct size in image space
scale = np.sqrt(np.prod(shp[-2:]))
gauss_noise_fc_real = np.random.normal(mean, std, size=shp)
gauss_noise_fc_imag = np.random.normal(mean, std, size=shp) * 1j
gauss_noise_fc = gauss_noise_fc_real[...] + gauss_noise_fc_imag[...]
gauss_noise_fc[np.nonzero(coeff_inp == 0.+0.j)] = 0.+0.j
gauss_noise_fc *= 1 / np.linalg.norm(gauss_noise_fc)
if epsilon != 0:
fc_scaling = np.linalg.norm(coeff_inp_full) / fraction
gauss_noise_fc *= fc_scaling
print(perturb, scale, perturb/scale, np.sum(np.abs(gauss_noise_fc)), np.sum(np.abs(coeff_inp)))
print('FC Scaling', fc_scaling)
else:
gauss_noise_fc = np.zeros(shape=gauss_noise_fc.shape, dtype=np.complex64)
noise_size = np.linalg.norm(gauss_noise_fc)
size_array.append(noise_size)
coeff_final = np.copy(coeff_inp) + np.copy(gauss_noise_fc)
tst_fc = sf.sos(sf.crop(sf.ifft2c(coeff_final), (320, 320)))
tst_fc_noise = sf.sos(sf.crop(sf.ifft2c(gauss_noise_fc), (320, 320)))
tst_fc_noise = tst_fc_noise[np.newaxis, ...]
print(np.sum(np.abs(tst_inp)), np.sum(np.abs(first_img)), np.sum(np.abs(tst_fc)))
tst_fc = tst_fc[np.newaxis, ...]
print('PIC SUMS', epsilon, np.sum(np.abs(tst_fc[...])), np.sum(np.abs(tst_rec_gauss)), np.sum(np.abs(tst_rec_adv[...])))
print('NOISE SUMS', np.sum(np.abs(tst_fc_noise)), np.sum(np.abs(gauss_noise)), np.sum(np.abs(tst_intr_adv)))
noise_transform_size = np.linalg.norm(tst_fc_noise)
size_tarray.append(noise_transform_size)
[fc_error, gaussian_error, adv_error] = calculate_sse(tst_org, tst_fc, tst_rec_gauss, tst_rec_adv)
#[ssim_fc, ssim_gauss, ssim_adv] = calculate_ssim(tst_org, tst_fc, tst_rec_gauss, tst_rec_adv)
if fraction != 0:
#image_grid_small('fc_visual', np.abs(coeff_inp), np.abs(gauss_noise_fc), np.abs(coeff_final))
fraction_original_to_perturb = 1 / fraction
else:
fraction_original_to_perturb = 0
save_results(filename + '.npy', img_shape, fc_error, gaussian_error, adv_error, fraction_original_to_perturb)
#save_results(filename + '_ssim.npy', img_shape, ssim_fc, ssim_gauss, ssim_adv, fraction_original_to_perturb)
if img_name:
save_images(img_name, tst_org, tst_inp, tst_fc, gauss_noise, tst_inp_gauss, tst_rec_gauss, tst_intr_adv, tst_inp_adv, tst_rec_adv)
}) #Update the generator, twice for good measure.
_, gLoss = sess.run([update_G, g_loss], feed_dict={z_in: zs})
if i % 1 == 0:
print("Gen Loss " + str(gLoss) + " Disc Loss: " + str(dLoss))
z2 = np.random.uniform(-1.0, 1.0,
size=[batch_size, z_size]).astype(
np.float32) #Generate another z batch
newZ = sess.run(Gz, feed_dict={
z_in: z2
}) #Use new z to get sample images from generator.
newZ_filt = helper.filt(newZ[0])
if not os.path.exists(sample_directory):
os.makedirs(sample_directory)
#Save sample generator images for viewing training progress.
#newZint = interpolate(newZ[0])
helper.save_images(np.reshape(newZ_filt, [1, 52, 52]), [1, 1],
sample_directory + '/can' + str(i) + '.png')
if i % 100 == 0 and i != 0:
if not os.path.exists(model_directory):
os.makedirs(model_directory)
saver.save(sess, model_directory + '/model-' + str(i) + '.cptk')
print("Saved Model")
# Loading previous network to generate additional images
sample_directory = './figs_cancer_synth' #Directory to save sample images from generator in.
model_directory = './models_cancer' #Directory to load trained model from.
batch_size_sample = 20
path = '/Users/michaelcraig/software/Generative-Adversarial-Network-Tutorial/models_cancer/'
init = tf.global_variables_initializer()
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(init)
def inference_model(a, external_emb=None):
if a.seed is None:
a.seed = random.randint(0, 2**31 - 1)
tf.set_random_seed(a.seed)
np.random.seed(a.seed)
random.seed(a.seed)
if not os.path.exists(a.output_dir):
os.makedirs(a.output_dir)
if a.checkpoint is None:
raise Exception("checkpoint required for test mode")
# Load some options from the checkpoint.
options = {"which_direction", "ngf", "ndf", "lab_colorization"}
with open(os.path.join(a.checkpoint, "options.json")) as f:
for key, val in json.loads(f.read()).items():
if key in options:
print("loaded", key, "=", val)
setattr(a, key, val)
# Disable these features in test mode.
a.scale_size = CROP_SIZE
a.flip = False
for k, v in a._get_kwargs():
print(k, "=", v)
with open(os.path.join(a.output_dir, "options.json"), "w") as f:
f.write(json.dumps(vars(a), sort_keys=True, indent=4))
examples = load_examples(a.input_dir,
"train",
a.lab_colorization,
a.which_direction,
a.flip,
a.scale_size,
a.batch_size,
a.png16bits,
"load_images",
mix_weight=True)
test_examples = load_examples(a.test_dir,
"train",
a.lab_colorization,
a.which_direction,
a.flip,
a.scale_size,
a.batch_size,
a.png16bits,
"load_images",
mix_weight=True)
# Inputs and targets are [batch_size, height, width, channels].
model = create_model(examples.inputs,
examples.targets,
test_examples.inputs,
test_examples.targets,
examples.labels,
test_examples.labels,
a,
examples.labels2,
examples.weight,
external_emb=(not external_emb is None))
# Undo colorization splitting on images that we use for display/output.
if a.lab_colorization:
if a.which_direction == "AtoB":
# inputs is brightness, this will be handled fine as a grayscale image
# need to augment targets and outputs with brightness
targets = augment(examples.targets, examples.inputs)
outputs = augment(model.outputs, examples.inputs)
# inputs can be deprocessed normally and handled as if they are single channel
# grayscale images
inputs = deprocess(examples.inputs)
elif a.which_direction == "BtoA":
# inputs will be color channels only, get brightness from targets
inputs = augment(examples.inputs, examples.targets)
targets = deprocess(examples.targets)
outputs = deprocess(model.outputs)
else:
raise Exception("invalid direction")
else:
inputs = deprocess(examples.inputs)
targets = deprocess(examples.targets)
outputs = deprocess(model.outputs)
def convert(image):
if a.aspect_ratio != 1.0:
# Upscale to correct aspect ratio.
size = [CROP_SIZE, int(round(CROP_SIZE * a.aspect_ratio))]
image = tf.image.resize_images(
image, size=size, method=tf.image.ResizeMethod.BICUBIC)
if a.png16bits:
return tf.image.convert_image_dtype(image,
dtype=tf.uint16,
saturate=True)
else:
return tf.image.convert_image_dtype(image,
dtype=tf.uint8,
saturate=True)
# reverse any processing on images so they can be written to disk or displayed to user
with tf.name_scope("convert_inputs"):
converted_inputs = convert(inputs)
with tf.name_scope("convert_targets"):
converted_targets = convert(targets)
with tf.name_scope("convert_outputs"):
converted_outputs = convert(outputs)
with tf.name_scope("encode_images"):
display_fetches = {
"paths":
examples.paths,
"inputs":
tf.map_fn(tf.image.encode_png,
converted_inputs,
dtype=tf.string,
name="input_pngs"),
"targets":
tf.map_fn(tf.image.encode_png,
converted_targets,
dtype=tf.string,
name="target_pngs"),
"outputs":
tf.map_fn(tf.image.encode_png,
converted_outputs,
dtype=tf.string,
name="output_pngs"),
}
with tf.name_scope("parameter_count"):
parameter_count = tf.reduce_sum(
[tf.reduce_prod(tf.shape(v)) for v in tf.trainable_variables()])
saver = tf.train.Saver(max_to_keep=5)
logdir = a.output_dir if (a.trace_freq > 0 or a.summary_freq > 0) else None
sv = tf.train.Supervisor(logdir=logdir, save_summaries_secs=0, saver=None)
with sv.managed_session() as sess:
print("parameter_count =", sess.run(parameter_count))
if a.checkpoint is not None:
print("loading model from checkpoint")
checkpoint = tf.train.latest_checkpoint(a.checkpoint)
saver.restore(sess, checkpoint)
max_steps = 2**32
if a.max_epochs is not None:
max_steps = examples.steps_per_epoch * a.max_epochs
if a.max_steps is not None:
max_steps = a.max_steps
# Testing: process the test data once
start = time.time()
max_steps = min(examples.steps_per_epoch, max_steps)
display_fetches["labels"] = model.labels
for step in range(max_steps):
if external_emb is None:
results = sess.run(display_fetches)
else:
results = sess.run(
display_fetches,
feed_dict={model.external_emb: external_emb})
print("labels: ", results["labels"])
filesets = save_images(results, output_dir=a.output_dir)
for i, f in enumerate(filesets):
print("evaluated image", f["name"])
index_path = append_index(filesets, output_dir=a.output_dir)
print("wrote index at", index_path)
print("rate", (time.time() - start) / max_steps, 's')
tf.reset_default_graph()