def main(device, train_data_loader, test_data_loader): # select which autoencoder to train # available autoencoders [MLP, CNN] is_mlp = False if constants.RUN_MLP_AUTOEN: is_mlp = True autoencoder = MLPAutoencoder() train_output_dir = constants.MLP_AUTO_EN_TRAIN_OUT_DIR test_output_dir = constants.MLP_AUTO_EN_TEST_OUT_DIR model_path = constants.MLP_AUTO_EN_MODEL_PATH else: autoencoder = ImprvdCnnAutoencoder() if constants.RUN_CNN_AUTOEN: autoencoder = CnnAutoencoder() train_output_dir = constants.CNN_AUTO_EN_TRAIN_OUT_DIR test_output_dir = constants.CNN_AUTO_EN_TEST_OUT_DIR model_path = constants.CNN_AUTO_EN_MODEL_PATH # Migrate all operations on GPU if available else on CPU autoencoder.to(device) # Get the loss (MSE) criterian = autoencoder.criterian optimizer = optim.Adam(autoencoder.parameters(), constants.LR) train_loss = 0 print("Training Initiated") for epoch in range(1, constants.NUM_EPOCHS + 1): for data in train_data_loader: images, _ = data optimizer.zero_grad() images = images.to(device) if is_mlp: images = images.view(-1, 28 * 28) output = autoencoder(images) loss = criterian(output, images.float()) loss.backward() optimizer.step() train_loss += loss.item() * images.size(0) train_loss = train_loss / len(train_data_loader) if epoch % constants.PRINT_EVERY == 0: print( f"epoch {epoch}/{constants.NUM_EPOCHS} loss is :{train_loss}") save_images(images[0], train_output_dir, epoch) save_images(output[0], train_output_dir, epoch, 'out') if not os.path.exists(model_path): os.makedirs(model_path) print(f"saving checkpoint for epoch: {epoch}") torch.save(autoencoder.state_dict(), os.path.join(model_path, f"epoch{epoch}.pth")) if epoch == constants.SAVE_CHECKPOINT: break # activating evaluation mode so better to switch off the autograd autoencoder.eval() with torch.no_grad(): print("Performing inference on test set") for data in test_data_loader: images, _ = data images = images.to(device) if is_mlp: images = images.view(-1, 28 * 28) output = autoencoder(images) for i in range(len(images)): save_images(images[i], test_output_dir, i) save_images(output[i], test_output_dir, i, image_name='out') print("Process completed!")
real_images = (np.reshape(real_images,[BATCH_SIZE,28,28,1]) - 0.5) * 2.0 real_images = np.lib.pad(real_images, ((0,0),(2,2),(2,2),(0,0)),'constant', constant_values=(-1, -1)) feed_dict = {noise_vector:noise_samples, image:real_images} loss_d, _ = sess.run([discriminator_loss, updateD], feed_dict = feed_dict) feed_dict = {noise_vector:noise_samples} loss_g, _ = sess.run([generator_loss, updateG], feed_dict = feed_dict) loss_g, _ = sess.run([generator_loss, updateG], feed_dict = feed_dict) logger.log_scalar(tag='Generator Loss',value=float(loss_g),step=epoch) logger.log_scalar(tag='Discriminator Loss',value=float(loss_d),step=epoch) if (epoch%10000) == 0: print("LossG: {} and LossD: {}".format(float(loss_g), float(loss_d))) test_noise = helper.sample_noise(BATCH_SIZE, NOISE_SIZE) images = sess.run(generated_image, feed_dict = {noise_vector:test_noise}) ImagesPath = os.path.join(os.getcwd(), LOG_FILE, 'ImagesPath') if not os.path.exists(ImagesPath): os.mkdir(ImagesPath) helper.save_images(np.reshape(images[0:36],[36,32,32]), [6,6], ImagesPath+'/fig'+str(epoch)+'.png') ModelPath = os.path.join(os.getcwd(), LOG_FILE, 'ModelPath') if not os.path.exists(ModelPath): os.mkdir(ModelPath) saver.save(sess,ModelPath+'/model'+str(epoch)+'.ckpt') helper.log_string('######'+str(epoch)+'######', LOG_FILE) helper.log_string('Generator Loss: '+str(loss_g), LOG_FILE) helper.log_string('Discriminator Loss: '+str(loss_d), LOG_FILE)
def main(): out_dir = './out' helper.check_and_create_dir(out_dir) batch_size = 100 from tensorflow.examples.tutorials.mnist import input_data mnist = input_data.read_data_sets('../MNIST_data', one_hot=True) train_step_per_epoch = mnist.train.num_examples / batch_size height, width, channel = 28, 28, 1 with tf.Graph().as_default(): session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) session_conf.gpu_options.allow_growth = True sess = tf.Session(config=session_conf) with sess.as_default(), tf.device('/gpu:0'): pixel_rnn = PixelRNN(sess, 28, 28, 1, 64, 1., 'pixel_rnn') sess.run(tf.global_variables_initializer()) for epoch in range(100000): total_train_costs = [] for idx in range(int(train_step_per_epoch)): x_mb, _ = mnist.train.next_batch(batch_size) x_mb = x_mb.reshape([batch_size, height, width, channel]) x_mb = (np.random.uniform(size=x_mb.shape) < x_mb).astype('float32') _, cost, outputs = sess.run( [pixel_rnn.optim, pixel_rnn.loss, pixel_rnn.output], feed_dict={pixel_rnn.inputs: x_mb}) total_train_costs.append(cost) test_mb, _ = mnist.test.next_batch(batch_size) test_mb = test_mb.reshape( [batch_size, height, width, channel]) test_mb = (np.random.uniform(size=test_mb.shape) < test_mb).astype('float32') test_cost = sess.run(pixel_rnn.loss, feed_dict={pixel_rnn.inputs: test_mb}) print( "Epoch: %d, iteration:%d, train loss: %f, test loss: %f" % (epoch, idx, cost, test_cost)) if idx % 100 == 0: avg_train_cost = np.mean(total_train_costs) #print("Epoch: %d, iteration:%d, train l: %f" % (epoch, idx, cost)) #print("Epoch: %d, iteration:%d, train l: %f" % (epoch, idx, avg_train_cost)) samples = pixel_rnn.generate() helper.save_images(samples, height, width, 10, 10, dir=out_dir, prefix='test_') avg_train_cost = np.mean(total_train_costs) samples = pixel_rnn.generate() helper.save_images(samples, height, width, 10, 10, dir=out_dir, prefix='test_') print("Epoch: %d, iteration:%d, train l: %f" % (epoch, idx, avg_train_cost))
def main(argv): filename = argv.filename img_name = argv.image full = argv.fullcoeff os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' os.environ['OMP_NUM_THREADS'] = '8' tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR) cwd = os.getcwd() config = tf.compat.v1.ConfigProto() # Model directory name directory = 'baselineModel' model_dir = cwd + '/' + directory load_chk_pt = tf.train.latest_checkpoint(model_dir) epsilons = np.asarray([0, 2.5, 5, 7.5, 10, 12.5, 15, 17.5, 20]) #epsilons = np.asarray([0, 2.5]) perturb_sizes = list(np.linspace(0, 30000, num=9)) epsilons_perturb = zip(epsilons, perturb_sizes) size_array = [] size_tarray = [] for epsilon, perturb in epsilons_perturb: tst_org, tst_inp = rd.get_validation_data(num_from=20, num_img=1, acc=4, full=full) tst_org_full, tst_inp_full = rd.get_validation_data(num_from=20, num_img=1, acc=4, full=True) mu = np.mean(tst_org, axis=(-1, -2), keepdims=True) st = np.std(tst_org, axis=(-1, -2), keepdims=True) tst_inp = (tst_inp - mu) / st tst_inp = np.clip(tst_inp, -6, 6) tst_org_adv = np.copy(tst_org) tst_inp_orig = np.copy(tst_inp) mu_full = np.mean(tst_org_full, axis=(-1, -2), keepdims=True) st_full = np.std(tst_org_full, axis=(-1, -2), keepdims=True) tst_inp_full = (tst_inp_full - mu_full) / st_full tst_inp_full = np.clip(tst_inp_full, -6, 6) # Load model and predict tst_org_adv = tst_org_adv[..., np.newaxis] tst_inp = tst_inp[..., np.newaxis] tst_inp_full = tst_inp_full[..., np.newaxis] tst_rec = np.zeros(tst_inp.shape, dtype=np.float32) tst_inp_adv = np.copy(tst_inp) tst_intr_adv = np.zeros(tst_inp_adv.shape, dtype=np.float32) tst_rec_adv = np.zeros(tst_inp_adv.shape, dtype=np.float32) # Initialize graph tf.compat.v1.reset_default_graph() with tf.compat.v1.Session(config=config) as sess: new_saver = tf.compat.v1.train.import_meta_graph(model_dir + '/modelTst.meta') new_saver.restore(sess, load_chk_pt) graph = tf.compat.v1.get_default_graph() predT = graph.get_tensor_by_name('predTst:0') senseT = graph.get_tensor_by_name('sense:0') # Run reconstruction for original reconstruction tst_rec[0] = sess.run(predT, feed_dict={senseT: tst_inp[[0]]}) # Get gradient of loss with respect to input origT = tf.convert_to_tensor(tst_org_adv) loss = tf.norm(origT - predT, ord='euclidean') g = tf.gradients(loss, senseT) grad = sess.run(g, feed_dict={senseT: tst_inp[[0]]}) # Create adversarial example and run reconstruction tst_intr_adv = epsilon * np.copy(grad)[0] if epsilon != 0: print('tst_inp, tst_inp_full', np.sum(np.abs(tst_inp_adv)), np.sum(np.abs(tst_inp_full))) tst_intr_adv *= perturb / (np.sum(np.abs(tst_intr_adv))) fraction = np.sum(np.abs(tst_inp_full)) / np.sum(np.abs(tst_intr_adv)) #fraction = np.linalg.norm(tst_inp_adv) / np.linalg.norm(tst_intr_adv) #fraction = np.linalg.norm(tst_inp_full) / np.linalg.norm(tst_intr_adv) else: fraction = 0 # Fraction of original image is perturbation print('Fraction', fraction) tst_inp_adv += np.copy(tst_intr_adv) tst_rec_adv[0] = sess.run(predT, feed_dict={senseT: tst_inp_adv[[0]]}) # Corrupt the input Fourier coefficients with Gaussian noise mean, std = mu[0][0][0], st gauss_noise = np.random.normal(mean, std, tst_inp_orig.shape) gauss_noise *= (np.linalg.norm(tst_intr_adv)) / np.linalg.norm(gauss_noise) img_shape = np.shape(tst_inp_orig) tst_inp_gauss = np.copy(tst_inp_orig) + np.copy(gauss_noise) tst_inp_gauss = tst_inp_gauss[..., np.newaxis] tst_rec_gauss = np.zeros(tst_inp_gauss.shape, dtype=np.float32) # Run reconstruction for Gaussian case tst_rec_gauss[0] = sess.run(predT, feed_dict={senseT: tst_inp_gauss[[0]]}) # Transform images into original shape tst_inp = tst_inp[..., 0] tst_inp_full = tst_inp_full[..., 0] tst_rec = tst_rec[..., 0] tst_inp_gauss = tst_inp_gauss[..., 0] tst_rec_gauss = tst_rec_gauss[..., 0] tst_inp_adv = tst_inp_adv[..., 0] tst_intr_adv = tst_intr_adv[..., 0] tst_rec_adv = tst_rec_adv[..., 0] tst_inp, tst_rec = tst_inp * st + mu, tst_rec * st + mu tst_inp_gauss, tst_rec_gauss = tst_inp_gauss * st + mu, tst_rec_gauss * st + mu tst_inp_adv, tst_rec_adv = tst_inp_adv * st + mu, tst_rec_adv * st + mu tst_inp_full = tst_inp_full * st_full + mu_full # Add noise to FC to see how it reacts coeff_inp = sf.fft2c(tst_inp) coeff_inp_full = sf.fft2c(tst_inp_full) first_img = sf.sos(sf.crop(sf.ifft2c(coeff_inp), (320, 320))) first_img = first_img[np.newaxis, ...] shp = list(coeff_inp.shape) # Scale should be 320, so dividing by it in the Fourier space will yield correct size in image space scale = np.sqrt(np.prod(shp[-2:])) gauss_noise_fc_real = np.random.normal(mean, std, size=shp) gauss_noise_fc_imag = np.random.normal(mean, std, size=shp) * 1j gauss_noise_fc = gauss_noise_fc_real[...] + gauss_noise_fc_imag[...] gauss_noise_fc[np.nonzero(coeff_inp == 0.+0.j)] = 0.+0.j gauss_noise_fc *= 1 / np.linalg.norm(gauss_noise_fc) if epsilon != 0: fc_scaling = np.linalg.norm(coeff_inp_full) / fraction gauss_noise_fc *= fc_scaling print(perturb, scale, perturb/scale, np.sum(np.abs(gauss_noise_fc)), np.sum(np.abs(coeff_inp))) print('FC Scaling', fc_scaling) else: gauss_noise_fc = np.zeros(shape=gauss_noise_fc.shape, dtype=np.complex64) noise_size = np.linalg.norm(gauss_noise_fc) size_array.append(noise_size) coeff_final = np.copy(coeff_inp) + np.copy(gauss_noise_fc) tst_fc = sf.sos(sf.crop(sf.ifft2c(coeff_final), (320, 320))) tst_fc_noise = sf.sos(sf.crop(sf.ifft2c(gauss_noise_fc), (320, 320))) tst_fc_noise = tst_fc_noise[np.newaxis, ...] print(np.sum(np.abs(tst_inp)), np.sum(np.abs(first_img)), np.sum(np.abs(tst_fc))) tst_fc = tst_fc[np.newaxis, ...] print('PIC SUMS', epsilon, np.sum(np.abs(tst_fc[...])), np.sum(np.abs(tst_rec_gauss)), np.sum(np.abs(tst_rec_adv[...]))) print('NOISE SUMS', np.sum(np.abs(tst_fc_noise)), np.sum(np.abs(gauss_noise)), np.sum(np.abs(tst_intr_adv))) noise_transform_size = np.linalg.norm(tst_fc_noise) size_tarray.append(noise_transform_size) [fc_error, gaussian_error, adv_error] = calculate_sse(tst_org, tst_fc, tst_rec_gauss, tst_rec_adv) #[ssim_fc, ssim_gauss, ssim_adv] = calculate_ssim(tst_org, tst_fc, tst_rec_gauss, tst_rec_adv) if fraction != 0: #image_grid_small('fc_visual', np.abs(coeff_inp), np.abs(gauss_noise_fc), np.abs(coeff_final)) fraction_original_to_perturb = 1 / fraction else: fraction_original_to_perturb = 0 save_results(filename + '.npy', img_shape, fc_error, gaussian_error, adv_error, fraction_original_to_perturb) #save_results(filename + '_ssim.npy', img_shape, ssim_fc, ssim_gauss, ssim_adv, fraction_original_to_perturb) if img_name: save_images(img_name, tst_org, tst_inp, tst_fc, gauss_noise, tst_inp_gauss, tst_rec_gauss, tst_intr_adv, tst_inp_adv, tst_rec_adv)
}) #Update the generator, twice for good measure. _, gLoss = sess.run([update_G, g_loss], feed_dict={z_in: zs}) if i % 1 == 0: print("Gen Loss " + str(gLoss) + " Disc Loss: " + str(dLoss)) z2 = np.random.uniform(-1.0, 1.0, size=[batch_size, z_size]).astype( np.float32) #Generate another z batch newZ = sess.run(Gz, feed_dict={ z_in: z2 }) #Use new z to get sample images from generator. newZ_filt = helper.filt(newZ[0]) if not os.path.exists(sample_directory): os.makedirs(sample_directory) #Save sample generator images for viewing training progress. #newZint = interpolate(newZ[0]) helper.save_images(np.reshape(newZ_filt, [1, 52, 52]), [1, 1], sample_directory + '/can' + str(i) + '.png') if i % 100 == 0 and i != 0: if not os.path.exists(model_directory): os.makedirs(model_directory) saver.save(sess, model_directory + '/model-' + str(i) + '.cptk') print("Saved Model") # Loading previous network to generate additional images sample_directory = './figs_cancer_synth' #Directory to save sample images from generator in. model_directory = './models_cancer' #Directory to load trained model from. batch_size_sample = 20 path = '/Users/michaelcraig/software/Generative-Adversarial-Network-Tutorial/models_cancer/' init = tf.global_variables_initializer() saver = tf.train.Saver() with tf.Session() as sess: sess.run(init)
def inference_model(a, external_emb=None): if a.seed is None: a.seed = random.randint(0, 2**31 - 1) tf.set_random_seed(a.seed) np.random.seed(a.seed) random.seed(a.seed) if not os.path.exists(a.output_dir): os.makedirs(a.output_dir) if a.checkpoint is None: raise Exception("checkpoint required for test mode") # Load some options from the checkpoint. options = {"which_direction", "ngf", "ndf", "lab_colorization"} with open(os.path.join(a.checkpoint, "options.json")) as f: for key, val in json.loads(f.read()).items(): if key in options: print("loaded", key, "=", val) setattr(a, key, val) # Disable these features in test mode. a.scale_size = CROP_SIZE a.flip = False for k, v in a._get_kwargs(): print(k, "=", v) with open(os.path.join(a.output_dir, "options.json"), "w") as f: f.write(json.dumps(vars(a), sort_keys=True, indent=4)) examples = load_examples(a.input_dir, "train", a.lab_colorization, a.which_direction, a.flip, a.scale_size, a.batch_size, a.png16bits, "load_images", mix_weight=True) test_examples = load_examples(a.test_dir, "train", a.lab_colorization, a.which_direction, a.flip, a.scale_size, a.batch_size, a.png16bits, "load_images", mix_weight=True) # Inputs and targets are [batch_size, height, width, channels]. model = create_model(examples.inputs, examples.targets, test_examples.inputs, test_examples.targets, examples.labels, test_examples.labels, a, examples.labels2, examples.weight, external_emb=(not external_emb is None)) # Undo colorization splitting on images that we use for display/output. if a.lab_colorization: if a.which_direction == "AtoB": # inputs is brightness, this will be handled fine as a grayscale image # need to augment targets and outputs with brightness targets = augment(examples.targets, examples.inputs) outputs = augment(model.outputs, examples.inputs) # inputs can be deprocessed normally and handled as if they are single channel # grayscale images inputs = deprocess(examples.inputs) elif a.which_direction == "BtoA": # inputs will be color channels only, get brightness from targets inputs = augment(examples.inputs, examples.targets) targets = deprocess(examples.targets) outputs = deprocess(model.outputs) else: raise Exception("invalid direction") else: inputs = deprocess(examples.inputs) targets = deprocess(examples.targets) outputs = deprocess(model.outputs) def convert(image): if a.aspect_ratio != 1.0: # Upscale to correct aspect ratio. size = [CROP_SIZE, int(round(CROP_SIZE * a.aspect_ratio))] image = tf.image.resize_images( image, size=size, method=tf.image.ResizeMethod.BICUBIC) if a.png16bits: return tf.image.convert_image_dtype(image, dtype=tf.uint16, saturate=True) else: return tf.image.convert_image_dtype(image, dtype=tf.uint8, saturate=True) # reverse any processing on images so they can be written to disk or displayed to user with tf.name_scope("convert_inputs"): converted_inputs = convert(inputs) with tf.name_scope("convert_targets"): converted_targets = convert(targets) with tf.name_scope("convert_outputs"): converted_outputs = convert(outputs) with tf.name_scope("encode_images"): display_fetches = { "paths": examples.paths, "inputs": tf.map_fn(tf.image.encode_png, converted_inputs, dtype=tf.string, name="input_pngs"), "targets": tf.map_fn(tf.image.encode_png, converted_targets, dtype=tf.string, name="target_pngs"), "outputs": tf.map_fn(tf.image.encode_png, converted_outputs, dtype=tf.string, name="output_pngs"), } with tf.name_scope("parameter_count"): parameter_count = tf.reduce_sum( [tf.reduce_prod(tf.shape(v)) for v in tf.trainable_variables()]) saver = tf.train.Saver(max_to_keep=5) logdir = a.output_dir if (a.trace_freq > 0 or a.summary_freq > 0) else None sv = tf.train.Supervisor(logdir=logdir, save_summaries_secs=0, saver=None) with sv.managed_session() as sess: print("parameter_count =", sess.run(parameter_count)) if a.checkpoint is not None: print("loading model from checkpoint") checkpoint = tf.train.latest_checkpoint(a.checkpoint) saver.restore(sess, checkpoint) max_steps = 2**32 if a.max_epochs is not None: max_steps = examples.steps_per_epoch * a.max_epochs if a.max_steps is not None: max_steps = a.max_steps # Testing: process the test data once start = time.time() max_steps = min(examples.steps_per_epoch, max_steps) display_fetches["labels"] = model.labels for step in range(max_steps): if external_emb is None: results = sess.run(display_fetches) else: results = sess.run( display_fetches, feed_dict={model.external_emb: external_emb}) print("labels: ", results["labels"]) filesets = save_images(results, output_dir=a.output_dir) for i, f in enumerate(filesets): print("evaluated image", f["name"]) index_path = append_index(filesets, output_dir=a.output_dir) print("wrote index at", index_path) print("rate", (time.time() - start) / max_steps, 's') tf.reset_default_graph()