class PixelDA(object): """ Paradigm of GAN (keras implementation) 1. Construct D 1a) Compile D 2. Construct G 3. Set D.trainable = False 4. Stack G and D, to construct GAN 4a) Compile GAN Approved by fchollet: "the process you describe is in fact correct." See issue #4674 keras: https://github.com/keras-team/keras/issues/4674 """ def __init__(self): # Input shape self.img_rows = 32 self.img_cols = 32 self.channels = 3 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.num_classes = 10 self.noise_size = (10, ) # Calculate output shape of D (PatchGAN) patch = int(self.img_rows / 2**4) self.disc_patch = (patch, patch, 1) # Number of residual blocks in the generator self.residual_blocks = 3 #6 def build_all_model(self): # Loss weights lambda_adv = 10 lambda_clf = 1 # optimizer = Adam(0.0002, 0.5) # optimizer = SGD(lr=0.0001) optimizer = RMSprop(lr=1e-5) # Number of filters in first layer of discriminator and classifier self.df = 64 self.cf = 64 # Build and compile the discriminators self.discriminator = self.build_discriminator() self.discriminator.name = "Discriminator" self.discriminator.compile(loss='mse', optimizer=optimizer, metrics=['accuracy']) # Build the generator self.generator = self.build_generator() self.generator.name = "Generator" # Build the task (classification) network self.clf = self.build_classifier() self.clf.name = "Classifier" # Input images from both domains img_A = Input(shape=self.img_shape, name='source_image') # Input noise noise = Input(shape=self.noise_size, name='noise_input') # Translate images from domain A to domain B fake_B = self.generator([img_A, noise]) # Classify the translated image class_pred = self.clf(fake_B) # For the combined model we will only train the generator and classifier self.discriminator.trainable = False # Discriminator determines validity of translated images valid = self.discriminator(fake_B) self.combined = Model([img_A, noise], [valid, class_pred]) self.combined.compile(loss=['mse', 'categorical_crossentropy'], loss_weights=[lambda_adv, lambda_clf], optimizer=optimizer, metrics=['accuracy']) def load_dataset(self): # Configure MNIST and MNIST-M data loader self.data_loader = DataLoader(img_res=(self.img_rows, self.img_cols)) def build_generator(self): """Resnet Generator""" def residual_block(layer_input): """Residual block described in paper""" d = Conv2D(64, kernel_size=3, strides=1, padding='same')(layer_input) d = BatchNormalization(momentum=0.8)(d) # TODO 6/5/2018 d = Activation('relu')(d) d = Conv2D(64, kernel_size=3, strides=1, padding='same')(d) d = BatchNormalization(momentum=0.8)(d) # TODO 6/5/2018 d = Add()([d, layer_input]) return d # Image input img = Input(shape=self.img_shape, name='image_input') ## Noise input noise = Input(shape=self.noise_size, name='noise_input') noise_layer = Dense(1024, activation="relu")(noise) noise_layer = Reshape((self.img_rows, self.img_cols, 1))(noise_layer) conditioned_img = keras.layers.concatenate([img, noise_layer]) # keras.layers.concatenate # l1 = Conv2D(64, kernel_size=3, padding='same', activation='relu')(img) l1 = Conv2D(64, kernel_size=3, strides=1, padding='same', activation='relu')(conditioned_img) # Propogate signal through residual blocks r = residual_block(l1) for _ in range(self.residual_blocks - 1): r = residual_block(r) output_img = Conv2D(self.channels, kernel_size=3, strides=1, padding='same', activation='tanh')(r) return Model([img, noise], output_img) def build_discriminator(self): def d_layer(layer_input, filters, f_size=4, normalization=True): """Discriminator layer""" d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input) d = LeakyReLU(alpha=0.2)(d) if normalization: d = InstanceNormalization()(d) return d img = Input(shape=self.img_shape) d1 = d_layer(img, self.df, normalization=False) d2 = d_layer(d1, self.df * 2, normalization=False) d3 = d_layer(d2, self.df * 4, normalization=False) d4 = d_layer(d3, self.df * 8, normalization=False) d5 = d_layer(d4, self.df * 16, normalization=False) # validity = Conv2D(1, kernel_size=4, strides=2, padding='same')(d5) validity = Dense(1, activation='sigmoid')(Flatten()(d5)) return Model(img, validity) def build_classifier(self): def clf_layer(layer_input, filters, f_size=4, normalization=True): """Classifier layer""" d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input) d = LeakyReLU(alpha=0.2)(d) if normalization: d = InstanceNormalization()(d) return d img = Input(shape=self.img_shape, name='image_input') c1 = clf_layer(img, self.cf, normalization=False) c2 = clf_layer(c1, self.cf * 2) c3 = clf_layer(c2, self.cf * 4) c4 = clf_layer(c3, self.cf * 8) c5 = clf_layer(c4, self.cf * 8) class_pred = Dense(self.num_classes, activation='softmax')(Flatten()(c5)) return Model(img, class_pred) def load_pretrained_weights(self, weights_path="../Weights/all_weights.h5"): print("Loading pretrained weights from path: {} ...".format( weights_path)) self.combined.load_weights(weights_path, by_name=True) print("+ Done.") def summary(self): print("=" * 50) print("Discriminator summary:") self.discriminator.summary() print("=" * 50) print("Generator summary:") self.generator.summary() print("=" * 50) print("Classifier summary:") self.clf.summary() print("=" * 50) print("All summary:") self.combined.summary() def train(self, epochs, batch_size=32, sample_interval=50, save_sample2dir="../samples/exp0", save_weights_path='../Weights/all_weights.h5', save_model=False): dirpath = "/".join(save_weights_path.split("/")[:-1]) if not os.path.exists(dirpath): os.makedirs(dirpath) # half_batch = batch_size #int(batch_size / 2) ### TODO half_batch = int(batch_size / 2) # Classification accuracy on 100 last batches of domain B test_accs = [] print("=" * 50) print("Discriminator summary:") self.discriminator.summary() print("=" * 50) print("Generator summary:") self.generator.summary() print("=" * 50) print("Classifier summary:") self.clf.summary() print("=" * 50) print("All summary:") self.combined.summary() ## Monitor to save model weights Lu best_test_cls_acc = 0 second_best_cls_acc = -1 for epoch in range(epochs): # --------------------- # Train Discriminator # --------------------- # n_sample = half_batch # imgs_A.shape[0] imgs_A, _ = self.data_loader.load_data(domain="A", batch_size=half_batch) imgs_B, _ = self.data_loader.load_data(domain="B", batch_size=half_batch) noise_prior = np.random.normal( 0, 1, (half_batch, self.noise_size[0])) # TODO # noise_prior = np.random.rand(half_batch, self.noise_size[0]) # TODO 6/5/2018 # Translate images from domain A to domain B fake_B = self.generator.predict([imgs_A, noise_prior]) valid = np.ones((half_batch, 1)) fake = np.zeros((half_batch, 1)) # fake = -valid # TODO 6/5/2018 NEW D_train_label = np.vstack([valid, fake]) # 6/5/2018 NEW D_train_images = np.vstack([imgs_B, fake_B]) # 6/5/2018 NEW # Train the discriminators (original images = real / translated = Fake) # d_loss_real = self.discriminator.train_on_batch(imgs_B, valid) # d_loss_fake = self.discriminator.train_on_batch(fake_B, fake) # d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) d_loss = self.discriminator.train_on_batch(D_train_images, D_train_label) # -------------------------------- # Train Generator and Classifier # -------------------------------- # Sample a batch of images from both domains imgs_A, labels_A = self.data_loader.load_data( domain="A", batch_size=batch_size) imgs_B, labels_B = self.data_loader.load_data( domain="B", batch_size=batch_size) # One-hot encoding of labels labels_A = to_categorical(labels_A, num_classes=self.num_classes) # The generators want the discriminators to label the translated images as real valid = np.ones((batch_size, 1)) # noise_prior = np.random.normal( 0, 1, (batch_size, self.noise_size[0])) # TODO # noise_prior = np.random.rand(batch_size, self.noise_size[0]) # TODO 6/5/2018 # Train the generator and classifier g_loss = self.combined.train_on_batch([imgs_A, noise_prior], [valid, labels_A]) #----------------------- # Evaluation (domain B) #----------------------- pred_B = self.clf.predict(imgs_B) test_acc = np.mean(np.argmax(pred_B, axis=1) == labels_B) # Add accuracy to list of last 100 accuracy measurements test_accs.append(test_acc) if len(test_accs) > 100: test_accs.pop(0) # Plot the progress # print ( "%d : [D - loss: %.5f, acc: %3d%%], [G - loss: %.5f], [clf - loss: %.5f, acc: %3d%%, test_acc: %3d%% (%3d%%)]" % \ # (epoch, d_loss[0], 100*float(d_loss[1]), # g_loss[1], g_loss[2], 100*float(g_loss[-1]), # 100*float(test_acc), 100*float(np.mean(test_accs)))) if epoch % 10 == 0: d_train_acc = 100 * float(d_loss[1]) gen_loss = g_loss[1] clf_train_acc = 100 * float(g_loss[-1]) clf_train_loss = g_loss[2] current_test_acc = 100 * float(test_acc) test_mean_acc = 100 * float(np.mean(test_accs)) if test_mean_acc > best_test_cls_acc: second_best_cls_acc = best_test_cls_acc best_test_cls_acc = test_mean_acc if save_model: self.combined.save(save_weights_path) else: self.combined.save_weights(save_weights_path) print( "{} : [D - loss: {:.5f}, acc: {:.2f}%], [G - loss: {:.5f}], [clf - loss: {:.5f}, acc: {:.2f}%, test_acc: {:.2f}% ({:.2f}%)] (latest)" .format(epoch, d_loss[0], d_train_acc, gen_loss, clf_train_loss, clf_train_acc, current_test_acc, test_mean_acc)) elif test_mean_acc > second_best_cls_acc: second_best_cls_acc = test_mean_acc if save_model: self.combined.save(save_weights_path) else: self.combined.save_weights(save_weights_path[:-3] + "_bis.h5") print( "{} : [D - loss: {:.5f}, acc: {:.2f}%], [G - loss: {:.5f}], [clf - loss: {:.5f}, acc: {:.2f}%, test_acc: {:.2f}% ({:.2f}%)] (before latest)" .format(epoch, d_loss[0], d_train_acc, gen_loss, clf_train_loss, clf_train_acc, current_test_acc, test_mean_acc)) else: print( "{} : [D - loss: {:.5f}, acc: {:.2f}%], [G - loss: {:.5f}], [clf - loss: {:.5f}, acc: {:.2f}%, test_acc: {:.2f}% ({:.2f}%)]" .format(epoch, d_loss[0], d_train_acc, gen_loss, clf_train_loss, clf_train_acc, current_test_acc, test_mean_acc)) # If at save interval => save generated image samples if epoch % sample_interval == 0: self.sample_images(epoch, save2dir=save_sample2dir) def sample_images(self, epoch, save2dir="../samples"): if not os.path.exists(save2dir): os.makedirs(save2dir) r, c = 2, 10 imgs_A, _ = self.data_loader.load_data(domain="A", batch_size=c) n_sample = imgs_A.shape[0] noise_prior = np.random.normal(0, 1, (n_sample, self.noise_size[0])) # TODO # noise_prior = np.random.rand(n_sample, self.noise_size[0]) # TODO 6/5/2018 # Translate images to the other domain fake_B = self.generator.predict([imgs_A, noise_prior]) gen_imgs = np.concatenate([imgs_A, fake_B]) # Rescale images 0 - 1 gen_imgs = 0.5 * gen_imgs + 0.5 #titles = ['Original', 'Translated'] fig, axs = plt.subplots(r, c, figsize=(20, 4)) cnt = 0 for i in range(r): for j in range(c): axs[i, j].imshow(gen_imgs[cnt]) #axs[i, j].set_title(titles[i]) axs[i, j].axis('off') cnt += 1 fig.savefig(os.path.join(save2dir, "{}.png".format(epoch))) plt.close() def deploy_transform(self, save2file="../domain_adapted/generated.npy", stop_after=None): dirpath = "/".join(save2file.split("/")[:-1]) if not os.path.exists(dirpath): os.makedirs(dirpath) dirname = "/".join(save2file.split("/")[:-1]) if stop_after is not None: predict_steps = int(stop_after / 32) else: predict_steps = stop_after noise_vec = np.random.normal(0, 1, self.noise_size[0]) assert 1 == 2 # np.random.rand(n_sample, self.noise_size[0]) # TODO 6/5/2018 print("Performing Pixel-level domain adaptation on original images...") adaptaed_images = self.generator.predict( [ self.data_loader.mnist_X[:32 * predict_steps], np.tile(noise_vec, (32 * predict_steps, 1)) ], batch_size=32) #, steps=predict_steps # self.data_loader.mnistm_X[:stop_after] print("+ Done.") print("Saving transformed images to file {}".format(save2file)) np.save(save2file, adaptaed_images) noise_vec_filepath = os.path.join(dirname, "noise_vectors.npy") print( "Saving random noise (seed) to file {}".format(noise_vec_filepath)) np.save(noise_vec_filepath, noise_vec) print("+ All done.") def deploy_debug(self, save2file="../domain_adapted/debug.npy", sample_size=9, seed=0): dirpath = "/".join(save2file.split("/")[:-1]) if not os.path.exists(dirpath): os.makedirs(dirpath) dirname = "/".join(save2file.split("/")[:-1]) np.random.seed(seed=seed) noise_vec = np.random.normal(0, 1, (sample_size, self.noise_size[0])) assert 1 == 2 # np.random.rand(n_sample, self.noise_size[0]) # TODO 6/5/2018 print("Performing Pixel-level domain adaptation on original images...") collections = [] for i in range(sample_size): adaptaed_images = self.generator.predict([ self.data_loader.mnist_X[:15], np.tile(noise_vec[i], (15, 1)) ], batch_size=15) collections.append(adaptaed_images) print("+ Done.") print("Saving transformed images to file {}".format(save2file)) np.save(save2file, np.stack(collections)) print("+ All done.") def deploy_classification(self, batch_size=32): print("Predicting ... ") pred_B = self.clf.predict(self.data_loader.mnistm_X, batch_size=batch_size) print("+ Done.") N_samples = len(pred_B) precision = (np.argmax(pred_B, axis=1) == self.data_loader.mnistm_y) Moy = np.mean(precision) Std = np.std(precision) lower_bound = Moy - 2.576 * Std / np.sqrt(N_samples) upper_bound = Moy + 2.576 * Std / np.sqrt(N_samples) print("=" * 50) print("Unsupervised MNIST-M classification accuracy : {}".format(Moy)) print("Confidence interval (99%) [{}, {}]".format( lower_bound, upper_bound)) print("Length of confidence interval 99%: {}".format(upper_bound - lower_bound)) print("=" * 50) print("+ All done.")
from __future__ import print_function, division import scipy import datetime import matplotlib.pyplot as plt import sys from data_processing import DataLoader import numpy as np import os # Configure MNIST and MNIST-M data loader data_loader = DataLoader(img_res=(32, 32)) mnist, _ = data_loader.load_data(domain="A", batch_size=25) mnistm, _ = data_loader.load_data(domain="B", batch_size=25) r, c = 5, 5 for img_i, imgs in enumerate([mnist, mnistm]): #titles = ['Original', 'Translated'] fig, axs = plt.subplots(r, c) cnt = 0 for i in range(r): for j in range(c): axs[i, j].imshow(imgs[cnt]) #axs[i, j].set_title(titles[i]) axs[i, j].axis('off') cnt += 1 fig.savefig("%d.png" % (img_i)) plt.close()
class PixelDA(object): """ Paradigm of GAN (keras implementation) 1. Construct D 1a) Compile D 2. Construct G 3. Set D.trainable = False 4. Stack G and D, to construct GAN (combined model) 4a) Compile GAN Approved by fchollet: "the process you describe is in fact correct." See issue #4674 keras: https://github.com/keras-team/keras/issues/4674 """ def __init__(self, noise_size=100, use_PatchGAN=False, use_Wasserstein=True): # Input shape self.img_rows = 32 self.img_cols = 32 self.channels = 3 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.num_classes = 10 self.noise_size = (noise_size, ) #(100,) # Calculate output shape of D (PatchGAN) patch = int(self.img_rows / 2**4) self.disc_patch = (patch, patch, 1) # Number of residual blocks in the generator self.residual_blocks = 6 self.use_PatchGAN = use_PatchGAN #False self.use_Wasserstein = use_Wasserstein if self.use_Wasserstein: self.critic_steps = 10 else: self.critic_steps = 1 self.GRADIENT_PENALTY_WEIGHT = 10 # As the paper def build_all_model(self, batch_size=32): self.batch_size = batch_size # Loss weights lambda_adv = 7 lambda_clf = 1 # optimizer = Adam(0.0002, 0.5) optimizer = Adam(0.0001, beta_1=0.5, beta_2=0.9) # optimizer = SGD(lr=0.0001) # optimizer = RMSprop(lr=1e-5) # Number of filters in first layer of discriminator and classifier self.df = 128 # NEW TODO #64 11/5/2018 self.cf = 64 # Build and compile the discriminators self.discriminator = self.build_discriminator() self.discriminator.name = "Discriminator" img_A = Input(shape=self.img_shape, name='source_image') # real A img_B = Input(shape=self.img_shape, name='target_image') # real B fake_img = Input(shape=self.img_shape, name="fake_image") # fake B # We also need to generate weighted-averages of real and generated samples, to use for the gradient norm penalty. avg_img = RandomWeightedAverage(batch_size=self.batch_size)( [img_B, fake_img]) real_img_rating = self.discriminator(img_B) # TODO img_A fake_img_rating = self.discriminator(fake_img) avg_img_output = self.discriminator(avg_img) # The gradient penalty loss function requires the input averaged samples to get gradients. However, # Keras loss functions can only have two arguments, y_true and y_pred. We get around this by making a partial() # of the function with the averaged samples here. partial_gp_loss = partial( gradient_penalty_loss, averaged_samples=avg_img, gradient_penalty_weight=self.GRADIENT_PENALTY_WEIGHT) partial_gp_loss.__name__ = 'gradient_penalty' # Functions need names or Keras will throw an error if self.use_Wasserstein: self.discriminator_model = Model( inputs=[img_B, fake_img], #, avg_img # loss_weights=[1,1,1], # useless, since we have multiply the penalization by GRADIENT_PENALTY_WEIGHT=10 outputs=[real_img_rating, fake_img_rating, avg_img_output]) self.discriminator_model.compile( loss=[wasserstein_loss, wasserstein_loss, partial_gp_loss], optimizer=optimizer, metrics=[my_critic_acc]) else: self.discriminator.compile(loss='mse', optimizer=optimizer, metrics=['accuracy']) # For the combined model we will only train the generator and classifier self.discriminator.trainable = False # Build the generator self.generator = self.build_generator() self.generator.name = "Generator" # Build the task (classification) network self.clf = self.build_classifier() self.clf.name = "Classifier" # Input images from both domains # Input noise noise = Input(shape=self.noise_size, name='noise_input') # Translate images from domain A to domain B fake_B = self.generator([img_A, noise]) # Classify the translated image class_pred = self.clf(fake_B) # Discriminator determines validity of translated images valid = self.discriminator(fake_B) # fake_B_rating if self.use_Wasserstein: self.combined = Model(inputs=[img_A, noise], outputs=[valid, class_pred]) self.combined.compile( optimizer=optimizer, loss=[wasserstein_loss, 'categorical_crossentropy'], loss_weights=[lambda_adv, lambda_clf], # TODO NEW metrics=['accuracy']) else: self.combined = Model([img_A, noise], [valid, class_pred]) self.combined.compile(loss=['mse', 'categorical_crossentropy'], loss_weights=[lambda_adv, lambda_clf], optimizer=optimizer, metrics=['accuracy']) def load_dataset(self): # Configure MNIST and MNIST-M data loader self.data_loader = DataLoader(img_res=(self.img_rows, self.img_cols)) def build_generator(self): """Resnet Generator""" def residual_block(layer_input): """Residual block described in paper""" d = Conv2D(64, kernel_size=3, strides=1, padding='same')(layer_input) d = BatchNormalization(momentum=0.8)(d) # TODO 6/5/2018 d = Activation('relu')(d) d = Conv2D(64, kernel_size=3, strides=1, padding='same')(d) d = BatchNormalization(momentum=0.8)(d) # TODO 6/5/2018 d = Add()([d, layer_input]) return d # Image input img = Input(shape=self.img_shape, name='image_input') ## Noise input noise = Input(shape=self.noise_size, name='noise_input') noise_layer = Dense(1024, activation="relu")(noise) noise_layer = Reshape((self.img_rows, self.img_cols, 1))(noise_layer) conditioned_img = keras.layers.concatenate([img, noise_layer]) # keras.layers.concatenate # l1 = Conv2D(64, kernel_size=3, padding='same', activation='relu')(img) l1 = Conv2D(64, kernel_size=3, padding='same', activation='relu')(conditioned_img) # Propogate signal through residual blocks r = residual_block(l1) for _ in range(self.residual_blocks - 1): r = residual_block(r) output_img = Conv2D(self.channels, kernel_size=3, padding='same', activation='tanh')(r) return Model([img, noise], output_img) # def build_discriminator(self): # model = Sequential() # model.add(Input(shape=self.img_shape, name='image')) # def d_layer(model, filters, f_size=4, normalization=True): # """Discriminator layer""" # model.add(Conv2D(filters, kernel_size=f_size, strides=2, padding='same')) # model.add(LeakyReLU(alpha=0.2)) # if normalization: # model.add(InstanceNormalization()) # return model # model = d_layer(model, self.df, normalization=False) # model = d_layer(model, self.df*2) # model = d_layer(model, self.df*4) # model = d_layer(model, self.df*8) # if self.use_PatchGAN: # NEW 7/5/2018 # model.add(Conv2D(1, kernel_size=4, strides=1, padding='same')) # else: # if self.use_Wasserstein: # NEW 8/5/2018 # model.add(Flatten()) # model.add(Dense(1, kernel_initializer='he_normal')) # he_normal ?? TODO # else: # model.add(Flatten()) # model.add(Dense(1, activation='sigmoid')) # return model def build_discriminator(self): def d_layer(layer_input, filters, f_size=4, normalization=True): """Discriminator layer""" d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input) d = LeakyReLU(alpha=0.2)(d) if normalization: d = InstanceNormalization()(d) return d img = Input(shape=self.img_shape, name="image") d1 = d_layer(img, self.df, normalization=False) d2 = d_layer(d1, self.df * 2, normalization=True) d3 = d_layer(d2, self.df * 4, normalization=True) d4 = d_layer(d3, self.df * 8, normalization=True) if self.use_PatchGAN: # NEW 7/5/2018 validity = Conv2D(1, kernel_size=4, strides=1, padding='same')(d4) else: if self.use_Wasserstein: # NEW 8/5/2018 validity = Dense(1, activation=None)( Flatten()(d4)) # he_normal ?? TODO else: validity = Dense(1, activation='sigmoid')(Flatten()(d4)) return Model(img, validity) def build_classifier(self): def clf_layer(layer_input, filters, f_size=4, normalization=True): """Classifier layer""" d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input) d = LeakyReLU(alpha=0.2)(d) if normalization: d = InstanceNormalization()(d) return d img = Input(shape=self.img_shape, name='image_input') c1 = clf_layer(img, self.cf, normalization=False) c2 = clf_layer(c1, self.cf * 2) c3 = clf_layer(c2, self.cf * 4) c4 = clf_layer(c3, self.cf * 8) c5 = clf_layer(c4, self.cf * 8) class_pred = Dense(self.num_classes, activation='softmax')(Flatten()(c5)) return Model(img, class_pred) def load_pretrained_weights(self, weights_path="../Weights/all_weights.h5"): print("Loading pretrained weights from path: {} ...".format( weights_path)) self.combined.load_weights(weights_path, by_name=True) print("+ Done.") def summary(self): print("=" * 50) print("Discriminator summary:") self.discriminator.summary() print("=" * 50) print("Generator summary:") self.generator.summary() print("=" * 50) print("Classifier summary:") self.clf.summary() if self.use_Wasserstein: print("=" * 50) print("Discriminator model summary:") self.discriminator_model.summary() print("=" * 50) print("Combined model summary:") self.combined.summary() def write_tensorboard_graph(self, to_dir="../logs", save_png2dir="../Model_graph"): if not os.path.exists(save_png2dir): os.makedirs(save_png2dir) tensorboard = keras.callbacks.TensorBoard(log_dir=to_dir, histogram_freq=0, write_graph=True, write_images=False) # tensorboard.set_model(self.combined) tensorboard.set_model(self.discriminator_model) try: plot_model(self.combined, to_file=os.path.join(save_png2dir, "Combined_model.png")) plot_model(self.discriminator_model, to_file=os.path.join(save_png2dir, "Discriminator_model.png")) except: pass def train(self, epochs, batch_size=32, sample_interval=50, save_sample2dir="../samples/exp0", save_weights_path='../Weights/all_weights.h5', save_model=False): dirpath = "/".join(save_weights_path.split("/")[:-1]) if not os.path.exists(dirpath): os.makedirs(dirpath) half_batch = batch_size #int(batch_size / 2) ### TODO # half_batch = int(batch_size / 2) # Classification accuracy on 100 last batches of domain B test_accs = [] ## Monitor to save model weights Lu best_test_cls_acc = 0 second_best_cls_acc = -1 for epoch in range(epochs): # --------------------- # Train Discriminator # --------------------- # n_sample = half_batch # imgs_A.shape[0] for _ in range(self.critic_steps): imgs_A, _ = self.data_loader.load_data(domain="A", batch_size=half_batch) imgs_B, _ = self.data_loader.load_data(domain="B", batch_size=half_batch) noise_prior = np.random.normal( 0, 1, (half_batch, self.noise_size[0])) # TODO # noise_prior = np.random.rand(half_batch, self.noise_size[0]) # TODO 6/5/2018 # Translate images from domain A to domain B fake_B = self.generator.predict([imgs_A, noise_prior]) if self.use_PatchGAN: valid = np.ones((half_batch, ) + self.disc_patch) fake = np.zeros((half_batch, ) + self.disc_patch) else: if self.use_Wasserstein: valid = np.ones((half_batch, 1)) fake = -valid #np.ones((half_batch, 1)) # = - valid ? TODO dummy_y = np.zeros((batch_size, 1)) # NEW else: valid = np.ones((half_batch, 1)) fake = np.zeros((half_batch, 1)) # fake = -valid # TODO 6/5/2018 NEW # Train the discriminators (original images = real / translated = Fake) # d_loss_real = self.discriminator.train_on_batch(imgs_B, valid) # d_loss_fake = self.discriminator.train_on_batch(fake_B, fake) # d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) if self.use_Wasserstein: d_loss = self.discriminator_model.train_on_batch( [imgs_B, fake_B], [valid, fake, dummy_y]) # d_loss = self.discriminator.train_on_batch(D_train_images, D_train_label, dummy_y) else: D_train_images = np.vstack([imgs_B, fake_B]) # 6/5/2018 NEW D_train_label = np.vstack([valid, fake]) # 6/5/2018 NEW d_loss = self.discriminator.train_on_batch( D_train_images, D_train_label) # -------------------------------- # Train Generator and Classifier # -------------------------------- # Sample a batch of images from both domains imgs_A, labels_A = self.data_loader.load_data( domain="A", batch_size=batch_size) imgs_B, labels_B = self.data_loader.load_data( domain="B", batch_size=batch_size) # One-hot encoding of labels labels_A = to_categorical(labels_A, num_classes=self.num_classes) # The generators want the discriminators to label the translated images as real if self.use_PatchGAN: valid = np.ones((batch_size, ) + self.disc_patch) else: valid = np.ones((batch_size, 1)) # noise_prior = np.random.normal( 0, 1, (batch_size, self.noise_size[0])) # TODO # noise_prior = np.random.rand(batch_size, self.noise_size[0]) # TODO 6/5/2018 # Train the generator and classifier g_loss = self.combined.train_on_batch([imgs_A, noise_prior], [valid, labels_A]) #----------------------- # Evaluation (domain B) #----------------------- pred_B = self.clf.predict(imgs_B) test_acc = np.mean(np.argmax(pred_B, axis=1) == labels_B) # Add accuracy to list of last 100 accuracy measurements test_accs.append(test_acc) if len(test_accs) > 100: test_accs.pop(0) # Plot the progress # print ( "%d : [D - loss: %.5f, acc: %3d%%], [G - loss: %.5f], [clf - loss: %.5f, acc: %3d%%, test_acc: %3d%% (%3d%%)]" % \ # (epoch, d_loss[0], 100*float(d_loss[1]), # g_loss[1], g_loss[2], 100*float(g_loss[-1]), # 100*float(test_acc), 100*float(np.mean(test_accs)))) if epoch % 10 == 0: with open(os.path.join(dirpath, "D_Losses.csv"), "ab") as csv_file: np.savetxt(csv_file, np.array(d_loss).reshape(1, -1), delimiter=",") with open(os.path.join(dirpath, "G_Losses.csv"), "ab") as csv_file: np.savetxt(csv_file, np.array(g_loss).reshape(1, -1), delimiter=",") if self.use_Wasserstein: d_train_acc = 100 * (float(d_loss[4]) + float(d_loss[5])) / 2 else: d_train_acc = 100 * float(d_loss[1]) gen_loss = g_loss[1] clf_train_acc = 100 * float(g_loss[-1]) clf_train_loss = g_loss[2] current_test_acc = 100 * float(test_acc) test_mean_acc = 100 * float(np.mean(test_accs)) if test_mean_acc > best_test_cls_acc: second_best_cls_acc = best_test_cls_acc best_test_cls_acc = test_mean_acc if save_model: self.combined.save(save_weights_path) else: self.combined.save_weights(save_weights_path) print( "{} : [D - loss: {:.5f}, acc: {:.2f}%], [G - loss: {:.5f}], [clf - loss: {:.5f}, acc: {:.2f}%, test_acc: {:.2f}% ({:.2f}%)] (latest)" .format(epoch, d_loss[0], d_train_acc, gen_loss, clf_train_loss, clf_train_acc, current_test_acc, test_mean_acc)) elif test_mean_acc > second_best_cls_acc: second_best_cls_acc = test_mean_acc if save_model: self.combined.save(save_weights_path) else: self.combined.save_weights(save_weights_path[:-3] + "_bis.h5") print( "{} : [D - loss: {:.5f}, acc: {:.2f}%], [G - loss: {:.5f}], [clf - loss: {:.5f}, acc: {:.2f}%, test_acc: {:.2f}% ({:.2f}%)] (before latest)" .format(epoch, d_loss[0], d_train_acc, gen_loss, clf_train_loss, clf_train_acc, current_test_acc, test_mean_acc)) else: print( "{} : [D - loss: {:.5f}, acc: {:.2f}%], [G - loss: {:.5f}], [clf - loss: {:.5f}, acc: {:.2f}%, test_acc: {:.2f}% ({:.2f}%)]" .format(epoch, d_loss[0], d_train_acc, gen_loss, clf_train_loss, clf_train_acc, current_test_acc, test_mean_acc)) # If at save interval => save generated image samples if epoch % sample_interval == 0: self.sample_images(epoch, save2dir=save_sample2dir) def sample_images(self, epoch, save2dir="../samples"): if not os.path.exists(save2dir): os.makedirs(save2dir) r, c = 5, 10 imgs_A, _ = self.data_loader.load_data(domain="A", batch_size=c) n_sample = imgs_A.shape[0] gen_imgs = imgs_A for i in range(r - 1): noise_prior = np.random.normal( 0, 1, (n_sample, self.noise_size[0])) # TODO # noise_prior = np.random.rand(n_sample, self.noise_size[0]) # TODO 6/5/2018 # Translate images to the other domain fake_B = self.generator.predict([imgs_A, noise_prior]) gen_imgs = np.concatenate([gen_imgs, fake_B]) # Rescale images to 0 - 1 gen_imgs = 0.5 * gen_imgs + 0.5 #titles = ['Original', 'Translated'] fig, axs = plt.subplots(r, c, figsize=(2 * c, 2 * r)) cnt = 0 for i in range(r): for j in range(c): axs[i, j].imshow(gen_imgs[cnt]) #axs[i, j].set_title(titles[i]) axs[i, j].axis('off') cnt += 1 fig.savefig(os.path.join(save2dir, "{}.png".format(epoch))) plt.close() def deploy_transform(self, save2file="../domain_adapted/generated.npy", stop_after=None): dirpath = "/".join(save2file.split("/")[:-1]) if not os.path.exists(dirpath): os.makedirs(dirpath) dirname = "/".join(save2file.split("/")[:-1]) if stop_after is not None: predict_steps = int(stop_after / 32) else: predict_steps = stop_after noise_vec = np.random.normal(0, 1, self.noise_size[0]) assert 1 == 2 # np.random.rand(n_sample, self.noise_size[0]) # TODO 6/5/2018 print("Performing Pixel-level domain adaptation on original images...") adaptaed_images = self.generator.predict( [ self.data_loader.mnist_X[:32 * predict_steps], np.tile(noise_vec, (32 * predict_steps, 1)) ], batch_size=32) #, steps=predict_steps # self.data_loader.mnistm_X[:stop_after] print("+ Done.") print("Saving transformed images to file {}".format(save2file)) np.save(save2file, adaptaed_images) noise_vec_filepath = os.path.join(dirname, "noise_vectors.npy") print( "Saving random noise (seed) to file {}".format(noise_vec_filepath)) np.save(noise_vec_filepath, noise_vec) print("+ All done.") def deploy_debug(self, save2file="../domain_adapted/debug.npy", sample_size=100, noise_number=128, seed=17): dirpath = "/".join(save2file.split("/")[:-1]) if not os.path.exists(dirpath): os.makedirs(dirpath) dirname = "/".join(save2file.split("/")[:-1]) np.random.seed(seed=seed) # np.random.rand(n_sample, self.noise_size[0]) # TODO 6/5/2018 print("Performing Pixel-level domain adaptation on original images...") # noise_vec = np.random.normal(0,1, (sample_size, self.noise_size[0])) # collections = [] # for i in range(sample_size): # adaptaed_images = self.generator.predict([self.data_loader.mnist_X[:15], np.tile(noise_vec[i], (15,1))], batch_size=15) # collections.append(adaptaed_images) collections = [] imgs_A, labels_A = self.data_loader.load_data(domain="A", batch_size=sample_size) for i in range(sample_size): noise_vec = np.random.normal(0, 3, (noise_number, self.noise_size[0])) adaptaed_images = self.generator.predict( [np.tile(imgs_A[i], (noise_number, 1, 1, 1)), noise_vec], batch_size=32) collections.append(adaptaed_images) print("+ Done.") print("Saving transformed images to file {}".format(save2file)) np.save(save2file, np.stack(collections)) print("+ All done.") def deploy_classification(self, batch_size=32): print("Predicting ... ") pred_B = self.clf.predict(self.data_loader.mnistm_X, batch_size=batch_size) print("+ Done.") N_samples = len(pred_B) precision = (np.argmax(pred_B, axis=1) == self.data_loader.mnistm_y) Moy = np.mean(precision) Std = np.std(precision) lower_bound = Moy - 2.576 * Std / np.sqrt(N_samples) upper_bound = Moy + 2.576 * Std / np.sqrt(N_samples) print("=" * 50) print("Unsupervised MNIST-M classification accuracy : {}".format(Moy)) print("Confidence interval (99%) [{}, {}]".format( lower_bound, upper_bound)) print("Length of confidence interval 99%: {}".format(upper_bound - lower_bound)) print("=" * 50) print("+ All done.")
class PixelDA(_DLalgo): """ Paradigm of GAN (keras implementation) 1. Construct D 1a) Compile D 2. Construct G 3. Set D.trainable = False 4. Stack G and D, to construct GAN (combined model) 4a) Compile GAN Approved by fchollet: "the process you describe is in fact correct." See issue #4674 keras: https://github.com/keras-team/keras/issues/4674 """ def __init__(self, noise_size=(100, ), use_PatchGAN=False, use_Wasserstein=True, batch_size=64, **kwargs): # Input shape self.dataset_name = "CT" #"MNIST" # "CT" if self.dataset_name == "MNIST": self.img_shape = (32, 32, 3) self.num_classes = 10 elif self.dataset_name == "CT": self.img_shape = (128, 128, 1) else: raise ValueError( "Only support two datasets for now. ('CT', 'MNIST')") self.img_rows, self.img_cols, self.channels = self.img_shape self.noise_size = noise_size #(100,) self.batch_size = batch_size # Loss weights self.lambda_adv = 5 #before Exp5:10 # Exp1: 20 #17 MNIST-M self.lambda_seg = 1 # Number of filters in first layer of discriminator and Segmenter self.df = 64 self.sf = 64 self.normalize_G = False self.normalize_D = False self.normalize_S = False # Number of residual blocks in the generator self.residual_blocks = 12 #before Exp5: 6 #17 # 6 # NEW TODO 14/5/2018 self.use_PatchGAN = use_PatchGAN #False self.use_Wasserstein = use_Wasserstein self.use_He_initialization = False self.my_initializer = lambda: he_normal( ) if self.use_He_initialization else "glorot_uniform" # TODO if self.use_PatchGAN: # Calculate output shape of D (PatchGAN) patch = int(self.img_rows / 2**4) self.disc_patch = (patch, patch, 1) if self.use_Wasserstein: self.critic_steps = 5 #5 #7 #10 else: self.critic_steps = 1 self.GRADIENT_PENALTY_WEIGHT = 10 #10#5 #10 As the paper ##### Set up the other attributes for key in kwargs: setattr(self, key, kwargs[key]) def build_all_model(self): # optimizer = Adam(0.0002, 0.5) optimizer = Adam(0.0001, beta_1=0.5, beta_2=0.9) # Exp4 # optimizer = Adam(0.0001, beta_1=0.0, beta_2=0.9) # Exp3 of CT2XperCT # Build and compile the discriminators self.discriminator = self.build_discriminator() self.discriminator.name = "Discriminator" img_A = Input(shape=self.img_shape, name='source_image') # real A img_B = Input(shape=self.img_shape, name='target_image') # real B fake_img = Input(shape=self.img_shape, name="fake_image") # fake B # We also need to generate weighted-averages of real and generated samples, to use for the gradient norm penalty. avg_img = RandomWeightedAverage()([img_B, fake_img]) real_img_rating = self.discriminator(img_B) fake_img_rating = self.discriminator(fake_img) avg_img_output = self.discriminator(avg_img) # The gradient penalty loss function requires the input averaged samples to get gradients. However, # Keras loss functions can only have two arguments, y_true and y_pred. We get around this by making a partial() # of the function with the averaged samples here. partial_gp_loss = partial( gradient_penalty_loss, averaged_samples=avg_img, gradient_penalty_weight=self.GRADIENT_PENALTY_WEIGHT) partial_gp_loss.__name__ = 'gradient_penalty' # Functions need names or Keras will throw an error if self.use_Wasserstein: self.discriminator_model = Model( inputs=[img_B, fake_img], #, avg_img # loss_weights=[1,1,1], # useless, since we have multiply the penalization by GRADIENT_PENALTY_WEIGHT=10 outputs=[real_img_rating, fake_img_rating, avg_img_output]) self.discriminator_model.compile( loss=[wasserstein_loss, wasserstein_loss, partial_gp_loss], optimizer=optimizer, metrics=[my_critic_acc]) else: self.discriminator.compile(loss='mse', optimizer=optimizer, metrics=['accuracy']) # For the combined model we will only train the generator and Segmenter self.discriminator.trainable = False # Build the generator self.generator = self.build_generator() self.generator.name = "Generator" # Build the task (segmentation) network self.seg = self.build_segmenter() self.seg.name = "Segmenter" # Input images from both domains # Input noise noise = Input(shape=self.noise_size, name='noise_input') # Translate images from domain A to domain B fake_B = self.generator([img_A, noise]) # Segment the translated image mask_pred = self.seg(fake_B) # Discriminator determines validity of translated images valid = self.discriminator(fake_B) # fake_B_rating if self.use_Wasserstein: self.combined = Model(inputs=[img_A, noise], outputs=[valid, mask_pred]) self.combined.compile( optimizer=optimizer, loss=[wasserstein_loss, dice_coef_loss], loss_weights=[self.lambda_adv, self.lambda_seg], metrics=['accuracy']) else: self.combined = Model([img_A, noise], [valid, mask_pred]) self.combined.compile( loss=['mse', dice_coef_loss], loss_weights=[self.lambda_adv, self.lambda_seg], optimizer=optimizer, metrics=['accuracy']) def load_dataset(self, dataset_name="CT", domain_A_folder="output8", domain_B_folder="output5_x_128"): self.dataset_name = dataset_name if self.dataset_name == "MNIST": # Configure MNIST and MNIST-M data loader self.data_loader = DataLoader(img_res=(self.img_rows, self.img_cols)) elif self.dataset_name == "CT": bodys_filepath_A = "/home/lulin/na4/src/output/{}/train/bodys.npy".format( domain_A_folder) masks_filepath_A = "/home/lulin/na4/src/output/{}/train/liver_masks.npy".format( domain_A_folder) self.Dataset_A = MyDataset( paths=[bodys_filepath_A, masks_filepath_A], batch_size=self.batch_size, augment=False, seed=17, domain="A") bodys_filepath_B = "/home/lulin/na4/src/output/{}/train/bodys.npy".format( domain_B_folder) masks_filepath_B = "/home/lulin/na4/src/output/{}/train/liver_masks.npy".format( domain_B_folder) self.Dataset_B = MyDataset( paths=[bodys_filepath_B, masks_filepath_B], batch_size=self.batch_size, augment=False, seed=17, domain="B") else: pass def build_generator(self): """Resnet Generator""" def residual_block(layer_input, normalization=self.normalize_G): """Residual block described in paper""" d = Conv2D(64, kernel_size=3, strides=1, padding='same', kernel_initializer=self.my_initializer())(layer_input) if normalization: d = InstanceNormalization()(d) # d = BatchNormalization(momentum=0.8)(d) # 6/5/2018 d = Activation('relu')(d) d = Conv2D(64, kernel_size=3, strides=1, padding='same')(d) if normalization: d = InstanceNormalization()(d) # d = BatchNormalization(momentum=0.8)(d) # 6/5/2018 d = Add()([d, layer_input]) return d # Image input img = Input(shape=self.img_shape, name='image_input') ## Noise input noise = Input(shape=self.noise_size, name='noise_input') noise_layer = Dense(self.img_rows * self.img_cols, activation="relu", kernel_initializer=self.my_initializer())(noise) noise_layer = Reshape((self.img_rows, self.img_cols, 1))(noise_layer) conditioned_img = keras.layers.concatenate([img, noise_layer]) # keras.layers.concatenate # l1 = Conv2D(64, kernel_size=3, padding='same', activation='relu')(img) l1 = Conv2D(64, kernel_size=3, padding='same', activation='relu', kernel_initializer=self.my_initializer())(conditioned_img) # Propogate signal through residual blocks r = residual_block(l1) for _ in range(self.residual_blocks - 1): r = residual_block(r) output_img = Conv2D(self.channels, kernel_size=3, padding='same', activation='tanh')(r) return Model([img, noise], output_img) def build_discriminator(self): def d_layer(layer_input, filters, f_size=4, normalization=self.normalize_D): """Discriminator layer""" d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input) d = LeakyReLU(alpha=0.2)(d) if normalization: d = InstanceNormalization()(d) return d img = Input(shape=self.img_shape, name="image") d1 = d_layer(img, self.df, normalization=False) d2 = d_layer(d1, self.df * 2, normalization=self.normalize_D) d3 = d_layer(d2, self.df * 4, normalization=self.normalize_D) d4 = d_layer(d3, self.df * 8, normalization=self.normalize_D) if self.use_PatchGAN: # NEW 7/5/2018 validity = Conv2D(1, kernel_size=4, strides=1, padding='same')(d4) else: if self.use_Wasserstein: # NEW 8/5/2018 validity = Dense(1, activation=None)( Flatten()(d4)) # he_normal ?? else: validity = Dense(1, activation='sigmoid')(Flatten()(d4)) return Model(img, validity) def build_segmenter(self): """Segmenter layer""" model = UNet(self.img_shape, depth=3, dropout=0.5, start_ch=32, upconv=False, batchnorm=self.normalize_S) return model def load_pretrained_weights(self, weights_path="../Weights/all_weights.h5"): print("Loading pretrained weights from path: {} ...".format( weights_path)) self.combined.load_weights(weights_path, by_name=True) print("+ Done.") def summary(self): print("=" * 50) print("Discriminator summary:") self.discriminator.summary() print("=" * 50) print("Generator summary:") self.generator.summary() print("=" * 50) print("Segmenter summary:") self.seg.summary() if self.use_Wasserstein: print("=" * 50) print("Discriminator model summary:") self.discriminator_model.summary() print("=" * 50) print("Combined model summary:") self.combined.summary() def write_tensorboard_graph(self, to_dir="../logs", save_png2dir="../Model_graph"): if not os.path.exists(save_png2dir): os.makedirs(save_png2dir) tensorboard = keras.callbacks.TensorBoard(log_dir=to_dir, histogram_freq=0, write_graph=True, write_images=False) # tensorboard.set_model(self.combined) tensorboard.set_model(self.discriminator_model) try: plot_model(self.combined, to_file=os.path.join(save_png2dir, "Combined_model.png")) plot_model(self.discriminator_model, to_file=os.path.join(save_png2dir, "Discriminator_model.png")) except: pass def train(self, iterations, sample_interval=50, save_sample2dir="../samples/exp0", save_weights_path='../Weights/all_weights.h5', save_model=False, time_monitor=True): dirpath = "/".join(save_weights_path.split("/")[:-1]) if not os.path.exists(dirpath): os.makedirs(dirpath) self.save_config(save2path=os.path.join(dirpath, "config.dill"), verbose=True) # segmentation accuracy on 100 last batches of domain B test_accs = [] ## Monitor to save model weights Lu best_test_cls_acc = 0 second_best_cls_acc = -1 st = time() elapsed_time = 0 for iteration in range(iterations): if time_monitor and (iteration % 10 == 0) and (iteration > 0): et = time() elapsed_time = et - st st = et # --------------------- # Train Discriminator # --------------------- # n_sample = half_batch # imgs_A.shape[0] for _ in range(self.critic_steps): if self.dataset_name == "MNIST": imgs_A, _ = self.data_loader.load_data( domain="A", batch_size=self.batch_size) imgs_B, _ = self.data_loader.load_data( domain="B", batch_size=self.batch_size) elif self.dataset_name == "CT": imgs_A, _ = self.Dataset_A.next() imgs_B, _ = self.Dataset_B.next() noise_prior = np.random.normal( 0, 1, (self.batch_size, self.noise_size[0])) # noise_prior = np.random.rand(half_batch, self.noise_size[0]) # 6/5/2018 # Translate images from domain A to domain B fake_B = self.generator.predict([imgs_A, noise_prior]) if self.use_PatchGAN: valid = np.ones((self.batch_size, ) + self.disc_patch) fake = np.zeros((self.batch_size, ) + self.disc_patch) else: if self.use_Wasserstein: valid = np.ones((self.batch_size, 1)) fake = -valid #np.ones((half_batch, 1)) # = - valid ? dummy_y = np.zeros((self.batch_size, 1)) # NEW else: valid = np.ones((self.batch_size, 1)) fake = np.zeros((self.batch_size, 1)) # Train the discriminators (original images = real / translated = Fake) # d_loss_real = self.discriminator.train_on_batch(imgs_B, valid) # d_loss_fake = self.discriminator.train_on_batch(fake_B, fake) # d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) if self.use_Wasserstein: d_loss = self.discriminator_model.train_on_batch( [imgs_B, fake_B], [valid, fake, dummy_y]) # d_loss = self.discriminator.train_on_batch(D_train_images, D_train_label, dummy_y) else: D_train_images = np.vstack([imgs_B, fake_B]) # 6/5/2018 NEW D_train_label = np.vstack([valid, fake]) # 6/5/2018 NEW d_loss = self.discriminator.train_on_batch( D_train_images, D_train_label) # -------------------------------- # Train Generator and Segmenter # -------------------------------- # Sample a batch of images from both domains if self.dataset_name == "MNIST": imgs_A, _, masks_A = self.data_loader.load_data( domain="A", batch_size=self.batch_size, return_mask=True) imgs_B, _, masks_B = self.data_loader.load_data( domain="B", batch_size=self.batch_size, return_mask=True) elif self.dataset_name == "CT": imgs_A, masks_A = self.Dataset_A.next() imgs_B, masks_B = self.Dataset_B.next() else: pass # One-hot encoding of labels # labels_A = to_categorical(labels_A, num_classes=self.num_classes) # The generators want the discriminators to label the translated images as real if self.use_PatchGAN: valid = np.ones((self.batch_size, ) + self.disc_patch) else: valid = np.ones((self.batch_size, 1)) # noise_prior = np.random.normal( 0, 1, (self.batch_size, self.noise_size[0])) # noise_prior = np.random.rand(batch_size, self.noise_size[0]) # 6/5/2018 # Train the generator and Segmenter g_loss = self.combined.train_on_batch([imgs_A, noise_prior], [valid, masks_A]) #----------------------- # Evaluation (domain B) #----------------------- pred_B = self.seg.predict(imgs_B) # test_acc = np.mean(np.argmax(pred_B, axis=1) == labels_B) _, test_acc = dice_predict(masks_B, pred_B) # Add accuracy to list of last 100 accuracy measurements test_accs.append(test_acc) if len(test_accs) > 100: test_accs.pop(0) if iteration % 10 == 0: if self.use_Wasserstein: d_real_acc = 100 * float(d_loss[4]) d_fake_acc = 100 * float(d_loss[5]) d_train_acc = 100 * (float(d_loss[4]) + float(d_loss[5])) / 2 else: d_train_acc = 100 * float(d_loss[1]) gen_loss = g_loss[1] clf_train_acc = 100 * float(g_loss[-1]) clf_train_loss = g_loss[2] current_test_acc = 100 * float(test_acc) test_mean_acc = 100 * float(np.mean(test_accs)) g_loss.append(current_test_acc) g_loss.append(test_mean_acc) with open(os.path.join(dirpath, "D_Losses.csv"), "ab") as csv_file: np.savetxt(csv_file, np.array(d_loss).reshape(1, -1), delimiter=",") with open(os.path.join(dirpath, "G_Losses.csv"), "ab") as csv_file: np.savetxt(csv_file, np.array(g_loss).reshape(1, -1), delimiter=",") message = "{} : [D - loss: {:.5f}, GP_loss: {:.5f}, (+) acc: {:.2f}%, (-) acc: {:.2f}%, acc: {:.2f}%], [G - loss: {:.5f}], [seg - loss: {:.5f}, acc: {:.2f}%, test_dice: {:.2f}% ({:.2f}%)]".format( iteration, d_loss[0], d_loss[3], d_real_acc, d_fake_acc, d_train_acc, gen_loss, clf_train_loss, clf_train_acc, current_test_acc, test_mean_acc) if test_mean_acc > best_test_cls_acc: second_best_cls_acc = best_test_cls_acc best_test_cls_acc = test_mean_acc if save_model: self.combined.save(save_weights_path) else: self.combined.save_weights(save_weights_path) message += " (best)" elif test_mean_acc > second_best_cls_acc: second_best_cls_acc = test_mean_acc if save_model: self.combined.save(save_weights_path) else: self.combined.save_weights(save_weights_path[:-3] + "_bis.h5") message += " (second best)" else: pass if time_monitor: message += "... {:.2f}s.".format(elapsed_time) print(message) # If at save interval => save generated image samples if iteration % sample_interval == 0: self.sample_images(iteration, save2dir=save_sample2dir) #### NEW 24/5/2018 self.combined.save_weights(save_weights_path[:-3] + "_final.h5") def sample_images(self, iterations, save2dir="../samples"): if not os.path.exists(save2dir): os.makedirs(save2dir) if self.dataset_name == "MNIST": r, c = 5, 10 imgs_A, _ = self.data_loader.load_data(domain="A", batch_size=c) elif self.dataset_name == "CT": r, c = 2, 5 assert r == 2 imgs_A, masks_A = self.Dataset_A.next() imgs_A = imgs_A[:c] masks_A = masks_A[:c] masks_A = np.squeeze(masks_A) # raise ValueError("Not implemented error.") else: pass n_sample = imgs_A.shape[0] # == c gen_imgs = imgs_A for i in range(r - 1): noise_prior = np.random.normal( 0, 1, (n_sample, self.noise_size[0])) # TODO # noise_prior = np.random.normal(0,3, (n_sample, self.noise_size[0])) # TODO # 16/5/2018 # noise_prior = np.random.rand(n_sample, self.noise_size[0]) # TODO 6/5/2018 # Translate images to the other domain fake_B = self.generator.predict([imgs_A, noise_prior]) # print(fake_B.shape) gen_imgs = np.concatenate([gen_imgs, fake_B]) if self.dataset_name == "MNIST": # Rescale images from (-1, 1) to (0, 1) gen_imgs = 0.5 * gen_imgs + 0.5 elif self.dataset_name == "CT": gen_imgs = np.squeeze(gen_imgs) # print(gen_imgs.shape) #titles = ['Original', 'Translated'] # TODO r = 4 fig, axs = plt.subplots(r, c, figsize=(3 * c, 3 * r)) cnt = 0 for i in range(2): # replace r by 2 for j in range(c): axs[i, j].imshow(gen_imgs[cnt], cmap="gray") #axs[i, j].set_title(titles[i]) axs[i, j].axis('off') cnt += 1 for j in range(c): ############ TODO ############ # visualize image with adaptive histogram axs[2, j].imshow(apply_adapt_hist()(gen_imgs[j + c * 1]), cmap="gray") axs[2, j].axis('off') # mask image with ground truth mask axs[3, j].imshow(apply_adapt_hist()(gen_imgs[j + c * 1]), cmap="gray") axs[3, j].imshow(masks_A[j], aspect="equal", cmap="Blues", alpha=0.4) axs[3, j].axis('off') fig.savefig(os.path.join(save2dir, "{}.png".format(iterations))) plt.close() def train_segmenter(self, iterations, batch_size=32, noise_range=5, save_weights_path=None): raise ValueError("Not modified yet.") if save_weights_path is not None: dirpath = "/".join(save_weights_path.split("/")[:-1]) if not os.path.exists(dirpath): os.makedirs(dirpath) optimizer = Adam(0.000001, beta_1=0.0, beta_2=0.9) # Input noise noise = Input(shape=self.noise_size, name='noise_input_seg') img_A = Input(shape=self.img_shape, name='source_image_seg') # Translate images from domain A to domain B fake_B = self.generator([img_A, noise]) # Segment the translated image mask_pred = self.seg(fake_B) self.generator.trainable = False self.segmentation_model = Model(inputs=[img_A, noise], outputs=[mask_pred]) self.segmentation_model.compile(loss=dice_coef_loss, optimizer=optimizer, metrics=["acc", dice_coef]) self.segmentation_model.name = "U-net (freeze Generator)" self.segmentation_model.summary() best_test_dice = 0.0 second_best_test_dice = -1.0 collections = [] for e in range(iterations): noise = (2 * np.random.random( (batch_size, self.noise_size[0])) - 1) * noise_range images_A, _, mask_A = self.data_loader.load_data( domain="A", batch_size=batch_size, return_mask=True) s_loss = self.segmentation_model.train_on_batch([images_A, noise], mask_A) if e % 100 == 0: images_B, _, mask_B = self.data_loader.load_data( domain="B", batch_size=batch_size, return_mask=True) pred_mask_B = self.seg.predict(images_B) _, current_test_dice = dice_predict(mask_B, pred_mask_B) if len(collections) >= 100: collections.pop(0) collections.append(current_test_dice) mean_dice = np.mean(collections) message = "{} dice loss: {:.3f}; acc: {:.5f}; mean dice (test): {:.3f}".format( e, 100 * s_loss[0], s_loss[1], 100 * mean_dice) if mean_dice > best_test_dice: best_test_dice = mean_dice message += " (best)" if save_weights_path is not None: self.segmentation_model.save_weights(save_weights_path) elif mean_dice > second_best_test_dice: second_best_test_dice = mean_dice message += " (second best)" if save_weights_path is not None: self.segmentation_model.save_weights( save_weights_path[:-3] + "_bis.h5") else: pass print(message) return def deploy_transform(self, save2file="../domain_adapted/generated.npy", stop_after=None): raise ValueError("Not modified yet.") dirpath = "/".join(save2file.split("/")[:-1]) if not os.path.exists(dirpath): os.makedirs(dirpath) dirname = "/".join(save2file.split("/")[:-1]) if stop_after is not None: predict_steps = int(stop_after / 32) else: predict_steps = stop_after noise_vec = np.random.normal(0, 1, self.noise_size[0]) assert 1 == 2 # np.random.rand(n_sample, self.noise_size[0]) # TODO 6/5/2018 print("Performing Pixel-level domain adaptation on original images...") adaptaed_images = self.generator.predict( [ self.data_loader.mnist_X[:32 * predict_steps], np.tile(noise_vec, (32 * predict_steps, 1)) ], batch_size=32) #, steps=predict_steps # self.data_loader.mnistm_X[:stop_after] print("+ Done.") print("Saving transformed images to file {}".format(save2file)) np.save(save2file, adaptaed_images) noise_vec_filepath = os.path.join(dirname, "noise_vectors.npy") print( "Saving random noise (seed) to file {}".format(noise_vec_filepath)) np.save(noise_vec_filepath, noise_vec) print("+ All done.") def deploy_debug(self, save2file="../domain_adapted/debug.npy", sample_size=100, noise_number=128, use_sobol=False, use_linear=False, use_sphere=False, use_uniform_linear=False, use_zeros=False, seed=17): raise ValueError("Not modified yet.") dirpath = "/".join(save2file.split("/")[:-1]) if not os.path.exists(dirpath): os.makedirs(dirpath) dirname = "/".join(save2file.split("/")[:-1]) np.random.seed(seed=seed) # np.random.rand(n_sample, self.noise_size[0]) # TODO 6/5/2018 print("Performing Pixel-level domain adaptation on original images...") # noise_vec = np.random.normal(0,1, (sample_size, self.noise_size[0])) # collections = [] # for i in range(sample_size): # adaptaed_images = self.generator.predict([self.data_loader.mnist_X[:15], np.tile(noise_vec[i], (15,1))], batch_size=15) # collections.append(adaptaed_images) collections = [] imgs_A, labels_A = self.data_loader.load_data(domain="A", batch_size=sample_size) for i in tqdm(range(sample_size)): if use_sobol: noise_vec = 5 * (2 * i4_sobol_generate( self.noise_size[0], noise_number, i * noise_number).T - 1) elif use_linear: tangents = 3.0 * (2 * np.random.random((noise_number, 1)) - 1) noise_vec = np.ones( (noise_number, self.noise_size[0])) * tangents elif use_sphere: noise_vec = 2 * (np.random.random( (noise_number, self.noise_size[0])) - 1) norm_vec = np.linalg.norm(noise_vec, axis=-1) noise_vec = noise_vec / norm_vec[:, np.newaxis] elif use_uniform_linear: tangents = 10.0 * np.linspace(-1, 1, noise_number)[:, np.newaxis] noise_vec = np.ones( (noise_number, self.noise_size[0])) * tangents elif use_zeros: noise_vec = np.zeros((noise_number, self.noise_size[0])) else: noise_vec = np.random.normal( 0, 3, (noise_number, self.noise_size[0])) adaptaed_images = self.generator.predict( [np.tile(imgs_A[i], (noise_number, 1, 1, 1)), noise_vec], batch_size=32) collections.append(adaptaed_images) print("+ Done.") print("Saving transformed images to file {}".format(save2file)) np.save(save2file, np.stack(collections)) print("+ All done.") def deploy_segmentation(self, batch_size=32): print("Predicting ... ") if self.dataset_name == "MNIST": pred_B = self.seg.predict(self.data_loader.mnistm_X, batch_size=batch_size) precision = (np.argmax(pred_B, axis=1) == self.data_loader.mnistm_y) Moy = np.mean(precision) Std = np.std(precision) elif self.dataset_name == "CT": pred_B = self.seg.predict(self.Dataset_B.X_train, batch_size=batch_size) gt_B = self.Dataset_B.Y_train dice_all, dice_mean = dice_predict(gt_B, pred_B) Moy = dice_mean Std = np.std(dice_all) print("+ Done.") N_samples = len(pred_B) lower_bound = Moy - 2.576 * Std / np.sqrt(N_samples) upper_bound = Moy + 2.576 * Std / np.sqrt(N_samples) print("=" * 50) print("Unsupervised MNIST-M segmentation accuracy : {}".format(Moy)) print("Confidence interval (99%) [{}, {}]".format( lower_bound, upper_bound)) print("Length of confidence interval 99%: {}".format(upper_bound - lower_bound)) print("=" * 50) print("+ All done.") def deploy_demo_only(self, save2file="../domain_adapted/WGAN_GP/Exp4/demo.npy", sample_size=25, noise_number=512, linspace_size=10.0): raise ValueError("Not modified yet.") collections = [] imgs_A, labels_A = self.data_loader.load_data(domain="A", batch_size=sample_size) tangents = linspace_size * np.linspace(-1, 1, noise_number)[:, np.newaxis] noise_vec = np.ones((noise_number, self.noise_size[0])) * tangents for i in tqdm(range(noise_number)): adaptaed_images = self.generator.predict( [imgs_A, np.tile(noise_vec[i], (sample_size, 1))], batch_size=sample_size) collections.append(adaptaed_images) print("+ Done.") print("Saving transformed images to file {}".format(save2file)) np.save(save2file, np.stack(collections)) print("+ All done.") def deploy_cherry_pick( self, save2file="../domain_adapted/WGAN_GP/Exp4/demo_cherry_picked.png", sample_size=25, noise_number=25, linspace_size=5.0): raise ValueError("Not modified yet.") collections = [] imgs_A, labels_A = self.data_loader.load_data(domain="A", batch_size=sample_size) assert noise_number == sample_size tangents = linspace_size * np.linspace(-1, 1, noise_number)[:, np.newaxis] noise_vec = np.ones((noise_number, self.noise_size[0])) * tangents np.random.shuffle(noise_vec) # shuffle background color ! adaptaed_images = self.generator.predict([imgs_A, noise_vec], batch_size=sample_size) adaptaed_images = (adaptaed_images + 1) / 2 print("+ Done.") print("Saving transformed images to file {}".format(save2file)) r = 5 c = 5 fig, axs = plt.subplots(r, c, figsize=(5 * c, 5 * r)) for j in range(r): for i in range(c): axs[j, i].imshow(adaptaed_images[c * j + i]) axs[j, i].axis('off') plt.savefig(save2file) plt.close() print("+ All done.")
class PixelDA(_DLalgo): """ Paradigm of GAN (keras implementation) 1. Construct D 1a) Compile D 2. Construct G 3. Set D.trainable = False 4. Stack G and D, to construct GAN (combined model) 4a) Compile GAN Approved by fchollet: "the process you describe is in fact correct." See issue #4674 keras: https://github.com/keras-team/keras/issues/4674 """ def __init__(self, noise_size=(100,), use_PatchGAN=False, use_Wasserstein=True, batch_size=64, **kwargs): # Input shape self.dataset_name = "MNIST" # "CT" if self.dataset_name == "MNIST": self.img_shape = (32, 32, 3) self.num_classes = 10 elif self.dataset_name == "CT": self.img_shape = (64, 64, 1) else: assert 1==2, "Only support two datasets for now. ('CT', 'MNIST')" self.img_rows, self.img_cols, self.channels = self.img_shape self.noise_size = noise_size #(100,) self.batch_size = batch_size # Loss weights self.lambda_adv = 7#10 # 7 self.lambda_clf = 1 # Number of filters in first layer of discriminator and classifier self.df = 64 # NEW TODO #64 11/5/2018 self.cf = 64 self.normalize_G = False self.normalize_D = False self.normalize_C = True self.shift_label = False # TODO NEW 31/5/2018 # Number of residual blocks in the generator self.residual_blocks = 17 # 6 # NEW TODO 14/5/2018 self.use_PatchGAN = use_PatchGAN #False self.use_Wasserstein = use_Wasserstein if self.use_PatchGAN: # Calculate output shape of D (PatchGAN) patch = int(self.img_rows / 2**4) self.disc_patch = (patch, patch, 1) if self.use_Wasserstein: self.critic_steps = 5#5 #7 #10 else: self.critic_steps = 1 self.GRADIENT_PENALTY_WEIGHT = 10#10#5 #10 As the paper ##### Set up the other attributes for key in kwargs: setattr(self, key, kwargs[key]) def freeze_layers_kernel(self, model): valid_name = ["dense", "conv2d"] layers_names = list(map(lambda layer:layer.name, model.layers)) num_layers = len(layers_names) valid_index = list(map(lambda layer_name:(layer_name.split('_')[0] in valid_name), layers_names)) valid_index = np.arange(num_layers)[valid_index] for i in valid_index: ## Remove 'kernel' from trainable weights before compile model ! model.layers[i].trainable_weights = model.layers[i].trainable_weights[1:] def build_all_model(self): # optimizer = Adam(0.0002, 0.5) optimizer = Adam(0.0001, beta_1=0.5, beta_2=0.9) # optimizer = SGD(lr=0.0001) # optimizer = RMSprop(lr=1e-5) # Build and compile the discriminators self.discriminator = self.build_discriminator() self.discriminator.name = "Discriminator" img_A = Input(shape=self.img_shape, name='source_image') # real A img_B = Input(shape=self.img_shape, name='target_image') # real B fake_img = Input(shape=self.img_shape, name="fake_image") # fake B # We also need to generate weighted-averages of real and generated samples, to use for the gradient norm penalty. avg_img = RandomWeightedAverage()([img_B, fake_img]) real_img_rating = self.discriminator(img_B) fake_img_rating = self.discriminator(fake_img) avg_img_output = self.discriminator(avg_img) # The gradient penalty loss function requires the input averaged samples to get gradients. However, # Keras loss functions can only have two arguments, y_true and y_pred. We get around this by making a partial() # of the function with the averaged samples here. partial_gp_loss = partial(gradient_penalty_loss, averaged_samples=avg_img, gradient_penalty_weight=self.GRADIENT_PENALTY_WEIGHT) partial_gp_loss.__name__ = 'gradient_penalty' # Functions need names or Keras will throw an error if self.use_Wasserstein: ### One time experimence: Freeze all Discriminator's 'kernel' # self.freeze_layers_kernel(self.discriminator) # TODO self.discriminator_model = Model(inputs=[img_B, fake_img], #, avg_img # loss_weights=[1,1,1], # useless, since we have multiply the penalization by GRADIENT_PENALTY_WEIGHT=10 outputs=[real_img_rating, fake_img_rating, avg_img_output]) self.discriminator_model.compile(loss=[wasserstein_loss, wasserstein_loss, partial_gp_loss], optimizer=optimizer, metrics=[my_critic_acc]) # print(len(self.discriminator_model._collected_trainable_weights)) else: self.discriminator.compile(loss='mse', optimizer=optimizer, metrics=['accuracy']) # For the combined model we will only train the generator and classifier self.discriminator.trainable = False # Build the generator self.generator = self.build_generator() self.generator.name = "Generator" # Build the task (classification) network self.clf = self.build_classifier() self.clf.name = "Classifier" # Input images from both domains # Input noise noise = Input(shape=self.noise_size, name='noise_input') # Translate images from domain A to domain B fake_B = self.generator([img_A, noise]) # Classify the translated image class_pred = self.clf(fake_B) # Discriminator determines validity of translated images valid = self.discriminator(fake_B) # fake_B_rating if self.use_Wasserstein: self.combined = Model(inputs=[img_A, noise], outputs=[valid, class_pred]) self.combined.compile(optimizer=optimizer, loss=[wasserstein_loss, 'categorical_crossentropy'], loss_weights=[self.lambda_adv, self.lambda_clf], metrics=['accuracy']) else: self.combined = Model([img_A, noise], [valid, class_pred]) self.combined.compile(loss=['mse', 'categorical_crossentropy'], loss_weights=[self.lambda_adv, self.lambda_clf], optimizer=optimizer, metrics=['accuracy']) def load_dataset(self): # Configure MNIST and MNIST-M data loader self.data_loader = DataLoader(img_res=(self.img_rows, self.img_cols)) def build_generator(self): """Resnet Generator""" def residual_block(layer_input, normalization=self.normalize_G): """Residual block described in paper""" d = Conv2D(64, kernel_size=3, strides=1, padding='same')(layer_input) if normalization: d = InstanceNormalization()(d) # d = BatchNormalization(momentum=0.8)(d) # TODO 6/5/2018 d = Activation('relu')(d) d = Conv2D(64, kernel_size=3, strides=1, padding='same')(d) if normalization: d = InstanceNormalization()(d) # d = BatchNormalization(momentum=0.8)(d) # TODO 6/5/2018 d = Add()([d, layer_input]) return d # Image input img = Input(shape=self.img_shape, name='image_input') ## Noise input noise = Input(shape=self.noise_size, name='noise_input') noise_layer = Dense(1024, activation="relu")(noise) noise_layer = Reshape((self.img_rows,self.img_cols, 1))(noise_layer) conditioned_img = keras.layers.concatenate([img, noise_layer]) # keras.layers.concatenate # l1 = Conv2D(64, kernel_size=3, padding='same', activation='relu')(img) l1 = Conv2D(64, kernel_size=3, padding='same', activation='relu')(conditioned_img) # Propogate signal through residual blocks r = residual_block(l1) for _ in range(self.residual_blocks - 1): r = residual_block(r) output_img = Conv2D(self.channels, kernel_size=3, padding='same', activation='tanh')(r) return Model([img, noise], output_img) def build_discriminator(self): def d_layer(layer_input, filters, f_size=4, normalization=self.normalize_D): """Discriminator layer""" d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input) d = LeakyReLU(alpha=0.2)(d) if normalization: d = InstanceNormalization()(d) return d img = Input(shape=self.img_shape, name="image") d1 = d_layer(img, self.df, normalization=False) d2 = d_layer(d1, self.df*2, normalization=self.normalize_D) d3 = d_layer(d2, self.df*4, normalization=self.normalize_D) d4 = d_layer(d3, self.df*8, normalization=self.normalize_D) if self.use_PatchGAN: # NEW 7/5/2018 validity = Conv2D(1, kernel_size=4, strides=1, padding='same')(d4) else: if self.use_Wasserstein: # NEW 8/5/2018 validity = Dense(1, activation=None)(Flatten()(d4)) # he_normal ?? TODO else: validity = Dense(1, activation='sigmoid')(Flatten()(d4)) return Model(img, validity) def build_classifier(self): def clf_layer(layer_input, filters, f_size=4, normalization=self.normalize_C): """Classifier layer""" d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input) d = LeakyReLU(alpha=0.2)(d) if normalization: d = InstanceNormalization()(d) return d img = Input(shape=self.img_shape, name='image_input') c1 = clf_layer(img, self.cf, normalization=False) c2 = clf_layer(c1, self.cf*2) c3 = clf_layer(c2, self.cf*4) c4 = clf_layer(c3, self.cf*8) c5 = clf_layer(c4, self.cf*8) class_pred = Dense(self.num_classes, activation='softmax')(Flatten()(c5)) return Model(img, class_pred) def load_pretrained_weights(self, weights_path="../Weights/all_weights.h5", only_cls=False): print("Loading pretrained weights from path: {} ...".format(weights_path)) if only_cls: # self.clf.load_weights(weights_path, by_name=True) # Don't work ! # See: https://github.com/keras-team/keras/issues/5348 # Solution: # Build a model, load (all) weights, save sub model weigths as np.array, kill(clear session) model # than finally, build new model, set sub model weights from pre-saved np.array ! self.combined.load_weights(weights_path, by_name=True) clf_weights = self.clf.get_weights() K.clear_session() self.build_all_model() self.clf.set_weights(clf_weights) else: self.combined.load_weights(weights_path, by_name=True) print("+ Done.") def summary(self): print("="*50) print("Discriminator summary:") self.discriminator.summary() print("="*50) print("Generator summary:") self.generator.summary() print("="*50) print("Classifier summary:") self.clf.summary() if self.use_Wasserstein: print("="*50) print("Discriminator model summary:") self.discriminator_model.summary() print("="*50) print("Combined model summary:") self.combined.summary() def write_tensorboard_graph(self, to_dir="../logs", save_png2dir="../Model_graph"): if not os.path.exists(save_png2dir): os.makedirs(save_png2dir) tensorboard = keras.callbacks.TensorBoard(log_dir=to_dir, histogram_freq=0, write_graph=True, write_images=False) # tensorboard.set_model(self.combined) tensorboard.set_model(self.discriminator_model) try: plot_model(self.combined, to_file=os.path.join(save_png2dir, "Combined_model.png")) plot_model(self.discriminator_model, to_file=os.path.join(save_png2dir, "Discriminator_model.png")) except: pass def train(self, epochs, sample_interval=50, save_sample2dir="../samples/exp0", save_weights_path='../Weights/all_weights.h5', save_model=False): dirpath = "/".join(save_weights_path.split("/")[:-1]) if not os.path.exists(dirpath): os.makedirs(dirpath) self.save_config(save2path=os.path.join(dirpath, "config.dill"), verbose=True) #half_batch = batch_size #int(batch_size / 2) ### TODO # half_batch = int(batch_size / 2) # Classification accuracy on 100 last batches of domain B test_accs = [] ## Monitor to save model weights Lu best_test_cls_acc = 0 second_best_cls_acc = -1 for epoch in range(epochs): # --------------------- # Train Discriminator # --------------------- # n_sample = half_batch # imgs_A.shape[0] for _ in range(self.critic_steps): imgs_A, _ = self.data_loader.load_data(domain="A", batch_size=self.batch_size) imgs_B, _ = self.data_loader.load_data(domain="B", batch_size=self.batch_size) noise_prior = np.random.normal(0,1, (self.batch_size, self.noise_size[0])) # noise_prior = np.random.rand(half_batch, self.noise_size[0]) # TODO 6/5/2018 # Translate images from domain A to domain B fake_B = self.generator.predict([imgs_A, noise_prior]) if self.use_PatchGAN: valid = np.ones((self.batch_size,) + self.disc_patch) fake = np.zeros((self.batch_size,) + self.disc_patch) else: if self.use_Wasserstein: valid = np.ones((self.batch_size, 1)) fake = - valid #np.ones((half_batch, 1)) # = - valid ? TODO dummy_y = np.zeros((self.batch_size, 1)) # NEW else: valid = np.ones((self.batch_size, 1)) fake = np.zeros((self.batch_size, 1)) # Train the discriminators (original images = real / translated = Fake) # d_loss_real = self.discriminator.train_on_batch(imgs_B, valid) # d_loss_fake = self.discriminator.train_on_batch(fake_B, fake) # d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) if self.use_Wasserstein: d_loss = self.discriminator_model.train_on_batch([imgs_B, fake_B], [valid, fake, dummy_y]) # d_loss = self.discriminator.train_on_batch(D_train_images, D_train_label, dummy_y) else: D_train_images = np.vstack([imgs_B, fake_B]) # 6/5/2018 NEW D_train_label = np.vstack([valid, fake]) # 6/5/2018 NEW d_loss = self.discriminator.train_on_batch(D_train_images, D_train_label) # -------------------------------- # Train Generator and Classifier # -------------------------------- # Sample a batch of images from both domains imgs_A, labels_A = self.data_loader.load_data(domain="A", batch_size=self.batch_size) imgs_B, labels_B = self.data_loader.load_data(domain="B", batch_size=self.batch_size) if self.shift_label: labels_A = np.mod(labels_A+1, 10) # == (labels_A+1) % 10 # One-hot encoding of labels labels_A = to_categorical(labels_A, num_classes=self.num_classes) # The generators want the discriminators to label the translated images as real if self.use_PatchGAN: valid = np.ones((self.batch_size,) + self.disc_patch) else: valid = np.ones((self.batch_size, 1)) # noise_prior = np.random.normal(0,1, (self.batch_size, self.noise_size[0])) # noise_prior = np.random.rand(batch_size, self.noise_size[0]) # TODO 6/5/2018 # Train the generator and classifier g_loss = self.combined.train_on_batch([imgs_A, noise_prior], [valid, labels_A]) #----------------------- # Evaluation (domain B) #----------------------- pred_B = self.clf.predict(imgs_B) test_acc = np.mean(np.argmax(pred_B, axis=1) == labels_B) # Add accuracy to list of last 100 accuracy measurements test_accs.append(test_acc) if len(test_accs) > 100: test_accs.pop(0) # Plot the progress # print ( "%d : [D - loss: %.5f, acc: %3d%%], [G - loss: %.5f], [clf - loss: %.5f, acc: %3d%%, test_acc: %3d%% (%3d%%)]" % \ # (epoch, d_loss[0], 100*float(d_loss[1]), # g_loss[1], g_loss[2], 100*float(g_loss[-1]), # 100*float(test_acc), 100*float(np.mean(test_accs)))) if epoch % 10 == 0: if self.use_Wasserstein: d_real_acc = 100*float(d_loss[4]) d_fake_acc = 100*float(d_loss[5]) d_train_acc = 100*(float(d_loss[4])+float(d_loss[5]))/2 else: d_train_acc = 100*float(d_loss[1]) gen_loss = g_loss[1] clf_train_acc = 100*float(g_loss[-1]) clf_train_loss = g_loss[2] current_test_acc = 100*float(test_acc) test_mean_acc = 100*float(np.mean(test_accs)) g_loss.append(current_test_acc) g_loss.append(test_mean_acc) with open(os.path.join(dirpath, "D_Losses.csv"), "ab") as csv_file: np.savetxt(csv_file, np.array(d_loss).reshape(1,-1), delimiter=",") with open(os.path.join(dirpath, "G_Losses.csv"), "ab") as csv_file: np.savetxt(csv_file, np.array(g_loss).reshape(1,-1), delimiter=",") message = "{} : [D - loss: {:.5f}, GP_loss: {:.5f}, (+) acc: {:.2f}%, (-) acc: {:.2f}%, acc: {:.2f}%], [G - loss: {:.5f}], [clf - loss: {:.5f}, acc: {:.2f}%, test_dice: {:.2f}% ({:.2f}%)]".format(epoch, d_loss[0], d_loss[3], d_real_acc, d_fake_acc, d_train_acc, gen_loss, clf_train_loss, clf_train_acc, current_test_acc, test_mean_acc) if test_mean_acc > best_test_cls_acc: second_best_cls_acc = best_test_cls_acc best_test_cls_acc = test_mean_acc if save_model: self.combined.save(save_weights_path) else: self.combined.save_weights(save_weights_path) message += " (best)" elif test_mean_acc > second_best_cls_acc: second_best_cls_acc = test_mean_acc if save_model: self.combined.save(save_weights_path) else: self.combined.save_weights(save_weights_path[:-3]+"_bis.h5") message += " (second best)" else: pass print(message) # If at save interval => save generated image samples if epoch % sample_interval == 0: self.sample_images(epoch, save2dir=save_sample2dir) def sample_images(self, epoch, save2dir="../samples"): if not os.path.exists(save2dir): os.makedirs(save2dir) r, c = 5, 10 imgs_A, _ = self.data_loader.load_data(domain="A", batch_size=c) n_sample = imgs_A.shape[0] gen_imgs = imgs_A for i in range(r-1): noise_prior = np.random.normal(0,1, (n_sample, self.noise_size[0])) # TODO # noise_prior = np.random.normal(0,3, (n_sample, self.noise_size[0])) # TODO # 16/5/2018 # noise_prior = np.random.rand(n_sample, self.noise_size[0]) # TODO 6/5/2018 # Translate images to the other domain fake_B = self.generator.predict([imgs_A, noise_prior]) gen_imgs = np.concatenate([gen_imgs, fake_B]) # Rescale images to 0 - 1 gen_imgs = 0.5 * gen_imgs + 0.5 #titles = ['Original', 'Translated'] fig, axs = plt.subplots(r, c, figsize=(2*c, 2*r)) cnt = 0 for i in range(r): for j in range(c): axs[i,j].imshow(gen_imgs[cnt]) #axs[i, j].set_title(titles[i]) axs[i,j].axis('off') cnt += 1 fig.savefig(os.path.join(save2dir, "{}.png".format(epoch))) plt.close() def train_segmenter(self, iterations, batch_size=32, noise_range=5, save_weights_path=None): if save_weights_path is not None: dirpath = "/".join(save_weights_path.split("/")[:-1]) if not os.path.exists(dirpath): os.makedirs(dirpath) optimizer = Adam(0.000001, beta_1=0.0, beta_2=0.9) # Input noise noise = Input(shape=self.noise_size, name='noise_input_seg') img_A = Input(shape=self.img_shape, name='source_image_seg') # Translate images from domain A to domain B fake_B = self.generator([img_A, noise]) # Segment the translated image mask_pred = self.seg(fake_B) self.generator.trainable = False self.segmentation_model = Model(inputs=[img_A, noise], outputs=[mask_pred]) self.segmentation_model.compile(loss=dice_coef_loss, optimizer=optimizer, metrics=["acc", dice_coef]) self.segmentation_model.name = "U-net (freeze Generator)" self.segmentation_model.summary() best_test_dice = 0.0 second_best_test_dice = -1.0 collections = [] for e in range(iterations): noise = (2*np.random.random((batch_size, self.noise_size[0]))-1)*noise_range images_A, _, mask_A = self.data_loader.load_data(domain="A", batch_size=batch_size, return_mask=True) s_loss = self.segmentation_model.train_on_batch([images_A, noise], mask_A) if e%100 == 0: images_B, _, mask_B = self.data_loader.load_data(domain="B", batch_size=batch_size, return_mask=True) pred_mask_B = self.seg.predict(images_B) _, current_test_dice = dice_predict(mask_B, pred_mask_B) if len(collections)>=100: collections.pop(0) collections.append(current_test_dice) mean_dice = np.mean(collections) message = "{} dice loss: {:.3f}; acc: {:.5f}; mean dice (test): {:.3f}".format(e, 100*s_loss[0], s_loss[1], 100*mean_dice) if mean_dice>best_test_dice: best_test_dice = mean_dice message += " (best)" if save_weights_path is not None: self.segmentation_model.save_weights(save_weights_path) elif mean_dice> second_best_test_dice: second_best_test_dice = mean_dice message += " (second best)" if save_weights_path is not None: self.segmentation_model.save_weights(save_weights_path[:-3]+"_bis.h5") else: pass print(message) return def deploy_transform(self, save2file="../domain_adapted/generated.npy", noise_range=5.0, separate_cls=True, stop_after=None, seed=None): dirpath = "/".join(save2file.split("/")[:-1]) if not os.path.exists(dirpath): os.makedirs(dirpath) dirname = "/".join(save2file.split("/")[:-1]) # noise_vec = np.random.normal(0,1, self.noise_size[0]) # np.random.rand(n_sample, self.noise_size[0]) # TODO 6/5/2018 print("Performing Pixel-level domain adaptation on original images...") if separate_cls: for mnist_cls in tqdm(range(10)): imgs_A = self.data_loader.mnist_X[self.data_loader.mnist_y==mnist_cls] if stop_after is not None: num_samples = int(stop_after) else: num_samples = len(imgs_A) np.random.seed(seed) noise_vec = (2*np.random.random((num_samples, self.noise_size[0]))-1)*noise_range adaptaed_images = self.generator.predict([imgs_A[:num_samples], noise_vec], batch_size=32) filename_by_cls = save2file[:-4]+"_{}.npy".format(mnist_cls) np.save(filename_by_cls, adaptaed_images) else: imgs_A = self.data_loader.mnist_X if stop_after is not None: num_samples = int(stop_after) else: num_samples = len(imgs_A) np.random.seed(seed) noise_vec = (2*np.random.random((num_samples, self.noise_size[0]))-1)*noise_range adaptaed_images = self.generator.predict([imgs_A[:num_samples], noise_vec], batch_size=32) print("Saving transformed images to file {}".format(save2file)) np.save(save2file, adaptaed_images) # np.save(save2file, adaptaed_images) print("+ All done.") def deploy_debug(self, save2file="../domain_adapted/debug.npy", sample_size=100, noise_number=128, use_sobol=False, use_linear=False, use_sphere=False, use_uniform_linear=False, use_zeros=False, seed = 17): dirpath = "/".join(save2file.split("/")[:-1]) if not os.path.exists(dirpath): os.makedirs(dirpath) dirname = "/".join(save2file.split("/")[:-1]) np.random.seed(seed=seed) # np.random.rand(n_sample, self.noise_size[0]) # TODO 6/5/2018 print("Performing Pixel-level domain adaptation on original images...") # noise_vec = np.random.normal(0,1, (sample_size, self.noise_size[0])) # collections = [] # for i in range(sample_size): # adaptaed_images = self.generator.predict([self.data_loader.mnist_X[:15], np.tile(noise_vec[i], (15,1))], batch_size=15) # collections.append(adaptaed_images) collections = [] # imgs_A, _ = self.data_loader.load_data(domain="A", batch_size=sample_size) imgs_A = np.load("/home/lulin/na4/src/output/output13_32x32/test/liver_masks.npy")[100:100+sample_size] imgs_A = np.repeat(imgs_A, 3, axis=-1) for i in tqdm(range(sample_size)): if use_sobol: noise_vec = 5*(2*i4_sobol_generate(self.noise_size[0], noise_number, i*noise_number).T-1) elif use_linear: tangents = 3.0*(2*np.random.random((noise_number, 1))-1) noise_vec = np.ones((noise_number, self.noise_size[0]))*tangents elif use_sphere: noise_vec = 2*(np.random.random((noise_number, self.noise_size[0]))-1) norm_vec = np.linalg.norm(noise_vec, axis=-1) noise_vec = noise_vec/ norm_vec[:, np.newaxis] elif use_uniform_linear: tangents = 10.0*np.linspace(-1,1,noise_number)[:, np.newaxis] noise_vec = np.ones((noise_number, self.noise_size[0]))*tangents elif use_zeros: noise_vec = np.zeros((noise_number, self.noise_size[0])) else: noise_vec = np.random.normal(0,3, (noise_number, self.noise_size[0])) adaptaed_images = self.generator.predict([np.tile(imgs_A[i], (noise_number,1,1,1)), noise_vec], batch_size=32) collections.append(adaptaed_images) print("+ Done.") print("Saving transformed images to file {}".format(save2file)) np.save(save2file, np.stack(collections)) print("+ All done.") def deploy_classification(self, batch_size=32): print("Predicting ... ") pred_B = self.clf.predict(self.data_loader.mnistm_X, batch_size=batch_size) print("+ Done.") N_samples = len(pred_B) precision = (np.argmax(pred_B, axis=1) == self.data_loader.mnistm_y) Moy = np.mean(precision) Std = np.std(precision) lower_bound = Moy - 2.576*Std/np.sqrt(N_samples) upper_bound = Moy + 2.576*Std/np.sqrt(N_samples) print("="*50) print("Unsupervised MNIST-M classification accuracy : {}".format(Moy)) print("Confidence interval (99%) [{}, {}]".format(lower_bound, upper_bound)) print("Length of confidence interval 99%: {}".format(upper_bound-lower_bound)) print("="*50) print("+ All done.") def deploy_demo_only(self, save2file="../domain_adapted/WGAN_GP/Exp4/demo.npy", sample_size=25, noise_number=512, linspace_size=10.0): collections = [] imgs_A, labels_A = self.data_loader.load_data(domain="A", batch_size=sample_size) tangents = linspace_size*np.linspace(-1,1,noise_number)[:, np.newaxis] noise_vec = np.ones((noise_number, self.noise_size[0]))*tangents for i in tqdm(range(noise_number)): adaptaed_images = self.generator.predict([imgs_A, np.tile(noise_vec[i],(sample_size, 1))], batch_size=sample_size) collections.append(adaptaed_images) print("+ Done.") print("Saving transformed images to file {}".format(save2file)) np.save(save2file, np.stack(collections)) print("+ All done.") def deploy_cherry_pick(self, save2file="../domain_adapted/WGAN_GP/Exp4/demo_cherry_picked.png", sample_size=25, noise_number=25, linspace_size=5.0): collections = [] imgs_A, labels_A = self.data_loader.load_data(domain="A", batch_size=sample_size) assert noise_number == sample_size tangents = linspace_size*np.linspace(-1,1,noise_number)[:, np.newaxis] noise_vec = np.ones((noise_number, self.noise_size[0]))*tangents np.random.shuffle(noise_vec) # shuffle background color ! adaptaed_images = self.generator.predict([imgs_A, noise_vec], batch_size=sample_size) adaptaed_images = (adaptaed_images+1)/2 print("+ Done.") print("Saving transformed images to file {}".format(save2file)) r = 5 c = 5 fig, axs = plt.subplots(r, c, figsize=(5*c, 5*r)) for j in range(r): for i in range(c): axs[j,i].imshow(adaptaed_images[c*j+i]) axs[j,i].axis('off') plt.savefig(save2file) plt.close() print("+ All done.")