def __init__(self, configtag): # Load hyperparmeters self.configtag = configtag self.init_params(configtag) # Import slices self.real_imgs = np.load('./data/' + self.dataname + '_train.npy') self.val_imgs = np.load('./data/' + self.dataname + '_val.npy') self.n_imgs = self.real_imgs.shape[0] # Build networks self.discrim = self.build_discriminator() self.genrtor = self.build_generator() # Compile discriminator so it can be trained separately self.discrim.compile(loss='binary_crossentropy', optimizer=keras.optimizers.Adam(lr=0.0002, beta_1=0.5), metrics=['accuracy']) # Stack generator and discriminator networks together and compile z = Input(shape=(1, self.noise_vect_len)) genimg = self.genrtor(z) self.discrim.trainable = False decision = self.discrim(genimg) self.stacked = Model(z, decision) self.stacked.compile(loss='binary_crossentropy', optimizer=keras.optimizers.Adam(lr=0.0002, beta_1=0.5)) # Setup tensorboard stuff self.TB_genimg = tboard.TboardImg('genimg') self.TB_pixhist = tboard.TboardImg('pixhist') self.TB_scalars = tboard.TboardScalars()
def __init__(self, configtag, expDir): # Load hyperparmeters self.configtag = configtag logging.info('Parameters:') self.init_params(configtag) self.expDir = expDir self.inits = { 'dense': keras.initializers.RandomNormal(mean=0.0, stddev=0.02), 'conv': keras.initializers.TruncatedNormal(mean=0.0, stddev=0.02), 'tconv': keras.initializers.RandomNormal(mean=0.0, stddev=0.02) } # Import slices self.real_imgs = np.load('./data/' + self.dataname + '_train.npy') self.val_imgs = np.load('./data/' + self.valname + '_val.npy') self.n_imgs = self.real_imgs.shape[0] if self.datafmt == 'channels_last': self.real_imgs = np.moveaxis(self.real_imgs, 1, -1) self.val_imgs = np.moveaxis(self.val_imgs, 1, -1) # Build networks self.discrim, self.genrtor = self.load_networks() # Compile discriminator so it can be trained separately loss_fns = self._get_loss() def mean_prob(y_true, y_pred): # metric to measure mean probability of D predictions (0=fake, 1=real) return K.mean(K.sigmoid(y_pred)) self.discrim.compile(loss=loss_fns['D'], optimizer=keras.optimizers.Adam(lr=self.D_lr, beta_1=0.5), metrics=[mean_prob]) # Stack generator and discriminator networks together and compile z = Input(shape=(1, self.noise_vect_len)) genimg = self.genrtor(z) self.discrim.trainable = False decision = self.discrim(genimg) self.stacked = Model(z, decision) self.stacked.compile(loss=loss_fns['G'], optimizer=keras.optimizers.Adam(lr=self.G_lr, beta_1=0.5)) # Setup tensorboard stuff self.TB_genimg = tboard.TboardImg('genimg') self.TB_pixhist = tboard.TboardImg('pixhist') self.TB_scalars = tboard.TboardScalars() if self.weight_hists: self.TB_Dwts = tboard.TboardHists('D') self.TB_Gwts = tboard.TboardHists('G') if self.grad_hists: self.TB_Ggrads = tboard.TboardHists('Ggrad') if self.specnorm and self.sigma_plot: self.TB_Dsigmas = tboard.TboardSigmas('Discriminator') self.TB_Gsigmas = tboard.TboardSigmas('Generator')
def __init__(self, configtag, expDir): # Load hyperparmeters self.configtag = configtag logging.info('Parameters:') self.init_params(configtag) self.expDir = expDir self.inits = { 'dense': keras.initializers.RandomNormal(mean=0.0, stddev=0.02), 'conv': keras.initializers.TruncatedNormal(mean=0.0, stddev=0.02), 'tconv': keras.initializers.RandomNormal(mean=0.0, stddev=0.02) } # Import slices self.real_imgs = np.load('./data/' + self.dataname + '_train.npy') self.val_imgs = np.load('./data/' + self.dataname + '_val.npy') self.n_imgs = self.real_imgs.shape[0] if self.datafmt == 'channels_first': self.real_imgs = np.moveaxis(self.real_imgs, -1, 1) self.val_imgs = np.moveaxis(self.val_imgs, -1, 1) self.set_transf_funcs() self.set_transf_funcs_tensor() self.real_imgs = self.transform(self.real_imgs) # Build networks self.discrim = self.build_discriminator() self.genrtor = self.build_generator() # Custom loss/metrics def mean_prob(y_true, y_pred): '''metric to measure mean probability of D predictions (0=fake, 1=real)''' return K.mean(K.sigmoid(y_pred)) def crossentropy_from_logits(y_true, y_pred): '''crossentropy loss from logits (circumvents default Keras crossentropy loss, which is unstable)''' return K.mean(K.binary_crossentropy(y_true, y_pred, from_logits=True), axis=-1) # Compile discriminator so it can be trained separately self.discrim.compile(loss=crossentropy_from_logits, optimizer=keras.optimizers.Adam(lr=self.D_lr, beta_1=0.5), metrics=[mean_prob]) # Stack generator and discriminator networks together and compile z = Input(shape=(1, self.noise_vect_len)) genimg = self.genrtor(z) self.discrim.trainable = False decision = self.discrim(genimg) self.stacked = Model(z, decision) self.stacked.compile(loss=crossentropy_from_logits, optimizer=keras.optimizers.Adam(lr=self.G_lr, beta_1=0.5)) # Setup tensorboard stuff self.TB_genimg = tboard.TboardImg('genimg') self.TB_pixhist = tboard.TboardImg('pixhist') self.TB_pspect = tboard.TboardImg('pspect') self.TB_scalars = tboard.TboardScalars()
def __init__(self, configtag, expDir, horovod_flag=False): # Load hyperparmeters self.configtag = configtag logging.info('Parameters:') self.init_params(configtag) self.expDir = expDir self.inits = {'dense':keras.initializers.RandomNormal(mean=0.0, stddev=0.02), 'conv':keras.initializers.TruncatedNormal(mean=0.0, stddev=0.02), 'tconv':keras.initializers.RandomNormal(mean=0.0, stddev=0.02)} # Import slices self.real_imgs = np.load(self.dataname+self.train_fname) self.val_imgs = np.load(self.dataname+self.val_fname)[:3000]### Large validation set slows things self.n_imgs = self.real_imgs.shape[0] if self.datafmt == 'channels_first': self.real_imgs = np.moveaxis(self.real_imgs, -1, 1) self.val_imgs = np.moveaxis(self.val_imgs, -1, 1) self.set_transf_funcs() self.set_transf_funcs_tensor() self.real_imgs = self.transform(self.real_imgs) print('Number of samples for training',self.n_imgs) # Custom loss/metrics def mean_prob(y_true, y_pred): '''metric to measure mean probability of D predictions (0=fake, 1=real)''' return K.mean(K.sigmoid(y_pred)) def crossentropy_from_logits(y_true, y_pred): '''crossentropy loss from logits (circumvents default Keras crossentropy loss, which is unstable)''' return K.mean(K.binary_crossentropy(y_true, y_pred, from_logits=True), axis=-1) # Build networks self.discrim = self.build_discriminator() self.genrtor = self.build_generator() if horovod_flag: # Horovod: adjust learning rate based on number of GPUs. # opt = keras.optimizers.Adadelta(1.0 * hvd.size()) opt=keras.optimizers.Adam(lr=self.D_lr, beta_1=0.5) # Horovod: add Horovod Distributed Optimizer. opt = hvd.DistributedOptimizer(opt) # Compile discriminator so it can be trained separately self.discrim.compile(loss=crossentropy_from_logits, optimizer=opt, metrics=[mean_prob]) callbacks = [hvd.callbacks.BroadcastGlobalVariablesCallback(0),] # Horovod: broadcast initial variable states from rank 0 to all other processes. # This is necessary to ensure consistent initialization of all workers when # training is started with random weights or restored from a checkpoint. # Horovod: save checkpoints only on worker 0 to prevent other workers from corrupting them. if hvd.rank() == 0: callbacks.append(keras.callbacks.ModelCheckpoint('./results_data/checkpoint-genr-{epoch}.h5')) else: self.discrim.compile(loss=crossentropy_from_logits, optimizer=keras.optimizers.Adam(lr=self.D_lr, beta_1=0.5), metrics=[mean_prob]) # Stack generator and discriminator networks together and compile z = Input(shape=(1,self.noise_vect_len)) genimg = self.genrtor(z) self.discrim.trainable = False decision = self.discrim(genimg) self.stacked = Model(z, decision) if horovod_flag: # Horovod: adjust learning rate based on number of GPUs. opt = keras.optimizers.Adam(lr=self.G_lr, beta_1=0.5) # keras.optimizers.Adadelta(1.0 * hvd.size()) # Horovod: add Horovod Distributed Optimizer. opt = hvd.DistributedOptimizer(opt) self.stacked.compile(loss=crossentropy_from_logits, optimizer=opt) callbacks = [hvd.callbacks.BroadcastGlobalVariablesCallback(0),] # Horovod: broadcast initial variable states from rank 0 to all other processes. # This is necessary to ensure consistent initialization of all workers when # training is started with random weights or restored from a checkpoint. # Horovod: save checkpoints only on worker 0 to prevent other workers from corrupting them. if hvd.rank() == 0: callbacks.append(keras.callbacks.ModelCheckpoint('./results_data/checkpoint-stacked-{epoch}.h5')) else: self.stacked.compile(loss=crossentropy_from_logits, optimizer=keras.optimizers.Adam(lr=self.G_lr, beta_1=0.5)) # Setup tensorboard stuff self.TB_genimg = tboard.TboardImg('genimg') self.TB_pixhist = tboard.TboardImg('pixhist') self.TB_pspect = tboard.TboardImg('pspect') self.TB_scalars = tboard.TboardScalars()
def __init__(self, configtag, expDir): # Load hyperparmeters self.configtag = configtag logging.info('Parameters:') self.init_params(configtag) self.expDir = expDir self.inits = {'dense':keras.initializers.RandomNormal(mean=0.0, stddev=0.02), 'conv':keras.initializers.TruncatedNormal(mean=0.0, stddev=0.02), 'tconv':keras.initializers.RandomNormal(mean=0.0, stddev=0.02)} # Import slices self.real_imgs = np.load('./data/'+self.dataname+'_train.npy') self.val_imgs = np.load('./data/'+self.valname+'_val.npy') self.n_imgs = self.real_imgs.shape[0] if self.datafmt == 'channels_last': self.real_imgs = np.moveaxis(self.real_imgs, 1, -1) self.val_imgs = np.moveaxis(self.val_imgs, 1, -1) # Build networks self.discrim, self.genrtor = self.load_networks() # Build critic self.genrtor.trainable = False real_img = Input(shape=self.imshape) z_disc = Input(shape=(1,self.noise_vect_len)) fake_img = self.genrtor(z_disc) interp_img = RandomWeightedAverage()([real_img, fake_img]) fake = self.discrim(fake_img) real = self.discrim(real_img) interp = self.discrim(interp_img) partial_gp_loss = partial(self.gradient_penalty_loss, averaged_samples=interp_img) partial_gp_loss.__name__ = 'gradient_penalty' self.critic = Model(inputs=[real_img, z_disc], outputs=[real, fake, interp]) self.critic.compile(loss=[self.mean_loss, self.mean_loss, partial_gp_loss], optimizer=keras.optimizers.RMSprop(lr=self.D_lr), loss_weights=[1,1,10]) ''' self.genrtor.trainable = False real_img = Input(shape=self.imshape) real = self.discrim(real_img) self.critic = Model(inputs=[real_img], outputs=[real]) self.critic.compile(loss=self.mean_loss, optimizer=keras.optimizers.RMSprop(lr=self.D_lr), metrics=['accuracy']) ''' # Build generator self.discrim.trainable = False self.genrtor.trainable = True z = Input(shape=(1,self.noise_vect_len)) genimg = self.genrtor(z) decision = self.discrim(genimg) self.stacked = Model(z, decision) self.stacked.compile(loss=self.mean_loss, optimizer=keras.optimizers.RMSprop(lr=self.G_lr)) # Setup tensorboard stuff self.TB_genimg = tboard.TboardImg('genimg') self.TB_pixhist = tboard.TboardImg('pixhist') self.TB_scalars = tboard.TboardScalars() if self.weight_hists: self.TB_Dwts = tboard.TboardHists('D') self.TB_Gwts = tboard.TboardHists('G') if self.grad_hists: self.TB_Ggrads = tboard.TboardHists('Ggrad') if self.specnorm and self.sigma_plot: self.TB_Dsigmas = tboard.TboardSigmas('Discriminator') self.TB_Gsigmas = tboard.TboardSigmas('Generator')
def __init__(self, configtag, expDir): # Load hyperparmeters self.configtag = configtag logging.info('Parameters:') self.init_params(configtag) self.expDir = expDir # Import slices self.real_imgs = np.load('./data/' + self.dataname + '_train.npy') self.val_imgs = np.load('./data/' + self.valname + '_val.npy') self.n_imgs = self.real_imgs.shape[0] if self.datafmt == 'channels_last': self.real_imgs = np.moveaxis(self.real_imgs, 1, -1) self.val_imgs = np.moveaxis(self.val_imgs, 1, -1) # Build networks self.discrim, self.genrtor = self.load_networks() # Compile discriminator so it can be trained separately loss_fns = self._get_loss() def mean_prob(y_true, y_pred): # metric to measure mean probability of D predictions (0=fake, 1=real) return K.mean(K.sigmoid(y_pred)) self.discrim.compile(loss=loss_fns['D'], optimizer=keras.optimizers.Adam(lr=self.D_lr, beta_1=0.5), metrics=[mean_prob]) # Set up generator to use Jacobian clamping self.lam_max = 15. self.lam_min = 5. self.eps = 1. z = Input(shape=(1, self.noise_vect_len)) eps = Input(shape=(1, self.noise_vect_len)) G_z = self.genrtor(z) zp = keras.layers.add([z, eps]) G_zp = self.genrtor(zp) logging.info('zp: ' + str(zp.shape)) dG = tf.norm(tf.squeeze(G_z - G_zp, axis=[-1]), axis=(1, 2)) # NHWC format logging.info('dG: ' + str(dG.shape)) dz = tf.norm(tf.squeeze(eps, axis=[1]), axis=-1) logging.info('dz: ' + str(dz.shape)) Q = Lambda(lambda inputs: inputs[0] / inputs[1])([dG, dz]) logging.info('Q: ' + str(Q.shape)) l_max = K.constant(self.lam_max, name='lam_max') l_min = K.constant(self.lam_min, name='lam_min') logging.info('lmin, lmax : ' + str(l_min.shape) + ', ' + str(l_max.shape)) l_max_diff = keras.layers.maximum([Q - l_max, tf.zeros_like(Q)]) l_min_diff = keras.layers.minimum([Q - l_min, tf.zeros_like(Q)]) logging.info('lmin_diff, lmax_diff: ' + str(l_min_diff.shape) + ', ' + str(l_max_diff.shape)) Lmax = K.pow(l_max_diff, 2.) Lmin = K.pow(l_min_diff, 2.) L = keras.layers.Add()([Lmax, Lmin]) logging.info('Lmax, Lmin, L: ' + str(Lmax.shape) + ', ' + str(Lmin.shape) + ', ' + str(L.shape)) self.discrim.trainable = False decision = self.discrim(G_z) self.stacked = Model(inputs=[z, eps], outputs=[decision]) self.stacked.summary() self.stacked.compile(loss=loss_fns['G'](JC_loss=L), optimizer=keras.optimizers.Adam(lr=self.G_lr, beta_1=0.5)) # Setup tensorboard stuff self.TB_genimg = tboard.TboardImg('genimg') self.TB_pixhist = tboard.TboardImg('pixhist') self.TB_scalars = tboard.TboardScalars() if self.weight_hists: self.TB_Dwts = tboard.TboardHists('D') self.TB_Gwts = tboard.TboardHists('G') if self.grad_hists: self.TB_Ggrads = tboard.TboardHists('Ggrad') if self.specnorm and self.sigma_plot: self.TB_Dsigmas = tboard.TboardSigmas('Discriminator') self.TB_Gsigmas = tboard.TboardSigmas('Generator')