def __init__(self, configtag):
# Load hyperparmeters
self.configtag = configtag
self.init_params(configtag)
# Import slices
self.real_imgs = np.load('./data/' + self.dataname + '_train.npy')
self.val_imgs = np.load('./data/' + self.dataname + '_val.npy')
self.n_imgs = self.real_imgs.shape[0]
# Build networks
self.discrim = self.build_discriminator()
self.genrtor = self.build_generator()
# Compile discriminator so it can be trained separately
self.discrim.compile(loss='binary_crossentropy',
optimizer=keras.optimizers.Adam(lr=0.0002,
beta_1=0.5),
metrics=['accuracy'])
# Stack generator and discriminator networks together and compile
z = Input(shape=(1, self.noise_vect_len))
genimg = self.genrtor(z)
self.discrim.trainable = False
decision = self.discrim(genimg)
self.stacked = Model(z, decision)
self.stacked.compile(loss='binary_crossentropy',
optimizer=keras.optimizers.Adam(lr=0.0002,
beta_1=0.5))
# Setup tensorboard stuff
self.TB_genimg = tboard.TboardImg('genimg')
self.TB_pixhist = tboard.TboardImg('pixhist')
self.TB_scalars = tboard.TboardScalars()
def __init__(self, configtag, expDir):
# Load hyperparmeters
self.configtag = configtag
logging.info('Parameters:')
self.init_params(configtag)
self.expDir = expDir
self.inits = {
'dense': keras.initializers.RandomNormal(mean=0.0, stddev=0.02),
'conv': keras.initializers.TruncatedNormal(mean=0.0, stddev=0.02),
'tconv': keras.initializers.RandomNormal(mean=0.0, stddev=0.02)
}
# Import slices
self.real_imgs = np.load('./data/' + self.dataname + '_train.npy')
self.val_imgs = np.load('./data/' + self.valname + '_val.npy')
self.n_imgs = self.real_imgs.shape[0]
if self.datafmt == 'channels_last':
self.real_imgs = np.moveaxis(self.real_imgs, 1, -1)
self.val_imgs = np.moveaxis(self.val_imgs, 1, -1)
# Build networks
self.discrim, self.genrtor = self.load_networks()
# Compile discriminator so it can be trained separately
loss_fns = self._get_loss()
def mean_prob(y_true, y_pred):
# metric to measure mean probability of D predictions (0=fake, 1=real)
return K.mean(K.sigmoid(y_pred))
self.discrim.compile(loss=loss_fns['D'],
optimizer=keras.optimizers.Adam(lr=self.D_lr,
beta_1=0.5),
metrics=[mean_prob])
# Stack generator and discriminator networks together and compile
z = Input(shape=(1, self.noise_vect_len))
genimg = self.genrtor(z)
self.discrim.trainable = False
decision = self.discrim(genimg)
self.stacked = Model(z, decision)
self.stacked.compile(loss=loss_fns['G'],
optimizer=keras.optimizers.Adam(lr=self.G_lr,
beta_1=0.5))
# Setup tensorboard stuff
self.TB_genimg = tboard.TboardImg('genimg')
self.TB_pixhist = tboard.TboardImg('pixhist')
self.TB_scalars = tboard.TboardScalars()
if self.weight_hists:
self.TB_Dwts = tboard.TboardHists('D')
self.TB_Gwts = tboard.TboardHists('G')
if self.grad_hists:
self.TB_Ggrads = tboard.TboardHists('Ggrad')
if self.specnorm and self.sigma_plot:
self.TB_Dsigmas = tboard.TboardSigmas('Discriminator')
self.TB_Gsigmas = tboard.TboardSigmas('Generator')
def __init__(self, configtag, expDir):
# Load hyperparmeters
self.configtag = configtag
logging.info('Parameters:')
self.init_params(configtag)
self.expDir = expDir
self.inits = {
'dense': keras.initializers.RandomNormal(mean=0.0, stddev=0.02),
'conv': keras.initializers.TruncatedNormal(mean=0.0, stddev=0.02),
'tconv': keras.initializers.RandomNormal(mean=0.0, stddev=0.02)
}
# Import slices
self.real_imgs = np.load('./data/' + self.dataname + '_train.npy')
self.val_imgs = np.load('./data/' + self.dataname + '_val.npy')
self.n_imgs = self.real_imgs.shape[0]
if self.datafmt == 'channels_first':
self.real_imgs = np.moveaxis(self.real_imgs, -1, 1)
self.val_imgs = np.moveaxis(self.val_imgs, -1, 1)
self.set_transf_funcs()
self.set_transf_funcs_tensor()
self.real_imgs = self.transform(self.real_imgs)
# Build networks
self.discrim = self.build_discriminator()
self.genrtor = self.build_generator()
# Custom loss/metrics
def mean_prob(y_true, y_pred):
'''metric to measure mean probability of D predictions (0=fake, 1=real)'''
return K.mean(K.sigmoid(y_pred))
def crossentropy_from_logits(y_true, y_pred):
'''crossentropy loss from logits (circumvents default Keras crossentropy loss, which is unstable)'''
return K.mean(K.binary_crossentropy(y_true,
y_pred,
from_logits=True),
axis=-1)
# Compile discriminator so it can be trained separately
self.discrim.compile(loss=crossentropy_from_logits,
optimizer=keras.optimizers.Adam(lr=self.D_lr,
beta_1=0.5),
metrics=[mean_prob])
# Stack generator and discriminator networks together and compile
z = Input(shape=(1, self.noise_vect_len))
genimg = self.genrtor(z)
self.discrim.trainable = False
decision = self.discrim(genimg)
self.stacked = Model(z, decision)
self.stacked.compile(loss=crossentropy_from_logits,
optimizer=keras.optimizers.Adam(lr=self.G_lr,
beta_1=0.5))
# Setup tensorboard stuff
self.TB_genimg = tboard.TboardImg('genimg')
self.TB_pixhist = tboard.TboardImg('pixhist')
self.TB_pspect = tboard.TboardImg('pspect')
self.TB_scalars = tboard.TboardScalars()
def __init__(self, configtag, expDir, horovod_flag=False):
# Load hyperparmeters
self.configtag = configtag
logging.info('Parameters:')
self.init_params(configtag)
self.expDir = expDir
self.inits = {'dense':keras.initializers.RandomNormal(mean=0.0, stddev=0.02),
'conv':keras.initializers.TruncatedNormal(mean=0.0, stddev=0.02),
'tconv':keras.initializers.RandomNormal(mean=0.0, stddev=0.02)}
# Import slices
self.real_imgs = np.load(self.dataname+self.train_fname)
self.val_imgs = np.load(self.dataname+self.val_fname)[:3000]### Large validation set slows things
self.n_imgs = self.real_imgs.shape[0]
if self.datafmt == 'channels_first':
self.real_imgs = np.moveaxis(self.real_imgs, -1, 1)
self.val_imgs = np.moveaxis(self.val_imgs, -1, 1)
self.set_transf_funcs()
self.set_transf_funcs_tensor()
self.real_imgs = self.transform(self.real_imgs)
print('Number of samples for training',self.n_imgs)
# Custom loss/metrics
def mean_prob(y_true, y_pred):
'''metric to measure mean probability of D predictions (0=fake, 1=real)'''
return K.mean(K.sigmoid(y_pred))
def crossentropy_from_logits(y_true, y_pred):
'''crossentropy loss from logits (circumvents default Keras crossentropy loss, which is unstable)'''
return K.mean(K.binary_crossentropy(y_true, y_pred, from_logits=True), axis=-1)
# Build networks
self.discrim = self.build_discriminator()
self.genrtor = self.build_generator()
if horovod_flag:
# Horovod: adjust learning rate based on number of GPUs.
# opt = keras.optimizers.Adadelta(1.0 * hvd.size())
opt=keras.optimizers.Adam(lr=self.D_lr, beta_1=0.5)
# Horovod: add Horovod Distributed Optimizer.
opt = hvd.DistributedOptimizer(opt)
# Compile discriminator so it can be trained separately
self.discrim.compile(loss=crossentropy_from_logits, optimizer=opt, metrics=[mean_prob])
callbacks = [hvd.callbacks.BroadcastGlobalVariablesCallback(0),]
# Horovod: broadcast initial variable states from rank 0 to all other processes.
# This is necessary to ensure consistent initialization of all workers when
# training is started with random weights or restored from a checkpoint.
# Horovod: save checkpoints only on worker 0 to prevent other workers from corrupting them.
if hvd.rank() == 0:
callbacks.append(keras.callbacks.ModelCheckpoint('./results_data/checkpoint-genr-{epoch}.h5'))
else:
self.discrim.compile(loss=crossentropy_from_logits, optimizer=keras.optimizers.Adam(lr=self.D_lr, beta_1=0.5), metrics=[mean_prob])
# Stack generator and discriminator networks together and compile
z = Input(shape=(1,self.noise_vect_len))
genimg = self.genrtor(z)
self.discrim.trainable = False
decision = self.discrim(genimg)
self.stacked = Model(z, decision)
if horovod_flag:
# Horovod: adjust learning rate based on number of GPUs.
opt = keras.optimizers.Adam(lr=self.G_lr, beta_1=0.5)
# keras.optimizers.Adadelta(1.0 * hvd.size())
# Horovod: add Horovod Distributed Optimizer.
opt = hvd.DistributedOptimizer(opt)
self.stacked.compile(loss=crossentropy_from_logits, optimizer=opt)
callbacks = [hvd.callbacks.BroadcastGlobalVariablesCallback(0),]
# Horovod: broadcast initial variable states from rank 0 to all other processes.
# This is necessary to ensure consistent initialization of all workers when
# training is started with random weights or restored from a checkpoint.
# Horovod: save checkpoints only on worker 0 to prevent other workers from corrupting them.
if hvd.rank() == 0:
callbacks.append(keras.callbacks.ModelCheckpoint('./results_data/checkpoint-stacked-{epoch}.h5'))
else:
self.stacked.compile(loss=crossentropy_from_logits, optimizer=keras.optimizers.Adam(lr=self.G_lr, beta_1=0.5))
# Setup tensorboard stuff
self.TB_genimg = tboard.TboardImg('genimg')
self.TB_pixhist = tboard.TboardImg('pixhist')
self.TB_pspect = tboard.TboardImg('pspect')
self.TB_scalars = tboard.TboardScalars()
def __init__(self, configtag, expDir):
# Load hyperparmeters
self.configtag = configtag
logging.info('Parameters:')
self.init_params(configtag)
self.expDir = expDir
self.inits = {'dense':keras.initializers.RandomNormal(mean=0.0, stddev=0.02),
'conv':keras.initializers.TruncatedNormal(mean=0.0, stddev=0.02),
'tconv':keras.initializers.RandomNormal(mean=0.0, stddev=0.02)}
# Import slices
self.real_imgs = np.load('./data/'+self.dataname+'_train.npy')
self.val_imgs = np.load('./data/'+self.valname+'_val.npy')
self.n_imgs = self.real_imgs.shape[0]
if self.datafmt == 'channels_last':
self.real_imgs = np.moveaxis(self.real_imgs, 1, -1)
self.val_imgs = np.moveaxis(self.val_imgs, 1, -1)
# Build networks
self.discrim, self.genrtor = self.load_networks()
# Build critic
self.genrtor.trainable = False
real_img = Input(shape=self.imshape)
z_disc = Input(shape=(1,self.noise_vect_len))
fake_img = self.genrtor(z_disc)
interp_img = RandomWeightedAverage()([real_img, fake_img])
fake = self.discrim(fake_img)
real = self.discrim(real_img)
interp = self.discrim(interp_img)
partial_gp_loss = partial(self.gradient_penalty_loss, averaged_samples=interp_img)
partial_gp_loss.__name__ = 'gradient_penalty'
self.critic = Model(inputs=[real_img, z_disc], outputs=[real, fake, interp])
self.critic.compile(loss=[self.mean_loss, self.mean_loss, partial_gp_loss],
optimizer=keras.optimizers.RMSprop(lr=self.D_lr),
loss_weights=[1,1,10])
'''
self.genrtor.trainable = False
real_img = Input(shape=self.imshape)
real = self.discrim(real_img)
self.critic = Model(inputs=[real_img], outputs=[real])
self.critic.compile(loss=self.mean_loss, optimizer=keras.optimizers.RMSprop(lr=self.D_lr), metrics=['accuracy'])
'''
# Build generator
self.discrim.trainable = False
self.genrtor.trainable = True
z = Input(shape=(1,self.noise_vect_len))
genimg = self.genrtor(z)
decision = self.discrim(genimg)
self.stacked = Model(z, decision)
self.stacked.compile(loss=self.mean_loss, optimizer=keras.optimizers.RMSprop(lr=self.G_lr))
# Setup tensorboard stuff
self.TB_genimg = tboard.TboardImg('genimg')
self.TB_pixhist = tboard.TboardImg('pixhist')
self.TB_scalars = tboard.TboardScalars()
if self.weight_hists:
self.TB_Dwts = tboard.TboardHists('D')
self.TB_Gwts = tboard.TboardHists('G')
if self.grad_hists:
self.TB_Ggrads = tboard.TboardHists('Ggrad')
if self.specnorm and self.sigma_plot:
self.TB_Dsigmas = tboard.TboardSigmas('Discriminator')
self.TB_Gsigmas = tboard.TboardSigmas('Generator')
def __init__(self, configtag, expDir):
# Load hyperparmeters
self.configtag = configtag
logging.info('Parameters:')
self.init_params(configtag)
self.expDir = expDir
# Import slices
self.real_imgs = np.load('./data/' + self.dataname + '_train.npy')
self.val_imgs = np.load('./data/' + self.valname + '_val.npy')
self.n_imgs = self.real_imgs.shape[0]
if self.datafmt == 'channels_last':
self.real_imgs = np.moveaxis(self.real_imgs, 1, -1)
self.val_imgs = np.moveaxis(self.val_imgs, 1, -1)
# Build networks
self.discrim, self.genrtor = self.load_networks()
# Compile discriminator so it can be trained separately
loss_fns = self._get_loss()
def mean_prob(y_true, y_pred):
# metric to measure mean probability of D predictions (0=fake, 1=real)
return K.mean(K.sigmoid(y_pred))
self.discrim.compile(loss=loss_fns['D'],
optimizer=keras.optimizers.Adam(lr=self.D_lr,
beta_1=0.5),
metrics=[mean_prob])
# Set up generator to use Jacobian clamping
self.lam_max = 15.
self.lam_min = 5.
self.eps = 1.
z = Input(shape=(1, self.noise_vect_len))
eps = Input(shape=(1, self.noise_vect_len))
G_z = self.genrtor(z)
zp = keras.layers.add([z, eps])
G_zp = self.genrtor(zp)
logging.info('zp: ' + str(zp.shape))
dG = tf.norm(tf.squeeze(G_z - G_zp, axis=[-1]),
axis=(1, 2)) # NHWC format
logging.info('dG: ' + str(dG.shape))
dz = tf.norm(tf.squeeze(eps, axis=[1]), axis=-1)
logging.info('dz: ' + str(dz.shape))
Q = Lambda(lambda inputs: inputs[0] / inputs[1])([dG, dz])
logging.info('Q: ' + str(Q.shape))
l_max = K.constant(self.lam_max, name='lam_max')
l_min = K.constant(self.lam_min, name='lam_min')
logging.info('lmin, lmax : ' + str(l_min.shape) + ', ' +
str(l_max.shape))
l_max_diff = keras.layers.maximum([Q - l_max, tf.zeros_like(Q)])
l_min_diff = keras.layers.minimum([Q - l_min, tf.zeros_like(Q)])
logging.info('lmin_diff, lmax_diff: ' + str(l_min_diff.shape) + ', ' +
str(l_max_diff.shape))
Lmax = K.pow(l_max_diff, 2.)
Lmin = K.pow(l_min_diff, 2.)
L = keras.layers.Add()([Lmax, Lmin])
logging.info('Lmax, Lmin, L: ' + str(Lmax.shape) + ', ' +
str(Lmin.shape) + ', ' + str(L.shape))
self.discrim.trainable = False
decision = self.discrim(G_z)
self.stacked = Model(inputs=[z, eps], outputs=[decision])
self.stacked.summary()
self.stacked.compile(loss=loss_fns['G'](JC_loss=L),
optimizer=keras.optimizers.Adam(lr=self.G_lr,
beta_1=0.5))
# Setup tensorboard stuff
self.TB_genimg = tboard.TboardImg('genimg')
self.TB_pixhist = tboard.TboardImg('pixhist')
self.TB_scalars = tboard.TboardScalars()
if self.weight_hists:
self.TB_Dwts = tboard.TboardHists('D')
self.TB_Gwts = tboard.TboardHists('G')
if self.grad_hists:
self.TB_Ggrads = tboard.TboardHists('Ggrad')
if self.specnorm and self.sigma_plot:
self.TB_Dsigmas = tboard.TboardSigmas('Discriminator')
self.TB_Gsigmas = tboard.TboardSigmas('Generator')