# training settings
lr = tf.get_variable(
'lr', shape=[], trainable=False,
initializer=tf.constant_initializer(1e-4))
d_optimizer = tf.train.AdamOptimizer(lr, beta1=0.5, beta2=0.9)
g_optimizer = d_optimizer
# gradient processor
if config.GRADIENT_CLIP:
gradient_processor = lambda grad_var: (
tf.clip_by_average_norm(grad_var[0], config.GRADIENT_CLIP_VALUE),
grad_var[1])
else:
gradient_processor = None
# log dir
log_prefix = 'model_logs/' + '_'.join([
ng.date_uid(), socket.gethostname(), config.DATASET,
'MASKED' if config.GAN_WITH_MASK else 'NORMAL',
config.GAN,config.LOG_DIR])
# train discriminator with secondary trainer, should initialize before
# primary trainer.
discriminator_training_callback = ng.callbacks.SecondaryTrainer(
pstep=1,
optimizer=d_optimizer,
var_list=d_vars,
max_iters=5,
graph_def=multigpu_graph_def,
graph_def_kwargs={
'model': model, 'data': data, 'config': config, 'loss_type': 'd', 'mask_index': mask_index, 'exclusionmask_index': exclusionmask_index},
)
# train generator with primary trainer
trainer = ng.train.Trainer(
shape=[],
trainable=False,
initializer=tf.constant_initializer(1e-4))
d_optimizer = tf.train.AdamOptimizer(lr, beta1=0.5, beta2=0.9)
g_optimizer = d_optimizer
# gradient processor
if config.GRADIENT_CLIP:
gradient_processor = lambda grad_var: (tf.clip_by_average_norm(
grad_var[0], config.GRADIENT_CLIP_VALUE), grad_var[1])
else:
gradient_processor = None
# log dir
log_prefix = 'model_logs/' + '_'.join([
ng.date_uid(),
socket.gethostname(), config.DATASET, 'MASKED'
if config.GAN_WITH_MASK else 'NORMAL', config.GAN, config.LOG_DIR
])
# train discriminator with secondary trainer, should initialize before primary trainer.
discriminator_training_callback = ng.callbacks.SecondaryTrainer(
pstep=1,
optimizer=d_optimizer,
var_list=d_vars,
max_iters=5,
graph_def=multigpu_graph_def,
graph_def_kwargs={
'model': model,
'data': data,
'config': config,
def train_gan(config):
"""main function for training progressive gan
Args:
config (ng.Config): all hyperparameters
Returns:
"""
logger.info('Start to train progressive gan.')
# get dataset
with open(config.DATA_FLIST[config.DATASET]) as f:
fnames = f.read().splitlines()
img_shapes = [config.CURRENT_RESOLUTION, config.CURRENT_RESOLUTION, 3]
data = ng.data.DataFromFNames(fnames, img_shapes)
# init model
model = progressive_model.ProgressiveGAN(1024, config)
g_vars, d_vars, losses = model.build_graph_with_losses(data,
config,
reuse=False)
lr = tf.get_variable('lr',
shape=[],
trainable=False,
initializer=tf.constant_initializer(1e-4))
g_optimizer = tf.train.AdamOptimizer(
lr,
beta1=0.5,
beta2=0.9,
# beta1=config.TRAIN['adam_beta1'],
# beta2=config.TRAIN['adam_beta2'],
# epsilon=config.TRAIN['adam_epsilon']
)
d_optimizer = g_optimizer
discriminator_training_callback = ng.callbacks.SecondaryTrainer(
pstep=1,
optimizer=d_optimizer,
var_list=d_vars,
max_iters=5,
# max_iters=config.TRAIN['D_training_repeats'],
graph_def=multigpu_graph_def,
graph_def_kwargs={
'model': model,
'data': data,
'config': config,
'loss_type': 'd'
},
)
log_prefix = 'model_logs/' + '_'.join([
str(ng.date_uid()),
socket.gethostname(), config.DATASET, config.LOG_DIR
])
trainer = ng.train.Trainer(
config=config,
optimizer=g_optimizer,
var_list=g_vars,
gpu_num=config.NUM_GPUS,
async_train=True,
graph_def=multigpu_graph_def,
graph_def_kwargs={
'model': model,
'data': data,
'config': config,
'loss_type': 'g'
},
spe=config.TRAIN_SPE,
max_iters=config.MAX_ITERS,
log_dir=log_prefix,
)
trainer.add_callbacks([
ng.callbacks.WeightsViewer(),
# ng.callbacks.ModelRestorer(trainer.context['saver'], dump_prefix=config.RESTORE_PREFIX, optimistic=True),
discriminator_training_callback,
ng.callbacks.ModelSaver(config.TRAIN_SPE, trainer.context['saver'],
log_prefix + '/snap'),
ng.callbacks.SummaryWriter((config.VAL_PSTEPS // 1),
trainer.context['summary_writer'],
tf.summary.merge_all()),
])
trainer.train()
