def main(args):
exp = expman.from_dir(args.run)
params = exp.params
batch_size = args.batch_size if args.batch_size else params.batch_size
is_object = params.category in objects
# get data
test_dataset, test_labels = get_test_data(params.category,
image_size=params.image_size,
patch_size=params.patch_size,
batch_size=batch_size)
# build models
generator = make_generator(params.latent_size,
channels=params.channels,
upsample_first=is_object,
upsample_type=params.ge_up,
bn=params.ge_bn,
act=params.ge_act)
encoder = make_encoder(params.patch_size,
params.latent_size,
channels=params.channels,
bn=params.ge_bn,
act=params.ge_act)
discriminator = make_discriminator(params.patch_size,
params.latent_size,
channels=params.channels,
bn=params.d_bn,
act=params.d_act)
# checkpointer
checkpoint = tf.train.Checkpoint(generator=generator,
encoder=encoder,
discriminator=discriminator)
ckpt_suffix = 'best' if args.best else 'last'
ckpt_path = exp.path_to(f'ckpt/ckpt_{params.category}_{ckpt_suffix}')
checkpoint.read(ckpt_path).expect_partial()
discriminator_features = get_discriminator_features_model(discriminator)
auc, balanced_accuracy = evaluate(generator,
encoder,
discriminator_features,
test_dataset,
test_labels,
patch_size=params.patch_size,
lambda_=args.lambda_)
# print(f'{params.category}: AUC={auc}, BalAcc={balanced_accuracy}')
index = pd.Index(args.lambda_, name='lambda')
table = pd.DataFrame({
'auc': auc,
'balanced_accuracy': balanced_accuracy
},
index=index)
print(table)
def __init__(self,
seqs,
adj,
nodes_features,
epochs,
key,
use_gcn,
batch_size,
use_gru=True):
self.epochs = epochs
self.seqs = seqs.astype('float32')
self.seqs_noised = seqs.copy().astype('float32')
self.max_s = seqs[key].max()
self.seqs_noised[key] = np.random.normal(
self.max_s / 2.0, self.max_s / 10.0,
size=(seqs.shape[0])).astype('float32')
self.key = key
self.gen_optimizer = SGD(lr, adam_beta_1)
self.desc_optimizer = SGD(lr, adam_beta_1)
self.adj = normalized_laplacian(adj.astype('float32'))
self.adj_expanded = tf.expand_dims(normalized_laplacian(
adj.astype('float32')),
axis=0)
self.nodes_features = nodes_features.astype('float32')
self.nodes_f_expanded = tf.expand_dims(
nodes_features.astype('float32'), axis=0)
self.generator = model.make_generator('generator', batch_size,
self.adj, self.nodes_features,
use_gcn, use_gru)
self.discriminator = model.make_discriminator('discriminator',
batch_size, self.adj,
self.nodes_features,
use_gcn, use_gru)
self.d_loss_fn, self.g_loss_fn = losses.get_wasserstein_losses_fn()
self.wsst_hist = []
self.cos_simi = []
self.var_hist = []
self.rmse_hist = []
self.mae_hist = []
self.r2_hist = []
self.g_loss_hist = []
self.d_loss_hist = []
def main(args):
# do not track lambda param, it can be changed after train
exp = Experiment(args, ignore=('lambda_', ))
print(exp)
if exp.found:
print('Already exists: SKIPPING')
exit(0)
np.random.seed(args.seed)
tf.random.set_seed(args.seed)
# get data
train_dataset = get_train_data(args.category,
image_size=args.image_size,
patch_size=args.patch_size,
batch_size=args.batch_size,
n_batches=args.n_batches,
rotation_range=args.rotation_range,
seed=args.seed)
test_dataset, test_labels = get_test_data(args.category,
image_size=args.image_size,
patch_size=args.patch_size,
batch_size=args.batch_size)
is_object = args.category in objects
# build models
generator = make_generator(args.latent_size,
channels=args.channels,
upsample_first=is_object,
upsample_type=args.ge_up,
bn=args.ge_bn,
act=args.ge_act)
encoder = make_encoder(args.patch_size,
args.latent_size,
channels=args.channels,
bn=args.ge_bn,
act=args.ge_act)
discriminator = make_discriminator(args.patch_size,
args.latent_size,
channels=args.channels,
bn=args.d_bn,
act=args.d_act)
# feature extractor model for evaluation
discriminator_features = get_discriminator_features_model(discriminator)
# build optimizers
generator_encoder_optimizer = O.Adam(args.lr,
beta_1=args.ge_beta1,
beta_2=args.ge_beta2)
discriminator_optimizer = O.Adam(args.lr,
beta_1=args.d_beta1,
beta_2=args.d_beta2)
# reference to the models to use in eval
generator_eval = generator
encoder_eval = encoder
# for smoothing generator and encoder evolution
if args.ge_decay > 0:
ema = tf.train.ExponentialMovingAverage(decay=args.ge_decay)
generator_ema = tf.keras.models.clone_model(generator)
encoder_ema = tf.keras.models.clone_model(encoder)
generator_eval = generator_ema
encoder_eval = encoder_ema
# checkpointer
checkpoint = tf.train.Checkpoint(
generator=generator,
encoder=encoder,
discriminator=discriminator,
generator_encoder_optimizer=generator_encoder_optimizer,
discriminator_optimizer=discriminator_optimizer)
best_ckpt_path = exp.ckpt(f'ckpt_{args.category}_best')
last_ckpt_path = exp.ckpt(f'ckpt_{args.category}_last')
# log stuff
log, log_file = exp.require_csv(f'log_{args.category}.csv.gz')
metrics, metrics_file = exp.require_csv(f'metrics_{args.category}.csv')
best_metric = 0.
best_recon = float('inf')
best_recon_file = exp.path_to(f'best_recon_{args.category}.png')
last_recon_file = exp.path_to(f'last_recon_{args.category}.png')
# animate generation during training
n_preview = 6
train_batch = next(iter(train_dataset))[:n_preview]
test_batch = next(iter(test_dataset))[0][:n_preview]
latent_batch = tf.random.normal([n_preview, args.latent_size])
if not is_object: # take random patches from test images
patch_location = np.random.randint(0,
args.image_size - args.patch_size,
(n_preview, 2))
test_batch = [
x[i:i + args.patch_size, j:j + args.patch_size, :]
for x, (i, j) in zip(test_batch, patch_location)
]
test_batch = K.stack(test_batch)
video_out = exp.path_to(f'{args.category}.mp4')
video_options = dict(fps=30, codec='libx265',
quality=4) # see imageio FFMPEG options
video_saver = VideoSaver(train_batch, test_batch, latent_batch, video_out,
**video_options)
video_saver.generate_and_save(generator, encoder)
# train loop
progress = tqdm(train_dataset, desc=args.category, dynamic_ncols=True)
try:
for step, image_batch in enumerate(progress, start=1):
if step == 1 or args.d_iter == 0: # only for JIT compilation (tf.function) to work
d_train = True
ge_train = True
elif args.d_iter:
n_iter = step % (abs(args.d_iter) + 1) # can be in [0, d_iter]
d_train = (n_iter != 0) if (args.d_iter > 0) else (
n_iter == 0) # True in [1, d_iter]
ge_train = not d_train # True when step == d_iter + 1
else: # d_iter == None: dynamic adjustment
d_train = (scores['fake_score'] > 0) or (scores['real_score'] <
0)
ge_train = (scores['real_score'] > 0) or (scores['fake_score']
< 0)
losses, scores = train_step(image_batch,
generator,
encoder,
discriminator,
generator_encoder_optimizer,
discriminator_optimizer,
d_train,
ge_train,
alpha=args.alpha,
gp_weight=args.gp_weight)
if (args.ge_decay > 0) and (step % 10 == 0):
ge_vars = generator.variables + encoder.variables
ema.apply(ge_vars) # update exponential moving average
# tensor to numpy
losses = {
n: l.numpy() if l is not None else l
for n, l in losses.items()
}
scores = {
n: s.numpy() if s is not None else s
for n, s in scores.items()
}
# log step metrics
entry = {
'step': step,
'timestamp': pd.to_datetime('now'),
**losses,
**scores
}
log = log.append(entry, ignore_index=True)
if step % 100 == 0:
if args.ge_decay > 0:
ge_ema_vars = generator_ema.variables + encoder_ema.variables
for v_ema, v in zip(ge_ema_vars, ge_vars):
v_ema.assign(ema.average(v))
preview = video_saver.generate_and_save(
generator_eval, encoder_eval)
if step % 1000 == 0:
log.to_csv(log_file, index=False)
checkpoint.write(file_prefix=last_ckpt_path)
auc, balanced_accuracy = evaluate(generator_eval,
encoder_eval,
discriminator_features,
test_dataset,
test_labels,
patch_size=args.patch_size,
lambda_=args.lambda_)
entry = {
'step': step,
'auc': auc,
'balanced_accuracy': balanced_accuracy
}
metrics = metrics.append(entry, ignore_index=True)
metrics.to_csv(metrics_file, index=False)
if auc > best_metric:
best_metric = auc
checkpoint.write(file_prefix=best_ckpt_path)
# save last image to inspect it during training
imageio.imwrite(last_recon_file, preview)
recon = losses['images_reconstruction_loss']
if recon < best_recon:
best_recon = recon
imageio.imwrite(best_recon_file, preview)
progress.set_postfix({
'AUC': f'{auc:.1%}',
'BalAcc': f'{balanced_accuracy:.1%}',
'BestAUC': f'{best_metric:.1%}',
})
except KeyboardInterrupt:
checkpoint.write(file_prefix=last_ckpt_path)
finally:
log.to_csv(log_file, index=False)
video_saver.close()
# score the test set
checkpoint.read(best_ckpt_path)
auc, balanced_accuracy = evaluate(generator,
encoder,
discriminator_features,
test_dataset,
test_labels,
patch_size=args.patch_size,
lambda_=args.lambda_)
print(f'{args.category}: AUC={auc}, BalAcc={balanced_accuracy}')
def main():
# Todo change face dataset. horse2zebra are for model test.
autotune = tf.data.experimental.AUTOTUNE
dataset, _ = tfds.load("cycle_gan/horse2zebra",
with_info=True,
as_supervised=True)
train_horses, train_zebras = dataset["trainA"], dataset["trainB"]
test_horses, test_zebras = dataset["testA"], dataset["testB"]
buffer_size = 256
batch_size = 1
# Apply the preprocessing operations to the training data
train_horses = (train_horses.map(
preprocess_train_image, num_parallel_calls=autotune).cache().shuffle(
buffer_size).batch(batch_size))
train_zebras = (train_zebras.map(
preprocess_train_image, num_parallel_calls=autotune).cache().shuffle(
buffer_size).batch(batch_size))
# Apply the preprocessing operations to the test data
test_horses = (test_horses.map(
preprocess_test_image, num_parallel_calls=autotune).cache().shuffle(
buffer_size).batch(batch_size))
test_zebras = (test_zebras.map(
preprocess_test_image, num_parallel_calls=autotune).cache().shuffle(
buffer_size).batch(batch_size))
disc_A = make_discriminator(input_img_size)
disc_B = make_discriminator(input_img_size)
gen_A2B = make_generator(input_img_size)
gen_B2A = make_generator(input_img_size)
cycle_gan_model = CycleGan(
generator_A2B=gen_A2B,
generator_B2A=gen_B2A,
discriminator_A=disc_A,
discriminator_B=disc_B,
)
# Compile the model
cycle_gan_model.compile(
gen_A2B_optimizer=tf.keras.optimizers.Adam(learning_rate=2e-4,
beta_1=0.5),
gen_B2A_optimizer=tf.keras.optimizers.Adam(learning_rate=2e-4,
beta_1=0.5),
disc_A_optimizer=tf.keras.optimizers.Adam(learning_rate=2e-4,
beta_1=0.5),
disc_B_optimizer=tf.keras.optimizers.Adam(learning_rate=2e-4,
beta_1=0.5),
gen_loss_fn=generator_loss_fn,
disc_loss_fn=discriminator_loss_fn,
)
# Callbacks
plotter = GANMonitor()
checkpoint_filepath = "./model_checkpoints/cyclegan_checkpoints.{epoch:03d}"
model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
filepath=checkpoint_filepath)
# Here we will train the model for just one epoch as each epoch takes around
# 7 minutes on a single P100 backed machine.
cycle_gan_model.fit(
tf.data.Dataset.zip((train_horses, train_zebras)),
epochs=1,
callbacks=[plotter, model_checkpoint_callback],
)
def main():
args = parsing()
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
BATCH_SIZE = 64
TRAINING_RATIO = 5
GRADIENT_PENALTY_WEIGHT = 10 # As per the paper
X_train = get_mnist()
generator = make_generator()
discriminator = make_discriminator()
# The generator_model is used when we want to train the generator layers.
# As such, we ensure that the discriminator layers are not trainable.
# Note that once we compile this model, updating .trainable will have no effect within
# it. As such, it won't cause problems if we later set discriminator.trainable = True
# for the discriminator_model, as long as we compile the generator_model first.
for layer in discriminator.layers:
layer.trainable = False
discriminator.trainable = False
generator_input = Input(shape=(100, ))
generator_layers = generator(generator_input)
discriminator_layers_for_generator = discriminator(generator_layers)
generator_model = Model(inputs=[generator_input],
outputs=[discriminator_layers_for_generator])
# We use the Adam paramaters from Gulrajani et al.
generator_model.compile(optimizer=Adam(0.0001, beta_1=0.5, beta_2=0.9),
loss=wloss)
# Now that the generator_model is compiled, we can make the discriminator
# layers trainable.
for layer in discriminator.layers:
layer.trainable = True
for layer in generator.layers:
layer.trainable = False
discriminator.trainable = True
generator.trainable = False
# The discriminator_model is more complex. It takes both real image samples and random
# noise seeds as input. The noise seed is run through the generator model to get
# generated images. Both real and generated images are then run through the
# discriminator. Although we could concatenate the real and generated images into a
# single tensor, we don't (see model compilation for why).
real_samples = Input(shape=X_train.shape[1:])
generator_input_for_discriminator = Input(shape=(100, ))
generated_samples_for_discriminator = generator(
generator_input_for_discriminator)
discriminator_output_from_generator = discriminator(
generated_samples_for_discriminator)
discriminator_output_from_real_samples = discriminator(real_samples)
# We also need to generate weighted-averages of real and generated samples,
# to use for the gradient norm penalty.
averaged_samples = RandomWeightedAverage(BATCH_SIZE)(
[real_samples, generated_samples_for_discriminator])
# We then run these samples through the discriminator as well. Note that we never
# really use the discriminator output for these samples - we're only running them to
# get the gradient norm for the gradient penalty loss.
averaged_samples_out = discriminator(averaged_samples)
# The gradient penalty loss function requires the input averaged samples to get
# gradients. However, Keras loss functions can only have two arguments, y_true and
# y_pred. We get around this by making a partial() of the function with the averaged
# samples here.
partial_gp_loss = partial(gradient_penalty_wloss,
averaged_samples=averaged_samples,
gradient_penalty_weight=GRADIENT_PENALTY_WEIGHT)
# Functions need names or Keras will throw an error
partial_gp_loss.__name__ = 'gradient_penalty'
# Keras requires that inputs and outputs have the same number of samples. This is why
# we didn't concatenate the real samples and generated samples before passing them to
# the discriminator: If we had, it would create an output with 2 * BATCH_SIZE samples,
# while the output of the "averaged" samples for gradient penalty
# would have only BATCH_SIZE samples.
# If we don't concatenate the real and generated samples, however, we get three
# outputs: One of the generated samples, one of the real samples, and one of the
# averaged samples, all of size BATCH_SIZE. This works neatly!
discriminator_model = Model(
inputs=[real_samples, generator_input_for_discriminator],
outputs=[
discriminator_output_from_real_samples,
discriminator_output_from_generator, averaged_samples_out
])
# We use the Adam paramaters from Gulrajani et al. We use the Wasserstein loss for both
# the real and generated samples, and the gradient penalty loss for the averaged samples
discriminator_model.compile(optimizer=Adam(0.0001, beta_1=0.5, beta_2=0.9),
loss=[wloss, wloss, partial_gp_loss])
# We make three label vectors for training. positive_y is the label vector for real
# samples, with value 1. negative_y is the label vector for generated samples, with
# value -1. The dummy_y vector is passed to the gradient_penalty loss function and
# is not used.
positive_y = np.ones((BATCH_SIZE, 1), dtype=np.float32)
negative_y = -positive_y
dummy_y = np.zeros((BATCH_SIZE, 1), dtype=np.float32)
d_loss = []
discriminator_loss = []
generator_loss = []
print('Training...')
for epoch in range(100):
print("---epoch: %d---" % epoch)
np.random.shuffle(X_train)
print("Epoch: ", epoch)
print("Number of batches: ", int(X_train.shape[0] // BATCH_SIZE))
minibatches_size = BATCH_SIZE * TRAINING_RATIO
for i in range(int(X_train.shape[0] // (BATCH_SIZE * TRAINING_RATIO))):
print("batch: ", i)
discriminator_minibatches = X_train[i * minibatches_size:(i + 1) *
minibatches_size]
for j in range(TRAINING_RATIO):
image_batch = discriminator_minibatches[j *
BATCH_SIZE:(j + 1) *
BATCH_SIZE]
noise = np.random.rand(BATCH_SIZE, 100).astype(np.float32)
discriminator_loss.append(
discriminator_model.train_on_batch(
[image_batch, noise],
[positive_y, negative_y, dummy_y]))
d_loss.append(np.mean(np.array(discriminator_loss[-5:]), axis=0))
generator_loss.append(
generator_model.train_on_batch(np.random.rand(BATCH_SIZE, 100),
positive_y))
generate_images(generator, args.output_dir, epoch)
loss_dict = {"g_loss": generator_loss, "d_loss": d_loss}
with open("loss.pkl", "wb") as fo:
pickle.dump(loss_dict, fo)
BUFFER_SIZE = 60000
BATCH_SIZE = 256
EPOCHS = 50
noise_dim = 100
num_examples_to_generate = 16
seed = tf.random.normal([num_examples_to_generate, noise_dim])
# Shuffle dataset
X_train = load_real_data()
training_data = tf.data.Dataset.from_tensor_slices(X_train).shuffle(
BUFFER_SIZE).batch(BATCH_SIZE)
print(training_data)
# Define models, setups checkpoints..
generator = make_generator(latent_dim=100)
discriminator = make_discriminator()
generator_optimizer = Adam(1e-4)
discriminator_optimizer = Adam(1e-4)
checkpoint_dir = "./training_checkpoints"
checkpoint_prefix = os.path.sep.join([checkpoint_dir, "ckpt"])
checkpoint = tf.train.Checkpoint(
generator_optimizer=generator_optimizer,
discriminator_optimizer=discriminator_optimizer,
generator=generator,
discriminator=discriminator)
train(training_data, EPOCHS)