def construct(self, classifier_path=None, classifier_weight=None):
# Build the discriminators
self.d_N = build_discriminator(self.img_shape, self.df)
self.d_P = build_discriminator(self.img_shape, self.df)
# Build the generators
self.g_NP = build_generator(self.img_shape, self.gf, self.channels)
self.g_PN = build_generator(self.img_shape, self.gf, self.channels)
self.build_combined(classifier_path, classifier_weight)
def train( num_generations, generation_size, starting_selfies, max_molecules_len,
disc_epochs_per_generation, disc_enc_type, disc_layers, training_start_gen,
device, properties_calc_ls, num_processors, beta,
max_fitness_collector, impose_time_adapted_pen):
# Collecting all generated molecules and counting occurances
smiles_all = []
selfies_all = []
smiles_all_counter = {}
#Preparing generator and discriminator
initial_population = du.sanitize_multiple_smiles([decoder(selfie) for selfie in starting_selfies])
reference_mols = dict.fromkeys(du.read_dataset_encoding(disc_enc_type), '')
discriminator, d_opt, d_loss = D.build_discriminator(disc_enc_type, disc_layers, max_molecules_len, device)
total_time = time.time()
for generation_index in range(1, num_generations+1):
print(f"###Generation {generation_index} of {num_generations}")
start_time = time.time()
# Molecules of previous population
smiles_here, selfies_here = G.get_prev_gen_mols(initial_population, starting_selfies, generation_size,
generation_index, selfies_all, smiles_all)
# Fitness of previous population
fitness_here, order, fitness_ordered, smiles_ordered, selfies_ordered = G.get_fitness( disc_enc_type, smiles_here, selfies_here,
properties_calc_ls, discriminator, generation_index,
max_molecules_len, device, generation_size,
num_processors, writer, beta,
image_dir, data_dir, max_fitness_collector, impose_time_adapted_pen
)
# Molecules to replace and to keep
to_replace, to_keep = G.cutoff_population(order, generation_size)
# Molecules of current population
smiles_mutated, selfies_mutated = G.get_next_gen_mols(order, to_replace, to_keep,
selfies_ordered, smiles_ordered, max_molecules_len)
# Results of generator
smiles_all, selfies_all, smiles_all_counter = G.update_gen_res(smiles_all, smiles_mutated, selfies_all, selfies_mutated, smiles_all_counter)
# Data for discriminator training
dataset_x, dataset_y = G.get_dis_data(disc_enc_type, reference_mols, smiles_mutated, selfies_mutated, max_molecules_len, num_processors, generation_index)
if generation_index >= training_start_gen:
discriminator = D.train_discriminator(dataset_x, dataset_y, discriminator, d_opt, d_loss , disc_epochs_per_generation, generation_index-1, device, writer, data_dir)
D.save_model(discriminator, generation_index-1, saved_models_dir)
print('Generation time: ', round((time.time()-start_time), 2), ' seconds')
print('Experiment time: ', round((time.time()-total_time)/60, 2), ' mins')
print('Number of unique molecules: ', len(smiles_all_counter))
return smiles_all_counter
def __init__(self):
self.height = 128
self.width = 128
self.channels = channels
self.img_shape = (self.height, self.width, self.channels)
optimizer = Adam(0.0002, 0.5)
self.d_A = discriminator.build_discriminator()
self.d_B = discriminator.build_discriminator()
self.d_A.compile(loss='mse', optimizer=optimizer, metrics=['accuracy'])
self.d_B.compile(loss='mse', optimizer=optimizer, metrics=['accuracy'])
self.g_AB = generator.build_generator()
self.g_BA = generator.build_generator()
img_A = Input(shape=self.img_shape)
img_B = Input(shape=self.img_shape)
fake_B = self.g_AB(img_A)
fake_A = self.g_BA(img_B)
reconstr_A = self.g_BA(fake_B)
reconstr_B = self.g_AB(fake_A)
img_A_id = self.g_BA(img_A)
img_B_id = self.g_AB(img_B)
self.d_A.trainable = False
self.d_B.trainable = False
valid_A = self.d_A(fake_A)
valid_B = self.d_B(fake_B)
self.combined = Model(inputs=[img_A, img_B],
outputs=[
valid_A, valid_B, reconstr_A, reconstr_B,
img_A_id, img_B_id
])
self.combined.compile(loss=['mse', 'mse', 'mae', 'mae', 'mae', 'mae'],
loss_weights=[1, 1, 10, 10, 1, 1],
optimizer=optimizer)
def make_discriminator(self,
img_shape,
optimizer,
the_loss='categorical_crossentropy',
metrics=[]):
# Build model
discriminator = build_discriminator(img_shape,
dropout_rate=0.25,
l2_reg=0.)
# Compile model
discriminator.compile(loss=the_loss,
metrics=metrics,
optimizer=optimizer)
# Show model
if self.cf.show_model:
print('Discriminator')
discriminator.summary()
plot(discriminator,
to_file=os.path.join(self.cf.savepath,
'model_discriminator.png'))
return discriminator
constant_values=0) for sentence in pure_sentences
]), np.stack([
np.pad(np.stack([embeddings[id] for id in sentence]),
[[0, sentence_length - min(len(sentence), sentence_length)],
[0, 0]],
'constant',
constant_values=0) for sentence in tweaked_sentences
])
x_data = tf.placeholder(dtype=tf.float32,
shape=[batch_size, sentence_length, embedding_size])
x_data_tweaked = tf.placeholder(
dtype=tf.float32, shape=[batch_size, sentence_length, embedding_size])
logits_data, logits_tweaked, _, _, _, _ = build_discriminator(
x_data, x_data_tweaked, batch_size, sentence_length, embedding_size)
var_list = tf.trainable_variables(
"discriminator/conv") + tf.trainable_variables(
"discriminator/discriminator_fc_1") + tf.trainable_variables(
"discriminator/discriminator_fc_2")
for var in var_list:
variable_summaries(var)
global_step = tf.Variable(0, name='global_step', trainable=False)
#d_loss = tf.reduce_mean(-(tf.log(y_data) + tf.log(1 - y_data_tweaked)))
d_loss = tf.reduce_sum(
tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=tf.concat([logits_data, logits_tweaked], axis=0),
labels=tf.constant([0] * batch_size + [1] * batch_size)))
def main():
object_name = "chair"
data_dir = "data/3DShapeNets/volumetric_data/" \
"{}/30/train/*.mat".format(object_name)
gen_learning_rate = 0.0025
dis_learning_rate = 10e-5
beta = 0.5
batch_size = 1
z_size = 200
epochs = 10
MODE = "train"
"""
Create models
"""
gen_optimizer = Adam(lr=gen_learning_rate, beta_1=beta)
dis_optimizer = Adam(lr=dis_learning_rate, beta_1=beta)
discriminator = build_discriminator()
discriminator.compile(loss='binary_crossentropy', optimizer=dis_optimizer)
generator = build_generator()
generator.compile(loss='binary_crossentropy', optimizer=gen_optimizer)
discriminator.trainable = False
input_layer = Input(shape=(1, 1, 1, z_size))
generated_volumes = generator(input_layer)
validity = discriminator(generated_volumes)
adversarial_model = Model(inputs=[input_layer], outputs=[validity])
adversarial_model.compile(loss='binary_crossentropy', optimizer=gen_optimizer)
print("Loading data...")
volumes = get_three_d_images(directory=data_dir)
volumes = volumes[..., np.newaxis].astype(np.float)
print("Data loaded...")
tensorboard = TensorBoard(log_dir="logs/{}".format(time.time()))
tensorboard.set_model(generator)
tensorboard.set_model(discriminator)
labels_real = np.reshape(np.ones((batch_size,)), (-1, 1, 1, 1, 1))
labels_fake = np.reshape(np.zeros((batch_size,)), (-1, 1, 1, 1, 1))
if MODE == 'train':
for epoch in range(epochs):
print("Epoch:", epoch)
gen_losses = []
dis_losses = []
number_of_batches = int(volumes.shape[0] / batch_size)
print("Number of batches:", number_of_batches)
for index in range(number_of_batches):
print("Batch:", index + 1)
z_sample = np.random.normal(0, 0.33, size=[batch_size, 1, 1, 1, z_size]).astype(np.float32)
volumes_batch = volumes[index * batch_size:(index + 1) * batch_size, :, :, :]
# Next, generate volumes using the generate network
gen_volumes = generator.predict_on_batch(z_sample)
"""
Train the discriminator network
"""
discriminator.trainable = True
if index % 2 == 0:
loss_real = discriminator.train_on_batch(volumes_batch, labels_real)
loss_fake = discriminator.train_on_batch(gen_volumes, labels_fake)
d_loss = 0.5 * np.add(loss_real, loss_fake)
print("d_loss:{}".format(d_loss))
else:
d_loss = 0.0
discriminator.trainable = False
"""
Train the generator network
"""
z = np.random.normal(0, 0.33, size=[batch_size, 1, 1, 1, z_size]).astype(np.float32)
g_loss = adversarial_model.train_on_batch(z, labels_real)
print("g_loss:{}".format(g_loss))
gen_losses.append(g_loss)
dis_losses.append(d_loss)
# Every 10th mini-batch, generate volumes and save them
if index % 10 == 0:
z_sample2 = np.random.normal(0, 0.33, size=[batch_size, 1, 1, 1, z_size]).astype(np.float32)
generated_volumes = generator.predict(z_sample2, verbose=3)
for i, generated_volume in enumerate(generated_volumes[:5]):
voxels = np.squeeze(generated_volume)
voxels[voxels < 0.5] = 0.
voxels[voxels >= 0.5] = 1.
save_voxels(voxels, "results/img_{}_{}_{}".format(epoch, index, i))
# Write losses to Tensorboard
log_this(tensorboard, 'g_loss', np.mean(gen_losses), epoch)
log_this(tensorboard, 'd_loss', np.mean(dis_losses), epoch)
"""
Save models
"""
generator.save_weights(os.path.join("models", "generator_weights.h5"))
discriminator.save_weights(os.path.join("models", "discriminator_weights.h5"))
if MODE == 'predict':
# Create models
generator = build_generator()
discriminator = build_discriminator()
# Load model weights
generator.load_weights(os.path.join("models", "generator_weights.h5"), True)
discriminator.load_weights(os.path.join("models", "discriminator_weights.h5"), True)
# Generate 3D models
z_sample = np.random.normal(0, 1, size=[batch_size, 1, 1, 1, z_size]).astype(np.float32)
generated_volumes = generator.predict(z_sample, verbose=3)
for i, generated_volume in enumerate(generated_volumes[:2]):
voxels = np.squeeze(generated_volume)
voxels[voxels < 0.5] = 0.
voxels[voxels >= 0.5] = 1.
save_voxels(voxels, "results/gen_{}".format(i))
target=tf.placeholder(tf.float32,shape=[None,None,None,3])
reflection=tf.placeholder(tf.float32,shape=[None,None,None,3])
issyn=tf.placeholder(tf.bool,shape=[])
# build the model
network=build(input)
transmission_layer, reflection_layer=tf.split(network, num_or_size_splits=2, axis=3)
# Perceptual Loss
loss_percep_t=compute_percep_loss(transmission_layer, target)
loss_percep_r=tf.where(issyn, compute_percep_loss(reflection_layer, reflection, reuse=True), 0.)
loss_percep=tf.where(issyn, loss_percep_t+loss_percep_r, loss_percep_t)
# Adversarial Loss
with tf.variable_scope("discriminator"):
predict_real,pred_real_dict = build_discriminator(input,target)
with tf.variable_scope("discriminator", reuse=True):
predict_fake,pred_fake_dict = build_discriminator(input,transmission_layer)
d_loss=(tf.reduce_mean(-(tf.log(predict_real + EPS) + tf.log(1 - predict_fake + EPS)))) * 0.5
g_loss=tf.reduce_mean(-tf.log(predict_fake + EPS))
# L1 loss on reflection image
loss_l1_r=tf.where(issyn,compute_l1_loss(reflection_layer, reflection),0)
# Gradient loss
loss_gradx,loss_grady=compute_exclusion_loss(transmission_layer,reflection_layer,level=3)
loss_gradxy=tf.reduce_sum(sum(loss_gradx)/3.)+tf.reduce_sum(sum(loss_grady)/3.)
loss_grad=tf.where(issyn,loss_gradxy/2.0,0)
loss=loss_l1_r+loss_percep*0.2+loss_grad
# Prepare and input variable:
# Input noise to the generator:
noise_tensor = tf.placeholder(tf.float32, [int(BATCH_SIZE * 0.5), 10 * 10],
name="noise")
gen_input = tf.reshape(noise_tensor, (tf.shape(noise_tensor)[0], 10, 10, 1))
# Placeholder for the discriminator input:
data_tensor = tf.placeholder(tf.float32, [int(BATCH_SIZE * 0.5), 784],
name='x')
# label_tensor = tf.placeholder(tf.float32, [BATCH_SIZE, 1], name='labels')
x = tf.reshape(data_tensor, (tf.shape(data_tensor)[0], 28, 28, 1))
# Build a residual network on top of it, nothing complicated:
# "Real" data logits:
real_logits = build_discriminator(input_tensor=x, reuse=False)
gen_images = build_generator(input_tensor=gen_input)
gen_images = tf.reshape(gen_images, (tf.shape(gen_images)[0], 28, 28, 1))
# "Fake" data logits:
fake_logits = build_discriminator(input_tensor=gen_images, reuse=True)
# Add a global step accounting for saving and restoring training:
with tf.name_scope("global_step") as scope:
global_step = tf.Variable(0,
dtype=tf.int32,
trainable=False,
name='global_step')
# Build the loss functions:
def train(num_epochs=200, batch_size=50, block_size=64):
data, labels = prep_data(['spectrograms/ravdess', 'spectrograms/savee'])
time_steps = data.shape[1]
features_size = data.shape[2]
time_block_count = time_steps // block_size
feature_block_count = features_size // block_size
generator = build_generator(block_size, block_size, time_block_count,
feature_block_count)
generator.summary()
discriminator = build_discriminator(block_size, block_size,
time_block_count, feature_block_count)
discriminator.summary()
gan = build_gan(generator, discriminator)
gan.summary()
train_stats = open('Discriminator_stats.csv', 'a')
train_stats.write('Epoch,D_Loss1,D_Loss2\n')
train_stats.close()
gen_stats = open('Generator_stats.csv', 'a')
gen_stats.write('Epoch,G_Loss,G_Acc\n')
gen_stats.close()
batch_stats = open('Batch_stats.csv', 'a')
batch_stats.write('Epoch,G_Loss,G_Acc\n')
batch_stats.close()
for epoch in range(num_epochs):
## the discriminator's weights are only updated every 5 epochs
if epoch % 5 == 0:
d_loss_1 = []
d_loss_2 = []
batch_count = data.shape[0] // batch_size
## enumerate batches over the training set
for batch_index in range(batch_count):
avg_d_loss1 = 0.0
avg_d_loss2 = 0.0
start = batch_index * batch_size
if batch_index < batch_count - 1:
batch_x = data[start:(start + batch_size), :, :]
batch_labels = labels[start:(start + batch_size)]
half_batch = int(batch_size / 2)
else:
batch_x = data[start:, :, :]
batch_labels = labels[start:]
half_batch = (data.shape[0] - start) // 2
x_real = batch_x[:half_batch, :, :]
labels_real = batch_labels[:half_batch]
for y in range(time_block_count):
Y = np.repeat(y, x_real.shape[0]).reshape(-1, 1)
for x in range(feature_block_count):
start_y = y * block_size
start_x = x * block_size
X = np.repeat(x, x_real.shape[0]).reshape(-1, 1)
## train the discriminator on half a batch of gound truth data
d_loss1, _ = discriminator.train_on_batch([
x_real[:, start_y:(start_y + block_size),
start_x:(start_x + block_size)], Y, X
], to_categorical(labels_real, num_classes=9))
## generate 'fake' examples with half the batch
x_fake, labels_fake = generate_fake_samples(
batch_x[half_batch:,
start_y:(start_y + block_size),
start_x:(start_x + block_size)],
batch_labels[half_batch:], Y, X, generator)
## update discriminator model weights using 'fake' samples
d_loss2, _ = discriminator.train_on_batch(
[x_fake, Y, X],
to_categorical(labels_fake, num_classes=9))
## summarize loss on this batch
avg_d_loss1 += (d_loss1 * x_real.shape[0])
avg_d_loss2 += (d_loss2 * x_fake.shape[0])
print('Epoch %d, Batch %d/%d, d1=%.3f, d2=%.3f' %
(epoch+1, batch_index+1, batch_count, avg_d_loss1 / (half_batch * time_block_count * feature_block_count), \
avg_d_loss2 / (half_batch * time_block_count * feature_block_count))) #avg_g_loss/ sample_count, accuracy
d_loss_1.append(
avg_d_loss1 /
(x_real.shape[0] * time_block_count * feature_block_count))
d_loss_2.append(
avg_d_loss2 /
(x_fake.shape[0] * time_block_count * feature_block_count))
dloss1 = sum(d_loss_1) / len(d_loss_1)
dloss2 = sum(d_loss_2) / len(d_loss_2)
train_stats = open('Discriminator_stats.csv', 'a')
train_stats.write('%d,%.6f,%.6f\n' % (epoch + 1, dloss1, dloss2))
train_stats.close()
## save the discriminator model
discriminator.save_weights(
'discriminator/discriminator_epoch_%d.h5' % (epoch + 1))
triplets = generate_triplets(data, labels)
np.random.shuffle(triplets)
batch_count = int(triplets.shape[0] // batch_size)
g_loss = []
g_acc = []
for batch_index in range(batch_count):
avg_g_loss = 0.0
avg_acc = 0.0
start = batch_index * batch_size
if batch_index < batch_count - 1:
batch_x = data[triplets[start:(start + batch_size), 0], :, :]
batch_labels = labels[triplets[start:(start + batch_size), 1]]
batch_targets = triplets[start:(start + batch_size), 2]
else:
batch_x = data[triplets[start:, 0], :, :]
batch_labels = labels[triplets[start:, 1]]
batch_targets = triplets[start:, 2]
for y in range(time_block_count):
Y = np.repeat(y, batch_x.shape[0]).reshape(-1, 1)
for x in range(feature_block_count):
start_y = y * block_size
start_x = x * block_size
X = np.repeat(x, batch_x.shape[0]).reshape(-1, 1)
# update the generator via the discriminator's error
loss, acc = gan.train_on_batch([
batch_x[:, start_y:(start_y + block_size),
start_x:(start_x + block_size)], Y, X,
batch_targets
], to_categorical(batch_targets, num_classes=9))
avg_g_loss += (loss * batch_x.shape[0])
avg_acc += (acc * batch_x.shape[0])
print('Epoch %d, Batch %d/%d, g_loss=%.3f, acc=%.3f' %
(epoch+1, batch_index+1, batch_count, avg_g_loss // (batch_x.shape[0] * time_block_count * feature_block_count), \
avg_acc / (batch_x.shape[0] * time_block_count * feature_block_count)))
batch_stats = open('Batch_stats.csv', 'a')
batch_stats.write('%d,%d,%f,%f\n'%(epoch+1, batch_index+1, avg_g_loss // (batch_x.shape[0] * time_block_count * feature_block_count), \
avg_acc / (batch_x.shape[0] * time_block_count * feature_block_count)))
batch_stats.close()
g_loss.append(
avg_g_loss /
(batch_x.shape[0] * time_block_count * feature_block_count))
g_acc.append(
avg_acc /
(batch_x.shape[0] * time_block_count * feature_block_count))
gen_loss = sum(g_loss) / len(g_loss)
gen_acc = sum(g_acc) / len(g_acc)
gen_stats = open('Generator_stats.csv', 'a')
gen_stats.write('%d,%.6f,%.6f\n' % (epoch + 1, gen_loss, gen_acc))
gen_stats.close()
# save the generator model
generator.save_weights('generator/generator_epoch_%d.h5' % (epoch + 1))
train_stats.close()
gen_stats.close()
batch_stats.close()
x_generated_ids, x_generated, _ = build_generator(z_prior, embeddings,
num_classes,
hidden_layer_size,
embedding_size, z_prior_size,
max_sentence_length)
# TODO i think my current gradient problem stems from here: we must output the generated embeddings
# instead of the looked-up embeddings. but we also should output the word ids so we actually
# know what the sentence was.
# TODO this only works b/c everything in the emit_ta tensorarray is the same length.
# x_generated = tf.map_fn(lambda sentence: tf.map_fn(lambda word_id: tf.nn.embedding_lookup(out_embeddings, word_id), sentence, dtype=tf.float32), emit_ta, dtype=tf.float32)
x_data = tf.placeholder(tf.float32,
[batch_size, max_sentence_length, embedding_size])
logits_data, logits_generated, encoding_data, encoding_generated, features_data, features_generated = build_discriminator(
x_data, x_generated, batch_size, max_sentence_length, embedding_size)
y_data, y_generated = tf.nn.softmax(logits_data), tf.nn.softmax(
logits_generated)
# TODO classifications come out as pairs of numbers; could instead come out as
# single numbers representing the probability that the sentence is real.
y_data, y_generated = y_data[:, 0], y_generated[:, 0]
# Loss, as described in Zhang 2017
# Lambda values meant to weight gan ~= recon > mmd
lambda_r, lambda_m = 1.0e-1, 1.0e-1
mmd_val = mmd.mix_rbf_mmd2(features_data,
features_generated,
sigmas=args.mmd_sigmas)
gan_val = tf.reduce_mean(tf.log(y_data)) + tf.reduce_mean(
tf.log(1 - y_generated))
sentences_as_ids = tf.placeholder(tf.int64, [batch_size, sentence_length])
sentences_as_embeddings = tf.map_fn(lambda sentence: tf.map_fn(
lambda id: tf.nn.embedding_lookup(embeddings, id)
if id != after_sentence_id else tf.zeros(embedding_size),
sentence,
dtype=tf.float32),
sentences_as_ids,
dtype=tf.float32)
# TODO this is dumb...have to rewrite the discriminator so that we don't have to
# input two sets of data.
empty = tf.zeros([batch_size, sentence_length, embedding_size],
dtype=tf.float32)
_, _, encoding, _, _, _ = build_discriminator(sentences_as_embeddings, empty,
batch_size, sentence_length,
embedding_size)
x_generated_ids, x_generated, total_log_probability = build_generator(
encoding,
embeddings,
num_classes,
hidden_layer_size,
embedding_size,
z_prior_size,
sentence_length,
after_sentence_id=after_sentence_id,
real_sentences=sentences_as_ids)
global_step = tf.Variable(0, name='global_step', trainable=False)
def train():
start_time = time.time()
dataset_dir = "data/*.*"
batch_size = 128
z_shape = 100
epochs = 10000
dis_learning_rate = 0.005
gen_learning_rate = 0.005
dis_momentum = 0.5
gen_momentum = 0.5
dis_nesterov = True
gen_nesterov = True
dis_optimizer = SGD(lr=dis_learning_rate,
momentum=dis_momentum,
nesterov=dis_nesterov)
gen_optimizer = SGD(lr=gen_learning_rate,
momentum=gen_momentum,
nesterov=gen_nesterov)
# Load images
all_images = []
for index, filename in enumerate(glob.glob(dataset_dir)):
all_images.append(imread(filename, flatten=False, mode="RGB"))
X = np.array(all_images)
X = (X - 127.5) / 127.5
X = X.astype(np.float32)
dis_model = build_discriminator()
dis_model.compile(loss="binary_crossentropy", optimizer=dis_optimizer)
gen_model = build_generator()
gen_model.compile(loss="mse", optimizer=gen_optimizer)
adversarial_model = build_adversarial_model(gen_model, dis_model)
adversarial_model.compile(loss="binary_crossentropy",
optimizer=gen_optimizer)
tensorboard = TensorBoard(
log_dir="logs/{}".format(time.time()),
write_images=True,
write_grads=True,
write_graph=True,
)
tensorboard.set_model(gen_model)
tensorboard.set_model(dis_model)
for epoch in range(epochs):
print("--------------------------")
print("Epoch:{}".format(epoch))
dis_losses = []
gen_losses = []
num_batches = int(X.shape[0] / batch_size)
print("Number of batches:{}".format(num_batches))
for index in range(num_batches):
print("Batch:{}".format(index))
z_noise = np.random.normal(0, 1, size=(batch_size, z_shape))
# z_noise = np.random.uniform(-1, 1, size=(batch_size, 100))
generated_images = gen_model.predict_on_batch(z_noise)
# visualize_rgb(generated_images[0])
"""
Train the discriminator model
"""
dis_model.trainable = True
image_batch = X[index * batch_size:(index + 1) * batch_size]
y_real = np.ones((batch_size, )) * 0.9
y_fake = np.zeros((batch_size, )) * 0.1
dis_loss_real = dis_model.train_on_batch(image_batch, y_real)
dis_loss_fake = dis_model.train_on_batch(generated_images, y_fake)
d_loss = (dis_loss_real + dis_loss_fake) / 2
print("d_loss:", d_loss)
dis_model.trainable = False
"""
Train the generator model(adversarial model)
"""
z_noise = np.random.normal(0, 1, size=(batch_size, z_shape))
# z_noise = np.random.uniform(-1, 1, size=(batch_size, 100))
g_loss = adversarial_model.train_on_batch(z_noise, y_real)
print("g_loss:", g_loss)
dis_losses.append(d_loss)
gen_losses.append(g_loss)
"""
Sample some images and save them
"""
if epoch % 100 == 0:
z_noise = np.random.normal(0, 1, size=(batch_size, z_shape))
gen_images1 = gen_model.predict_on_batch(z_noise)
for img in gen_images1[:2]:
save_rgb_img(img, "results/one_{}.png".format(epoch))
print("Epoch:{}, dis_loss:{}".format(epoch, np.mean(dis_losses)))
print("Epoch:{}, gen_loss: {}".format(epoch, np.mean(gen_losses)))
"""
Save losses to Tensorboard after each epoch
"""
write_log(tensorboard, "discriminator_loss", np.mean(dis_losses),
epoch)
write_log(tensorboard, "generator_loss", np.mean(gen_losses), epoch)
"""
Save models
"""
gen_model.save("generator_model.h5")
dis_model.save("generator_model.h5")
print("Time:", (time.time() - start_time))
def train_gan(epochs = 1, batch_size=512, smaller=1.0, save_dir='results', load=False, examples=1,
imformat=df.read_image, txtformat=df.format_text, on_epoch=None):
"""
Traing a GAN
:param epochs: Number of epochs to do
:param batch_size: The batch size when running
:param smaller: The amount by which to make the dataset smaller
:param save_dir: The directory to save the results
:param load: Whether to load the model
:param examples: The number of examples to save
:param imformat: The function to format images
:param txtformat: The function to format text
:param on_epoch: A function to run on every epoch
:return:
"""
# For saved models
dp.safe_mkdir(save_dir)
dp.safe_mkdir(os.path.join(save_dir, 'examples'))
# Load data
train, val, _, vocab = dp.prepare(smaller=smaller)
print('TRAIN SIZE %d' % (len(train),))
print('VAL SIZE %d' % (len(val),))
# Check values
num_batch = int(np.ceil(len(train) / batch_size))
num_val = int(np.ceil(len(val) / batch_size))
assert len(train) >= batch_size
# Prepare examples
assert examples <= len(val)
assert examples >= 0
for i in range(examples):
dp.safe_mkdir(os.path.join(save_dir, 'examples', '%d' % i))
with open(os.path.join(save_dir, 'examples', '%d' % i, 'caption.txt'), 'w') as f:
f.write(val[i][1])
imageio.imwrite(os.path.join(save_dir, 'examples', '%d' % i, 'correct.jpg'), imageio.imread(val[i][0]))
# Print that we're starting
print('Training on %d images for %d epochs' % (len(train), epochs))
print('Will save %d examples every epoch' % (examples,))
# Create GAN
generator = keras.models.load_model(os.path.join(save_dir, 'generator.hdf5')) if load else gen.build_generator()
discriminator = keras.models.load_model(os.path.join(save_dir, 'discriminator.hdf5')) if load else dis.build_discriminator()
gan = build_gan(generator, discriminator)
# For every epoch, for every batch
for e in range(1, epochs + 1):
print("Epoch %d" % e)
idx = 0
for _ in tqdm(range(num_batch)):
# Get the input for this batch
l = idx * batch_size # low bound of batch
h = (idx + 1) * batch_size # high bound of batch
idx += 1
# Get text, np images
current_train = train[l:h]
batch_im = np.array(list(imformat(impth) for impth, _ in current_train))
batch_cap = np.array(list(txtformat(caption, vocab) for _, caption in current_train))
# Generate images to be compared
gen_im = generator.predict(batch_cap)
# Training data for discriminator
batch_im = batch_im.reshape((len(current_train), *consts.IMAGE_INPUT_SHAPE))
all_data = np.concatenate([batch_im, gen_im])
del batch_im
all_labels = np.zeros(2 * len(current_train))
all_labels[:len(current_train)] = 1 # Almost one
all_labels = keras.utils.to_categorical(all_labels, 2)
# Train discriminator
discriminator.trainable = True
dloss, dacc = discriminator.train_on_batch(all_data, all_labels)
del all_data
del all_labels
# Train generator now on the original data
discriminator.trainable = False
batch_labels = keras.utils.to_categorical(np.ones(len(current_train)), 2) # Trick into thinking generated images are the real ones
gloss, gacc = gan.train_on_batch(batch_cap, batch_labels)
del batch_cap
del batch_labels
# Save models after each epoch
generator.save(os.path.join(save_dir, 'generator.hdf5'))
discriminator.save(os.path.join(save_dir, 'discriminator.hdf5'))
# Save the losses from the end of the last epoch
with open(os.path.join(save_dir, 'discriminator_loss.csv'), 'w' if e == 1 else 'a') as f:
if e == 1:
f.write('Epoch,Loss,Accuracy\n')
f.write('%d,%f,%f\n' % (e, dloss, dacc))
with open(os.path.join(save_dir, 'gan_loss.csv'), 'w' if e == 1 else 'a') as f:
if e == 1:
f.write('Epoch,Loss,Accuracy\n')
f.write('%d,%f,%f\n' % (e, gloss, gacc))
# Save an example
for i in range(examples):
inp = np.array([df.format_text(val[i][1], vocab)])
ex = generator.predict(inp).reshape((128, 128, 3))
to_save = df.arr_to_image(ex)
imageio.imwrite(os.path.join(save_dir, 'examples', '%d' % i, '%d.jpg' % e), to_save)
# Other ops
if on_epoch is not None:
on_epoch(generator=generator, discriminator=discriminator, train=train, val=val, test=None, vocab=vocab)
del batch_labels
# Save models after each epoch
generator.save(os.path.join(save_dir, 'generator.hdf5'))
discriminator.save(os.path.join(save_dir, 'discriminator.hdf5'))
# Save the losses from the end of the last epoch
with open(os.path.join(save_dir, 'discriminator_loss.csv'), 'w' if e == 1 else 'a') as f:
if e == 1:
f.write('Epoch,Loss,Accuracy\n')
f.write('%d,%f,%f\n' % (e, dloss, dacc))
with open(os.path.join(save_dir, 'gan_loss.csv'), 'w' if e == 1 else 'a') as f:
if e == 1:
f.write('Epoch,Loss,Accuracy\n')
f.write('%d,%f,%f\n' % (e, gloss, gacc))
# Save an example
for i in range(examples):
inp = np.array([df.format_text(val[i][1], vocab)])
ex = generator.predict(inp).reshape((128, 128, 3))
to_save = df.arr_to_image(ex)
imageio.imwrite(os.path.join(save_dir, 'examples', '%d' % i, '%d.jpg' % e), to_save)
# Other ops
if on_epoch is not None:
on_epoch(generator=generator, discriminator=discriminator, train=train, val=val, test=None, vocab=vocab)
if __name__ == '__main__':
# Test script to see the GAN model summary
model = build_gan(gen.build_generator(), dis.build_discriminator())
model.summary()
)
sys.exit(1)
if not os.path.exists(sys.argv[1] + "/" + sys.argv[2] + "_resized"):
os.mkdir(sys.argv[1] + "/" + sys.argv[2] + "_resized")
if not os.path.exists(sys.argv[1] + "/" + sys.argv[2] + "_saved_models"):
os.mkdir(sys.argv[1] + "/" + sys.argv[2] + "_saved_models")
if not os.path.exists(sys.argv[1] + "/" + sys.argv[2] + "_samples"):
os.mkdir(sys.argv[1] + "/" + sys.argv[2] + "_samples")
resize_img(sys.argv[1] + "/" + sys.argv[2],
sys.argv[1] + "/" + sys.argv[2] + "_resized", IMG_WIDTH,
IMG_HEIGHT, IMG_CHANNEL)
images_path = glob(sys.argv[1] + "/" + sys.argv[2] + "_resized/*")
d_model = build_discriminator()
g_model = build_generator(latent_dim)
d_model.summary()
g_model.summary()
gan = GAN(d_model, g_model, latent_dim)
bce_loss_fn = tf.keras.losses.BinaryCrossentropy(from_logits=True,
label_smoothing=0.1)
d_optimizer = tf.keras.optimizers.Adam(learning_rate=0.0002, beta_1=0.5)
g_optimizer = tf.keras.optimizers.Adam(learning_rate=0.0002, beta_1=0.5)
gan.compile(d_optimizer, g_optimizer, bce_loss_fn)
images_dataset = tf_dataset(images_path, batch_size)
trial = 1
saved_pdf = 'fgan_test_%d.pdf' % trial
pp = PdfPages(saved_pdf)
def build_fgan(generator, discriminator):
z = layers.Input(shape=(z_dim, ))
protected_attrib = layers.Input(shape=(1, ))
img, outcome = generator([z, protected_attrib])
SJ_XY, SX_X, C_X, C_Y = discriminator([img, outcome])
model = Model([z, protected_attrib], [SJ_XY, SX_X, C_X, C_Y])
return model
discriminator = build_discriminator(img_shape, nums_classes)
opt = Adam()
discriminator.compile(loss=[
'binary_crossentropy', 'binary_crossentropy', 'binary_crossentropy',
'binary_crossentropy'
],
optimizer=opt)
discriminator.trainable = False
generator = build_generator(z_dim, nums_classes)
fgan = build_fgan(generator, discriminator)
fgan.compile(loss=[
'binary_crossentropy', 'binary_crossentropy', 'binary_crossentropy',
'binary_crossentropy'
],
optimizer=opt)
from keras.optimizers import Adam
from generator import build_generator
from discriminator import build_discriminator
from gan import build_gan
from tqdm import tqdm
img_rows = 28
img_cols = 28
channels = 1
img_shape = (img_rows, img_cols, channels)
z_dim = 100
discriminator = build_discriminator(img_shape)
discriminator.compile(loss='binary_crossentropy',
optimizer=Adam(),
metrics=['accuracy'])
generator = build_generator(z_dim)
discriminator.trainable = False
gan = build_gan(generator, discriminator)
gan.compile(loss='binary_crossentropy', optimizer=Adam())
losses = []
accuracies = []
iteration_checkpoints = []
latent_dim = 64
height = 64
width = 64
channels = 3
iterations = 10000
batch_size = 10
#Define directories
baseDir = r"D:\Arnaud\data_croutinet\ottawa\data"
trainDir = os.path.join(baseDir, "train/train.csv")
validationDir = os.path.join(baseDir, "validation/validation.csv")
save_dir = os.path.join(baseDir, "testgan")
roads_dir = os.path.join(baseDir, "trueskill20Duels/top2k")
generator = build_generator(latent_dim, channels)
discriminator = build_discriminator(height, width, channels)
gan = build_gan(latent_dim, generator, discriminator)
imagesNames = [f for f in os.listdir(os.path.join(save_dir, roads_dir))]
pictures = []
# Here we load in a big array all pictures as arrays (a & b are just to print the % of loading)
i = 0
a = 0
b = 0
print("loading pictures")
for name in imagesNames:
a = np.floor(i * 100 / len(imagesNames))
if a != b: