def train(**kwargs):
"""
Train model
Load the whole train data in memory for faster operations
args: **kwargs (dict) keyword arguments that specify the model hyperparameters
"""
# Roll out the parameters
batch_size = kwargs["batch_size"]
n_batch_per_epoch = kwargs["n_batch_per_epoch"]
nb_epoch = kwargs["nb_epoch"]
model_name = kwargs["model_name"]
generator = kwargs["generator"]
image_data_format = kwargs["image_data_format"]
img_dim = kwargs["img_dim"]
patch_size = kwargs["patch_size"]
bn_mode = kwargs["bn_mode"]
label_smoothing = kwargs["use_label_smoothing"]
label_flipping = kwargs["label_flipping"]
dset = kwargs["dset"]
use_mbd = kwargs["use_mbd"]
lastLayerActivation=kwargs["lastLayerActivation"]
PercentageOfTrianable=kwargs["PercentageOfTrianable"]
SpecificPathStr=kwargs["SpecificPathStr"]
epoch_size = n_batch_per_epoch * batch_size
# Setup environment (logging directory etc)
#general_utils.setup_logging(model_name)
# Load and rescale data
#X_full_train, X_sketch_train, X_full_val, X_sketch_val = data_utils.load_data(dset, image_data_format)
img_dim = (256,256,3) # Manual entry
# Get the number of non overlapping patch and the size of input image to the discriminator
nb_patch, img_dim_disc = data_utils.get_nb_patch(img_dim, patch_size, image_data_format)
try:
# Create optimizers
opt_dcgan = Adam(lr=1E-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
# opt_discriminator = SGD(lr=1E-3, momentum=0.9, nesterov=True)
opt_discriminator = Adam(lr=1E-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
# Load generator model
"""
generator_model = models.load("generator_unet_%s" % generator,
img_dim,
nb_patch,
bn_mode,
use_mbd,
batch_size)
"""
generator_model=CreatErrorMapModel(input_shape=img_dim,lastLayerActivation=lastLayerActivation, PercentageOfTrianable=PercentageOfTrianable)
# Load discriminator model
discriminator_model = models.load("DCGAN_discriminator",
img_dim_disc,
nb_patch,
bn_mode,
use_mbd,
batch_size)
generator_model.compile(loss='mae', optimizer=opt_discriminator)
#-------------------------------------------------------------------------------
logpath=os.path.join('../../log','DepthMapWith'+lastLayerActivation+str(PercentageOfTrianable)+'UnTr'+SpecificPathStr)
modelPath=os.path.join('../../models','DepthMapwith'+lastLayerActivation+str(PercentageOfTrianable)+'Untr'+SpecificPathStr)
os.makedirs(logpath, exist_ok=True)
os.makedirs(modelPath, exist_ok=True)os.makedirs(modelPath, exist_ok=True)
#-----------------------PreTraining Depth Map-------------------------------------
nb_train_samples = 2000
nb_validation_samples =
epochs = 20
history=whole_model.fit_generator(data_utils.facades_generator(img_dim,batch_size=batch_size), samples_per_epoch=nb_train_samples,epochs=epochs,validation_data=data_utils.facades_generator(img_dim,batch_size=batch_size),nb_val_samples=nb_validation_ samples, callbacks=[
keras.callbacks.ModelCheckpoint(os.path.join(modelPath,'DepthMap_weightsBestLoss.h5'), monitor='val_loss', verbose=1, save_best_only=True),
keras.callbacks.ModelCheckpoint(os.path.join(modelPath,'DepthMap_weightsBestAcc.h5'), monitor='acc', verbose=1, save_best_only=True),
keras.callbacks.ReduceLROnPlateau(monitor='loss', factor=0.1, patience=2, verbose=1, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0),
keras.callbacks.TensorBoard(log_dir=logpath, histogram_freq=0, batch_size=batchSize, write_graph=True, write_grads=False, write_images=True, embeddin gs_freq=0, embeddings_layer_names=None, embeddings_metadata=None)],)
#------------------------------------------------------------------------------------
discriminator_model.trainable = False
DCGAN_model = models.DCGAN(generator_model,
discriminator_model,
img_dim,
patch_size,
image_data_format)
loss = [l1_loss, 'binary_crossentropy']
loss_weights = [1E1, 1]
DCGAN_model.compile(loss=loss, loss_weights=loss_weights, optimizer=opt_dcgan)
discriminator_model.trainable = True
discriminator_model.compile(loss='binary_crossentropy', optimizer=opt_discriminator)
gen_loss = 100
disc_loss = 100
# Start training
print("Start training")
for e in range(nb_epoch):
# Initialize progbar and batch counter
progbar = generic_utils.Progbar(epoch_size)
batch_counter = 1
start = time.time()
for X_full_batch, X_sketch_batch in data_utils.facades_generator(img_dim,batch_size=batch_size):
# Create a batch to feed the discriminator model
X_disc, y_disc = data_utils.get_disc_batch(X_full_batch,
X_sketch_batch,
generator_model,
batch_counter,
patch_size,
image_data_format,
label_smoothing=label_smoothing,
label_flipping=label_flipping)
# Update the discriminator
disc_loss = discriminator_model.train_on_batch(X_disc, y_disc) # X_disc, y_disc
# Create a batch to feed the generator model
X_gen_target, X_gen = next(data_utils.facades_generator(img_dim,batch_size=batch_size))
y_gen = np.zeros((X_gen.shape[0], 2), dtype=np.uint8)
y_gen[:, 1] = 1
# Freeze the discriminator
discriminator_model.trainable = False
gen_loss = DCGAN_model.train_on_batch(X_gen, [X_gen_target, y_gen])
# Unfreeze the discriminator
discriminator_model.trainable = True
batch_counter += 1
progbar.add(batch_size, values=[("D logloss", disc_loss),
("G tot", gen_loss[0]),
("G L1", gen_loss[1]),
("G logloss", gen_loss[2])])
# Save images for visualization
if batch_counter % (n_batch_per_epoch / 2) == 0:
# Get new images from validation
figure_name = "training_"+str(e)
data_utils.plot_generated_batch(X_full_batch, X_sketch_batch, generator_model,
batch_size, image_data_format, figure_name)
if batch_counter >= n_batch_per_epoch:
break
print("")
print('Epoch %s/%s, Time: %s' % (e + 1, nb_epoch, time.time() - start))
if e % 5 == 0:
gen_weights_path = os.path.join('../../models/%s/gen_weights_epoch%s.h5' % (model_name, e))
generator_model.save_weights(gen_weights_path, overwrite=True)
disc_weights_path = os.path.join('../../models/%s/disc_weights_epoch%s.h5' % (model_name, e))
discriminator_model.save_weights(disc_weights_path, overwrite=True)
DCGAN_weights_path = os.path.join('../../models/%s/DCGAN_weights_epoch%s.h5' % (model_name, e))
DCGAN_model.save_weights(DCGAN_weights_path, overwrite=True)
def train(**kwargs):
"""
Train model
Load the whole train data in memory for faster operations
args: **kwargs (dict) keyword arguments that specify the model hyperparameters
"""
# Roll out the parameters
batch_size = kwargs["batch_size"]
n_batch_per_epoch = kwargs["n_batch_per_epoch"]
nb_epoch = kwargs["nb_epoch"]
model_name = kwargs["model_name"]
generator = kwargs["generator"]
image_data_format = kwargs["image_data_format"]
img_dim = kwargs["img_dim"]
patch_size = kwargs["patch_size"]
bn_mode = kwargs["bn_mode"]
label_smoothing = kwargs["use_label_smoothing"]
label_flipping = kwargs["label_flipping"]
dset = kwargs["dset"]
use_mbd = kwargs["use_mbd"]
epoch_size = n_batch_per_epoch * batch_size
# Setup environment (logging directory etc)
#general_utils.setup_logging(model_name)
# Load and rescale data
#X_full_train, X_sketch_train, X_full_val, X_sketch_val = data_utils.load_data(dset, image_data_format)
img_dim = (256, 256, 3) # Manual entry
# Get the number of non overlapping patch and the size of input image to the discriminator
nb_patch, img_dim_disc = data_utils.get_nb_patch(img_dim, patch_size,
image_data_format)
try:
# Create optimizers
opt_dcgan = Adam(lr=1E-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
# opt_discriminator = SGD(lr=1E-3, momentum=0.9, nesterov=True)
opt_discriminator = Adam(lr=1E-3,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08)
# Load generator model
generator_model = models.load("generator_unet_%s" % generator, img_dim,
nb_patch, bn_mode, use_mbd, batch_size)
# Load discriminator model
discriminator_model = models.load("DCGAN_discriminator", img_dim_disc,
nb_patch, bn_mode, use_mbd,
batch_size)
generator_model.compile(loss="mae", optimizer=opt_discriminator)
discriminator_model.trainable = False
DCGAN_model = models.DCGAN(generator_model, discriminator_model,
img_dim, patch_size, image_data_format)
loss = [l1_loss, 'binary_crossentropy']
loss_weights = [1E1, 1]
DCGAN_model.compile(loss=loss,
loss_weights=loss_weights,
optimizer=opt_dcgan)
discriminator_model.trainable = True
discriminator_model.compile(loss='binary_crossentropy',
optimizer=opt_discriminator)
gen_loss = 100
disc_loss = 100
best_loss = [100] * 3
# Start training
print("Start training")
for e in range(nb_epoch):
# Initialize progbar and batch counter
progbar = generic_utils.Progbar(epoch_size)
batch_counter = 1
start = time.time()
for X_full_batch, X_sketch_batch in data_utils.facades_generator(
img_dim, batch_size=batch_size):
X_gen, X_gen_target = next(
data_utils.facades_generator(img_dim,
batch_size=batch_size))
generator_model.train_on_batch(X_gen, X_gen_target)
# Create a batch to feed the discriminator model
X_disc, y_disc = data_utils.get_disc_batch(
X_full_batch,
X_sketch_batch,
generator_model,
batch_counter,
patch_size,
image_data_format,
label_smoothing=label_smoothing,
label_flipping=label_flipping)
# Update the discriminator
disc_loss = discriminator_model.train_on_batch(
X_disc, y_disc) # X_disc, y_disc
# Create a batch to feed the generator model
X_gen, X_gen_target = next(
data_utils.facades_generator(img_dim,
batch_size=batch_size))
y_gen = np.zeros((X_gen.shape[0], 2), dtype=np.uint8)
y_gen[:, 1] = 1
# Freeze the discriminator
discriminator_model.trainable = False
gen_loss = DCGAN_model.train_on_batch(X_gen,
[X_gen_target, y_gen])
# Unfreeze the discriminator
discriminator_model.trainable = True
batch_counter += 1
progbar.add(batch_size,
values=[("D logloss", disc_loss),
("G tot", gen_loss[0]),
("G L1", gen_loss[1]),
("G logloss", gen_loss[2])])
# Save images for visualization
if batch_counter % (n_batch_per_epoch / 2) == 0:
# Get new images from validation
figure_name = "training_" + str(e)
data_utils.plot_generated_batch(
X_full_batch, X_sketch_batch, generator_model,
batch_size, image_data_format, figure_name)
if batch_counter >= n_batch_per_epoch:
break
print("")
print(('Epoch %s/%s, Time: %s' %
(e + 1, nb_epoch, time.time() - start)))
if e % 5 == 0:
gen_weights_path = os.path.join(
'../../models/%s/gen_weights_epoch%s.h5' % (model_name, e))
generator_model.save_weights(gen_weights_path, overwrite=True)
disc_weights_path = os.path.join(
'../../models/%s/disc_weights_epoch%s.h5' %
(model_name, e))
discriminator_model.save_weights(disc_weights_path,
overwrite=True)
DCGAN_weights_path = os.path.join(
'../../models/%s/DCGAN_weights_epoch%s.h5' %
(model_name, e))
DCGAN_model.save_weights(DCGAN_weights_path, overwrite=True)
Best_gen_L1_weights_path = os.path.join(
'../../models/%s/best_gen_L1_weights_epoch.h5' %
(model_name))
if (gen_loss[1] <= best_loss[1]):
generator_model.save_weights(Best_gen_L1_weights_path,
overwrite=True)
best_loss[1] = gen_loss[1]
Best_gen_Totweights_path = os.path.join(
'../../models/%s/best_gen_Totweights_epoch.h5' %
(model_name))
if (gen_loss[0] <= best_loss[0]):
generator_model.save_weights(Best_gen_Totweights_path,
overwrite=True)
best_loss[0] = gen_loss[0]
except KeyboardInterrupt:
pass
def train(**kwargs):
"""
Train model
Load the whole train data in memory for faster operations
args: **kwargs (dict) keyword arguments that specify the model hyperparameters
"""
# Roll out the parameters
img_dim = kwargs["img_dim"]
patch_size = kwargs["patch_size"]
image_data_format = kwargs["image_data_format"]
generator_type = kwargs["generator_type"]
dset = kwargs["dset"]
use_identity_image = kwargs["use_identity_image"]
batch_size = kwargs["batch_size"]
n_batch_per_epoch = kwargs["n_batch_per_epoch"]
nb_epoch = kwargs["nb_epoch"]
augment_data = kwargs["augment_data"]
model_name = kwargs["model_name"]
save_weights_every_n_epochs = kwargs["save_weights_every_n_epochs"]
visualize_images_every_n_epochs = kwargs["visualize_images_every_n_epochs"]
save_only_last_n_weights = kwargs["save_only_last_n_weights"]
use_mbd = kwargs["use_mbd"]
label_smoothing = kwargs["use_label_smoothing"]
label_flipping_prob = kwargs["label_flipping_prob"]
use_l1_weighted_loss = kwargs["use_l1_weighted_loss"]
use_vgg_loss = kwargs["use_vgg_loss"]
vgg_model = kwargs["vgg_model"]
vgg_pooling = kwargs["vgg_pooling"]
prev_model = kwargs["prev_model"]
change_model_name_to_prev_model = kwargs["change_model_name_to_prev_model"]
discriminator_optimizer = kwargs["discriminator_optimizer"]
n_run_of_gen_for_1_run_of_disc = kwargs["n_run_of_gen_for_1_run_of_disc"]
load_all_data_at_once = kwargs["load_all_data_at_once"]
MAX_FRAMES_PER_GIF = kwargs["MAX_FRAMES_PER_GIF"]
dont_train = kwargs["dont_train"]
# batch_size = args.batch_size
# n_batch_per_epoch = args.n_batch_per_epoch
# nb_epoch = args.nb_epoch
# save_weights_every_n_epochs = args.save_weights_every_n_epochs
# generator_type = args.generator_type
# patch_size = args.patch_size
# label_smoothing = False
# label_flipping_prob = False
# dset = args.dset
# use_mbd = False
if dont_train:
# Get the number of non overlapping patch and the size of input image to the discriminator
nb_patch, img_dim_disc = data_utils.get_nb_patch(
img_dim, patch_size, image_data_format)
if use_identity_image:
gen_input_img_dim = [img_dim[0], 2 * img_dim[1], img_dim[2]]
else:
gen_input_img_dim = img_dim
generator_model = models.load("generator_unet_%s" % generator_type,
gen_input_img_dim, nb_patch, use_mbd,
batch_size, model_name)
generator_model.compile(loss='mae', optimizer='adam')
return generator_model
# Check and make the dataset
# If .h5 file of dset is not present, try making it
if load_all_data_at_once:
if not os.path.exists("../../data/processed/%s_data.h5" % dset):
print("dset %s_data.h5 not present in '../../data/processed'!" %
dset)
if not os.path.exists("../../data/%s/" % dset):
print(
"dset folder %s not present in '../../data'!\n\nERROR: Dataset .h5 file not made, and dataset not available in '../../data/'.\n\nQuitting."
% dset)
return
else:
if not os.path.exists(
"../../data/%s/train" % dset) or not os.path.exists(
"../../data/%s/val" % dset) or not os.path.exists(
"../../data/%s/test" % dset):
print(
"'train', 'val' or 'test' folders not present in dset folder '../../data/%s'!\n\nERROR: Dataset must contain 'train', 'val' and 'test' folders.\n\nQuitting."
% dset)
return
else:
print("Making %s dataset" % dset)
subprocess.call([
'python3', '../data/make_dataset.py',
'../../data/%s' % dset, '3'
])
print("Done!")
else:
if not os.path.exists(dset):
print("dset does not exist! Given:", dset)
return
if not os.path.exists(os.path.join(dset, 'train')):
print("dset does not contain a 'train' dir! Given dset:", dset)
return
if not os.path.exists(os.path.join(dset, 'val')):
print("dset does not contain a 'val' dir! Given dset:", dset)
return
epoch_size = n_batch_per_epoch * batch_size
init_epoch = 0
if prev_model:
print('\n\nLoading prev_model from', prev_model, '...\n\n')
prev_model_latest_gen = sorted(
glob.glob(
os.path.join('../../models/', prev_model,
'*gen*epoch*.h5')))[-1]
print(prev_model_latest_gen)
# Find prev model name, epoch
if change_model_name_to_prev_model:
model_name = prev_model_latest_gen.split('models')[-1].split(
'/')[1]
init_epoch = int(prev_model_latest_gen.split('epoch')[1][:5]) + 1
# img_dim = X_target_train.shape[-3:]
# img_dim = (256, 256, 3)
# Get the number of non overlapping patch and the size of input image to the discriminator
nb_patch, img_dim_disc = data_utils.get_nb_patch(img_dim, patch_size,
image_data_format)
if use_identity_image:
gen_input_img_dim = [img_dim[0], 2 * img_dim[1], img_dim[2]]
else:
gen_input_img_dim = img_dim
try:
# Create optimizer
opt_generator = Adam(lr=1E-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
# Load generator model
generator_model = models.load("generator_unet_%s" % generator_type,
gen_input_img_dim, nb_patch, use_mbd,
batch_size, model_name)
if use_vgg_loss:
load_vgg(model=vgg_model,
input_shape=gen_input_img_dim,
pooling=vgg_pooling)
if use_l1_weighted_loss and use_identity_image:
loss = vgg_l1_weighted_identity_loss
elif use_l1_weighted_loss and not use_identity_image:
loss = vgg_l1_weighted_loss
elif not use_l1_weighted_loss and use_identity_image:
loss = vgg_l1_identity_loss
else:
loss = vgg_l1_loss
else:
if use_l1_weighted_loss and use_identity_image:
loss = l1_weighted_identity_loss
elif use_l1_weighted_loss and not use_identity_image:
loss = l1_weighted_loss
else:
loss = l1_loss
generator_model.compile(loss=loss, optimizer=opt_generator)
# Load prev_model
if prev_model:
generator_model.load_weights(prev_model_latest_gen)
# Load .h5 data all at once
print('\n\nLoading data...\n\n')
check_this_process_memory()
if load_all_data_at_once:
X_target_train, X_sketch_train, X_target_val, X_sketch_val = data_utils.load_data(
dset, image_data_format)
check_this_process_memory()
print('X_target_train: %.4f' % (X_target_train.nbytes / 2**30),
"GB")
print('X_sketch_train: %.4f' % (X_sketch_train.nbytes / 2**30),
"GB")
print('X_target_val: %.4f' % (X_target_val.nbytes / 2**30), "GB")
print('X_sketch_val: %.4f' % (X_sketch_val.nbytes / 2**30), "GB")
# To generate training data
X_target_batch_gen_train, X_sketch_batch_gen_train = data_utils.data_generator(
X_target_train,
X_sketch_train,
batch_size,
augment_data=augment_data)
X_target_batch_gen_val, X_sketch_batch_gen_val = data_utils.data_generator(
X_target_val, X_sketch_val, batch_size, augment_data=False)
# Load data from images through an ImageDataGenerator
else:
if use_identity_image:
X_batch_gen_train = data_utils.data_generator_from_dir(
os.path.join(dset, 'train'),
target_size=(img_dim[0], 3 * img_dim[1]),
batch_size=batch_size)
X_batch_gen_val = data_utils.data_generator_from_dir(
os.path.join(dset, 'val'),
target_size=(img_dim[0], 3 * img_dim[1]),
batch_size=batch_size)
else:
X_batch_gen_train = data_utils.data_generator_from_dir(
os.path.join(dset, 'train'),
target_size=(img_dim[0], 2 * img_dim[1]),
batch_size=batch_size)
X_batch_gen_val = data_utils.data_generator_from_dir(
os.path.join(dset, 'val'),
target_size=(img_dim[0], 2 * img_dim[1]),
batch_size=batch_size)
check_this_process_memory()
if dont_train:
raise KeyboardInterrupt
# Setup environment (logging directory etc)
general_utils.setup_logging(**kwargs)
# Losses
gen_losses = []
# Start training
print("\n\nStarting training...\n\n")
# For each epoch
for e in range(nb_epoch):
# Initialize progbar and batch counter
# progbar = generic_utils.Progbar(epoch_size)
batch_counter = 0
gen_loss_epoch = 0
start = time.time()
# For each batch
# for X_target_batch, X_sketch_batch in data_utils.gen_batch(X_target_train, X_sketch_train, batch_size):
for batch in range(n_batch_per_epoch):
# Create a batch to feed the generator model
if load_all_data_at_once:
X_gen_target, X_gen_sketch = next(
X_target_batch_gen_train), next(
X_sketch_batch_gen_train)
else:
X_gen_target, X_gen_sketch = data_utils.load_data_from_data_generator_from_dir(
X_batch_gen_train,
img_dim=img_dim,
augment_data=augment_data,
use_identity_image=use_identity_image)
# Train generator
gen_loss = generator_model.train_on_batch(
X_gen_sketch, X_gen_target)
# Add losses
gen_loss_epoch += gen_loss
print("Epoch", str(init_epoch + e + 1), "batch",
str(batch + 1), "G_loss", gen_loss)
# Append loss
gen_losses.append(gen_loss_epoch / n_batch_per_epoch)
# Save images for visualization
if (e + 1) % visualize_images_every_n_epochs == 0:
data_utils.plot_generated_batch(X_gen_target, X_gen_sketch,
generator_model, batch_size,
image_data_format, model_name,
"training", init_epoch + e + 1,
MAX_FRAMES_PER_GIF)
# Get new images for validation
if load_all_data_at_once:
X_target_batch_val, X_sketch_batch_val = next(
X_target_batch_gen_val), next(X_sketch_batch_gen_val)
else:
X_target_batch_val, X_sketch_batch_val = data_utils.load_data_from_data_generator_from_dir(
X_batch_gen_val,
img_dim=img_dim,
augment_data=False,
use_identity_image=use_identity_image)
# Predict and validate
data_utils.plot_generated_batch(
X_target_batch_val, X_sketch_batch_val, generator_model,
batch_size, image_data_format, model_name, "validation",
init_epoch + e + 1, MAX_FRAMES_PER_GIF)
# Plot losses
data_utils.plot_gen_losses(gen_losses, model_name, init_epoch)
# Save weights
if (e + 1) % save_weights_every_n_epochs == 0:
# Delete all but the last n weights
purge_weights(save_only_last_n_weights, model_name)
# Save gen weights
gen_weights_path = os.path.join(
'../../models/%s/gen_weights_epoch%05d_genLoss%.04f.h5' %
(model_name, init_epoch + e, gen_losses[-1]))
print("Saving", gen_weights_path)
generator_model.save_weights(gen_weights_path, overwrite=True)
check_this_process_memory()
print(
'[{0:%Y/%m/%d %H:%M:%S}] Epoch {1:d}/{2:d} END, Time taken: {3:.4f} seconds'
.format(datetime.datetime.now(), init_epoch + e + 1,
init_epoch + nb_epoch,
time.time() - start))
print(
'------------------------------------------------------------------------------------'
)
except KeyboardInterrupt:
if dont_train:
return generator_model
else:
pass
# SAVE THE MODEL
# Save the model as it is, so that it can be loaded using -
# ```from keras.models import load_model; gen = load_model('generator_latest.h5')```
gen_weights_path = '../../models/%s/generator_latest.h5' % (model_name)
print("Saving", gen_weights_path)
if use_l1_weighted_loss:
generator_model.compile(loss='mae', optimizer=opt_generator)
generator_model.save(gen_weights_path, overwrite=True)
# Save model as json string
generator_model_json_string = generator_model.to_json()
print("Saving", '../../models/%s/generator_latest.txt' % model_name)
with open('../../models/%s/generator_latest.txt' % model_name,
'w') as outfile:
a = outfile.write(generator_model_json_string)
# Save model as json
generator_model_json_data = json.loads(generator_model_json_string)
print("Saving", '../../models/%s/generator_latest.json' % model_name)
with open('../../models/%s/generator_latest.json' % model_name,
'w') as outfile:
json.dump(generator_model_json_data, outfile)
print("Done.")
return generator_model
def train(**kwargs):
"""
Train model
Load the whole train data in memory for faster operations
args: **kwargs (dict) keyword arguments that specify the model hyperparameters
"""
# Roll out the parameters
patch_size = kwargs["patch_size"]
image_data_format = kwargs["image_data_format"]
generator_type = kwargs["generator_type"]
dset = kwargs["dset"]
batch_size = kwargs["batch_size"]
n_batch_per_epoch = kwargs["n_batch_per_epoch"]
nb_epoch = kwargs["nb_epoch"]
model_name = kwargs["model_name"]
save_weights_every_n_epochs = kwargs["save_weights_every_n_epochs"]
visualize_images_every_n_epochs = kwargs["visualize_images_every_n_epochs"]
use_mbd = kwargs["use_mbd"]
label_smoothing = kwargs["use_label_smoothing"]
label_flipping_prob = kwargs["label_flipping_prob"]
use_l1_weighted_loss = kwargs["use_l1_weighted_loss"]
prev_model = kwargs["prev_model"]
discriminator_optimizer = kwargs["discriminator_optimizer"]
n_run_of_gen_for_1_run_of_disc = kwargs["n_run_of_gen_for_1_run_of_disc"]
MAX_FRAMES_PER_GIF = kwargs["MAX_FRAMES_PER_GIF"]
# batch_size = args.batch_size
# n_batch_per_epoch = args.n_batch_per_epoch
# nb_epoch = args.nb_epoch
# save_weights_every_n_epochs = args.save_weights_every_n_epochs
# generator_type = args.generator_type
# patch_size = args.patch_size
# label_smoothing = False
# label_flipping_prob = False
# dset = args.dset
# use_mbd = False
# Check and make the dataset
# If .h5 file of dset is not present, try making it
if not os.path.exists("../../data/processed/%s_data.h5" % dset):
print("dset %s_data.h5 not present in '../../data/processed'!" % dset)
if not os.path.exists("../../data/%s/" % dset):
print(
"dset folder %s not present in '../../data'!\n\nERROR: Dataset .h5 file not made, and dataset not available in '../../data/'.\n\nQuitting."
% dset)
return
else:
if not os.path.exists(
"../../data/%s/train" % dset) or not os.path.exists(
"../../data/%s/val" % dset) or not os.path.exists(
"../../data/%s/test" % dset):
print(
"'train', 'val' or 'test' folders not present in dset folder '../../data/%s'!\n\nERROR: Dataset must contain 'train', 'val' and 'test' folders.\n\nQuitting."
% dset)
return
else:
print("Making %s dataset" % dset)
subprocess.call([
'python3', '../data/make_dataset.py',
'../../data/%s' % dset, '3'
])
print("Done!")
epoch_size = n_batch_per_epoch * batch_size
init_epoch = 0
if prev_model:
print('\n\nLoading prev_model from', prev_model, '...\n\n')
prev_model_latest_gen = sorted(
glob.glob(os.path.join('../../models/', prev_model,
'*gen*.h5')))[-1]
prev_model_latest_disc = sorted(
glob.glob(os.path.join('../../models/', prev_model,
'*disc*.h5')))[-1]
prev_model_latest_DCGAN = sorted(
glob.glob(os.path.join('../../models/', prev_model,
'*DCGAN*.h5')))[-1]
# Find prev model name, epoch
model_name = prev_model_latest_DCGAN.split('models')[-1].split('/')[1]
init_epoch = int(prev_model_latest_DCGAN.split('epoch')[1][:5]) + 1
# Setup environment (logging directory etc), if no prev_model is mentioned
general_utils.setup_logging(model_name)
# img_dim = X_full_train.shape[-3:]
img_dim = (256, 256, 3)
# Get the number of non overlapping patch and the size of input image to the discriminator
nb_patch, img_dim_disc = data_utils.get_nb_patch(img_dim, patch_size,
image_data_format)
try:
# Create optimizers
opt_dcgan = Adam(lr=1E-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
if discriminator_optimizer == 'sgd':
opt_discriminator = SGD(lr=1E-3, momentum=0.9, nesterov=True)
elif discriminator_optimizer == 'adam':
opt_discriminator = Adam(lr=1E-3,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08)
# Load generator model
generator_model = models.load("generator_unet_%s" % generator_type,
img_dim, nb_patch, use_mbd, batch_size,
model_name)
generator_model.compile(loss='mae', optimizer=opt_discriminator)
# Load discriminator model
discriminator_model = models.load("DCGAN_discriminator", img_dim_disc,
nb_patch, use_mbd, batch_size,
model_name)
discriminator_model.trainable = False
DCGAN_model = models.DCGAN(generator_model, discriminator_model,
img_dim, patch_size, image_data_format)
if use_l1_weighted_loss:
loss = [l1_weighted_loss, 'binary_crossentropy']
else:
loss = [l1_loss, 'binary_crossentropy']
loss_weights = [1E1, 1]
DCGAN_model.compile(loss=loss,
loss_weights=loss_weights,
optimizer=opt_dcgan)
discriminator_model.trainable = True
discriminator_model.compile(loss='binary_crossentropy',
optimizer=opt_discriminator)
# Load prev_model
if prev_model:
generator_model.load_weights(prev_model_latest_gen)
discriminator_model.load_weights(prev_model_latest_disc)
DCGAN_model.load_weights(prev_model_latest_DCGAN)
# Load and rescale data
print('\n\nLoading data...\n\n')
X_full_train, X_sketch_train, X_full_val, X_sketch_val = data_utils.load_data(
dset, image_data_format)
check_this_process_memory()
print('X_full_train: %.4f' % (X_full_train.nbytes / 2**30), "GB")
print('X_sketch_train: %.4f' % (X_sketch_train.nbytes / 2**30), "GB")
print('X_full_val: %.4f' % (X_full_val.nbytes / 2**30), "GB")
print('X_sketch_val: %.4f' % (X_sketch_val.nbytes / 2**30), "GB")
# Losses
disc_losses = []
gen_total_losses = []
gen_L1_losses = []
gen_log_losses = []
# Start training
print("\n\nStarting training\n\n")
for e in range(nb_epoch):
# Initialize progbar and batch counter
# progbar = generic_utils.Progbar(epoch_size)
batch_counter = 0
gen_total_loss_epoch = 0
gen_L1_loss_epoch = 0
gen_log_loss_epoch = 0
start = time.time()
for X_full_batch, X_sketch_batch in data_utils.gen_batch(
X_full_train, X_sketch_train, batch_size):
# Create a batch to feed the discriminator model
X_disc, y_disc = data_utils.get_disc_batch(
X_full_batch,
X_sketch_batch,
generator_model,
batch_counter,
patch_size,
image_data_format,
label_smoothing=label_smoothing,
label_flipping_prob=label_flipping_prob)
# Update the discriminator
disc_loss = discriminator_model.train_on_batch(X_disc, y_disc)
# Create a batch to feed the generator model
X_gen_target, X_gen = next(
data_utils.gen_batch(X_full_train, X_sketch_train,
batch_size))
y_gen = np.zeros((X_gen.shape[0], 2), dtype=np.uint8)
y_gen[:, 1] = 1
# Freeze the discriminator
discriminator_model.trainable = False
# Train generator
for _ in range(n_run_of_gen_for_1_run_of_disc - 1):
gen_loss = DCGAN_model.train_on_batch(
X_gen, [X_gen_target, y_gen])
gen_total_loss_epoch += gen_loss[
0] / n_run_of_gen_for_1_run_of_disc
gen_L1_loss_epoch += gen_loss[
1] / n_run_of_gen_for_1_run_of_disc
gen_log_loss_epoch += gen_loss[
2] / n_run_of_gen_for_1_run_of_disc
X_gen_target, X_gen = next(
data_utils.gen_batch(X_full_train, X_sketch_train,
batch_size))
gen_loss = DCGAN_model.train_on_batch(X_gen,
[X_gen_target, y_gen])
# Add losses
gen_total_loss_epoch += gen_loss[
0] / n_run_of_gen_for_1_run_of_disc
gen_L1_loss_epoch += gen_loss[
1] / n_run_of_gen_for_1_run_of_disc
gen_log_loss_epoch += gen_loss[
2] / n_run_of_gen_for_1_run_of_disc
# Unfreeze the discriminator
discriminator_model.trainable = True
# Progress
# progbar.add(batch_size, values=[("D logloss", disc_loss),
# ("G tot", gen_loss[0]),
# ("G L1", gen_loss[1]),
# ("G logloss", gen_loss[2])])
print("Epoch", str(init_epoch + e + 1), "batch",
str(batch_counter + 1), "D_logloss", disc_loss, "G_tot",
gen_loss[0], "G_L1", gen_loss[1], "G_log", gen_loss[2])
batch_counter += 1
if batch_counter >= n_batch_per_epoch:
break
gen_total_loss = gen_total_loss_epoch / n_batch_per_epoch
gen_L1_loss = gen_L1_loss_epoch / n_batch_per_epoch
gen_log_loss = gen_log_loss_epoch / n_batch_per_epoch
disc_losses.append(disc_loss)
gen_total_losses.append(gen_total_loss)
gen_L1_losses.append(gen_L1_loss)
gen_log_losses.append(gen_log_loss)
check_this_process_memory()
print('Epoch %s/%s, Time: %.4f' % (init_epoch + e + 1, init_epoch +
nb_epoch, time.time() - start))
# Save images for visualization
if (e + 1) % visualize_images_every_n_epochs == 0:
data_utils.plot_generated_batch(X_full_batch, X_sketch_batch,
generator_model, batch_size,
image_data_format, model_name,
"training", init_epoch + e + 1,
MAX_FRAMES_PER_GIF)
# Get new images from validation
X_full_batch, X_sketch_batch = next(
data_utils.gen_batch(X_full_val, X_sketch_val, batch_size))
data_utils.plot_generated_batch(X_full_batch, X_sketch_batch,
generator_model, batch_size,
image_data_format, model_name,
"validation",
init_epoch + e + 1,
MAX_FRAMES_PER_GIF)
# Plot losses
data_utils.plot_losses(disc_losses, gen_total_losses,
gen_L1_losses, gen_log_losses,
model_name, init_epoch)
# Save weights
if (e + 1) % save_weights_every_n_epochs == 0:
gen_weights_path = os.path.join(
'../../models/%s/gen_weights_epoch%05d_discLoss%.04f_genTotL%.04f_genL1L%.04f_genLogL%.04f.h5'
% (model_name, init_epoch + e, disc_losses[-1],
gen_total_losses[-1], gen_L1_losses[-1],
gen_log_losses[-1]))
generator_model.save_weights(gen_weights_path, overwrite=True)
disc_weights_path = os.path.join(
'../../models/%s/disc_weights_epoch%05d_discLoss%.04f_genTotL%.04f_genL1L%.04f_genLogL%.04f.h5'
% (model_name, init_epoch + e, disc_losses[-1],
gen_total_losses[-1], gen_L1_losses[-1],
gen_log_losses[-1]))
discriminator_model.save_weights(disc_weights_path,
overwrite=True)
DCGAN_weights_path = os.path.join(
'../../models/%s/DCGAN_weights_epoch%05d_discLoss%.04f_genTotL%.04f_genL1L%.04f_genLogL%.04f.h5'
% (model_name, init_epoch + e, disc_losses[-1],
gen_total_losses[-1], gen_L1_losses[-1],
gen_log_losses[-1]))
DCGAN_model.save_weights(DCGAN_weights_path, overwrite=True)
except KeyboardInterrupt:
pass
def train(**kwargs):
"""
Train model
Load the whole train data in memory for faster operations
args: **kwargs (dict) keyword arguments that specify the model hyperparameters
"""
# Roll out the parameters
batch_size = kwargs["batch_size"]
n_batch_per_epoch = kwargs["n_batch_per_epoch"]
nb_epoch = kwargs["nb_epoch"]
model_name = kwargs["model_name"]
generator = kwargs["generator"]
image_data_format = kwargs["image_data_format"]
img_dim = kwargs["img_dim"]
patch_size = kwargs["patch_size"]
bn_mode = kwargs["bn_mode"]
label_smoothing = kwargs["use_label_smoothing"]
label_flipping = kwargs["label_flipping"]
dset = kwargs["dset"]
use_mbd = kwargs["use_mbd"]
epoch_size = n_batch_per_epoch * batch_size
# Setup environment (logging directory etc)
#general_utils.setup_logging(model_name)
# Load and rescale data
#X_full_train, X_sketch_train, X_full_val, X_sketch_val = data_utils.load_data(dset, image_data_format)
img_dim = (256,256,3) # Manual entry
# Get the number of non overlapping patch and the size of input image to the discriminator
nb_patch, img_dim_disc = data_utils.get_nb_patch(img_dim, patch_size, image_data_format)
try:
# Create optimizers
opt_dcgan = Adam(lr=1E-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
# opt_discriminator = SGD(lr=1E-3, momentum=0.9, nesterov=True)
opt_discriminator = Adam(lr=1E-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
# Load generator model
generator_model = models.load("generator_unet_%s" % generator,
img_dim,
nb_patch,
bn_mode,
use_mbd,
batch_size)
# Load discriminator model
discriminator_model = models.load("DCGAN_discriminator",
img_dim_disc,
nb_patch,
bn_mode,
use_mbd,
batch_size)
generator_model.compile(loss="mae", optimizer=opt_discriminator)
discriminator_model.trainable = False
DCGAN_model = models.DCGAN(generator_model,
discriminator_model,
img_dim,
patch_size,
image_data_format)
loss = [l1_loss, 'binary_crossentropy']
loss_weights = [1E1, 1]
DCGAN_model.compile(loss=loss, loss_weights=loss_weights, optimizer=opt_dcgan)
discriminator_model.trainable = True
discriminator_model.compile(loss='binary_crossentropy', optimizer=opt_discriminator)
gen_loss = 100
disc_loss = 100
best_loss=[100]*3
# Start training
print("Start training")
for e in range(nb_epoch):
# Initialize progbar and batch counter
progbar = generic_utils.Progbar(epoch_size)
batch_counter = 1
start = time.time()
for X_full_batch, X_sketch_batch in data_utils.facades_generator(img_dim,batch_size=batch_size):
X_gen, X_gen_target = next(data_utils.facades_generator(img_dim,batch_size=batch_size))
generator_model.train_on_batch(X_gen, X_gen_target)
# Create a batch to feed the discriminator model
X_disc, y_disc = data_utils.get_disc_batch(X_full_batch,
X_sketch_batch,
generator_model,
batch_counter,
patch_size,
image_data_format,
label_smoothing=label_smoothing,
label_flipping=label_flipping)
# Update the discriminator
disc_loss = discriminator_model.train_on_batch(X_disc, y_disc) # X_disc, y_disc
# Create a batch to feed the generator model
X_gen, X_gen_target = next(data_utils.facades_generator(img_dim,batch_size=batch_size))
y_gen = np.zeros((X_gen.shape[0], 2), dtype=np.uint8)
y_gen[:, 1] = 1
# Freeze the discriminator
discriminator_model.trainable = False
gen_loss = DCGAN_model.train_on_batch(X_gen, [X_gen_target, y_gen])
# Unfreeze the discriminator
discriminator_model.trainable = True
batch_counter += 1
progbar.add(batch_size, values=[("D logloss", disc_loss),
("G tot", gen_loss[0]),
("G L1", gen_loss[1]),
("G logloss", gen_loss[2])])
# Save images for visualization
if batch_counter % (n_batch_per_epoch / 2) == 0:
# Get new images from validation
figure_name = "training_"+str(e)
data_utils.plot_generated_batch(X_full_batch, X_sketch_batch, generator_model,
batch_size, image_data_format, figure_name)
if batch_counter >= n_batch_per_epoch:
break
print("")
print('Epoch %s/%s, Time: %s' % (e + 1, nb_epoch, time.time() - start))
if e % 5 == 0:
gen_weights_path = os.path.join('../../models/%s/gen_weights_epoch%s.h5' % (model_name, e))
generator_model.save_weights(gen_weights_path, overwrite=True)
disc_weights_path = os.path.join('../../models/%s/disc_weights_epoch%s.h5' % (model_name, e))
discriminator_model.save_weights(disc_weights_path, overwrite=True)
DCGAN_weights_path = os.path.join('../../models/%s/DCGAN_weights_epoch%s.h5' % (model_name, e))
DCGAN_model.save_weights(DCGAN_weights_path, overwrite=True)
Best_gen_L1_weights_path = os.path.join('../../models/%s/best_gen_L1_weights_epoch.h5' % (model_name))
if(gen_loss[1]<=best_loss[1]):
generator_model.save_weights(Best_gen_L1_weights_path, overwrite=True)
best_loss[1]=gen_loss[1]
Best_gen_Totweights_path = os.path.join('../../models/%s/best_gen_Totweights_epoch.h5' % (model_name))
if(gen_loss[0]<=best_loss[0]):
generator_model.save_weights(Best_gen_Totweights_path, overwrite=True)
best_loss[0]=gen_loss[0]
except KeyboardInterrupt:
pass
def train(**kwargs):
"""
Train model
Load the whole train data in memory for faster operations
args: **kwargs (dict) keyword arguments that specify the model hyperparameters
"""
# Roll out the parameters
batch_size = kwargs["batch_size"]
n_batch_per_epoch = kwargs["n_batch_per_epoch"]
nb_epoch = kwargs["nb_epoch"]
model_name = kwargs["model_name"]
generator = kwargs["generator"]
image_data_format = kwargs["image_data_format"]
img_dim = kwargs["img_dim"]
patch_size = kwargs["patch_size"]
bn_mode = kwargs["bn_mode"]
label_smoothing = kwargs["use_label_smoothing"]
label_flipping = kwargs["label_flipping"]
dset = kwargs["dset"]
use_mbd = kwargs["use_mbd"]
epoch_size = n_batch_per_epoch * batch_size
# Setup environment (logging directory etc)
general_utils.setup_logging(model_name)
# Load and rescale data
X_full_train, X_sketch_train, X_full_val, X_sketch_val, target_train, target_val = data_utils.load_data(
dset, image_data_format)
img_dim = X_full_train.shape[-3:]
# Get the number of non overlapping patch and the size of input image to the discriminator
nb_patch, img_dim_disc = data_utils.get_nb_patch(img_dim, patch_size,
image_data_format)
try:
# Create optimizers
opt_dcgan = Adam(lr=1E-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
# opt_discriminator = SGD(lr=1E-3, momentum=0.9, nesterov=True)
opt_discriminator = Adam(lr=1E-3,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08)
# DCGAN_model = models.DCGAN(generator_model,
# discriminator_model,
# img_dim,
# patch_size,
# image_data_format)
##########################################################################
classifier_model = models.Pereira_classifier(img_dim)
#classifier_model = models.MyResNet18(img_dim)
#classifier_model = models.MyDensNet121(img_dim)
#classifier_model = models.MyNASNetMobile(img_dim)
#########################################################################
loss = [keras.losses.categorical_crossentropy]
loss_weights = [1]
classifier_model.compile(loss=loss,
loss_weights=loss_weights,
optimizer=opt_dcgan)
class_loss = 100
disc_loss = 100
max_accval = 0
# Start training
print("Start training")
for e in range(nb_epoch):
# Initialize progbar and batch counter
progbar = generic_utils.Progbar(epoch_size)
batch_counter = 1
start = time.time()
for X_full_batch, X_sketch_batch, Y_target in data_utils.gen_batch(
X_full_train, X_sketch_train, target_train, batch_size):
class_loss = classifier_model.train_on_batch(
X_sketch_batch, Y_target)
# Unfreeze the discriminator
batch_counter += 1
progbar.add(batch_size, values=[("class_loss", class_loss)])
# Save images for visualization
if batch_counter >= n_batch_per_epoch:
X_full_batch, X_sketch_batch, Y_target_val = next(
data_utils.gen_batch(X_full_val, X_sketch_val,
target_val,
int(X_sketch_val.shape[0])))
y_pred = classifier_model.predict(X_sketch_batch)
y_predd = np.argmax(y_pred, axis=1)
y_true = np.argmax(Y_target_val, axis=1)
#print(y_true.shape)
accval = (sum(
(y_predd == y_true)) / y_predd.shape[0] * 100)
if (accval > max_accval):
max_accval = accval
print('valacc=%.2f' % (accval))
print('max_accval=%.2f' % (max_accval))
break
print("")
print('Epoch %s/%s, Time: %s' %
(e + 1, nb_epoch, time.time() - start))
except KeyboardInterrupt:
pass
use_mbd = False
do_plot = False
logging_dir = './pix2pix/logging_dir_pix2pix/'
epoch_size = n_batch_per_epoch * batch_size
# Setup environment (logging directory etc)
setup_logging(model_name, logging_dir=logging_dir)
# Load and rescale data
X_full_train, X_sketch_train, X_full_val, X_sketch_val = load_data(
data_folder, dset, image_data_format)
img_dim = X_full_train.shape[-3:]
# Get the number of non overlapping patch and the size of input image to the discriminator
nb_patch, img_dim_disc = get_nb_patch(img_dim, patch_size, image_data_format)
try:
# Create optimizers
opt_dcgan = Adam(lr=1E-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
opt_discriminator = Adam(lr=1E-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
# Load generator model
generator_model = load_model("generator_unet_%s" % generator, img_dim,
nb_patch, bn_mode, use_mbd, batch_size,
do_plot)
# Load discriminator model
discriminator_model = load_model("DCGAN_discriminator", img_dim_disc,
nb_patch, bn_mode, use_mbd, batch_size,
do_plot)
def train(**kwargs):
"""
Train model
Load the whole train data in memory for faster operations
args: **kwargs (dict) keyword arguments that specify the model hyperparameters
"""
# Roll out the parameters
batch_size = kwargs["batch_size"]
n_batch_per_epoch = kwargs["n_batch_per_epoch"]
nb_epoch = kwargs["nb_epoch"]
model_name = kwargs["model_name"]
generator = kwargs["generator"]
image_data_format = kwargs["image_data_format"]
img_dim = kwargs["img_dim"]
patch_size = kwargs["patch_size"]
bn_mode = kwargs["bn_mode"]
label_smoothing = kwargs["use_label_smoothing"]
label_flipping = kwargs["label_flipping"]
dset = kwargs["dset"]
use_mbd = kwargs["use_mbd"]
# right strip '/' to avoid empty '/' dir
save_dir = kwargs["save_dir"].rstrip('/')
# join name with current datetime
save_dir = '_'.join(
[save_dir,
datetime.datetime.now().strftime("%I:%M%p-%B%d-%Y/")])
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
# save the config in save dir
with open('{0}job_config.json'.format(save_dir), 'w') as fp:
json.dump(kwargs, fp, sort_keys=True, indent=4)
epoch_size = n_batch_per_epoch * batch_size
# Setup environment (logging directory etc)
general_utils.setup_logging(model_name)
# Load and rescale data
X_full_train, X_sketch_train, X_full_val, X_sketch_val = data_utils.load_data(
dset, image_data_format)
img_dim = X_full_train.shape[-3:]
# Get the number of non overlapping patch and the size of input image to the discriminator
nb_patch, img_dim_disc = data_utils.get_nb_patch(img_dim, patch_size,
image_data_format)
try:
# Create optimizers
opt_dcgan = Adam(lr=1E-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
# opt_discriminator = SGD(lr=1E-3, momentum=0.9, nesterov=True)
opt_discriminator = Adam(lr=1E-3,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08)
# Load generator model
generator_model = models.load("generator_unet_%s" % generator, img_dim,
nb_patch, bn_mode, use_mbd, batch_size)
# Load discriminator model
discriminator_model = models.load("DCGAN_discriminator", img_dim_disc,
nb_patch, bn_mode, use_mbd,
batch_size)
generator_model.compile(loss='mae', optimizer=opt_discriminator)
discriminator_model.trainable = False
DCGAN_model = models.DCGAN(generator_model, discriminator_model,
img_dim, patch_size, image_data_format)
loss = [l1_loss, 'binary_crossentropy']
loss_weights = [1E1, 1]
DCGAN_model.compile(loss=loss,
loss_weights=loss_weights,
optimizer=opt_dcgan)
discriminator_model.trainable = True
discriminator_model.compile(loss='binary_crossentropy',
optimizer=opt_discriminator)
gen_loss = 100
disc_loss = 100
# Start training
print("Start training")
for e in range(nb_epoch):
# Initialize progbar and batch counter
progbar = generic_utils.Progbar(epoch_size)
batch_counter = 1
start = time.time()
for X_full_batch, X_sketch_batch in data_utils.gen_batch(
X_full_train, X_sketch_train, batch_size):
# Create a batch to feed the discriminator model
X_disc, y_disc = data_utils.get_disc_batch(
X_full_batch,
X_sketch_batch,
generator_model,
batch_counter,
patch_size,
image_data_format,
label_smoothing=label_smoothing,
label_flipping=label_flipping)
# Update the discriminator
disc_loss = discriminator_model.train_on_batch(X_disc, y_disc)
# Create a batch to feed the generator model
X_gen_target, X_gen = next(
data_utils.gen_batch(X_full_train, X_sketch_train,
batch_size))
y_gen = np.zeros((X_gen.shape[0], 2), dtype=np.uint8)
y_gen[:, 1] = 1
# Freeze the discriminator
discriminator_model.trainable = False
gen_loss = DCGAN_model.train_on_batch(X_gen,
[X_gen_target, y_gen])
# Unfreeze the discriminator
discriminator_model.trainable = True
batch_counter += 1
progbar.add(batch_size,
values=[("D logloss", disc_loss),
("G tot", gen_loss[0]),
("G L1", gen_loss[1]),
("G logloss", gen_loss[2])])
# Save images for visualization
if batch_counter % (n_batch_per_epoch / 2) == 0:
# Get new images from validation
data_utils.plot_generated_batch(
X_full_batch, X_sketch_batch, generator_model,
batch_size, image_data_format,
"{:03}_EPOCH_TRAIN".format(e + 1), save_dir)
X_full_batch, X_sketch_batch = next(
data_utils.gen_batch(X_full_val, X_sketch_val,
batch_size))
data_utils.plot_generated_batch(
X_full_batch, X_sketch_batch, generator_model,
batch_size, image_data_format,
"{:03}_EPOCH_VALID".format(e + 1), save_dir)
if batch_counter >= n_batch_per_epoch:
break
print("")
print('Epoch %s/%s, Time: %s' %
(e + 1, nb_epoch, time.time() - start))
if e % 5 == 0:
pass
# save models
# gen_weights_path = os.path.join('../../models/%s/gen_weights_epoch%s.h5' % (model_name, e))
# generator_model.save_weights(gen_weights_path, overwrite=True)
# disc_weights_path = os.path.join('../../models/%s/disc_weights_epoch%s.h5' % (model_name, e))
# discriminator_model.save_weights(disc_weights_path, overwrite=True)
# DCGAN_weights_path = os.path.join('../../models/%s/DCGAN_weights_epoch%s.h5' % (model_name, e))
# DCGAN_model.save_weights(DCGAN_weights_path, overwrite=True)
except KeyboardInterrupt:
pass
# save models
DCGAN_model.save(save_dir + 'DCGAN.h5')
generator_model.save(save_dir + 'GENERATOR.h5')
discriminator_model.save(save_dir + 'DISCRIMINATOR.h5')
def train(**kwargs):
"""
Train model
Load the whole train data in memory for faster operations
args: **kwargs (dict) keyword arguments that specify the model hyperparameters
"""
# Roll out the parameters
batch_size = kwargs["batch_size"]
n_batch_per_epoch = kwargs["n_batch_per_epoch"]
nb_epoch = kwargs["nb_epoch"]
model_name = kwargs["model_name"]
generator = kwargs["generator"]
image_dim_ordering = kwargs["image_dim_ordering"]
img_dim = kwargs["img_dim"]
patch_size = kwargs["patch_size"]
bn_mode = kwargs["bn_mode"]
label_smoothing = kwargs["use_label_smoothing"]
label_flipping = kwargs["label_flipping"]
dset = kwargs["dset"]
use_mbd = kwargs["use_mbd"]
epoch_size = n_batch_per_epoch * batch_size
# Setup environment (logging directory etc)
general_utils.setup_logging(model_name)
print "hi"
# Load and rescale data
X_full_train, X_sketch_train, X_full_val, X_sketch_val = data_utils.load_data(
dset, image_dim_ordering)
img_dim = X_full_train.shape[-3:]
print "data loaded in memory"
# Get the number of non overlapping patch and the size of input image to the discriminator
nb_patch, img_dim_disc = data_utils.get_nb_patch(img_dim, patch_size,
image_dim_ordering)
try:
# Create optimizers
opt_dcgan = Adam(lr=1E-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
# opt_discriminator = SGD(lr=1E-3, momentum=0.9, nesterov=True)
opt_discriminator = Adam(lr=1E-3,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08)
# Load generator model
generator_model = models.load("generator_unet_%s" % generator, img_dim,
nb_patch, bn_mode, use_mbd, batch_size)
# Load discriminator model
discriminator_model = models.load("DCGAN_discriminator", img_dim_disc,
nb_patch, bn_mode, use_mbd,
batch_size)
generator_model.compile(loss='mae', optimizer=opt_discriminator)
discriminator_model.trainable = False
DCGAN_model = models.DCGAN(generator_model, discriminator_model,
img_dim, patch_size, image_dim_ordering)
loss = [l1_loss, 'binary_crossentropy']
loss_weights = [1E1, 1]
DCGAN_model.compile(loss=loss,
loss_weights=loss_weights,
optimizer=opt_dcgan)
discriminator_model.trainable = True
discriminator_model.compile(loss='binary_crossentropy',
optimizer=opt_discriminator)
gen_loss = None
disc_loss = None
iter_num = 102
weights_path = "/home/abhik/pix2pix/src/model/weights/gen_weights_iter%s_epoch30.h5" % (
str(iter_num - 1))
print weights_path
generator_model.load_weights(weights_path)
#discriminator_model.load_weights("disc_weights1.2.h5")
#DCGAN_model.load_weights("DCGAN_weights1.2.h5")
print("Weights Loaded for iter - %d" % iter_num)
# Running average
losses_list = list()
# loss_list = list()
# prev_avg = 0
# Start training
print("Start training")
for e in range(nb_epoch):
# Initialize progbar and batch counter
progbar = generic_utils.Progbar(epoch_size)
batch_counter = 1
start = time.time()
# global disc_n, disc_prev_avg, gen1_n, gen1_prev_avg, gen2_n, gen2_prev_avg, gen3_n, gen3_prev_avg
# disc_n = 1
# disc_prev_avg = 0
# gen1_n = 1
# gen1_prev_avg = 0
# gen2_n = 1
# gen2_prev_avg = 0
# gen3_n = 1
# gen3_prev_avg = 0
for X_full_batch, X_sketch_batch in data_utils.gen_batch(
X_full_train, X_sketch_train, batch_size):
# Create a batch to feed the discriminator model
X_disc, y_disc = data_utils.get_disc_batch(
X_full_batch,
X_sketch_batch,
generator_model,
batch_counter,
patch_size,
image_dim_ordering,
label_smoothing=label_smoothing,
label_flipping=label_flipping)
# Update the discriminator
disc_loss = discriminator_model.train_on_batch(X_disc, y_disc)
# Create a batch to feed the generator model
X_gen_target, X_gen = next(
data_utils.gen_batch(X_full_train, X_sketch_train,
batch_size))
y_gen = np.zeros((X_gen.shape[0], 2), dtype=np.uint8)
y_gen[:, 1] = 1
# Freeze the discriminator
discriminator_model.trainable = False
gen_loss = DCGAN_model.train_on_batch(X_gen,
[X_gen_target, y_gen])
# Unfreeze the discriminator
discriminator_model.trainable = True
# Running average
# loss_list.append(disc_loss)
# loss_list_n = len(loss_list)
# new_avg = ((loss_list_n-1)*prev_avg + disc_loss)/loss_list_n
# prev_avg = new_avg
# disc_avg, gen1_avg, gen2_avg, gen3_avg = running_avg(disc_loss, gen_loss[0], gen_loss[1], gen_loss[2])
# print("running disc loss", new_avg)
# print(disc_loss, gen_loss)
# print ("all losses", disc_avg, gen1_avg, gen2_avg, gen3_avg)
# print("")
batch_counter += 1
progbar.add(batch_size,
values=[("D logloss", disc_loss),
("G tot", gen_loss[0]),
("G L1", gen_loss[1]),
("G logloss", gen_loss[2])])
# Saving data for plotting
# losses = [e+1, batch_counter, disc_loss, gen_loss[0], gen_loss[1], gen_loss[2], disc_avg, gen1_avg, gen2_avg, gen3_avg, iter_num]
# losses_list.append(losses)
# Save images for visualization
if batch_counter % (n_batch_per_epoch / 2) == 0:
# Get new images from validation
data_utils.plot_generated_batch(
X_full_batch, X_sketch_batch, generator_model,
batch_size, image_dim_ordering, "training", iter_num)
X_full_batch, X_sketch_batch = next(
data_utils.gen_batch(X_full_val, X_sketch_val,
batch_size))
data_utils.plot_generated_batch(
X_full_batch, X_sketch_batch, generator_model,
batch_size, image_dim_ordering, "validation", iter_num)
if batch_counter >= n_batch_per_epoch:
break
print("")
print('Epoch %s/%s, Time: %s' %
(e + 1, nb_epoch, time.time() - start))
#Running average
disc_avg, gen1_avg, gen2_avg, gen3_avg = running_avg(
disc_loss, gen_loss[0], gen_loss[1], gen_loss[2])
#Validation loss
y_gen_val = np.zeros((X_sketch_batch.shape[0], 2), dtype=np.uint8)
y_gen_val[:, 1] = 1
val_loss = DCGAN_model.test_on_batch(X_full_batch,
[X_sketch_batch, y_gen_val])
# print "val_loss ===" + str(val_loss)
#logging
# Saving data for plotting
losses = [
e + 1, iter_num, disc_loss, gen_loss[0], gen_loss[1],
gen_loss[2], disc_avg, gen1_avg, gen2_avg, gen3_avg,
val_loss[0], val_loss[1], val_loss[2]
]
losses_list.append(losses)
if (e + 1) % 5 == 0:
gen_weights_path = os.path.join(
'../../models/%s/gen_weights_iter%s_epoch%s.h5' %
(model_name, iter_num, e + 1))
generator_model.save_weights(gen_weights_path, overwrite=True)
disc_weights_path = os.path.join(
'../../models/%s/disc_weights_iter%s_epoch%s.h5' %
(model_name, iter_num, e + 1))
discriminator_model.save_weights(disc_weights_path,
overwrite=True)
DCGAN_weights_path = os.path.join(
'../../models/%s/DCGAN_weights_iter%s_epoch%s.h5' %
(model_name, iter_num, e + 1))
DCGAN_model.save_weights(DCGAN_weights_path, overwrite=True)
loss_array = np.asarray(losses_list)
print(loss_array.shape) # 10 element vector
loss_path = os.path.join(
'../../losses/loss_iter%s_epoch%s.csv' % (iter_num, e + 1))
np.savetxt(loss_path, loss_array, fmt='%.5f', delimiter=',')
np.savetxt('test.csv', loss_array, fmt='%.5f', delimiter=',')
except KeyboardInterrupt:
pass
def train(**kwargs):
"""
Train model
Load the whole train data in memory for faster operations
args: **kwargs (dict) keyword arguments that specify the model hyperparameters
"""
# Roll out the parameters
batch_size = kwargs["batch_size"]
n_batch_per_epoch = kwargs["n_batch_per_epoch"]
nb_epoch = kwargs["nb_epoch"]
model_name = kwargs["model_name"]
generator = kwargs["generator"]
image_dim_ordering = kwargs["image_dim_ordering"]
img_dim = kwargs["img_dim"]
patch_size = kwargs["patch_size"]
bn_mode = kwargs["bn_mode"]
label_smoothing = kwargs["use_label_smoothing"]
label_flipping = kwargs["label_flipping"]
dset = kwargs["dset"]
use_mbd = kwargs["use_mbd"]
epoch_size = n_batch_per_epoch * batch_size
# Setup environment (logging directory etc)
general_utils.setup_logging(model_name)
# Load and rescale data
X_full_train, X_sketch_train, X_full_val, X_sketch_val = data_utils.load_data(
dset, image_dim_ordering) # Initial order, going from ???
# For reverse
X_full_train, X_sketch_train = X_sketch_train, X_full_train
X_full_val, X_sketch_val = X_sketch_val, X_full_val
img_dim = X_full_train.shape[-3:]
# Get the number of non overlapping patch and the size of input image to the discriminator
nb_patch, img_dim_disc = data_utils.get_nb_patch(img_dim, patch_size,
image_dim_ordering)
try:
# Create optimizers
opt_dcgan = Adam(lr=1E-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
# opt_discriminator = SGD(lr=1E-3, momentum=0.9, nesterov=True)
opt_generator = Adam(lr=1E-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
opt_discriminator = Adam(lr=1E-2,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08)
# Load generator model
# generator_model = models.load("generator_unet_upsampling",
# img_dim,
# nb_patch,
# bn_mode,
# use_mbd,
# batch_size)
generator_model = models.load("generator_fire_upsampling", img_dim,
nb_patch, bn_mode, use_mbd, batch_size)
# generator_model = models.load("generator_fire_squeezenet_reverse",
# img_dim,
# nb_patch,
# bn_mode,
# use_mbd,
# batch_size)
# generator_model = models.load("generator_bullshit",
# img_dim,
# nb_patch,
# bn_mode,
# use_mbd,
# batch_size)
# Load discriminator model
discriminator_model = models.load("DCGAN_discriminator", img_dim_disc,
nb_patch, bn_mode, use_mbd,
batch_size)
generator_model.compile(loss='mae', optimizer=opt_generator)
discriminator_model.trainable = False
DCGAN_model = models.DCGAN(generator_model, discriminator_model,
img_dim, patch_size, image_dim_ordering)
loss = [l1_loss, 'binary_crossentropy']
loss_weights = [1E1, 1]
DCGAN_model.compile(loss=loss,
loss_weights=loss_weights,
optimizer=opt_dcgan)
discriminator_model.trainable = True
discriminator_model.compile(loss='binary_crossentropy',
optimizer=opt_discriminator)
gen_loss = 100
disc_loss = 100
# Start training
print("Start training")
for e in range(nb_epoch):
# Initialize progbar and batch counter
progbar = generic_utils.Progbar(epoch_size)
batch_counter = 1
start = time.time()
for X_full_batch, X_sketch_batch in data_utils.gen_batch(
X_full_train, X_sketch_train, batch_size):
# Create a batch to feed the discriminator model
X_disc, y_disc = data_utils.get_disc_batch(
X_full_batch,
X_sketch_batch,
generator_model,
batch_counter,
patch_size,
image_dim_ordering,
label_smoothing=label_smoothing,
label_flipping=label_flipping)
# Update the discriminator
disc_loss = discriminator_model.train_on_batch(X_disc, y_disc)
# Create a batch to feed the generator model
X_gen_target, X_gen = next(
data_utils.gen_batch(X_full_train, X_sketch_train,
batch_size))
y_gen = np.zeros((X_gen.shape[0], 2), dtype=np.uint8)
y_gen[:, 1] = 1
# Freeze the discriminator
discriminator_model.trainable = False
gen_loss = DCGAN_model.train_on_batch(X_gen,
[X_gen_target, y_gen])
# Unfreeze the discriminator
discriminator_model.trainable = True
batch_counter += 1
progbar.add(batch_size,
values=[("D logloss", disc_loss),
("G tot", gen_loss[0]),
("G L1", gen_loss[1]),
("G logloss", gen_loss[2])])
# Save images for visualization
# The images are in the order of input, output, ground truth
if batch_counter % (n_batch_per_epoch / 2) == 0:
# print "Saving images for visualization"
# Get new images from validation
data_utils.plot_generated_batch(
X_full_batch, X_sketch_batch, generator_model,
batch_size, image_dim_ordering,
str(e) + "_" + str(batch_counter) + "training")
X_full_batch, X_sketch_batch = next(
data_utils.gen_batch(X_full_val, X_sketch_val,
batch_size))
data_utils.plot_generated_batch(
X_full_batch, X_sketch_batch, generator_model,
batch_size, image_dim_ordering,
str(e) + "_" + str(batch_counter) + "validation")
if batch_counter >= n_batch_per_epoch:
break
print("")
print('Epoch %s/%s, Time: %s' %
(e + 1, nb_epoch, time.time() - start))
if e % 50 == 0:
gen_weights_path = os.path.join(
'../../models/%s/gen_weights_epoch%s.h5' % (model_name, e))
generator_model.save_weights(gen_weights_path, overwrite=True)
disc_weights_path = os.path.join(
'../../models/%s/disc_weights_epoch%s.h5' %
(model_name, e))
discriminator_model.save_weights(disc_weights_path,
overwrite=True)
DCGAN_weights_path = os.path.join(
'../../models/%s/DCGAN_weights_epoch%s.h5' %
(model_name, e))
DCGAN_model.save_weights(DCGAN_weights_path, overwrite=True)
except KeyboardInterrupt:
pass