def train(model = model(),checkpoint = "latest.hdf5"):
dataset = build_dataset()
callback = TensorBoard("_tensor_board")
callback.set_model(model)
labels = interested_words
for i in range(10000):
save_model(model, "checkpoint.model")
print("Chunk "+str(i)+" of 10000...")
X,Y = get_batch(dataset,batchsize=1000)
for j in range(10):
logs = model.train_on_batch(np.array(X),np.array(Y))
print("loss:",logs)
write_log(callback, ["training loss"], [logs], i*10+j)
X,Y = get_batch(dataset,batchsize=100,batchtype="test")
results = model.predict(X)
accuracy = 0
for result,actual in zip(results,Y):
#print("running test")
x =np.argmax(result)
j =np.argmax(actual)
try:
print("expected "+labels[j]," got "+labels[x])
except:
pass
if x == j: accuracy += 1
write_log(callback,["test accuracy"],[accuracy],i)
optimizer = Adam(lr=1e-5)
optimizer_classifier = Adam(lr=1e-5)
model_rpn.compile(optimizer=optimizer, loss=[losses.rpn_loss_cls(num_anchors), losses.rpn_loss_regr(num_anchors)])
model_classifier.compile(optimizer=optimizer_classifier, loss=[losses.class_loss_cls, losses.class_loss_regr(len(classes_count)-1)], metrics={'dense_class_{}'.format(len(classes_count)): 'accuracy'})
model_all.compile(optimizer='sgd', loss='mae')
# Tensorboard log폴더 생성
log_path = './logs'
if not os.path.isdir(log_path):
os.mkdir(log_path)
# Tensorboard log모델 연결
callback = TensorBoard(log_path)
callback.set_model(model_all)
epoch_length = 1000
num_epochs = int(options.num_epochs)
iter_num = 0
train_step = 0
losses = np.zeros((epoch_length, 5))
rpn_accuracy_rpn_monitor = []
rpn_accuracy_for_epoch = []
start_time = time.time()
best_loss = np.Inf
class_mapping_inv = {v: k for k, v in class_mapping.items()}
print('Starting training')
def create_keras_callbacks(
model,
modelname,
output_dir,
):
"""
creates callbacks for keras training process
creates:
+ tensorboard callback: logs are written to <output_dir>/logs/<modelname>
+ checkpoint model: saves best validation score model to <output_dir>/<modelname>.hdf5
+ early stopping: stops training when validation hasn't improved for 25 epochs
+ learning rate schedular: reduces learning rate by half every 20 epochs
returns:
list of callbacks which can be passed directly to keras.fit
TODO:
TFRunMetaData: captures GPU usage, memory stats etc, but requires at least series 10 GPU
WeightWriter: custom callback to output weights; needs integrating
ValidationOutput: custom callback to output results of validation steps; needs integrating
"""
tensorboard_dir = join(output_dir, "logs", modelname)
logger.info("tensorboard logs writing to '%s'" % tensorboard_dir)
checkpoint_filepath = join(output_dir, modelname + ".hdf5")
logger.info("writing checkpoint file to '%s'" % checkpoint_filepath)
# weight_dir = join(output_dir, modelname, "weights")
# logger.info("weightwriter writing to '%s'" % weight_dir)
# validation_dir = join(output_dir, model_name, "validation")
# logger.info("validation data writing to '%s'" % validation_dir)
for dir in [output_dir, tensorboard_dir]:
try:
makedirs(dir)
except FileExistsError:
logger.warning("'%s' already exists" % dir)
continue
except OSError:
logger.exception("creating '%s' failed with OSError" % dir)
continue
tensorboard_callback = TensorBoardCallback(log_dir=tensorboard_dir,
histogram_freq=0,
write_graph=True,
write_grads=False,
write_images=False,
embeddings_freq=0,
embeddings_layer_names=None,
embeddings_metadata=None)
tensorboard_callback.set_model(model)
return [
tensorboard_callback,
ModelCheckpointCallback(filepath=checkpoint_filepath,
verbose=0,
save_best_only=True),
EarlyStoppingCallback(monitor="val_loss",
min_delta=0.00000001,
patience=25,
mode="min"),
LearningRateScheduler(step_decay(20, 0.5))
#ReduceLROnPlateauCallback(
# monitor="val_loss",
# factor=0.5,
# patience=4,
# mode="min",
# epsilon=0.01,
# min_lr=0.0000001
#),
#TFRunMetaData(tensorboard_callback),
#WeightWriter(weight_dir),
#ValidationOutput(
# validation_dir,
# kwargs["validation_generator"],
# kwargs["validation_steps"]
#)
]
def __init__(self,
data=(),
nb_class=16,
opts='adam',
lossfunc='sparse_categorical_crossentropy',
model='dnn_1',
batch_size=64,
val_data=(),
learning_rate=0.001,
savepath=None,
callback_type=None,
epoch=100,
restore=False,
logger=None):
self.model = model
self.x_train, self.y_train = data
self.nb_class = nb_class
self.opts = opts
self.lossfunc = lossfunc
self.batch_size = batch_size
self.x_val, self.y_val = val_data
self.learning_rate = learning_rate
self.savepath = savepath
self.callback_type = callback_type
self.epoch = epoch
self.restore = restore
input_shape = self.x_train.shape[1]
nb_class = self.nb_class
# model choose
if self.model == 'dnn_1':
model = dnn_1(input_shape, nb_class)
elif self.model == 'bilstm':
model = bilstm(input_shape, nb_class)
elif self.model == 'advanced_dnn':
model = sample_dnn(input_shape, nb_class)
elif self.model == 'rwn':
model = RWN(self, input_shape)
elif self.model == 'semi_gan':
model = SemiGan(self, input_shape)
else:
pass
# loss funtion choose
if self.lossfunc == 'focal_loss':
lossfunc = focal_loss(alpha=1)
elif self.lossfunc == 'binary_crossentropy':
lossfunc = 'binary_crossentropy'
elif self.lossfunc == 'hinge':
lossfunc = 'hinge'
elif self.lossfunc == 'squared_hinge':
lossfunc = 'squared_hinge'
elif self.lossfunc == 'categorical_crossentropy':
lossfunc = 'categorical_crossentropy'
elif self.lossfunc == 'sparse_categorical_crossentropy':
lossfunc = 'sparse_categorical_crossentropy'
elif self.lossfunc == 'kulback_leibler_divergence':
lossfunc = 'kulback_leibler_divergence'
else:
pass
# optimizer choose
if self.opts == 'adam':
opts = Adam(lr=self.learning_rate, decay=1e-6)
elif self.opts == 'rmsprop':
opts = RMSprop(lr=self.learning_rate, epsilon=1.0)
elif self.opts == 'adagrad':
opts = Adam(lr=self.learning_rate, decay=1e-6, adagrad=True)
elif self.opts == 'sgd':
opts = SGD(lr=self.learning_rate,
momentum=0.9,
decay=1e-6,
nestrov=True)
elif self.opts == 'amsgrad':
opts = Adam(lr=self.learning_rate, decay=1e-6, amsgrad=True)
elif self.opts == 'adagrad':
opts = Adagrad(lr=self.learning_rate, decay=1e-6)
elif self.opts == 'nadam':
opts = Nadam(lr=self.learning_rate,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
schedule_decay=0.004)
elif self.opts == 'adadelta':
opts = Adadelta(lr=1.0, rho=0.95, epsilon=None, decay=0.0)
elif self.opts == 'radam':
opts = RectifiedAdam(lr=1e-3,
total_steps=self.epoch,
warmup_proportion=0.1,
min_lr=1e-6)
elif self.opts == 'lookahead':
radam = RectifiedAdam()
opts = Lookahead(radam, sync_period=6, slow_step_size=0.5)
elif self.opts == 'lazyadam':
opts = LazyAdam(lr=self.learning_rate)
elif self.opts == 'conditionalgradient':
opts = ConditionalGradient(lr=0.99949, lambda_=203)
else:
pass
print("loss func is {}".format(lossfunc))
auc = tf.keras.metrics.AUC()
recall = tf.keras.metrics.Recall()
precision = tf.keras.metrics.Precision()
model.compile(optimizer=opts,
loss=lossfunc,
metrics=['acc', auc, recall, precision])
# x_train = self.x_train
# x_val = self.x_val
# y_train = self.y_train
# y_val = self.y_val
x_train = np.asarray(self.x_train)
x_val = np.asarray(self.x_val)
y_train = np.asarray(self.y_train)
y_val = np.asarray(self.y_val)
MODEL_SAVE_FOLDER_PATH = os.path.join(self.savepath, self.model)
if not os.path.exists(MODEL_SAVE_FOLDER_PATH):
os.mkdir(MODEL_SAVE_FOLDER_PATH)
def lrdropping(self):
initial_lrate = self.learning_rate
drop = 0.9
epochs_drop = 3.0
lrate = initial_lrate * math.pow(
drop, math.floor((1 + epoch) / epochs_drop))
return lrate
callbacks = []
if 'checkpoint' in self.callback_type:
checkpoint = ModelCheckpoint(os.path.join(
MODEL_SAVE_FOLDER_PATH, 'checkpoint-{epoch:02d}.h5'),
monitor='val_auc',
save_best_only=False,
mode='max')
callbacks.append(checkpoint)
else:
pass
if 'elarystopping' in self.callback_type:
earlystopping = EarlyStopping(monitor='val_auc',
patience=5,
verbose=1,
mode='max')
callbacks.append(earlystopping)
else:
pass
if 'tensorboard' in self.callback_type:
logger.info("tensorboard path : {}".format(MODEL_SAVE_FOLDER_PATH))
tensorboard = TensorBoard(log_dir=os.path.join(
"tensorboard", MODEL_SAVE_FOLDER_PATH),
histogram_freq=0,
write_graph=True,
write_images=True)
tensorboard.set_model(model)
train_summary_writer = tf.summary.create_file_writer(
os.path.join("tensorboard", MODEL_SAVE_FOLDER_PATH))
callbacks.append(tensorboard)
else:
pass
if 'rateschedule' in self.callback_type:
lrd = LearningRateScheduler(lrdropping)
callbacks.append(lrd)
else:
pass
if 'interval_check' in self.callback_type:
inter = IntervalEvaluation(
validation_data=(x_val, y_val),
interval=1,
savedir=os.path.join(self.savepath, self.model),
file='Evaluation_{}.csv'.format(self.model),
logger=logger)
callbacks.append(inter)
else:
pass
if self.opts == 'conditionalgradient':
def frobenius_norm(m):
total_reduce_sum = 0
for i in range(len(m)):
total_reduce_sum = total_reduce_sum + tf.math.reduce_sum(
m[i]**2)
norm = total_reduce_sum**0.5
return norm
CG_frobenius_norm_of_weight = []
CG_get_weight_norm = LambdaCallback(
on_epoch_end=lambda batch, logs: CG_frobenius_norm_of_weight.
append(frobenius_norm(model.trainable_weights).np()))
cgnorm = CG_frobenius_norm_of_weight
callbacks.append(cgnorm)
else:
pass
model_json = model.to_json()
with open(os.path.join(MODEL_SAVE_FOLDER_PATH, 'model.json'),
'w') as json_file:
json_file.write(model_json)
if self.restore:
chkp = last_checkpoint(MODEL_SAVE_FOLDER_PATH)
model.load_weight(chkp)
init_epoch = int(os.path.basename(chkp).split('-')[1])
print("================== restore checkpoint ==================")
else:
init_epoch = 0
print("================== restore failed ==================")
cw = class_weight.compute_class_weight(class_weight='balanced',
classes=np.unique(y_train),
y=y_train)
logger.info("callback is {}".format(callbacks))
hist = model.fit(x=x_train,
y=y_train,
epochs=self.epoch,
verbose=1,
validation_data=(x_val, y_val),
shuffle=True,
callbacks=callbacks,
class_weight=cw)
is_show_detail=True,
is_adapt=False)
#==========================================
# training data shaper
#==========================================
data_shaper = shaper.Shape_data(LEFT_CONTEXT,
RIGHT_CONTEXT,
TRUNCATE_GRAD,
NUMBER_OF_SKIP_FRAME )
#==========================================
# set tensorboard
#==========================================
callback = TensorBoard(LOG_PATH)
callback.set_model(mask_estimator)
#==========================================
# get features
#==========================================
feature_extractor = feature.Feature(SAMPLING_FREQUENCY, FFTL, SHIFT)
noise_generator = augment.Generate_random_noise(noise_list, SAMPLING_FREQUENCY)
reverbarent_generator = augment.RIR_convolve(SAMPLING_FREQUENCY)
#==========================================
# go training
#==========================================
utterance_count = 0
TRIM = np.int(0.05 * SAMPLING_FREQUENCY) # beginning and ending of uttearnce is not used for training
val_loss = np.array([])
test_loss = np.array([])
def train_vsrganplus(self,
epochs=None,
batch_size=16,
modelname=None,
datapath_train=None,
datapath_validation=None,
steps_per_validation=1000,
datapath_test=None,
workers=4,
max_queue_size=10,
first_epoch=0,
print_frequency=1,
crops_per_image=2,
log_weight_frequency=None,
log_weight_path='./model/',
log_tensorboard_path='./data/logs/',
log_tensorboard_update_freq=10,
log_test_frequency=500,
log_test_path="./images/samples/",
media_type='i'):
"""Train the VSRGANplus network
:param int epochs: how many epochs to train the network for
:param str modelname: name to use for storing model weights etc.
:param str datapath_train: path for the image files to use for training
:param str datapath_test: path for the image files to use for testing / plotting
:param int print_frequency: how often (in epochs) to print progress to terminal. Warning: will run validation inference!
:param int log_weight_frequency: how often (in epochs) should network weights be saved. None for never
:param int log_weight_path: where should network weights be saved
:param int log_test_frequency: how often (in epochs) should testing & validation be performed
:param str log_test_path: where should test results be saved
:param str log_tensorboard_path: where should tensorflow logs be sent
"""
# Create data loaders
train_loader = DataLoader(datapath_train, batch_size, self.height_hr,
self.width_hr, self.upscaling_factor,
crops_per_image, media_type, self.channels,
self.colorspace)
# Validation data loader
validation_loader = None
if datapath_validation is not None:
validation_loader = DataLoader(datapath_validation, batch_size,
self.height_hr, self.width_hr,
self.upscaling_factor,
crops_per_image, 'i', self.channels,
self.colorspace)
test_loader = None
if datapath_test is not None:
test_loader = DataLoader(datapath_test, 1, self.height_hr,
self.width_hr, self.upscaling_factor, 1,
'i', self.channels, self.colorspace)
# Use several workers on CPU for preparing batches
enqueuer = OrderedEnqueuer(train_loader,
use_multiprocessing=True,
shuffle=True)
enqueuer.start(workers=workers, max_queue_size=max_queue_size)
output_generator = enqueuer.get()
# Callback: tensorboard
if log_tensorboard_path:
tensorboard = TensorBoard(log_dir=os.path.join(
log_tensorboard_path, modelname),
histogram_freq=0,
write_graph=True,
update_freq=log_tensorboard_update_freq)
tensorboard.set_model(self.vsrganplus)
else:
print(
">> Not logging to tensorboard since no log_tensorboard_path is set"
)
# Learning rate scheduler
def lr_scheduler(epoch, lr):
factor = 0.1
decay_step = [50000, 100000, 200000, 300000]
if epoch in decay_step and epoch:
return lr * factor
return lr
lr_scheduler_gan = LearningRateScheduler(lr_scheduler, verbose=1)
lr_scheduler_gan.set_model(self.vsrganplus)
lr_scheduler_gen = LearningRateScheduler(lr_scheduler, verbose=0)
lr_scheduler_gen.set_model(self.generator)
lr_scheduler_dis = LearningRateScheduler(lr_scheduler, verbose=0)
lr_scheduler_dis.set_model(self.discriminator)
lr_scheduler_ra = LearningRateScheduler(lr_scheduler, verbose=0)
lr_scheduler_ra.set_model(self.ra_discriminator)
# Callback: format input value
def named_logs(model, logs):
"""Transform train_on_batch return value to dict expected by on_batch_end callback"""
result = {}
for l in zip(model.metrics_names, logs):
result[l[0]] = l[1]
return result
# Shape of output from discriminator
disciminator_output_shape = list(self.ra_discriminator.output_shape)
disciminator_output_shape[0] = batch_size
disciminator_output_shape = tuple(disciminator_output_shape)
# VALID / FAKE targets for discriminator
real = np.ones(disciminator_output_shape)
fake = np.zeros(disciminator_output_shape)
# Each epoch == "update iteration" as defined in the paper
print_losses = {"GAN": [], "D": []}
start_epoch = datetime.datetime.now()
# Random images to go through
#idxs = np.random.randint(0, len(train_loader), epochs)
# Loop through epochs / iterations
for epoch in range(first_epoch, int(epochs) + first_epoch):
lr_scheduler_gan.on_epoch_begin(epoch)
lr_scheduler_ra.on_epoch_begin(epoch)
lr_scheduler_dis.on_epoch_begin(epoch)
lr_scheduler_gen.on_epoch_begin(epoch)
print("\nEpoch {}/{}:".format(epoch + 1, epochs + first_epoch))
# Start epoch time
if epoch % print_frequency == 0:
start_epoch = datetime.datetime.now()
# Train discriminator
self.discriminator.trainable = True
self.ra_discriminator.trainable = True
imgs_lr, imgs_hr = next(output_generator)
generated_hr = self.generator.predict(imgs_lr)
real_loss = self.ra_discriminator.train_on_batch(
[imgs_hr, generated_hr], real)
#print("Real: ",real_loss)
fake_loss = self.ra_discriminator.train_on_batch(
[generated_hr, imgs_hr], fake)
#print("Fake: ",fake_loss)
discriminator_loss = 0.5 * np.add(real_loss, fake_loss)
# Train generator
self.discriminator.trainable = False
self.ra_discriminator.trainable = False
#for _ in tqdm(range(10),ncols=60,desc=">> Training generator"):
imgs_lr, imgs_hr = next(output_generator)
gan_loss = self.vsrganplus.train_on_batch([imgs_lr, imgs_hr],
[imgs_hr, real, imgs_hr])
# Callbacks
logs = named_logs(self.vsrganplus, gan_loss)
tensorboard.on_epoch_end(epoch, logs)
# Save losses
print_losses['GAN'].append(gan_loss)
print_losses['D'].append(discriminator_loss)
# Show the progress
if epoch % print_frequency == 0:
g_avg_loss = np.array(print_losses['GAN']).mean(axis=0)
d_avg_loss = np.array(print_losses['D']).mean(axis=0)
print(">> Time: {}s\n>> GAN: {}\n>> Discriminator: {}".format(
(datetime.datetime.now() - start_epoch).seconds,
", ".join([
"{}={:.4f}".format(k, v) for k, v in zip(
self.vsrganplus.metrics_names, g_avg_loss)
]), ", ".join([
"{}={:.4f}".format(k, v) for k, v in zip(
self.discriminator.metrics_names, d_avg_loss)
])))
print_losses = {"GAN": [], "D": []}
# Run validation inference if specified
if datapath_validation:
validation_losses = self.generator.evaluate_generator(
validation_loader,
steps=steps_per_validation,
use_multiprocessing=workers > 1,
workers=workers)
print(">> Validation Losses: {}".format(", ".join([
"{}={:.4f}".format(k, v) for k, v in zip(
self.generator.metrics_names, validation_losses)
])))
# If test images are supplied, run model on them and save to log_test_path
if datapath_test and epoch % log_test_frequency == 0:
plot_test_images(self.generator,
test_loader,
datapath_test,
log_test_path,
epoch,
modelname,
channels=self.channels,
colorspace=self.colorspace)
# Check if we should save the network weights
if log_weight_frequency and epoch % log_weight_frequency == 0:
# Save the network weights
self.save_weights(os.path.join(log_weight_path, modelname))
stage1_gen = build_stage1_generator(embedding_compressor_model)
stage1_gen.compile(loss='binary_crossentropy', optimizer=gen_optimizer)
stage1_disc = build_stage1_discriminator(embedding_compressor_model)
stage1_disc.compile(loss=tf.nn.sigmoid_cross_entropy_with_logits,
optimizer=disc_optimizer)
adversarial_model = build_adversarial_model(stage1_gen, stage1_disc)
adversarial_model.compile(
loss=[tf.nn.sigmoid_cross_entropy_with_logits, KL_loss],
optimizer=gen_optimizer,
metrics=None)
tensorboard = TensorBoard(log_dir='logs/'.format(time.time()))
tensorboard.set_model(stage1_gen)
tensorboard.set_model(stage1_disc)
# tensorboard.set_model(ca_model)
# tensorboard.set_model(embedding_compressor_model)
real_labels = np.ones((batch_size, 1), dtype=np.float32) * 0.9
fake_labels = np.zeros((batch_size, 1), dtype=np.float32) + 0.1
summary_writer = tf.summary.create_file_writer('./logs')
for epoch in range(epochs):
print('===========================================')
print('Epoch is:', epoch)
print('Number of batches:', int(X_train.shape[0] / batch_size))
gen_losses = []
class GAN():
def __init__(self, model_yaml, train_yaml):
"""
Args:
model_yaml: dictionnary with the model parameters
train_yaml: dictionnary the tran parameters
"""
self.sigma_val = 0
self.model_yaml = model_yaml
self.img_rows = 28
self.img_cols = 28
self.channels = 1
self.img_shape = (self.img_rows, self.img_cols, self.channels)
if "dict_band_x" not in train_yaml:
self.dict_band_X = None
self.dict_band_label = None
self.dict_rescale_type = None
else:
self.dict_band_X = train_yaml["dict_band_x"]
self.dict_band_label = train_yaml["dict_band_label"]
self.dict_rescale_type = train_yaml["dict_rescale_type"]
self.s1bands = train_yaml["s1bands"]
self.s2bands = train_yaml["s2bands"]
# self.latent_dim = 100
# PATH
self.model_name = model_yaml["model_name"]
self.model_dir = train_yaml["training_dir"] + self.model_name + "/"
self.this_training_dir = self.model_dir + "training_{}/".format(
train_yaml["training_number"])
self.saving_image_path = self.this_training_dir + "saved_training_images/"
self.saving_logs_path = self.this_training_dir + "logs/"
self.checkpoint_dir = self.this_training_dir + "checkpoints/"
self.previous_checkpoint = train_yaml["load_model"]
# TRAIN PARAMETER
self.normalization = train_yaml["normalization"]
self.epoch = train_yaml["epoch"]
self.batch_size = train_yaml["batch_size"]
# self.sess = sess
self.learning_rate = train_yaml["lr"]
self.fact_g_lr = train_yaml["fact_g_lr"]
self.beta1 = train_yaml["beta1"]
self.val_directory = train_yaml["val_directory"]
self.fact_s2 = train_yaml["s2_scale"]
self.fact_s1 = train_yaml["s1_scale"]
self.data_X, self.data_y, self.scale_dict_train = load_data(
train_yaml["train_directory"],
x_shape=model_yaml["input_shape"],
label_shape=model_yaml["dim_gt_image"],
normalization=self.normalization,
dict_band_X=self.dict_band_X,
dict_band_label=self.dict_band_label,
dict_rescale_type=self.dict_rescale_type,
fact_s2=self.fact_s2,
fact_s1=self.fact_s1,
s2_bands=self.s2bands,
s1_bands=self.s1bands,
lim=train_yaml["lim_train_tile"])
self.val_X, self.val_Y, scale_dict_val = load_data(
self.val_directory,
x_shape=model_yaml["input_shape"],
label_shape=model_yaml["dim_gt_image"],
normalization=self.normalization,
dict_band_X=self.dict_band_X,
dict_band_label=self.dict_band_label,
dict_rescale_type=self.dict_rescale_type,
dict_scale=self.scale_dict_train,
fact_s2=self.fact_s2,
fact_s1=self.fact_s1,
s2_bands=self.s2bands,
s1_bands=self.s1bands,
lim=train_yaml["lim_val_tile"])
print("Loading the data done dataX {} dataY {}".format(
self.data_X.shape, self.data_y.shape))
self.gpu = train_yaml["n_gpu"]
self.num_batches = self.data_X.shape[0] // self.batch_size
self.model_yaml = model_yaml
self.im_saving_step = train_yaml["im_saving_step"]
self.w_saving_step = train_yaml["weights_saving_step"]
self.val_metric_step = train_yaml["metric_step"]
# REDUCE THE DISCRIMINATOR PERFORMANCE
self.val_lambda = train_yaml["lambda"]
self.real_label_smoothing = tuple(train_yaml["real_label_smoothing"])
self.fake_label_smoothing = tuple(train_yaml["fake_label_smoothing"])
self.sigma_init = train_yaml["sigma_init"]
self.sigma_step = train_yaml['sigma_step']
self.sigma_decay = train_yaml["sigma_decay"]
self.ite_train_g = train_yaml["train_g_multiple_time"]
self.max_im = 10
self.strategy = tf.distribute.MirroredStrategy()
print('Number of devices: {}'.format(
self.strategy.num_replicas_in_sync))
self.buffer_size = self.data_X.shape[0]
self.global_batch_size = self.batch_size * self.strategy.num_replicas_in_sync
with self.strategy.scope():
self.d_optimizer = Adam(self.learning_rate, self.beta1)
self.g_optimizer = Adam(self.learning_rate * self.fact_g_lr,
self.beta1)
self.build_model()
self.model_writer = tf.summary.create_file_writer(
self.saving_logs_path)
#self.strategy = tf.distribute.MirroredStrategy()
def build_model(self):
# strategy = tf.distribute.MirroredStrategy()
# print('Number of devices: {}'.format(strategy.num_replicas_in_sync))
# We use the discriminator
self.discriminator = self.build_discriminator(self.model_yaml)
self.discriminator.compile(loss='binary_crossentropy',
optimizer=self.d_optimizer,
metrics=['accuracy'])
self.generator = self.build_generator(self.model_yaml,
is_training=True)
print("Input G")
g_input = Input(shape=(self.data_X.shape[1], self.data_X.shape[2],
self.data_X.shape[3]),
name="g_build_model_input_data")
G = self.generator(g_input)
print("G", G)
# For the combined model we will only train the generator
self.discriminator.trainable = False
D_input = tf.concat([G, g_input], axis=-1)
print("INPUT DISCRI ", D_input)
# The discriminator takes generated images as input and determines validity
D_output_fake = self.discriminator(D_input)
# print(D_output)
# The combined model (stacked generator and discriminator)
# TO TRAIN WITH MULTIPLE GPU
self.combined = Model(g_input, [D_output_fake, G],
name="Combined_model")
self.combined.compile(loss=['binary_crossentropy', L1_loss],
loss_weights=[1, self.val_lambda],
optimizer=self.g_optimizer)
print("[INFO] combined model loss are : ".format(
self.combined.metrics_names))
def build_generator(self, model_yaml, is_training=True):
def build_resnet_block(input, id=0):
"""Define the ResNet block"""
x = Conv2D(model_yaml["dim_resnet"],
model_yaml["k_resnet"],
padding=model_yaml["padding"],
strides=tuple(model_yaml["stride"]),
name="g_block_{}_conv1".format(id))(input)
x = BatchNormalization(momentum=model_yaml["bn_momentum"],
trainable=is_training,
name="g_block_{}_bn1".format(id))(x)
x = ReLU(name="g_block_{}_relu1".format(id))(x)
x = Dropout(rate=model_yaml["do_rate"],
name="g_block_{}_do".format(id))(x)
x = Conv2D(model_yaml["dim_resnet"],
model_yaml["k_resnet"],
padding=model_yaml["padding"],
strides=tuple(model_yaml["stride"]),
name="g_block_{}_conv2".format(id))(x)
x = BatchNormalization(momentum=model_yaml["bn_momentum"],
trainable=is_training,
name="g_block_{}_bn2".format(id))(x)
x = Add(name="g_block_{}_add".format(id))([x, input])
x = ReLU(name="g_block_{}_relu2".format(id))(x)
return x
img_input = Input(shape=(self.data_X.shape[1], self.data_X.shape[2],
self.data_X.shape[3]),
name="g_input_data")
if model_yaml["last_activation"] == "tanh":
print("use tanh keras")
last_activ = lambda x: tf.keras.activations.tanh(x)
else:
last_activ = model_yaml["last_activation"]
x = img_input
for i, param_lay in enumerate(
model_yaml["param_before_resnet"]
): # build the blocks before the Resnet Blocks
x = Conv2D(param_lay[0],
param_lay[1],
strides=tuple(model_yaml["stride"]),
padding=model_yaml["padding"],
name="g_conv{}".format(i))(x)
x = BatchNormalization(momentum=model_yaml["bn_momentum"],
trainable=is_training,
name="g_{}_bn".format(i))(x)
x = ReLU(name="g_{}_lay_relu".format(i))(x)
for j in range(model_yaml["nb_resnet_blocs"]): # add the Resnet blocks
x = build_resnet_block(x, id=j)
for i, param_lay in enumerate(model_yaml["param_after_resnet"]):
x = Conv2D(param_lay[0],
param_lay[1],
strides=tuple(model_yaml["stride"]),
padding=model_yaml["padding"],
name="g_conv_after_resnetblock{}".format(i))(x)
x = BatchNormalization(
momentum=model_yaml["bn_momentum"],
trainable=is_training,
name="g_after_resnetblock{}_bn2".format(i))(x)
x = ReLU(name="g_after_resnetblock_relu_{}".format(i))(x)
# The last layer
x = Conv2D(model_yaml["last_layer"][0],
model_yaml["last_layer"][1],
strides=tuple(model_yaml["stride"]),
padding=model_yaml["padding"],
name="g_final_conv",
activation=last_activ)(x)
model_gene = Model(img_input, x, name="Generator")
model_gene.summary()
return model_gene
def build_discriminator(self, model_yaml, is_training=True):
discri_input = Input(shape=tuple([256, 256, 12]), name="d_input")
if model_yaml["d_activation"] == "lrelu":
d_activation = lambda x: tf.nn.leaky_relu(
x, alpha=model_yaml["lrelu_alpha"])
else:
d_activation = model_yaml["d_activation"]
if model_yaml["add_discri_noise"]:
x = GaussianNoise(self.sigma_val,
input_shape=self.model_yaml["dim_gt_image"],
name="d_GaussianNoise")(discri_input)
else:
x = discri_input
for i, layer_index in enumerate(model_yaml["dict_discri_archi"]):
layer_val = model_yaml["dict_discri_archi"][layer_index]
layer_key = model_yaml["layer_key"]
layer_param = dict(zip(layer_key, layer_val))
pad = layer_param["padding"]
vpadding = tf.constant([[0, 0], [pad, pad], [pad, pad],
[0, 0]]) # the last dimension is 12
x = tf.pad(
x,
vpadding,
model_yaml["discri_opt_padding"],
name="{}_padding_{}".format(
model_yaml["discri_opt_padding"],
layer_index)) # the type of padding is defined the yaml,
# more infomration in https://www.tensorflow.org/api_docs/python/tf/pad
#
# x = ZeroPadding2D(
# padding=(layer_param["padding"], layer_param["padding"]), name="d_pad_{}".format(layer_index))(x)
x = Conv2D(layer_param["nfilter"],
layer_param["kernel"],
padding="valid",
activation=d_activation,
strides=(layer_param["stride"], layer_param["stride"]),
name="d_conv{}".format(layer_index))(x)
if i > 0:
x = BatchNormalization(momentum=model_yaml["bn_momentum"],
trainable=is_training,
name="d_bn{}".format(layer_index))(x)
# x = Flatten(name="flatten")(x)
# for i, dlayer_idx in enumerate(model_yaml["discri_dense_archi"]):
# dense_layer = model_yaml["discri_dense_archi"][dlayer_idx]
# x = Dense(dense_layer, activation=d_activation, name="dense_{}".format(dlayer_idx))(x)
if model_yaml["d_last_activ"] == "sigmoid":
x_final = tf.keras.layers.Activation('sigmoid',
name="d_last_activ")(x)
else:
x_final = x
model_discri = Model(discri_input, x_final, name="discriminator")
model_discri.summary()
return model_discri
def produce_noisy_input(self, input, sigma_val):
if self.model_yaml["add_discri_white_noise"]:
# print("[INFO] On each batch GT label we add Gaussian Noise before training discri on labelled image")
new_gt = GaussianNoise(sigma_val,
input_shape=self.model_yaml["dim_gt_image"],
name="d_inputGN")(input)
if self.model_yaml["add_relu_after_noise"]:
new_gt = tf.keras.layers.Activation(
lambda x: tf.keras.activations.tanh(x),
name="d_before_activ")(new_gt)
else:
new_gt = input
return new_gt
def define_callback(self):
# Define Tensorboard callbacks
self.g_tensorboard_callback = TensorBoard(
log_dir=self.saving_logs_path,
histogram_freq=0,
batch_size=self.batch_size,
write_graph=True,
write_grads=True)
self.g_tensorboard_callback.set_model(self.combined)
def train_gpu(self):
valid = np.ones(
(self.batch_size, 30, 30, 1)) # because of the shape of the discri
fake = np.zeros((self.batch_size, 30, 30, 1))
print("valid shape {}".format(valid.shape))
if self.previous_checkpoint is not None:
print("LOADING the model from step {}".format(
self.previous_checkpoint))
start_epoch = int(self.previous_checkpoint) + 1
self.load_from_checkpoint(self.previous_checkpoint)
else:
# create_safe_directory(self.saving_logs_path)
create_safe_directory(self.saving_image_path)
train_dataset = tf.data.Dataset.from_tensor_slices(
(self.data_X, self.data_y)).shuffle(self.batch_size).batch(
self.global_batch_size)
train_dist_dataset = self.strategy.experimental_distribute_dataset(
train_dataset)
def train(self):
# Adversarial ground truths
valid = np.ones((self.global_batch_size, 30, 30,
1)) # because of the shape of the discri
fake = np.zeros((self.global_batch_size, 30, 30, 1))
#print("valid shape {}".format(valid.shape))
if self.previous_checkpoint is not None:
print("LOADING the model from step {}".format(
self.previous_checkpoint))
start_epoch = int(self.previous_checkpoint) + 1
self.load_from_checkpoint(self.previous_checkpoint)
else:
# create_safe_directory(self.saving_logs_path)
create_safe_directory(self.saving_image_path)
start_epoch = 0
train_dataset = tf.data.Dataset.from_tensor_slices(
(self.data_X, self.data_y)).shuffle(self.batch_size).batch(
self.global_batch_size)
# loop for epoch
sigma_val = self.sigma_init
# dict_metric={"epoch":[],"d_loss_real":[],"d_loss_fake":[],"d_loss":[],"g_loss":[]}
d_loss_real = [100, 100] # init losses
d_loss_fake = [100, 100]
d_loss = [100, 100]
l_val_name_metrics, l_val_value_metrics = [], []
start_time = time.time()
for epoch in range(start_epoch, self.epoch):
# print("starting epoch {}".format(epoch))
for idx, (batch_input, batch_gt) in enumerate(train_dataset):
#print(batch_input)
## TRAIN THE DISCRIMINATOR
d_noise_real = random.uniform(
self.real_label_smoothing[0],
self.real_label_smoothing[1]) # Add noise on the loss
d_noise_fake = random.uniform(
self.fake_label_smoothing[0],
self.fake_label_smoothing[1]) # Add noise on the loss
# Create a noisy gt images
batch_new_gt = self.produce_noisy_input(batch_gt, sigma_val)
# Generate a batch of new images
# print("Make a prediction")
gen_imgs = self.generator.predict(
batch_input) # .astype(np.float32)
D_input_real = tf.concat([batch_new_gt, batch_input], axis=-1)
D_input_fake = tf.concat([gen_imgs, batch_input], axis=-1)
print("shape d train")
print(valid.shape, D_input_fake.shape)
d_loss_real = self.discriminator.train_on_batch(
D_input_real, d_noise_real * valid)
d_loss_fake = self.discriminator.train_on_batch(
D_input_fake, d_noise_fake * fake)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
g_loss = self.combined.train_on_batch(batch_input,
[valid, batch_gt])
# Plot the progress
print("%d iter %d [D loss: %f, acc.: %.2f%%] [G loss: %f %f]" %
(epoch, self.num_batches * epoch + idx, d_loss[0],
100 * d_loss[1], g_loss[0], g_loss[1]))
if epoch % self.im_saving_step == 0 and idx < self.max_im: # to save some generated_images
gen_imgs = self.generator.predict(batch_input)
save_images(gen_imgs, self.saving_image_path, ite=idx)
# LOGS to print in Tensorboard
if epoch % self.val_metric_step == 0:
l_val_name_metrics, l_val_value_metrics = self.val_metric()
name_val_metric = [
"val_{}".format(name) for name in l_val_name_metrics
]
name_logs = self.combined.metrics_names + [
"g_loss_tot", "d_loss_real", "d_loss_fake",
"d_loss_tot", "d_acc_real", "d_acc_fake", "d_acc_tot"
]
val_logs = g_loss + [
g_loss[0] + 100 * g_loss[1], d_loss_real[0],
d_loss_fake[0], d_loss[0], d_loss_real[1],
d_loss_fake[1], d_loss[1]
]
# The metrics
#print(type(batch_gt),type(gen_imgs))
l_name_metrics, l_value_metrics = compute_metric(
batch_gt.numpy(), gen_imgs)
assert len(val_logs) == len(
name_logs
), "The name and value list of logs does not have the same lenght {} vs {}".format(
name_logs, val_logs)
write_log_tf2(
self.model_writer, name_logs + l_name_metrics +
name_val_metric + ["time_in_sec"],
val_logs + l_value_metrics + l_val_value_metrics +
[start_time - time.time()], epoch)
if epoch % self.sigma_step == 0: # update simga
sigma_val = sigma_val * self.sigma_decay
# save the models
if epoch % self.w_saving_step == 0:
self.save_model(epoch)
def save_model(self, step):
print("Saving model at {} step {}".format(self.checkpoint_dir, step))
checkpoint_dir = self.checkpoint_dir
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
if not os.path.isfile("{}model_generator.yaml".format(
self.checkpoint_dir)):
gene_yaml = self.generator.to_yaml()
with open("{}model_generator.yaml".format(self.checkpoint_dir),
"w") as yaml_file:
yaml_file.write(gene_yaml)
if not os.path.isfile("{}model_combined.yaml".format(
self.checkpoint_dir)):
comb_yaml = self.combined.to_yaml()
with open("{}model_combined.yaml".format(self.checkpoint_dir),
"w") as yaml_file:
yaml_file.write(comb_yaml)
if not os.path.isfile("{}model_discri.yaml".format(
self.checkpoint_dir)):
discri_yaml = self.discriminator.to_yaml()
with open("{}model_discri.yaml".format(self.checkpoint_dir),
"w") as yaml_file:
yaml_file.write(discri_yaml)
self.generator.save_weights("{}model_gene_i{}.h5".format(
self.checkpoint_dir, step))
self.discriminator.save_weights("{}model_discri_i{}.h5".format(
self.checkpoint_dir, step))
self.combined.save_weights("{}model_combined_i{}.h5".format(
self.checkpoint_dir, step))
def load_from_checkpoint(self, step):
assert os.path.isfile("{}model_discri_i{}.h5".format(
self.checkpoint_dir,
step)), "No file at {}".format("{}model_discri_i{}.h5".format(
self.checkpoint_dir, step))
self.discriminator.load_weights("{}model_discri_i{}.h5".format(
self.checkpoint_dir, step))
self.generator.load_weights("{}model_gene_i{}.h5".format(
self.checkpoint_dir, step))
self.combined.load_weights("{}model_combined_i{}.h5".format(
self.checkpoint_dir, step))
def load_generator(self, path_yaml, path_weight):
# load YAML and create model
yaml_file = open(path_yaml, 'r')
loaded_model_yaml = yaml_file.read()
yaml_file.close()
loaded_model = model_from_yaml(loaded_model_yaml)
# load weights into new model
loaded_model.load_weights(path_weight)
print("Loaded model from disk")
return loaded_model
def val_metric(self):
test_dataset = tf.data.Dataset.from_tensor_slices(
(self.val_X, self.val_Y)).batch(self.val_X.shape[0])
#test_dist_dataset = self.strategy.experimental_distribute_dataset(test_dataset)
for i, (x, y) in enumerate(test_dataset):
#print("eval on {}".format(i))
val_pred = self.generator.predict(x)
#print("type {} {}".format(type(y),type(val_pred)))
label = y
return compute_metric(label.numpy(), val_pred)
def predict_on_iter(self,
batch,
path_save,
l_image_id=None,
un_rescale=True):
"""given an iter load the model at this iteration, returns the a predicted_batch but check if image have been saved at this directory
:param dataset:
:param batch could be a string : path to the dataset or an array corresponding to the batch we are going to predict on
"""
if type(batch) == type(
"u"
): # the param is an string we load the bathc from this directory
#print("We load our data from {}".format(batch))
l_image_id = find_image_indir(batch + XDIR, "npy")
batch, _ = load_data(batch,
x_shape=self.model_yaml["input_shape"],
label_shape=self.model_yaml["dim_gt_image"],
normalization=self.normalization,
dict_band_X=self.dict_band_X,
dict_band_label=self.dict_band_label,
dict_rescale_type=self.dict_rescale_type,
dict_scale=self.scale_dict_train,
fact_s2=self.fact_s2,
fact_s1=self.fact_s1,
s2_bands=self.s2bands,
s1_bands=self.s1bands,
clip_s2=False)
else:
if l_image_id is None:
print("We defined our own index for image name")
l_image_id = [i for i in range(batch.shape[0])]
assert len(l_image_id) == batch.shape[
0], "Wrong size of the name of the images is {} should be {} ".format(
len(l_image_id), batch.shape[0])
if os.path.isdir(path_save):
print(
"[INFO] the directory where to store the image already exists")
data_array, path_tile, _ = load_from_dir(
path_save, self.model_yaml["dim_gt_image"])
return data_array
else:
create_safe_directory(path_save)
batch_res = self.generator.predict(batch)
# if un_rescale: # remove the normalization made on the data
# _, batch_res, _ = rescale_array(batch, batch_res, dict_group_band_X=self.dict_band_X,
# dict_group_band_label=self.dict_band_label,
# dict_rescale_type=self.dict_rescale_type,
# dict_scale=self.scale_dict_train, invert=True, fact_scale2=self.fact_s2,
# fact_scale1=self.fact_s1,clip_s2=False)
assert batch_res.shape[0] == batch.shape[
0], "Wrong prediction should have shape {} but has shape {}".format(
batch_res.shape, batch.shape)
if path_save is not None:
# we store the data at path_save
for i in range(batch_res.shape[0]):
np.save(
"{}_image_{}".format(path_save,
l_image_id[i].split("/")[-1]),
batch_res[i, :, :, :])
return batch_res
class AGZeroModel:
def __init__(self,
N,
batch_size=32,
archive_fit_samples=64,
use_tpu=None,
log_path='logs/tensorboard'):
self.N = N
self.batch_size = batch_size
self.model = None
self.archive_fit_samples = archive_fit_samples
self.position_archive = []
self.tpu_grpc_url = use_tpu
tpu_name_environ_key = 'TPU_NAME'
# Check has server got TPU
if use_tpu is not False and tpu_name_environ_key in os.environ:
tpu_name = os.environ[tpu_name_environ_key].strip()
if tpu_name != "":
self.is_tpu = True
self.tpu_grpc_url = TPUClusterResolver(
tpu=[os.environ[tpu_name_environ_key]]).get_master()
# TODO write an if condition to validate and resolve the TPU url provided
self.__loss_functions = [
'categorical_crossentropy', 'binary_crossentropy'
]
self.model_name = time.strftime('GM{0}-%y%m%dT%H%M%S').format('%02d' %
N)
# print(self.model_name)
log_path = os.path.join(log_path, self.model_name)
if not os.path.exists(log_path):
os.makedirs(log_path)
self.callback = TensorBoard(log_path)
def create(self):
bn_axis = 3
N = self.N
position = Input((N, N, 6))
resnet = ResNet(n_stages=N)
resnet.create(N, N, 6)
x = resnet.model(position)
dist = Conv2D(2, (1, 1))(x)
dist = BatchNormalization(axis=bn_axis)(dist)
dist = Activation('relu')(dist)
dist = Flatten()(dist)
dist = Dense(N * N + 1, activation='softmax',
name='distribution')(dist)
res = Conv2D(1, (1, 1))(x)
res = BatchNormalization(axis=bn_axis)(res)
res = Activation('relu')(res)
res = Flatten()(res)
res = Dense(256, activation='relu')(res)
res = Dense(1, activation='sigmoid', name='result')(res)
self.model = Model(position, [dist, res])
self.model.compile(Adam(lr=2e-2), self.__loss_functions)
self.callback.set_model(self.model)
# check if TPU available
if self.tpu_grpc_url is not None:
self.model = tf.contrib.tpu.keras_to_tpu_model(
self.model,
strategy=tf.contrib.tpu.TPUDistributionStrategy(
tf.contrib.cluster_resolver.TPUClusterResolver(
self.tpu_grpc_url)))
self.model.summary()
def fit_game(self, X_positions, result):
X_posres = []
for pos, dist in X_positions:
X_posres.append((pos, dist, result))
result = -result
if len(self.position_archive) >= self.archive_fit_samples:
archive_samples = random.sample(self.position_archive,
self.archive_fit_samples)
else:
# initial case
archive_samples = self.position_archive
self.position_archive.extend(X_posres)
# I'm going to some lengths to avoid the potentially overloaded + operator
X_fit_samples = list(itertools.chain(X_posres, archive_samples))
self.__fit_model(X_fit_samples, self.batch_size)
def retrain_position_archive(self, batch_size=None):
self.__fit_model(self.position_archive,
batch_size if batch_size else self.batch_size * 8)
def reduce_position_archive(self, ratio=0.5):
try:
self.position_archive = random.sample(
self.position_archive, int(len(self.position_archive) * ratio))
except:
pass
def __fit_model(self, X_fit_samples, batch_size):
batch_no = 1
X, y_dist, y_res = [], [], []
X_shuffled = random.sample(X_fit_samples, len(X_fit_samples))
X_shuffled = create_batches(X_shuffled, batch_size)
for batch in X_shuffled:
for pos, dist, res in batch:
X.append(pos)
y_dist.append(dist)
y_res.append(float(res) / 2 + 0.5)
logs = self.model.train_on_batch(
np.array(X),
[np.array(y_dist), np.array(y_res)])
self.write_log(self.__loss_functions, logs, batch_no)
batch_no += 1
X, y_dist, y_res = [], [], []
def write_log(self, names, logs, batch_no):
for name, value in zip(names, logs):
summary = tf.Summary()
summary_value = summary.value.add()
summary_value.simple_value = value
summary_value.tag = name
self.callback.writer.add_summary(summary, batch_no)
self.callback.writer.flush()
def predict(self, X_positions):
dist, res = self.model.predict(X_positions)
res = np.array([r[0] * 2 - 1 for r in res])
return [dist, res]
def save(self, snapshot_id, save_archive=False):
self.model.save_weights('%s.weights.h5' % (snapshot_id, ))
if save_archive:
joblib.dump(self.position_archive,
'%s.archive.joblib' % (snapshot_id, ),
compress=5)
def load(self, snapshot_id):
self.model.load_weights('%s.weights.h5' % (snapshot_id, ))
pos_fname = '%s.archive.joblib' % (snapshot_id, )
try:
self.position_archive = joblib.load(pos_fname)
except:
print('Warning: Could not load position archive %s' %
(pos_fname, ))
def unload_pos_archive(self):
self.position_archive = []
def load_pos_archive(self, archive_file):
try:
print('Attempting to load position archive %s' % (archive_file, ))
self.position_archive = joblib.load(archive_file)
print('Successfully loaded position archive %s' % (archive_file, ))
return True
except:
import traceback
traceback.print_exc()
print('Warning: Could not load position archive %s' %
(archive_file, ))
return False
def load_averaged(self, weights, log=None):
new_weights = []
loaded_weights = []
for weight in weights:
self.model.load_weights(weight)
loaded_weights.append(self.model.get_weights())
print("Read weight: {0}".format(weight))
if log is not None:
log("Read weight: {0}".format(weight), self.model_name)
if len(loaded_weights) > 0:
for weights_list_tuple in zip(*loaded_weights):
new_weights.append([
np.array(weights_).mean(axis=0)
for weights_ in zip(*weights_list_tuple)
])
self.model.set_weights(new_weights)
else:
print(
"No weights to load. Initializing the model with random weights!"
)
class NeuralNetworkLayout(object):
def __init__(self,gtrate,dtrate,gbeta1,dbeta1,dimensions,noise,echeck,overrides,layoutDir='layouts',fileDir='out'):
self.dir_output=fileDir
self.gan_training_rate=gtrate
self.d_training_rate=dtrate
self.gan_beta1=gbeta1
self.d_beta1=dbeta1
self.dimensionality=dimensions
self.noise=noise
self.epochCheck=echeck
self.logdir=self.dir_output+'/model/logs/'
self.layout_dir=layoutDir
self.overrides=overrides
self.d=None
self.g=None
self.gan=None
def compileComponent(self,verbose=False,component_type=None):
if(component_type=='discriminator'):
stream = open(self.layout_dir+"/discriminator_layout.yaml","r+")
elif(component_type=='generator'):
stream = open(self.layout_dir+"/generator_layout.yaml","r+")
data = yaml.load(stream, Loader=yaml.FullLoader)
g1_instructions= data['join'][0]
g2_instructions= data['join'][1]
gc_instructions = data['layers']
if(component_type=='discriminator'):
g_in_1 = Input((1,))
elif(component_type=='generator'):
g_in_1 = Input((self.noise,))
g_in_2=[]
for i in range(self.dimensionality):
g_in_2.append(Input((1,)))
g1 = (g_in_1)
for layer_info in g1_instructions['layers']:
if(self.overrides.get('activation')):
activation = self.overrides.get('activation')
else:
activation = layer_info.get('activation')
if(component_type=='discriminator' and self.overrides.get('d_nodes')):
units = self.overrides.get('d_nodes')
elif(component_type=='generator' and self.overrides.get('g_nodes')):
units = self.overrides.get('g_nodes')
else:
units = layer_info.get('nodes')
if(self.overrides.get('dropout_amount')):
rate = self.overrides.get('dropout_amount')
else:
rate = layer_info.get('dropout_amount')
if(self.overrides.get('leaky_amount')):
alpha = self.overrides.get('leaky_amount')
else:
alpha = layer_info.get('leaky_amount')
if(layer_info['layer_type']=='dense'):
g1=Dense(units=units,activation=activation)(g1)
elif(layer_info['layer_type']=='dropout'):
g1=Dropout(rate=rate)(g1)
elif(layer_info['layer_type']=='selu'):
g1=LeakyReLU(alpha=alpha)(g1)
elif(layer_info['layer_type']=='batchnorm'):
g1=BatchNormalization()(g1)
g2=[]
for i in range(self.dimensionality):
g2_current = (g_in_2[i])
for layer_info in g2_instructions['layers']:
if(self.overrides.get('activation')):
activation = self.overrides.get('activation')
else:
activation = layer_info.get('activation')
if(component_type=='discriminator' and self.overrides.get('d_nodes')):
units = self.overrides.get('d_nodes')
elif(component_type=='generator' and self.overrides.get('g_nodes')):
units = self.overrides.get('g_nodes')
else:
units = layer_info.get('nodes')
if(self.overrides.get('dropout_amount')):
rate = self.overrides.get('dropout_amount')
else:
rate = layer_info.get('dropout_amount')
if(self.overrides.get('leaky_amount')):
alpha = self.overrides.get('leaky_amount')
else:
alpha = layer_info.get('leaky_amount')
if(layer_info['layer_type']=='dense'):
g2_current=Dense(units=units,activation=activation)(g2_current)
elif(layer_info['layer_type']=='dropout'):
g2_current=Dropout(rate=rate)(g2_current)
elif(layer_info['layer_type']=='selu'):
g2_current=LeakyReLU(alpha=alpha)(g2_current)
elif(layer_info['layer_type']=='batchnorm'):
g2_current=BatchNormalization()(g2_current)
g2.append(g2_current)
gc = Concatenate()([g1]+g2)
for layer_info in gc_instructions:
if(self.overrides.get('activation')):
activation = self.overrides.get('activation')
else:
activation = layer_info.get('activation')
if(component_type=='discriminator' and self.overrides.get('d_nodes')):
units = self.overrides.get('d_nodes')
elif(component_type=='generator' and self.overrides.get('g_nodes')):
units = self.overrides.get('g_nodes')
else:
units = layer_info.get('nodes')
if(self.overrides.get('dropout_amount')):
rate = self.overrides.get('dropout_amount')
else:
rate = layer_info.get('dropout_amount')
if(self.overrides.get('leaky_amount')):
alpha = self.overrides.get('leaky_amount')
else:
alpha = layer_info.get('leaky_amount')
if(layer_info['layer_type']=='dense'):
gc=Dense(units=units,activation=activation)(gc)
elif(layer_info['layer_type']=='dropout'):
gc=Dropout(rate=rate)(gc)
elif(layer_info['layer_type']=='selu'):
gc=LeakyReLU(alpha=alpha)(gc)
elif(layer_info['layer_type']=='batchnorm'):
gc=BatchNormalization()(gc)
if(component_type=='generator'):
gc = Dense(1, activation="sigmoid")(gc)
gc = Model(name="Generator", inputs=[g_in_1]+g_in_2, outputs=[gc])
elif(component_type=='discriminator'):
gc = Dense(2, activation="softmax")(gc)
gc = Model(name="Discriminator", inputs=[g_in_1]+g_in_2, outputs=[gc])
gc.compile(loss="categorical_crossentropy", optimizer=Adam(self.d_training_rate, beta_1=self.d_beta1), metrics=["accuracy"])
if(verbose):
gc.summary()
return gc
def compileGAN(self,verbose=False):
self.d=self.compileComponent(verbose,component_type='discriminator')
self.g=self.compileComponent(verbose,component_type='generator')
hyper_in=[]
for i in range(self.dimensionality):
hyper_in.append(Input((1,)))
noise_in = Input((self.noise,))
inputs=[noise_in]+hyper_in
inputs2=[self.g(inputs)]+hyper_in
gan_out = self.d(inputs2)
gan = Model(inputs, gan_out, name="GAN")
self.d.trainable = False
gan.compile(loss="categorical_crossentropy", optimizer=Adam(self.gan_training_rate, beta_1=self.gan_beta1), metrics=["accuracy"])
if(verbose):
gan.summary()
self.gan=gan
def visualiseData(self,save=True):
pass
def saveHyperParameters(self):
if not exists(self.dir_output+'/layouts/'):
makedirs(self.dir_output+'/layouts/')
copy_tree('./layouts/', self.dir_output+'/layouts/')
if not exists(self.dir_output+'/model/'):
makedirs(self.dir_output+'/model/')
makedirs(self.dir_output+'/model/logs/')
self.tensorboard = TensorBoard(
log_dir=self.logdir,
histogram_freq=self.epochCheck,
batch_size=self.noise,
write_graph=True,
write_grads=True
)
self.tensorboard.set_model(self.gan)
with open(self.dir_output+'/model/hyperparameters.gan4ds',"w") as f:
allParams=[a for a in dir(self) if not a.startswith('__') and not callable(getattr(self, a))]
for param in allParams:
f.writelines(param+"\t"+str(getattr(self,param))+"\n")
class OperatorNetwork:
def __init__(self, x_batch_size, mask_batch_size, tensorboard_logs_dir="", add_mopt_perf_metric=True,
use_early_stopping=True):
self.batch_size = mask_batch_size * x_batch_size
self.mask_batch_size = mask_batch_size
self.x_batch_size = x_batch_size
self.losses_per_sample = None
# self.losses_per_sample = []
self.tr_loss_history = []
self.te_loss_history = []
self.tf_logs = tensorboard_logs_dir
self.epoch_counter = 0
self.add_mopt_perf_metric = add_mopt_perf_metric
self.useEarlyStopping = use_early_stopping
def create_dense_model(self, input_shape, dense_arch, last_activation="linear"):
self.x_shape = input_shape
self.y_shape = dense_arch[-1]
input_data_layer = Input(shape=input_shape)
x = Flatten()(input_data_layer)
input_mask_layer = Input(shape=input_shape)
mask = Flatten()(input_mask_layer)
# x = K.concatenate([x,mask])
x = tf.keras.layers.Concatenate(axis=1)([x, mask])
for units in dense_arch[:-1]:
x = Dense(units, activation="sigmoid")(x)
x = Dense(dense_arch[-1], activation=last_activation)(x)
self.model = Model(inputs=[input_data_layer, input_mask_layer], outputs=x)
print("Object network model built:")
#self.model.summary()
def create_1ch_conv_model(self, input_shape, image_shape, filter_sizes, kernel_sizes, dense_arch, padding,
last_activation="softmax"): # only for grayscale
self.x_shape = input_shape
self.y_shape = dense_arch[-1]
input_data_layer = Input(shape=input_shape)
in1 = Reshape(target_shape=(1,) + image_shape)(input_data_layer)
input_mask_layer = Input(shape=input_shape)
in2 = Reshape(target_shape=(1,) + image_shape)(input_mask_layer)
x = tf.keras.layers.Concatenate(axis=1)([in1, in2])
for i in range(len(filter_sizes)):
x = Conv2D(filters=filter_sizes[i], kernel_size=kernel_sizes[i], data_format="channels_first",
activation="relu", padding=padding)(x)
x = MaxPool2D(pool_size=(2, 2), padding=padding, data_format="channels_first")(x)
x = Flatten()(x)
for units in dense_arch[:-1]:
x = Dense(units, activation="relu")(x)
x = Dense(dense_arch[-1], activation=last_activation)(x)
self.model = Model(inputs=[input_data_layer, input_mask_layer], outputs=x)
print("Object network model built:")
#self.model.summary()
def create_2ch_conv_model(self, input_shape, image_shape, filter_sizes, kernel_sizes, dense_arch, padding,
last_activation="softmax"): # only for grayscale
self.x_shape = input_shape
self.y_shape = dense_arch[-1]
input_data_layer = Input(shape=input_shape)
ch_data = Reshape(target_shape=(1,) + image_shape)(input_data_layer)
input_mask_layer = Input(shape=input_shape)
ch_mask = Reshape(target_shape=(1,) + image_shape)(input_mask_layer)
for i in range(len(filter_sizes)):
ch_data = Conv2D(filters=filter_sizes[i], kernel_size=kernel_sizes[i], data_format="channels_last",
activation="relu", padding=padding)(ch_data)
ch_data = MaxPool2D(pool_size=(2, 2), padding=padding, data_format="channels_last")(ch_data)
ch_mask = Conv2D(filters=filter_sizes[i], kernel_size=kernel_sizes[i], data_format="channels_last",
activation="relu", padding=padding)(ch_mask)
ch_mask = MaxPool2D(pool_size=(2, 2), padding=padding, data_format="channels_last")(ch_mask)
ch_mask = Flatten()(ch_mask)
ch_data = Flatten()(ch_data)
x = tf.keras.layers.Concatenate(axis=1)([ch_mask, ch_data])
for units in dense_arch[:-1]:
x = Dense(units, activation="relu")(x)
x = Dense(dense_arch[-1], activation=last_activation)(x)
self.model = Model(inputs=[input_data_layer, input_mask_layer], outputs=x)
print("Object network model built:")
#self.model.summary()
def create_batch(self, x, masks, y):
"""
x = [[1,2],[3,4]] -> [[1,2],[1,2],[1,2],[3,4],[3,4],[3,4]]
masks = [[0,0],[1,0],[1,1]] -> [[0,0],[1,0],[1,1],[0,0],[1,0],[1,1]]
y = [1,3] -> [1 ,1 ,1 ,3 ,3 ,3 ]
"""
# assert len(masks) == self.mask_size
x_prim = np.repeat(x, len(masks), axis=0)
y_prim = np.repeat(y, len(masks), axis=0)
masks_prim = np.tile(masks, (len(x), 1))
x_prim *= masks_prim # MASKING
# assert len(x_prim) == self.batch_size
return x_prim, masks_prim, y_prim
def named_logs(self, model, logs, mode="train"):
result = {}
try:
iterator = iter(logs)
except TypeError:
logs = [logs]
metricNames = (mode + "_" + i for i in model.metrics_names)
for l in zip(metricNames, logs):
result[l[0]] = l[1]
return result
def compile_model(self, loss_per_sample, combine_losses, combine_mask_losses, metrics=None):
self.mask_loss_combine_function = combine_mask_losses
if self.add_mopt_perf_metric is True:
if metrics is None:
metrics = [self.get_mopt_perf_metric()]
else:
metrics.append(self.get_mopt_perf_metric())
def logging_loss_function(y_true, y_pred):
losses = loss_per_sample(y_true, y_pred)
self.losses_per_sample = losses
return combine_losses(losses)
self.model.compile(loss=logging_loss_function, optimizer='nadam', metrics=metrics, run_eagerly=True)
if self.tf_logs != "":
log_path = './logs'
self.tb_clbk = TensorBoard(self.tf_logs)
self.tb_clbk.set_model(self.model)
def get_per_mask_loss(self, used_target_shape=None):
if used_target_shape is None:
used_target_shape = (self.x_batch_size, self.mask_batch_size)
losses = tf.reshape(self.losses_per_sample, used_target_shape)
# losses = np.apply_along_axis(self.mask_loss_combine_function,0,losses)
losses = self.mask_loss_combine_function(losses)
return losses
def get_per_mask_loss_with_custom_batch(self, losses, new_x_batch_size, new_mask_batch_size):
losses = np.reshape(losses, newshape=(new_x_batch_size, new_mask_batch_size))
losses = np.apply_along_axis(self.mask_loss_combine_function, 0, losses)
return losses
def train_one(self, x, masks, y):
x_prim, masks_prim, y_prim = self.create_batch(x, masks, y)
curr_loss = self.model.train_on_batch(x=[x_prim, masks_prim], y=y_prim)
self.tr_loss_history.append(curr_loss)
self.epoch_counter += 1
if self.tf_logs != "":
self.tb_clbk.on_epoch_end(self.epoch_counter, self.named_logs(self.model, curr_loss))
return x_prim, masks_prim, y_prim
def validate_one(self, x, masks, y):
x_prim, masks_prim, y_prim = self.create_batch(x, masks, y)
# print("ON: x: "+str(x_prim))
# print("ON: m: "+str(masks_prim))
# print("ON: y_true: "+str(y_prim))
curr_loss = self.model.test_on_batch(x=[x_prim, masks_prim], y=y_prim)
self.te_loss_history.append(curr_loss)
if self.tf_logs != "":
self.tb_clbk.on_epoch_end(self.epoch_counter, self.named_logs(self.model, curr_loss, "val"))
if self.useEarlyStopping is True:
self.check_ES()
# print("ON y_pred:" +str(np.squeeze(y_pred)))
# print("ON loss per sample:" +str(np.squeeze(self.losses_per_sample.numpy())))
return x_prim, masks_prim, y_prim, self.losses_per_sample.numpy()
def test_one(self, x, masks, y):
x_prim, masks_prim, y_prim = self.create_batch(x, masks, y)
curr_loss = self.model.test_on_batch(x=[x_prim, masks_prim], y=y_prim)
self.te_loss_history.append(curr_loss)
return curr_loss
def get_mopt_perf_metric(self):
# used_target_shape = (self.x_batch_size,self.mask_batch_size)
def m_opt_loss(y_pred, y_true):
if (self.losses_per_sample.shape[0] % self.mask_batch_size != 0): # when testing happens, not used anymore
return 0.0
else: # for training and validation batches
losses = tf.reshape(self.losses_per_sample, (-1, self.mask_batch_size))
self.last_m_opt_perf = np.mean(losses[:, int(0.5 * self.mask_batch_size)])
return self.last_m_opt_perf
return m_opt_loss
def set_early_stopping_params(self, starting_epoch, patience_batches=800, minimize=True):
self.ES_patience = patience_batches
self.ES_minimize = minimize
if (minimize is True):
self.ES_best_perf = 1000000.0
else:
self.ES_best_perf = -1000000.0
self.ES_best_epoch = starting_epoch
self.ES_stop_training = False
self.ES_start_epoch = starting_epoch
self.ES_best_weights = None
return
def check_ES(self, ):
if self.epoch_counter >= self.ES_start_epoch:
if self.ES_minimize is True:
if self.last_m_opt_perf < self.ES_best_perf:
self.ES_best_perf = self.last_m_opt_perf
self.ES_best_epoch = self.epoch_counter
self.ES_best_weights = self.model.get_weights()
else:
if self.last_m_opt_perf > self.ES_best_perf:
self.ES_best_perf = self.last_m_opt_perf
self.ES_best_epoch = self.epoch_counter
self.ES_best_weights = self.model.get_weights()
# print("ES patience left: "+str(self.epoch_counter-self.ES_best_epoch))
if (self.epoch_counter - self.ES_best_epoch > self.ES_patience):
self.ES_stop_training = True
class AngleEstimator(Model):
def __init__(self,
number_sides: int = 2,
number_epochs: int = 1000,
batch_size: int = 32,
optimizer: Optimizer = None,
tensorboard_dir: Path = None):
super().__init__()
if optimizer is None:
optimizer = Adam(learning_rate=tf.Variable(0.1, dtype=tf.float64),
beta_1=tf.Variable(0.9, dtype=tf.float64),
beta_2=tf.Variable(0.999, dtype=tf.float64),
epsilon=tf.Variable(1e-7, dtype=tf.float64))
optimizer.iterations
self.optimizer = optimizer
self.number_sides = number_sides
self.input_layer = Dense(2,
activation=linear,
use_bias=False,
dtype=tf.float64)
self.hidden_layer = Dense(number_sides + 10,
activation=linear,
use_bias=False,
dtype=tf.float64)
self.output_layer = Dense(1,
activation=linear,
use_bias=False,
dtype=tf.float64)
self.batch_size = batch_size
self.number_epochs = number_epochs
self.tensorboard_dir = tensorboard_dir
self.build((None, 2))
self._setup_tensorboard_callback()
def _setup_tensorboard_callback(self):
if self.tensorboard_dir is not None:
self.tensorboard_callback = TensorBoard(self.tensorboard_dir)
self.tensorboard_callback.set_model(self)
def call(self, inputs):
x = self.input_layer(inputs)
x = self.hidden_layer(x)
y = self.output_layer(x)
return y
def _train_step(self, inputs, output):
with tf.GradientTape() as g:
z = self.call(inputs)
loss = mse(output, z)
gradients = g.gradient(loss, self.trainable_variables)
self.optimizer.apply_gradients(zip(gradients,
self.trainable_variables))
return tf.reduce_sum(loss)
def get_side_angles(self, x_y, z):
data = np.concatenate((x_y, np.reshape(z, (-1, 1))), axis=1)
transformer = MinMaxTransformer(column_wise=False)
transformer.transform(data)
x_y, z = data[:, 0:2], data[:, 2]
input_splits, output_splits = create_train_test_splits(x_y,
z,
ratio=0.0)
buffer_size = input_splits[0].shape[0]
dataset = Dataset.from_tensor_slices(
(input_splits[0],
output_splits[0])).shuffle(buffer_size=buffer_size)
dataset = dataset.batch(self.batch_size)
steps_per_epoch = max(buffer_size // self.batch_size, 1)
self._train(dataset=dataset, steps_per_epoch=steps_per_epoch)
def _train(self, dataset, steps_per_epoch):
for epoch in range(self.number_epochs):
for (batch, (input,
target)) in enumerate(dataset.take(steps_per_epoch)):
loss = self._train_step(input, target)
print("Epoch: {} Batch: {} Loss: {}".format(
epoch, batch, loss))
def plot_3d_prediction_plot(self, data, plane_steps: int = 50):
transformer = MinMaxTransformer(column_wise=False)
transformer.transform(data)
plane_input = self._get_prediction_array(x_range=(np.min(data[:, 0]),
np.max(data[:, 0])),
y_range=(np.min(data[:, 1]),
np.max(data[:, 1])),
steps=plane_steps)
z_hat = self.call(plane_input).numpy()
plane_data = np.concatenate((plane_input, z_hat), axis=1)
return get_3d_scatter_with_prediction_planes(data, plane_data)
def _get_prediction_array(self,
x_range: Tuple,
y_range: Tuple,
steps: int = 50):
prediction_array = np.zeros((steps**2, 2))
x_stepsize = self._get_step_size(x_range, steps)
y_stepsize = self._get_step_size(y_range, steps)
x = x_range[0]
k = 0
for i in range(steps):
y = y_range[0]
for j in range(steps):
prediction_array[k, 0] = x
prediction_array[k, 1] = y
k += 1
y += y_stepsize
x += x_stepsize
return prediction_array
@staticmethod
def _get_step_size(axis_range: Tuple, steps: int):
return (axis_range[1] - axis_range[0]) / steps
def train(self, z_noise=None):
start_time = time.time()
# log writer
logdir = self.log_dir + '/' + self.model_dir
tensorboard_callback = TensorBoard(log_dir=logdir)
tensorboard_callback.set_model(self.gan)
scalar_names = ['d_loss', 'g_loss']
for epoch in range(self.start_epoch, self.epoch):
# get batch data
for idx in range(self.start_batch_id, self.iteration):
# train generator
noise = np.random.normal(0, 1, (self.batch_size, self.z_dim))
y_gen = np.ones(self.batch_size)
g_loss = self.gan.train_on_batch(noise, y_gen)
# train discriminator
half_batch = self.batch_size // 2
real_x, _ = next(self.train_generator)
while real_x.shape[0] != half_batch:
real_x, _ = next(self.train_generator)
noise = np.random.normal(0, 1, (half_batch, self.z_dim))
fake_x = self.generator.predict(noise)
real_y = np.ones(half_batch)
real_y[:] = 0.9
fake_y = np.zeros(half_batch)
d_loss_real = self.discriminator.train_on_batch(real_x, real_y)
d_loss_fake = self.discriminator.train_on_batch(fake_x, fake_y)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
# write summary
# self.write_log(tensorboard_callback, scalar_names, [d_loss, g_loss], self.counter)
# display training status
self.counter += 1
print("Epoch: [%2d] [%5d/%5d] time: %4.4f, d_loss: %.8f, g_loss: %.8f" \
% (epoch, idx, self.iteration, time.time() - start_time, d_loss, g_loss))
# save training results for every n steps
if type(z_noise) is np.ndarray:
sample_num = z_noise.shape[0]
else:
sample_num = self.sample_num
z_noise = np.random.normal(0, 1, (sample_num, self.z_dim))
if np.mod(idx+1, self.print_freq) == 0:
sample_imgs = self.generator.predict(z_noise)
manifold = int(np.ceil(np.sqrt(sample_num)))
save_images_plt(sample_imgs, [manifold, manifold],
f'{self.sample_dir}/{self.model_name}_train_{epoch:02d}_{idx+1:05d}', mode='sample')
if np.mod(idx+1, self.save_freq) == 0:
self.save(self.checkpoint_dir, self.counter)
# After an epoch, start_batch_id is set to zero
# non-zero value is only for the first epoch after loading pre-trained model
self.start_batch_id = 0
# save model
self.save(self.checkpoint_dir, self.counter)
# save model for the final step
self.save(self.checkpoint_dir, self.counter)
earlystop.set_model(model)
earlystop.on_train_begin()
modelcheckpoint = ModelCheckpoint(filepath="weights/",
monitor="val_loss",
verbose=1,
save_best_only=True)
modelcheckpoint.set_model(model)
modelcheckpoint.on_train_begin()
reduce_lr = ReduceLROnPlateau(monitor="val_loss", patience=10, verbose=1)
reduce_lr.set_model(model)
reduce_lr.on_train_begin()
tensorboard = TensorBoard(log_dir="logs/")
tensorboard.set_model(model)
tensorboard.on_train_begin()
epochs = 3
train_logs_dict = {}
test_logs_dict = {}
for epoch in range(epochs):
training_acc, testing_acc, training_loss, testing_loss = [], [], [], []
print("\nStart of epoch %d" % (epoch + 1, ))
# Iterate over the batches of the dataset.
modelcheckpoint.on_epoch_begin(epoch)
earlystop.on_epoch_begin(epoch)
reduce_lr.on_epoch_begin(epoch)
tensorboard.on_epoch_begin(epoch)
for x_batch_train, y_batch_train in get_batch(batch_size, x_train,
y_train):
class GAN_train(object):
def __init__(self, dataset_common_folder_path, dataset):
self.dataset_folder_path = dataset_common_folder_path + dataset + "_noglasses"
self.dataset_path_images = self.dataset_folder_path + '/natural/face_before_inpainting'
self.dataset_path_GAN = self.dataset_folder_path + '/natural/GAN'
self.dataset_path_GAN_model = self.dataset_path_GAN + '/model'
self.dataset_path_GAN_samples = self.dataset_path_GAN + '/samples'
if not os.path.exists(self.dataset_path_GAN):
os.mkdir(self.dataset_path_GAN)
if not os.path.exists(self.dataset_path_GAN_model):
os.mkdir(self.dataset_path_GAN_model)
if not os.path.exists(self.dataset_path_GAN_samples):
os.mkdir(self.dataset_path_GAN_samples)
print(self.dataset_path_images)
# Training image setting
self.data = glob(os.path.join(self.dataset_path_images, 'face_*.png'))
self.num_total_data = len(self.data)
print(self.num_total_data)
sample_idx = np.random.randint(0, self.num_total_data, size=1)
self.x_train = PRL_data_image_load(self.data, sample_idx=sample_idx)
print(self.x_train.shape)
self.dataset = dataset
self.img_rows = self.x_train.shape[1]
self.img_cols = self.x_train.shape[2]
self.channel = self.x_train.shape[3]
# For GAN
self.noise_dim = 100
self.GAN = LSGAN_Model(self.img_rows, self.img_cols, self.channel,
self.noise_dim, dataset)
self.discriminator = self.GAN.discriminator()
self.generator = self.GAN.generator()
self.discriminator_cost = self.GAN.discriminator_model(
self.discriminator)
self.adversarial_cost = self.GAN.adversarial_model(
self.generator, self.discriminator)
# For tensorboard callbacks
now = datetime.now()
log_path = "tensorboard/GAN_" + now.strftime("%Y%m%d-%H%M%S")
self.callback = TensorBoard(log_path)
self.callback.set_model(self.adversarial_cost)
self.train_names = ['d_loss_real', 'd_loss_fake', 'd_loss', 'a_loss']
def train(self, num_epoch=2000, batch_size=256, save_interval=0):
# Initial Update Data Generation --------------------------------------
sample_noise_input = None
if save_interval > 0:
# sample_noise_input = np.random.normal(0.0, 1.0, size=[36, self.noise_dim])
sample_noise_input = np.random.uniform(-1.0,
1.0,
size=[36, self.noise_dim])
sample_idx = np.random.permutation(self.num_total_data)
sample_idx = sample_idx[0:batch_size]
images_train = PRL_data_image_load(self.data, sample_idx=sample_idx)
# noise = np.random.normal(0.0, 1.0, size=[batch_size, self.noise_dim])
noise = np.random.uniform(-1.0, 1.0, size=[batch_size, self.noise_dim])
images_fake = self.generator.predict(noise)
# Initial Update Discriminator
set_trainability(self.discriminator, True)
d_loss_real = self.discriminator_cost.train_on_batch(
images_train, np.ones([batch_size, 1]))
d_loss_fake = self.discriminator_cost.train_on_batch(
images_fake, np.zeros([batch_size, 1]))
d_loss = d_loss_real + d_loss_fake
# TRAINING STEPS ------------------------------------------------------
print('========= Main LSGAN Training ==========')
num_batch = self.num_total_data // batch_size
for e in range(num_epoch):
shuffled_sample_idx = np.random.permutation(self.num_total_data)
for b in tqdm(range(num_batch)):
batch_sample_idx = shuffled_sample_idx[b * batch_size:(b + 1) *
batch_size]
images_train = PRL_data_image_load(self.data,
sample_idx=batch_sample_idx)
# noise = np.random.normal(0.0, 1.0, size=[batch_size, self.noise_dim])
noise = np.random.uniform(-1.0,
1.0,
size=[batch_size, self.noise_dim])
images_fake = self.generator.predict(noise)
# Update Discriminator
set_trainability(self.discriminator, True)
d_loss_real = self.discriminator_cost.train_on_batch(
images_train, np.ones([batch_size, 1]))
d_loss_fake = self.discriminator_cost.train_on_batch(
images_fake, np.zeros([batch_size, 1]))
d_loss = d_loss_real + d_loss_fake
# Update Generator
y = np.ones([batch_size, 1])
# noise = np.random.normal(0.0, 1.0, size=[batch_size, self.noise_dim])
noise = np.random.uniform(-1.0,
1.0,
size=[batch_size, self.noise_dim])
set_trainability(self.discriminator, False)
a_loss = self.adversarial_cost.train_on_batch(noise, y)
# noise = np.random.normal(0.0, 1.0, size=[batch_size, self.noise_dim])
noise = np.random.uniform(-1.0,
1.0,
size=[batch_size, self.noise_dim])
set_trainability(self.discriminator, False)
a_loss = self.adversarial_cost.train_on_batch(noise, y)
# Log messages
write_log(self.callback, 'D_Loss', d_loss, e * num_batch + b)
write_log(self.callback, 'A_Loss', a_loss, e * num_batch + b)
if save_interval > 0:
if (e + 1) % save_interval == 0:
log_mesg = "Epoch %d [D loss real: %f, acc real: %f]" % (
e + 1, d_loss_real[0], d_loss_real[1])
log_mesg = "%s: [D loss fake: %f, acc fake: %f]" % (
log_mesg, d_loss_fake[0], d_loss_fake[1])
log_mesg = "%s: [D loss: %f, acc: %f]" % (
log_mesg, d_loss[0], d_loss[1])
log_mesg = "%s [A loss: %f, acc: %f]" % (
log_mesg, a_loss[0], a_loss[1])
print(log_mesg)
GAN_plot_images(generator=self.generator,
x_train=self.x_train,
dataset=self.dataset,
save2file=True,
samples=sample_noise_input.shape[0],
noise=sample_noise_input,
step=(e + 1),
folder_path=self.dataset_path_GAN_samples)
GAN_plot_images(generator=self.generator,
x_train=self.x_train,
dataset=self.dataset,
save2file=False,
samples=sample_noise_input.shape[0],
noise=sample_noise_input,
step=(e + 1))
# Save trained models
self.adversarial_cost.save(
self.dataset_path_GAN_model + "/GAN_" + str(e + 1) + "_" +
self.dataset + "_forganECCV_adversarial_model_uniform.h5")
self.discriminator.save(self.dataset_path_GAN_model + "/GAN_" +
str(e + 1) + "_" + self.dataset +
"_forganECCV_discriminator_uniform.h5")
self.generator.save(self.dataset_path_GAN_model + "/GAN_" +
str(e + 1) + "_" + self.dataset +
"_forganECCV_generator_uniform.h5")
class Agent(Portfolio):
def __init__(self, state_dim, balance, is_eval=False, model_name=""):
super().__init__(balance=balance)
self.model_type = 'DQN'
self.state_dim = state_dim
self.action_dim = 3 # hold, buy, sell
self.memory = deque(maxlen=100)
self.buffer_size = 60
self.gamma = 0.95
self.epsilon = 1.0 # initial exploration rate
self.epsilon_min = 0.01 # minimum exploration rate
self.epsilon_decay = 0.995 # decrease exploration rate as the agent becomes good at trading
self.is_eval = is_eval
self.model = load_model('saved_models/{}.h5'.format(model_name)) if is_eval else self.model()
self.tensorboard = TensorBoard(log_dir='./logs/DQN_tensorboard', update_freq=90)
self.tensorboard.set_model(self.model)
def model(self):
model = Sequential()
model.add(Dense(units=64, input_dim=self.state_dim, activation='relu'))
model.add(Dense(units=32, activation='relu'))
model.add(Dense(units=8, activation='relu'))
model.add(Dense(self.action_dim, activation='softmax'))
model.compile(loss='mse', optimizer=Adam(lr=0.001))
return model
def reset(self):
self.reset_portfolio()
self.epsilon = 1.0 # reset exploration rate
def remember(self, state, actions, reward, next_state, done):
self.memory.append((state, actions, reward, next_state, done))
def act(self, state):
if not self.is_eval and np.random.rand() <= self.epsilon:
return random.randrange(self.action_dim)
options = self.model.predict(state)
return np.argmax(options[0])
def experience_replay(self):
# retrieve recent buffer_size long memory
mini_batch = []
l = len(self.memory)
for i in range(l - self.buffer_size + 1, l):
mini_batch.append(self.memory[i])
for state, actions, reward, next_state, done in mini_batch:
if not done:
Q_target_value = reward + self.gamma * np.amax(self.model.predict(next_state)[0])
else:
Q_target_value = reward
next_actions = self.model.predict(state)
next_actions[0][np.argmax(actions)] = Q_target_value
history = self.model.fit(state, next_actions, epochs=1, verbose=0)
if self.epsilon > self.epsilon_min:
self.epsilon *= self.epsilon_decay
return history.history['loss'][0]
class DQNAgent:
def __init__(self, state_size, action_space, train=True):
self.t = 0
self.max_Q = 0
self.train = train
# Get size of state and action
self.state_size = state_size
self.action_space = action_space
self.action_size = action_space
# These are hyper parameters for the DQN
self.discount_factor = 0.99
self.learning_rate = 1e-4
if self.train:
self.epsilon = 0.5
self.initial_epsilon = 0.5
else:
self.epsilon = 1e-6
self.initial_epsilon = 1e-6
self.epsilon_min = 0.02
self.batch_size = 64
self.train_start = 100
self.explore = 10000
# Create replay memory using deque
self.memory = deque(maxlen=10000)
# Create main model and target model
if MODEL_TYPE == 1:
self.model = self.build_transfer_model_has_batchnorm()
self.target_model = self.build_transfer_model_has_batchnorm()
elif MODEL_TYPE == 2:
self.model = self.build_transfer_model_has_no_batchnorm()
self.target_model = self.build_transfer_model_has_no_batchnorm()
elif MODEL_TYPE == 3:
self.model = self.build_transfer_model_has_batchnorm_with_canny()
self.target_model = self.build_transfer_model_has_batchnorm_with_canny(
)
elif MODEL_TYPE == 4:
self.model = self.build_dc_model_without_transfer()
self.target_model = self.build_dc_model_without_transfer()
else:
self.model = self.build_og_dc_model_from_repo()
self.target_model = self.build_og_dc_model_from_repo()
# Copy the model to target model
# --> initialize the target model so that the parameters of model & target model to be same
self.update_target_model()
self.tensorboard = TensorBoard(log_dir='./logs/',
histogram_freq=0,
write_graph=True,
write_grads=True)
self.tensorboard.set_model(self.model)
def build_transfer_model_has_batchnorm(self):
source_model = load_model('./files/my_model_new.h5')
model = Sequential()
for layer in source_model.layers[:-1]:
if 'batch' not in layer.name:
model.add(layer)
for layer in model.layers:
layer.trainable = True
# model.add(Dense(512, activation="relu", name='dense'))
# model.add(Dense(512, activation="relu", name='dense_1'))
model.add(Dense(15, activation="linear", name='dense_2'))
adam = Adam(lr=self.learning_rate)
model.compile(loss='mse', optimizer=adam)
return model
def build_transfer_model_has_no_batchnorm(self):
source_model = load_model('./files/my_model_gs.h5')
model = Sequential()
for layer in source_model.layers[:-3]:
model.add(layer)
for layer in model.layers:
layer.trainable = True
model.add(Dense(512, activation="relu", name='dense'))
model.add(Dense(512, activation="relu", name='dense_1'))
model.add(Dense(15, activation="linear", name='dense_2'))
# print(model.summary())
adam = Adam(lr=self.learning_rate)
model.compile(loss='mse', optimizer=adam)
return model
def build_transfer_model_has_batchnorm_with_canny(self):
# source_model = load_model('./files/my_model_canny.h5')
# model = Sequential()
# for layer in source_model.layers[:-3]:
# model.add(layer)
# for layer in model.layers:
# layer.trainable = True
# model.add(Dense(512, activation="relu", name='dense'))
# model.add(Dense(512, activation="relu", name='dense_1'))
# model.add(Dense(15, activation="linear", name='dense_2'))
# # print(model.summary())
# adam = Adam(lr=self.learning_rate)
# model.compile(loss='mse', optimizer=adam)
# return model
#
# def build_og_dc_model_from_repo(self):
# model = Sequential()
# model.add(Conv2D(24, (5, 5), strides=(2, 2), padding="same",
# input_shape=(IMG_ROWS, IMG_COLS, IMG_CHANNELS))) # 80*80*4
# model.add(Activation('relu'))
# model.add(Conv2D(32, (5, 5), strides=(2, 2), padding="same"))
# model.add(Activation('relu'))
# model.add(Conv2D(64, (5, 5), strides=(2, 2), padding="same"))
# model.add(Activation('relu'))
# model.add(Conv2D(64, (3, 3), strides=(2, 2), padding="same"))
# model.add(Activation('relu'))
# model.add(Conv2D(64, (3, 3), strides=(1, 1), padding="same"))
# model.add(Activation('relu'))
# model.add(Flatten())
# model.add(Dense(512))
# model.add(Activation('relu'))
#
# # 15 categorical bins for Steering angles
# model.add(Dense(15, activation="linear"))
#
# adam = Adam(lr=self.learning_rate)
# model.compile(loss='mse', optimizer=adam)
#
# return model
return
def build_dc_model_without_transfer(self):
Input_1 = Input(shape=(64, 64, 3), name='Input_1')
Convolution2D_1 = Conv2D(4,
kernel_size=3,
padding='same',
activation='relu')(Input_1)
Convolution2D_2 = Conv2D(4,
kernel_size=3,
padding='same',
activation='relu')(Convolution2D_1)
# Convolution2D_2 = BatchNormalization()(Convolution2D_2)
MaxPooling2D_1 = MaxPooling2D()(Convolution2D_2)
Convolution2D_5 = Conv2D(8,
kernel_size=3,
padding='same',
activation='relu')(MaxPooling2D_1)
Convolution2D_6 = Conv2D(8,
kernel_size=3,
padding='same',
activation='relu')(Convolution2D_5)
# Convolution2D_6 = BatchNormalization()(Convolution2D_6)
MaxPooling2D_2 = MaxPooling2D()(Convolution2D_6)
Convolution2D_7 = Conv2D(16,
kernel_size=3,
padding='same',
activation='relu')(MaxPooling2D_2)
Convolution2D_8 = Conv2D(16,
kernel_size=3,
padding='same',
activation='relu')(Convolution2D_7)
Convolution2D_11 = Conv2D(16,
kernel_size=3,
padding='same',
activation='relu')(Convolution2D_8)
# Convolution2D_11 = BatchNormalization()(Convolution2D_11)
MaxPooling2D_3 = MaxPooling2D()(Convolution2D_11)
Convolution2D_9 = Conv2D(32,
kernel_size=3,
padding='same',
activation='relu')(MaxPooling2D_3)
Convolution2D_10 = Conv2D(32,
kernel_size=3,
padding='same',
activation='relu')(Convolution2D_9)
Convolution2D_12 = Conv2D(16,
kernel_size=3,
padding='same',
activation='relu')(Convolution2D_10)
# Convolution2D_12 = BatchNormalization()(Convolution2D_12)
MaxPooling2D_4 = MaxPooling2D(name='MaxPooling2D_4')(Convolution2D_12)
Convolution2D_13 = Conv2D(32,
kernel_size=3,
padding='same',
activation='relu')(MaxPooling2D_4)
Convolution2D_14 = Conv2D(32,
kernel_size=3,
padding='same',
activation='relu')(Convolution2D_13)
Convolution2D_16 = Conv2D(16,
kernel_size=3,
padding='same',
activation='relu')(Convolution2D_14)
# Convolution2D_16 = BatchNormalization()(Convolution2D_16)
MaxPooling2D_5 = MaxPooling2D(name='MaxPooling2D_5')(Convolution2D_16)
Flatten_1 = Flatten()(MaxPooling2D_5)
Dense_1 = Dense(512, activation='relu')(Flatten_1)
# Dropout_1 = Dropout(0.2)(Dense_1)
Dense_2 = Dense(512, activation='relu')(Dense_1)
# Dropout_2 = Dropout(0.2)(Dense_2)
Dense_3 = Dense(15, activation='linear')(Dense_2)
model = Model([Input_1], [Dense_3])
model.compile(optimizer='adam', loss='mse')
return model
def rgb2gray(self, rgb):
'''
take a numpy rgb image return a new single channel image converted to greyscale
'''
return np.dot(rgb[..., :3], [0.299, 0.587, 0.114])
def process_image(self, obs):
# obs = self.rgb2gray(obs)
obs = cv2.resize(obs, (IMG_ROWS, IMG_COLS))
return obs
def process_image_for_canny(self, obs):
# obs = self.rgb2gray(obs)
obs = cv2.resize(obs, (IMG_ROWS, IMG_COLS))
obs1 = cv2.Canny(obs, 100, 200)
return obs1
def update_target_model(self):
self.target_model.set_weights(self.model.get_weights())
# Get action from model using epsilon-greedy policy
def get_action(self, s_t):
if np.random.rand() <= self.epsilon:
return self.action_space.sample()[0]
else:
print('Max Q')
# print("Return Max Q Prediction")
q_value = self.model.predict(s_t)
# Convert q array to steering value
return linear_unbin(q_value[0])
def replay_memory(self, state, action, reward, next_state, done):
self.memory.append((state, action, reward, next_state, done))
def update_epsilon(self):
if self.epsilon > self.epsilon_min:
self.epsilon -= (self.initial_epsilon -
self.epsilon_min) / self.explore
def named_logs(self, model, logs):
result = {}
for l in zip(model.metrics_names, logs):
result[l[0]] = l[1]
return result
def train_replay(self, ep_num):
if len(self.memory) < self.train_start:
return
batch_size = min(self.batch_size, len(self.memory))
minibatch = random.sample(self.memory, batch_size)
state_t, action_t, reward_t, state_t1, terminal = zip(*minibatch)
state_t = np.concatenate(state_t)
state_t1 = np.concatenate(state_t1)
targets = self.model.predict(state_t)
self.max_Q = np.max(targets[0])
target_val = self.model.predict(state_t1)
target_val_ = self.target_model.predict(state_t1)
for i in range(batch_size):
if terminal[i]:
targets[i][action_t[i]] = reward_t[i]
else:
a = np.argmax(target_val[i])
targets[i][action_t[i]] = reward_t[i] + \
self.discount_factor * (target_val_[i][a])
logs = self.model.train_on_batch(state_t, targets)
return logs
# self.tensorboard.on_epoch_end(ep_num, self.named_logs(self.model, [logs]))
def update_tensorboard(self, e, log, rew):
self.tensorboard.on_epoch_end(e, self.named_logs(self.model, [rew]))
def load_model(self, name):
self.model.load_weights(name)
# Save the model which is under training
def save_model(self, name):
self.model.save_weights(name)
class MLP:
def __init__(self,
x_batch_size,
mask_batch_size,
tensorboard_logs_dir="",
add_mopt_perf_metric=True,
use_early_stopping=True):
self.batch_size = mask_batch_size * x_batch_size
self.mask_batch_size = mask_batch_size
self.x_batch_size = x_batch_size
self.losses_per_sample = None
self.tr_loss_history = []
self.te_loss_history = []
self.tf_logs = tensorboard_logs_dir
self.epoch_counter = 0
self.add_mopt_perf_metric = add_mopt_perf_metric
self.useEarlyStopping = use_early_stopping
def create_dense_model(self,
input_shape,
dense_arch,
last_activation="linear"):
self.x_shape = input_shape
self.y_shape = dense_arch[-1]
input_data_layer = Input(shape=input_shape)
x = Flatten()(input_data_layer)
input_mask_layer = Input(shape=input_shape)
mask = Flatten()(input_mask_layer)
x = tf.keras.layers.Concatenate(axis=1)([x, mask])
for units in dense_arch[:-1]:
x = Dense(units, activation="sigmoid")(x)
x = Dense(dense_arch[-1], activation=last_activation)(x)
self.model = Model(inputs=[input_data_layer, input_mask_layer],
outputs=x)
# self.model.summary()
def create_batch(self, x, masks, y):
"""
x = [[1,2],[3,4]] -> [[1,2],[1,2],[1,2],[3,4],[3,4],[3,4]]
masks = [[0,0],[1,0],[1,1]] -> [[0,0],[1,0],[1,1],[0,0],[1,0],[1,1]]
y = [1,3] -> [1 ,1 ,1 ,3 ,3 ,3 ]
"""
x_prim = np.repeat(x, len(masks), axis=0)
y_prim = np.repeat(y, len(masks), axis=0)
masks_prim = np.tile(masks, (len(x), 1))
x_prim *= masks_prim
return x_prim, masks_prim, y_prim
def named_logs(self, model, logs, mode="train"):
result = {}
try:
iterator = iter(logs)
except TypeError:
logs = [logs]
metricNames = (mode + "_" + i for i in model.metrics_names)
for l in zip(metricNames, logs):
result[l[0]] = l[1]
return result
def compile_model(self,
loss_per_sample,
combine_losses,
combine_mask_losses,
metrics=None):
self.mask_loss_combine_function = combine_mask_losses
if self.add_mopt_perf_metric is True:
if metrics is None:
metrics = [self.get_mopt_perf_metric()]
else:
metrics.append(self.get_mopt_perf_metric())
def logging_loss_function(y_true, y_pred):
losses = loss_per_sample(y_true, y_pred)[:, 0]
# print(losses)
self.losses_per_sample = losses
return combine_losses(losses)
self.model.compile(loss=logging_loss_function,
optimizer='nadam',
metrics=metrics,
run_eagerly=True)
if self.tf_logs != "":
log_path = './logs'
self.tb_clbk = TensorBoard(self.tf_logs)
self.tb_clbk.set_model(self.model)
def get_per_mask_loss(self, used_target_shape=None):
if used_target_shape is None:
used_target_shape = (self.x_batch_size, self.mask_batch_size)
losses = tf.reshape(self.losses_per_sample, used_target_shape)
losses = self.mask_loss_combine_function(losses)
return losses
def get_per_mask_loss_with_custom_batch(self, losses, new_x_batch_size,
new_mask_batch_size):
losses = np.reshape(losses,
newshape=(new_x_batch_size, new_mask_batch_size))
losses = np.apply_along_axis(self.mask_loss_combine_function, 0,
losses)
return losses
def train_one(self, x, masks, y):
x_prim, masks_prim, y_prim = self.create_batch(x, masks, y)
curr_loss = self.model.train_on_batch(x=[x_prim, masks_prim], y=y_prim)
self.tr_loss_history.append(curr_loss)
self.epoch_counter += 1
if self.tf_logs != "":
self.tb_clbk.on_epoch_end(self.epoch_counter,
self.named_logs(self.model, curr_loss))
return x_prim, masks_prim, y_prim
def test_one(self, x, masks, y):
x_prim, masks_prim, y_prim = self.create_batch(x, masks, y)
feature = self.model.predict(x=[x_prim, masks_prim])
return feature
def get_mopt_perf_metric(self):
def m_opt_loss(y_pred, y_true):
if self.losses_per_sample.shape[0] % self.mask_batch_size != 0:
return 0.0
else:
losses = tf.reshape(self.losses_per_sample,
(-1, self.mask_batch_size))
self.last_m_opt_perf = np.mean(
losses[:, int(0.5 * self.mask_batch_size)])
return self.last_m_opt_perf
return m_opt_loss
def set_early_stopping_params(self,
starting_epoch,
patience_batches=800,
minimize=True):
self.ES_patience = patience_batches
self.ES_minimize = minimize
if minimize is True:
self.ES_best_perf = 1000000.0
else:
self.ES_best_perf = -1000000.0
self.ES_best_epoch = starting_epoch
self.ES_stop_training = False
self.ES_start_epoch = starting_epoch
self.ES_best_weights = None
return
#Define the adversarial model
gan_model = networks.define_adversarial_model(genA2B,
genB2A,
discA,
discB,
train_optimizer,
lambda_cyc=lambda_cyc,
lambda_idt=lambda_idt)
#Setup the tensorboard to store and visualise losses
tensorboard = TensorBoard(log_dir="logs\{}".format(time.time()),
write_images=True,
write_grads=True,
write_graph=True)
tensorboard.set_model(genA2B)
tensorboard.set_model(genB2A)
tensorboard.set_model(discA)
tensorboard.set_model(discB)
print("Batch Size: {}".format(batch_size))
print("Num of ResNet Blocks: {}".format(num_resnet_blocks))
print(
"Starting training for {0} epochs with lambda_cyc = {1}, lambda_idt = {2}, num_resnet_blocks = {3}"
.format(epochs, lambda_cyc, lambda_idt, num_resnet_blocks))
#Start training
for epoch in range(epochs):
print("Epoch:{}".format(epoch))
start_time = time.time()
dis_losses = []
gen_losses = []
class Agent(Portfolio):
def __init__(self, state_dim, balance, is_eval=False, model_name=""):
super().__init__(balance=balance)
self.model_type = 'DQN'
self.state_dim = state_dim
self.action_dim = 3 # hold, buy, sell
self.memory = deque(maxlen=100)
self.buffer_size = 60
self.tau = 0.0001
self.gamma = 0.95
self.epsilon = 1.0 # initial exploration rate
self.epsilon_min = 0.01 # minimum exploration rate
self.epsilon_decay = 0.995 # decrease exploration rate as the agent becomes good at trading
self.is_eval = is_eval
self.model = load_model(
f'saved_models/{model_name}.h5') if is_eval else self.model()
self.model_target = load_model(
f'saved_models/{model_name}_target.h5') if is_eval else self.model
self.model_target.set_weights(self.model.get_weights(
)) # hard copy model parameters to target model parameters
self.tensorboard = TensorBoard(log_dir='./logs/DDQN_tensorboard',
update_freq=90)
self.tensorboard.set_model(self.model)
def update_model_target(self):
model_weights = self.model.get_weights()
model_target_weights = self.model_target.get_weights()
for i in range(len(model_weights)):
model_target_weights[i] = self.tau * model_weights[i] + (
1 - self.tau) * model_target_weights[i]
self.model_target.set_weights(model_target_weights)
def model(self):
model = Sequential()
model.add(Dense(units=64, input_dim=self.state_dim, activation='relu'))
model.add(Dense(units=32, activation='relu'))
model.add(Dense(units=8, activation='relu'))
model.add(Dense(self.action_dim, activation='softmax'))
model.compile(loss='mse', optimizer=Adam(lr=0.001))
return model
def reset(self):
self.reset_portfolio()
self.epsilon = 1.0
def remember(self, state, actions, reward, next_state, done):
self.memory.append((state, actions, reward, next_state, done))
def act(self, state):
if not self.is_eval and np.random.rand() <= self.epsilon:
return random.randrange(self.action_dim)
options = self.model.predict(state)
return np.argmax(options[0])
def experience_replay(self):
# sample random buffer_size long memory
mini_batch = random.sample(self.memory, self.buffer_size)
for state, actions, reward, next_state, done in mini_batch:
Q_expected = reward + (1 - done) * self.gamma * np.amax(
self.model_target.predict(next_state)[0])
next_actions = self.model.predict(state)
next_actions[0][np.argmax(actions)] = Q_expected
history = self.model.fit(state, next_actions, epochs=1, verbose=0)
self.update_model_target()
if self.epsilon > self.epsilon_min:
self.epsilon *= self.epsilon_decay
return history.history['loss'][0]
# create model
input_tensor = Input(shape=(img_height, img_widht, 3))
input_rois = Input(shape=(None, 5))
model_body = get_model_body(input_tensor, net='vgg16', trainable=True)
# rpn model
rpn_model = get_rpn_model(model_body, anchors_num)
# rfcn model
rfcn_model = get_rfcn_model(model_body, input_rois)
log_path = './logs'
if not os.path.exists(log_path):
os.mkdir(log_path)
rpn_callback = TensorBoard(os.path.join(log_path, '000'))
rpn_callback.set_model(rpn_model)
rfcn_callback = TensorBoard(os.path.join(log_path, '000'))
rfcn_callback.set_model(rfcn_model)
rpn_model_path = os.path.join(log_path, 'rpn_weights.h5')
rfcn_model_path = os.path.join(log_path, 'rfcn_model_weights.h5')
if os.path.exists(rpn_model_path):
rpn_model.load_weights(rpn_model_path)
if os.path.exists(rfcn_model_path):
rfcn_model.load_weights(rfcn_model_path)
rpn_optimizer = Adam(lr=1e-5)
rfcn_optimizer = Adam(lr=1e-5)
rpn_model.compile(optimizer=rpn_optimizer, loss=[rpn_cls_loss, rpn_reg_loss])
embedding_dim=50,
feed_forward_units=50,
head_num=1,
block_num=2,
dropout_rate=0.2)
optimizer = AdamOptimizer(0.001)
tbcb = TensorBoard(log_dir='/logs', histogram_freq=1, write_graph=True, write_grads=True, write_images=True, embeddings_freq=1)
loss_history = []
cos_loss_history = []
T = 0.0
t0 = time.time()
tbcb.set_model(model)
tbcb.on_train_begin()
f = open('log.txt', 'w')
try:
for epoch in range(200):
tbcb.on_epoch_begin(epoch)
cos_loss = CosineSimilarity()
for step in (range(num_batch)):
tbcb.on_train_batch_begin(step)
print('========== step: {:03d} / {:03d} ============\r'.format(step, num_batch), end='')
class Trainer_adv:
"""Class object to setup and carry the training.
Takes as input a generator that produces SR images.
Conditionally, also a discriminator network and a feature extractor
to build the components of the perceptual loss.
Compiles the model(s) and trains in a GANS fashion if a discriminator is provided, otherwise
carries a regular ISR training.
Args:
generator: Keras model, the super-scaling, or generator, network.
discriminator: Keras model, the discriminator network for the adversarial
component of the perceptual loss.
feature_extractor: Keras model, feature extractor network for the deep features
component of perceptual loss function.
lr_train_dir: path to the directory containing the Low-Res images for training.
hr_train_dir: path to the directory containing the High-Res images for training.
lr_valid_dir: path to the directory containing the Low-Res images for validation.
hr_valid_dir: path to the directory containing the High-Res images for validation.
learning_rate: float.
loss_weights: dictionary, use to weigh the components of the loss function.
Contains 'generator' for the generator loss component, and can contain 'discriminator' and 'feature_extractor'
for the discriminator and deep features components respectively.
logs_dir: path to the directory where the tensorboard logs are saved.
weights_dir: path to the directory where the weights are saved.
dataname: string, used to identify what dataset is used for the training session.
weights_generator: path to the pre-trained generator's weights, for transfer learning.
weights_discriminator: path to the pre-trained discriminator's weights, for transfer learning.
n_validation:integer, number of validation samples used at training from the validation set.
flatness: dictionary. Determines determines the 'flatness' threshold level for the training patches.
See the TrainerHelper class for more details.
lr_decay_frequency: integer, every how many epochs the learning rate is reduced.
lr_decay_factor: 0 < float <1, learning rate reduction multiplicative factor.
Methods:
train: combines the networks and triggers training with the specified settings.
"""
def __init__(
self,
generator,
discriminator,
feature_extractor,
lr_train_dir,
hr_train_dir,
lr_valid_dir,
hr_valid_dir,
loss_weights={
'generator': 1.0,
'discriminator': 0.003,
'feature_extractor': 1 / 12
},
log_dirs={
'logs': 'logs',
'weights': 'weights'
},
fallback_save_every_n_epochs=2,
dataname=None,
weights_generator=None,
weights_discriminator=None,
n_validation=None,
flatness={
'min': 0.0,
'increase_frequency': None,
'increase': 0.0,
'max': 0.0
},
learning_rate={
'initial_value': 0.0004,
'decay_frequency': 100,
'decay_factor': 0.5
},
adam_optimizer={
'beta1': 0.9,
'beta2': 0.999,
'epsilon': None
},
losses={
'generator': 'mae',
'discriminator': 'binary_crossentropy',
'feature_extractor': 'mse',
},
metrics={'generator': 'PSNR_Y'},
):
self.generator = generator
self.discriminator = discriminator
self.feature_extractor = feature_extractor
self.scale = generator.scale
self.lr_patch_size = generator.patch_size
self.learning_rate = learning_rate
self.loss_weights = loss_weights
self.weights_generator = weights_generator
self.weights_discriminator = weights_discriminator
self.adam_optimizer = adam_optimizer
self.dataname = dataname
self.flatness = flatness
self.n_validation = n_validation
self.losses = losses
self.log_dirs = log_dirs
self.metrics = metrics
if self.metrics['generator'] == 'PSNR_Y':
self.metrics['generator'] = PSNR_Y
elif self.metrics['generator'] == 'PSNR':
self.metrics['generator'] = PSNR
self._parameters_sanity_check()
self.mag_num = len(hr_train_dir)
if int(np.log2(self.scale)) != self.mag_num:
print('scale does not match hr_train_dir')
return
self.model = self._combine_networks()
self.settings = {}
self.settings['training_parameters'] = locals()
self.settings['training_parameters'][
'lr_patch_size'] = self.lr_patch_size
self.settings = self.update_training_config(self.settings)
self.logger = get_logger(__name__)
self.helper = TrainerHelper(
generator=self.generator,
weights_dir=log_dirs['weights'],
logs_dir=log_dirs['logs'],
lr_train_dir=lr_train_dir,
feature_extractor=self.feature_extractor,
discriminator=self.discriminator,
dataname=dataname,
weights_generator=self.weights_generator,
weights_discriminator=self.weights_discriminator,
fallback_save_every_n_epochs=fallback_save_every_n_epochs,
)
self.train_dh = {}
for index in range(self.mag_num):
self.train_dh[index] = DataHandler(
lr_dir=lr_train_dir,
hr_dir=hr_train_dir[index],
patch_size=self.lr_patch_size,
scale=int(np.exp2((index + 1))),
n_validation_samples=None,
)
self.valid_dh = {}
for index in range(self.mag_num):
if index == 0:
self.valid_dh[index] = DataHandler(
lr_dir=lr_valid_dir,
hr_dir=hr_valid_dir[index],
patch_size=self.lr_patch_size,
scale=int(np.exp2((index + 1))),
n_validation_samples=32)
else:
self.valid_dh[index] = DataHandler(
lr_dir=lr_valid_dir,
hr_dir=hr_valid_dir[index],
patch_size=self.lr_patch_size,
scale=int(np.exp2((index + 1))),
n_validation_samples=32,
index=self.valid_dh[0].index)
#self.discriminator.model.summary()
#self.generator.model.summary()
#self.feature_extractor.model.summary()
self.model.summary()
def _parameters_sanity_check(self):
""" Parameteres sanity check. """
if self.discriminator:
assert self.lr_patch_size * self.scale == self.discriminator.patch_size
self.adam_optimizer
#if self.feature_extractor:
# assert self.lr_patch_size * self.scale == self.feature_extractor.patch_size
check_parameter_keys(
self.learning_rate,
needed_keys=['initial_value'],
optional_keys=['decay_factor', 'decay_frequency'],
default_value=None,
)
check_parameter_keys(
self.flatness,
needed_keys=[],
optional_keys=['min', 'increase_frequency', 'increase', 'max'],
default_value=0.0,
)
check_parameter_keys(
self.adam_optimizer,
needed_keys=['beta1', 'beta2'],
optional_keys=['epsilon'],
default_value=None,
)
check_parameter_keys(self.log_dirs, needed_keys=['logs', 'weights'])
def _combine_networks(self):
"""
Constructs the combined model which contains the generator network,
as well as discriminator and geature extractor, if any are defined.
"""
losses = []
loss_weights = []
lr = Input(shape=(self.lr_patch_size, ) * 2 + (3, ))
sr = self.generator.model(lr)
outputs = sr
losses.extend([self.losses['generator']] * len(sr))
loss_weights.extend([self.loss_weights['generator']] * len(sr))
final_sr = sr[-1]
n = 0
if self.discriminator:
self.discriminator.model.trainable = False
validity = self.discriminator.model(final_sr)
outputs.append(validity)
n = 1
losses.append(self.losses['discriminator'])
loss_weights.append(self.loss_weights['discriminator'])
if self.feature_extractor:
self.feature_extractor.model.trainable = False
for i in range(len(sr) - n):
sr_feats = self.feature_extractor.model(sr[i])
outputs.extend([*sr_feats])
losses.extend([self.losses['feature_extractor']] *
len(sr_feats))
loss_weights.extend([
self.loss_weights['feature_extractor'] / len(sr_feats) *
(1 + i / 3.0)
] * len(sr_feats))
combined = Model(inputs=lr, outputs=outputs)
# https://stackoverflow.com/questions/42327543/adam-optimizer-goes-haywire-after-200k-batches-training-loss-grows
optimizer = Adam(
beta_1=self.adam_optimizer['beta1'],
beta_2=self.adam_optimizer['beta2'],
lr=self.learning_rate['initial_value'],
epsilon=self.adam_optimizer['epsilon'],
)
combined.compile(loss=losses,
loss_weights=loss_weights,
optimizer=optimizer,
metrics=self.metrics)
return combined
def _lr_scheduler(self, epoch):
""" Scheduler for the learning rate updates. """
n_decays = epoch // self.learning_rate['decay_frequency']
lr = self.learning_rate['initial_value'] * (
self.learning_rate['decay_factor']**n_decays)
# no lr below minimum control 10e-7
return max(1e-7, lr)
def _flatness_scheduler(self, epoch):
if self.flatness['increase']:
n_increases = epoch // self.flatness['increase_frequency']
else:
return self.flatness['min']
f = self.flatness['min'] + n_increases * self.flatness['increase']
return min(self.flatness['max'], f)
def _load_weights(self):
"""
Loads the pretrained weights from the given path, if any is provided.
If a discriminator is defined, does the same.
"""
if self.weights_generator:
self.model.get_layer('generator').load_weights(
str(self.weights_generator))
if self.discriminator:
if self.weights_discriminator:
self.model.get_layer('discriminator').load_weights(
str(self.weights_discriminator))
self.discriminator.model.load_weights(
str(self.weights_discriminator))
def _format_losses(self, prefix, losses, model_metrics):
""" Creates a dictionary for tensorboard tracking. """
return dict(zip([prefix + m for m in model_metrics], losses))
def update_training_config(self, settings):
""" Summarizes training setting. """
_ = settings['training_parameters'].pop('weights_generator')
_ = settings['training_parameters'].pop('self')
_ = settings['training_parameters'].pop('generator')
_ = settings['training_parameters'].pop('discriminator')
_ = settings['training_parameters'].pop('feature_extractor')
settings['generator'] = {}
settings['generator']['name'] = self.generator.name
settings['generator']['parameters'] = self.generator.params
settings['generator']['weights_generator'] = self.weights_generator
_ = settings['training_parameters'].pop('weights_discriminator')
if self.discriminator:
settings['discriminator'] = {}
settings['discriminator']['name'] = self.discriminator.name
settings['discriminator'][
'weights_discriminator'] = self.weights_discriminator
else:
settings['discriminator'] = None
if self.discriminator:
settings['feature_extractor'] = {}
settings['feature_extractor']['name'] = self.feature_extractor.name
settings['feature_extractor'][
'layers'] = self.feature_extractor.layers_to_extract
else:
settings['feature_extractor'] = None
return settings
def train(self, epochs, steps_per_epoch, batch_size, monitored_metrics):
"""
Carries on the training for the given number of epochs.
Sends the losses to Tensorboard.
Args:
epochs: how many epochs to train for.
steps_per_epoch: how many batches epoch.
batch_size: amount of images per batch.
monitored_metrics: dictionary, the keys are the metrics that are monitored for the weights
saving logic. The values are the mode that trigger the weights saving ('min' vs 'max').
"""
self.settings['training_parameters'][
'steps_per_epoch'] = steps_per_epoch
self.settings['training_parameters']['batch_size'] = batch_size
starting_epoch = self.helper.initialize_training(
self) # load_weights, creates folders, creates basename
self.tensorboard = TensorBoard(
log_dir=str(self.helper.callback_paths['logs']))
self.tensorboard.set_model(self.model)
# validation data
validation_set = {}
top_left_fronts = {}
transform_before = []
for index in range(self.mag_num):
if index == 0:
validation_set[
index], top_left_fronts, transform_before = self.valid_dh[
index].get_validation_set(batch_size)
else:
validation_set[index], _, _ = self.valid_dh[
index].get_validation_set(batch_size, top_left_fronts,
transform_before)
y_validation = [
validation_set[index]['hr'] for index in range(self.mag_num)
]
if self.discriminator:
discr_out_shape = list(
self.discriminator.model.outputs[0].shape)[1:4]
valid = np.ones([batch_size] + discr_out_shape)
fake = np.zeros([batch_size] + discr_out_shape)
validation_valid = np.ones(
[len(validation_set[self.mag_num - 1]['hr'])] +
discr_out_shape)
y_validation.append(validation_valid)
if self.feature_extractor:
for index in range(self.mag_num):
validation_feats = self.feature_extractor.model.predict(
validation_set[index]['hr'])
y_validation.extend([*validation_feats])
for epoch in range(starting_epoch, epochs):
self.logger.info('Epoch {e}/{tot_eps}'.format(e=epoch,
tot_eps=epochs))
K.set_value(self.model.optimizer.lr,
self._lr_scheduler(epoch=epoch))
self.logger.info('Current learning rate: {}'.format(
K.eval(self.model.optimizer.lr)))
flatness = self._flatness_scheduler(epoch)
if flatness:
self.logger.info(
'Current flatness treshold: {}'.format(flatness))
epoch_start = time()
for step in tqdm(range(steps_per_epoch)):
batch = {}
top_left_front = {}
temp = {}
idx = None
transform_before = []
for index in range(self.mag_num):
if index == 0:
batch[
index], top_left_front, idx, transform_before = self.train_dh[
index].get_batch(batch_size, flatness=flatness)
else:
batch[index], temp, idx, _ = self.train_dh[
index].get_batch(batch_size,
idx,
flatness=flatness,
top_left_front=top_left_front,
transform_before=transform_before)
y_train = [batch[index]['hr'] for index in range(self.mag_num)]
training_losses = {}
## Discriminator training
if self.discriminator:
sr = self.generator.model.predict(batch[0]['lr'])
d_loss_real = self.discriminator.model.train_on_batch(
batch[self.mag_num - 1]['hr'], valid)
d_loss_fake = self.discriminator.model.train_on_batch(
sr[-1], fake)
d_loss_fake = self._format_losses(
'train_d_fake_', d_loss_fake,
self.discriminator.model.metrics_names)
d_loss_real = self._format_losses(
'train_d_real_', d_loss_real,
self.discriminator.model.metrics_names)
training_losses.update(d_loss_real)
training_losses.update(d_loss_fake)
y_train.append(valid)
## Generator training
if self.feature_extractor:
for index in range(self.mag_num):
hr_feats = self.feature_extractor.model.predict(
batch[index]['hr'])
y_train.extend([*hr_feats])
model_losses = self.model.train_on_batch(
batch[0]['lr'], y_train)
model_losses = self._format_losses('train_', model_losses,
self.model.metrics_names)
training_losses.update(model_losses)
self.tensorboard.on_epoch_end(epoch * steps_per_epoch + step,
training_losses)
self.logger.debug('Losses at step {s}:\n {l}'.format(
s=step, l=training_losses))
elapsed_time = time() - epoch_start
self.logger.info('Epoch {} took {:10.1f}s'.format(
epoch, elapsed_time))
validation_losses = self.model.evaluate(validation_set[0]['lr'],
y_validation,
batch_size=batch_size)
validation_losses = self._format_losses('val_', validation_losses,
self.model.metrics_names)
if epoch == starting_epoch:
remove_metrics = []
for metric in monitored_metrics:
if (metric not in training_losses) and (
metric not in validation_losses):
msg = ' '.join([
metric,
'is NOT among the model metrics, removing it.'
])
self.logger.error(msg)
remove_metrics.append(metric)
for metric in remove_metrics:
_ = monitored_metrics.pop(metric)
# should average train metrics
end_losses = {}
end_losses.update(validation_losses)
end_losses.update(training_losses)
self.helper.on_epoch_end(
epoch=epoch,
losses=end_losses,
generator=self.model.get_layer('generator'),
discriminator=self.discriminator,
metrics=monitored_metrics,
)
self.tensorboard.on_epoch_end(epoch, validation_losses)
self.tensorboard.on_train_end(None)
def train(self, x_train, y_train, batch_size, epochs):
# Select sampled real dataset (label O, balanced class)
x_train, y_train, x_super, y_super = self.select_supervised_sampled(
x_train, y_train)
x_train, y_train = x_train.values, y_train.values
# Calculate the number of batches per training epoch
bat_per_epo = int(x_train.shape[0] / batch_size)
# Calulate the number of trainging iterations
n_steps = bat_per_epo * epochs
half_batch = batch_size // 2
tensorboard = TensorBoard(log_dir="./logs",
histogram_freq=3,
write_graph=True,
write_images=True)
tensorboard.set_model(self.gan_model)
train_log_dir = "./logs"
train_summary_writer = tf.summary.create_file_writer(train_log_dir)
metrics = {}
print('epochs=%d, batch_size=%d, 1/2=%d, b/e=%d, steps=%d' %
(epochs, batch_size, half_batch, bat_per_epo, n_steps))
for step in range(n_steps):
# update supervised discriminator (c)
[Xsup_real,
labels], _ = self.generate_real_samples([x_super, y_super],
half_batch)
metrics["c_loss"], metrics["c_acc"], metrics["c_recall"], metrics[
"c_precision"] = self.c_model.train_on_batch(
Xsup_real, labels)
with train_summary_writer.as_default():
tf.summary.scalar("c_loss", metrics["c_loss"], step=step)
tf.summary.scalar("c_acc", metrics["c_acc"], step=step)
tf.summary.scalar("c_recall", metrics["c_recall"], step=step)
tf.summary.scalar("c_precision",
metrics["c_precision"],
step=step)
# update unsupervised discriminator (d)
[X_real,
_], y_real = self.generate_real_samples([x_train, y_train],
half_batch)
d_loss1 = self.d_model.train_on_batch(X_real, y_real)
X_fake, y_fake = self.generate_fake_samples(half_batch)
d_loss2 = self.d_model.train_on_batch(X_fake, y_fake)
# update generator (g)
z_input = np.random.randn(self.random_noise * half_batch)
z_input = z_input.reshape(half_batch, self.random_noise)
X_gan = z_input
y_gan = np.ones((X_gan.shape[0], 1))
g_loss = self.gan_model.train_on_batch(X_gan, y_gan)
if step % 10 == 0:
print(
'>%d, c[loss: %.3f, acc: %.3f, recall : %.3f, precision : %.3f], d[loss1: %.3f, loss2: %.3f], g_loss[%.3f]'
% (step + 1, metrics["c_loss"], metrics["c_acc"] * 100,
metrics["c_recall"] * 100, metrics["c_precision"] * 100,
d_loss1, d_loss2, g_loss))
def train(train_lbld_trios,
val_lbls_trios,
network,
weights,
model_path,
n_epochs,
init_lr,
optmzr_name,
imagenet=False,
freeze_until=None):
"""Training function: train a model of type 'network' over the data.
Args:
network (str): String identifying the network architecture to use.
weights (str): Path string to a .cpkt weights file.
model_path (str): Path string to a directory to save models in.
n_epochs (int): Integer representing the number of epochs
to run training.
"""
# Create a folder for saving trained models
if os.path.isdir(model_path) is False:
logging.info("Creating a folder to save models at: " + str(model_path))
os.mkdir(model_path)
starting_epoch = 0
if network == 'SiameseNetTriplet':
siamese_net = SiameseNetTriplet((128, 128, 3),
arch='resnet18',
sliding=True,
imagenet=imagenet,
freeze_until=freeze_until)
optimizer = Adam(lr=0.0006)
model = siamese_net.build_model()
loss_model = siamese_net.loss_model
single_model = siamese_net.single_model
if weights:
print("Loading model at: " + str(weights))
starting_epoch = int(weights.split('-')[1]) + 1
model.load_weights(weights)
model.compile(loss=triplet_loss,
optimizer=optimizer,
metrics=[cos_sim_pos, cos_sim_neg])
# Load data and create data generator:
train_ds = tf.data.Dataset.from_generator(
image_trio_generator,
args=[train_lbld_trios, True, False, False, imagenet],
output_types=((tf.float32, tf.float32, tf.float32), tf.float32,
tf.string),
output_shapes=(((TARGET_WIDTH, TARGET_HEIGHT, 3), (TARGET_WIDTH,
TARGET_HEIGHT, 3),
(TARGET_WIDTH, TARGET_HEIGHT, 3)), (1, None), (3)))
batched_train_ds = train_ds.batch(BATCH_SIZE) # shuffle(10000).batch
valid_ds = tf.data.Dataset.from_generator(
image_trio_generator,
args=[val_lbld_trios, False, False, False, imagenet],
output_types=((tf.float32, tf.float32, tf.float32), tf.float32,
tf.string),
# output_shapes=(((TARGET_WIDTH, TARGET_HEIGHT, 3), (TARGET_WIDTH, TARGET_HEIGHT, 3), (TARGET_WIDTH, TARGET_HEIGHT, 3)), (1, None)))
output_shapes=(((TARGET_WIDTH, TARGET_HEIGHT, 3), (TARGET_WIDTH,
TARGET_HEIGHT, 3),
(TARGET_WIDTH, TARGET_HEIGHT, 3)), (1, None), (3)))
batched_valid_ds = valid_ds.batch(BATCH_SIZE)
logdir = "logs/scalars/" + datetime.now().strftime("%Y%m%d-%H%M%S")
tensorboard_callback = TensorBoard(log_dir=logdir,
histogram_freq=0,
batch_size=BATCH_SIZE,
write_graph=True,
write_grads=True)
tensorboard_callback.set_model(model)
def named_logs(metrics_names, logs):
result = {}
for l in zip(metrics_names, logs):
result[l[0]] = l[1]
return result
# Train model:
steps_per_epoch = len(train_lbld_trios) // BATCH_SIZE
val_steps_per_epoch = len(
val_lbld_trios) // BATCH_SIZE # steps_per_epoch//3
best_val_loss = 1000
best_train_loss = 1000
batched_train_iter = iter(batched_train_ds)
batched_val_iter = iter(batched_valid_ds)
# batched_train_iter = batched_train_ds.make_one_shot_iterator()
# batched_val_iter = batched_valid_ds.make_one_shot_iterator()
for epoch in range(starting_epoch, n_epochs):
cumm_csim_pos_tr = 0
cumm_csim_neg_tr = 0
cumm_tr_loss = 0
cumm_csim_pos_val = 0
cumm_csim_neg_val = 0
cumm_val_loss = 0
print('Epoch #' + str(epoch) + ':')
for step in tqdm(range(steps_per_epoch)):
train_inputs, train_y, train_seq_ids = next(batched_train_iter)
#train_inputs, train_y = batched_train_iter.get_next()
# tinrain_x1 = train_inputs['input_1']
# train_x2 = train_inputs['input_2']
train_x1 = train_inputs[0]
train_x2 = train_inputs[1]
train_x3 = train_inputs[2]
# train_y = train_y['output']
X_dict = {}
seq_ids = []
for idx, row in enumerate(train_seq_ids):
for idx2, class_id in enumerate(row):
class_id = class_id.numpy().decode('utf8')
if class_id not in seq_ids:
seq_ids.append(class_id)
if class_id in X_dict:
X_dict[class_id].append(train_inputs[idx2][idx])
else:
X_dict[class_id] = []
X_dict[class_id].append(train_inputs[idx2][idx])
triplets = get_batch_hard(model, train_inputs, seq_ids, BATCH_SIZE)
loss, csim_pos_tr, csim_neg_tr = model.train_on_batch(
[triplets[0], triplets[1], triplets[2]],
train_y[:BATCH_SIZE // 2])
cumm_tr_loss += loss
cumm_csim_pos_tr += csim_pos_tr
cumm_csim_neg_tr += csim_neg_tr
cumm_csim_pos_tr = cumm_csim_pos_tr / steps_per_epoch
cumm_csim_neg_tr = cumm_csim_neg_tr / steps_per_epoch
cumm_tr_loss = cumm_tr_loss / steps_per_epoch
# evaluate
for step in tqdm(range(val_steps_per_epoch)):
valid_inputs, val_y, val_seq_ids = next(batched_val_iter)
# valid_inputs, valid_y = batched_val_iter.get_next()
valid_x1 = valid_inputs[0]
valid_x2 = valid_inputs[1]
valid_x3 = valid_inputs[2]
val_loss, csim_pos_val, csim_neg_val = model.test_on_batch(
[valid_x1, valid_x2, valid_x3], val_y)
cumm_val_loss += val_loss
cumm_csim_pos_val += csim_pos_val
cumm_csim_neg_val += csim_neg_val
cumm_csim_pos_val = cumm_csim_pos_val / val_steps_per_epoch
cumm_csim_neg_val = cumm_csim_neg_val / val_steps_per_epoch
cumm_val_loss = cumm_val_loss / val_steps_per_epoch
print('Training loss: ' + str(cumm_tr_loss))
print('Validation loss: ' + str(cumm_val_loss))
print('* Cosine sim positive (train) for this epoch: %0.2f' %
(cumm_csim_pos_tr))
print('* Cosine sim negative (train) for this epoch: %0.2f' %
(cumm_csim_neg_tr))
print('* Cosine sim positive (valid) for this epoch: %0.2f' %
(cumm_csim_pos_val))
print('* Cosine sim negative (valid) for this epoch: %0.2f' %
(cumm_csim_neg_val))
metrics_names = [
'tr_loss', 'tr_csim_pos', 'tr_csim_neg', 'val_loss',
'val_csim_pos', 'val_csim_neg'
]
tensorboard_callback.on_epoch_end(
epoch,
named_logs(metrics_names, [
cumm_tr_loss, cumm_csim_pos_tr, cumm_csim_neg_tr,
cumm_val_loss, cumm_csim_pos_val, cumm_csim_neg_val
]))
model_filepath = os.path.join(
model_path, "model-{epoch:03d}-{val_loss:.4f}.hdf5".format(
epoch=epoch, val_loss=cumm_val_loss))
if cumm_val_loss < best_val_loss * 1.5:
if cumm_val_loss < best_val_loss:
best_val_loss = cumm_val_loss
model.save(model_filepath) # OR model.save_weights()
print("Best model w/ val loss {} saved to {}".format(
cumm_val_loss, model_filepath))
tensorboard_callback.on_train_end(None)
return
def train(self, epochs=10, batch_size=1, print_interval=50):
# Tensorboard callback
train_callback = TensorBoard(os.path.join(self.save_folder, 'train'))
train_callback.set_model(self.cycle_vae)
val_callback = TensorBoard(os.path.join(self.save_folder, 'val'))
val_callback.set_model(self.cycle_vae)
metrics_names = [
'discri_loss_source', 'discri_accuracy_source', 'discri_loss_targ',
'discri_accuracy_targ', 'g_total_loss', 'g_reconst', 'g_kl',
'g_cyclic_reconst', 'g_cyclic_kl', 'g_gan', 'g_vgg'
]
# Dataset
X_day_train, X_night_train = load_day_and_night('train')
self.X_day_val, self.X_night_val = load_day_and_night('val')
n_train = len(X_day_train)
n_val = len(self.X_day_val)
print('Training examples: {}'.format(n_train))
print('Validation examples: {}'.format(n_val))
discri_dim = self.D_source.output.get_shape()[1]
valid = np.ones((batch_size, discri_dim))
fake = np.zeros((batch_size, discri_dim))
val_loss_min = float('inf')
iter_nb = 0
for e in range(epochs):
n_iter = n_train // batch_size
indices = np.random.permutation(n_train)
for i in range(n_iter):
idx = np.random.choice(indices, size=batch_size, replace=False)
#indices[i*batch_size : (i+1)*batch_size]
source_batch = load_batch(X_day_train[idx], self.image_size)
targ_batch = load_batch(X_night_train[idx], self.image_size)
# Discriminator
d_loss_source, d_loss_targ = self.discri_train_on_batch(
source_batch, targ_batch, valid, fake)
# Generator
g_loss = self.cycle_vae.train_on_batch(
[source_batch, targ_batch], [valid, valid])
if i % print_interval == 0:
print('Iteration {:4d}: Training loss:'.format(i))
logs = self.get_logs(d_loss_source, d_loss_targ, g_loss)
print(
'Discri source loss: {:.2f}, accuracy: {:.1f}%'.format(
logs[0], logs[1]))
print('Discri targ loss: {:.2f}, accuracy: {:.1f}%'.format(
logs[2], logs[3]))
print('Gen loss: {:.2f}'.format(logs[4]))
print('Reconst: {:.2f}, kl: {:.2f}, cyclic_reconst: {:.2f}, cyclic_kl: {:.2f},gan: {:.2f}, vgg: {:.2f}'\
.format(*logs[5:]))
write_log(train_callback, metrics_names, logs, iter_nb)
if i % 100 == 0:
self.predict(e + 1, iter=iter_nb, save_fig=True)
print('Figure saved.')
iter_nb += 1
# Calculate validation loss
val_loss = 0.0
n_iter_val = n_val // batch_size
indices = np.arange(n_val)
logs = np.zeros(len(metrics_names))
for i in range(n_iter_val):
idx = indices[i * batch_size:(i + 1) * batch_size]
source_batch = load_batch(self.X_day_val[idx], self.image_size)
targ_batch = load_batch(self.X_night_val[idx], self.image_size)
# Discriminator
d_loss_source, d_loss_targ = self.discri_train_on_batch(
source_batch, targ_batch, valid, fake)
# Generator
g_loss = self.cycle_vae.train_on_batch(
[source_batch, targ_batch], [valid, valid])
logs += self.get_logs(d_loss_source, d_loss_targ, g_loss)
val_loss += g_loss[0]
val_loss /= n_iter_val
logs /= n_iter_val
print('\n Epoch {} - Validation loss: {:.2f}'.format(
e + 1, val_loss))
print('Discri source loss: {:.2f}, accuracy: {:.1f}%'.format(
logs[0], logs[1]))
print('Discri targ loss: {:.2f}, accuracy: {:.1f}%'.format(
logs[2], logs[3]))
print('Gen loss: {:.2f}'.format(logs[4]))
print('Reconst: {:.2f}, kl: {:.2f}, cyclic_reconst: {:.2f}, cyclic_kl: {:.2f},gan: {:.2f}, vgg: {:.2f}' \
.format(*logs[5:]))
write_log(val_callback, metrics_names, logs, iter_nb)
if val_loss < val_loss_min:
val_loss_min = val_loss
print('Saving model\n')
weights_filename = os.path.join(
self.save_folder,
'cycle_vae.epoch{:02d}-val_loss{:.2f}.h5'.format(
e + 1, val_loss))
self.cycle_vae.save_weights(weights_filename)
def train(self, epochs, batch_size=1, sample_interval=50):
start_time = datetime.datetime.now()
self.data_loader.preprocess()
tensorboard = TensorBoard(log_dir="logs/".format(time.time()))
tensorboard.set_model(self.generator)
tensorboard.set_model(self.discriminator)
# Checpoint management
start_epoch = 0
#print("##### BEFORE", self.discriminator.get_weights())
if self.ckpt_manager.latest_checkpoint:
start_epoch = int(self.ckpt_manager.latest_checkpoint.split('-')[-1])
print("Checkpoint found. Starting at epoch {}".format(start_epoch))
# restoring the latest checkpoint in checkpoint_path
self.ckpt.restore(self.ckpt_manager.latest_checkpoint)
#print("##### AFTER", self.discriminator.get_weights())
for epoch in tqdm.trange(start_epoch, epochs):
# ----------------------
# Train Discriminator
# ----------------------
# Sample images and their conditioning counterparts
imgs_hr, imgs_lr = self.data_loader.load_data(batch_size)
# From low res. image generate high res. version
fake_hr = self.generator.predict(imgs_lr)
valid = np.ones((batch_size,) + self.disc_patch)
fake = np.zeros((batch_size,) + self.disc_patch)
# Train the discriminators (original images = real / generated = Fake)
d_loss_real = self.discriminator.train_on_batch(imgs_hr, valid)
d_loss_fake = self.discriminator.train_on_batch(fake_hr, fake)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
print("d_loss:", d_loss)
# ------------------
# Train Generator
# ------------------
# Sample images and their conditioning counterparts
imgs_hr, imgs_lr = self.data_loader.load_data(batch_size)
# The generators want the discriminators to label the generated images as real
valid = np.ones((batch_size,) + self.disc_patch)
# Extract ground truth image features using pre-trained VGG19 model
image_features = self.vgg.predict(imgs_hr)
# Train the generators
g_loss = self.combined.train_on_batch([imgs_lr, imgs_hr], [valid, image_features])
print("g_loss:", g_loss)
elapsed_time = datetime.datetime.now() - start_time
# Plot the progress
print ("%d time: %s" % (epoch, elapsed_time))
# If at save interval => save generated image samples
if epoch % sample_interval == 0:
self.sample_images(epoch)
if epoch % 2000 == 0:
# Save models
#self.generator.save_weights("{}/generator-{}.h5".format(CHECKPOINT_PATH, epoch))
#self.discriminator.save_weights("{}/discriminator-{}.h5".format(CHECKPOINT_PATH, epoch))
ckpt_save_path = self.ckpt_manager.save()
print ('Saving checkpoint for epoch {} at {}'.format(epoch+1,
ckpt_save_path))
features = vgg(generated_high_resolution_images)
# Make the discriminator network as non-trainable
discriminator.trainable = False
# Get the probability of generated high-resolution images
probs = discriminator(generated_high_resolution_images)
# Create and compile an adversarial model
adversarial_model = Model([input_low_resolution, input_high_resolution], [probs, features])
adversarial_model.compile(loss=['binary_crossentropy', 'mse'], loss_weights=[1e-3, 1], optimizer=common_optimizer)
# Add Tensorboard
tensorboard = TensorBoard(log_dir="logs/training")
train_summary_writer = tf.summary.create_file_writer("logs/training")
tensorboard.set_model(generator)
tensorboard.set_model(discriminator)
for epoch in range(epochs):
print("Epoch:{}".format(epoch))
"""
Train the discriminator network
"""
# Sample a batch of images
high_resolution_images, low_resolution_images = sample_images(data_dir=data_dir, batch_size=batch_size,
low_resolution_shape=low_resolution_shape,
high_resolution_shape=high_resolution_shape)
# Normalize images
high_resolution_images = high_resolution_images / 127.5 - 1.
class FIR_Network:
def __init__(self, mask_batch_size, tensorboard_logs_dir=""):
self.batch_size = mask_batch_size
self.mask_batch_size = mask_batch_size
self.tr_loss_history = []
self.te_loss_history = []
self.y_pred_std_history = []
self.y_true_std_history = []
self.tf_logs = tensorboard_logs_dir
self.epoch_counter = 0
self.data_masks = None
self.data_targets = None
self.best_performing_mask = None
self.sample_weights = None
def create_dense_model(self, input_shape, dense_arch):
input_mask_layer = Input(shape=input_shape)
x = Flatten()(input_mask_layer)
for i in range(len(dense_arch[:-1])):
x = Dense(dense_arch[i], activation="sigmoid")(x)
x = Dense(dense_arch[-1], activation="linear")(x)
self.model = Model(inputs=[input_mask_layer], outputs=x)
# self.model.summary()
def named_logs(self, model, logs):
result = {}
try:
iterator = iter(logs)
except TypeError:
logs = [logs]
for l in zip(model.metrics_names, logs):
result[l[0]] = l[1]
return result
def compile_model(self):
self.model.compile(loss='mae',
optimizer='adam',
metrics=[
self.get_y_std_metric(True),
self.get_y_std_metric(False)
])
if self.tf_logs != "":
self.tb_clbk = TensorBoard(self.tf_logs)
self.tb_clbk.set_model(self.model)
def train_one(self, epoch_number, apply_weights):
if apply_weights == False:
curr_loss = self.model.train_on_batch(x=self.data_masks,
y=self.data_targets)
else:
curr_loss = self.model.train_on_batch(
x=self.data_masks,
y=self.data_targets,
sample_weight=self.sample_weights)
self.best_performing_mask = self.data_masks[np.argmin(
self.data_targets, axis=0)]
self.tr_loss_history.append(curr_loss)
self.epoch_counter = epoch_number
if self.tf_logs != "":
self.tb_clbk.on_epoch_end(self.epoch_counter,
self.named_logs(self.model, curr_loss))
self.data_masks = None
self.data_targets = None
def append_data(self, x, y):
if self.data_masks is None:
self.data_masks = x
self.data_targets = y
else:
self.data_masks = np.concatenate([self.data_masks, x], axis=0)
self.data_targets = tf.concat([self.data_targets, y], axis=0)
def test_one(self, x, y):
y_pred = self.model.predict(x=x)
curr_loss = self.model.test_on_batch(x=x, y=y)
self.te_loss_history.append(curr_loss)
return curr_loss
def predict(self, x):
y_pred = self.model.predict(x=x)
return y_pred
def get_y_std_metric(self, ifpred=True):
def y_pred_std_metric(y_true, y_pred):
y_pred_std = K.std(y_pred)
self.y_pred_std_history.append(y_pred_std)
return y_pred_std
def y_true_std_metric(y_true, y_pred):
y_true_std = K.std(y_true)
self.y_true_std_history.append(y_true_std)
return y_true_std
if ifpred:
return y_pred_std_metric
else:
return y_true_std_metric
def _run(game, network_params, memory_params, ops):
"""Sets up and runs the gaming simulation.
Initializes TensorFlow, the training agent, and the game environment.
The agent plays the game from the starting state for a number of
episodes set by the user.
Args:
args: The arguments from the command line parsed by_parse_arguments.
"""
# Setup TensorBoard Writer.
trial_id = json.loads(os.environ.get('TF_CONFIG',
'{}')).get('task',
{}).get('trial', '')
output_path = ops.job_dir if not trial_id else ops.job_dir + '/'
tensorboard = TensorBoard(log_dir=output_path)
hpt = hypertune.HyperTune()
graph = tf.Graph()
with graph.as_default():
env = gym.make(game)
agent = _create_agent(env, network_params, memory_params)
rewards = []
tensorboard.set_model(agent.policy)
def _train_or_evaluate(print_score, training=False):
"""Runs a gaming simulation and writes results for tensorboard.
Args:
print_score (bool): True to print a score to the console.
training (bool): True if the agent is training, False to eval.
Returns:
loss if training, else reward for evaluating.
"""
reward = _play(agent, env, training)
if print_score:
print(
'Train - ',
'Episode: {}'.format(episode),
'Total reward: {}'.format(reward),
)
return reward
for episode in range(1, ops.episodes + 1):
print_score = ops.print_rate and episode % ops.print_rate == 0
get_summary = ops.eval_rate and episode % ops.eval_rate == 0
rewards.append(_train_or_evaluate(print_score, training=True))
if get_summary:
avg_reward = sum(rewards) / len(rewards)
summary = {'eval_reward': avg_reward}
tensorboard.on_epoch_end(episode, summary)
hpt.report_hyperparameter_tuning_metric(
hyperparameter_metric_tag='avg_reward',
metric_value=avg_reward,
global_step=episode)
print(
'Eval - ',
'Episode: {}'.format(episode),
'Average Reward: {}'.format(avg_reward),
)
rewards = []
tensorboard.on_train_end(None)
_record_video(env, agent, output_path)
agent.policy.save(output_path, save_format='tf')
def find_best_lr(path_desc='', combined_lobes=False, records_add=''):
min_lr = 1e-7
max_lr = 1e-1
num_classes = 9
if combined_lobes:
num_classes = 6
max_conv_blocks = 3
iteration = 0
change_background = True
for weighted_loss in [True, False]:
for atrous in [True, False]:
for layer in [2, 3]:
for growth_rate in [4]:
for filters in [8]:
for num_conv_blocks in [
2
]: # Always start with 1 before downsampling
for conv_lambda in [1]:
base_path, morfeus_drive = return_paths()
run_data = {
'layers': layer,
'max_conv_blocks': max_conv_blocks,
'filters': filters,
'num_conv_blocks': num_conv_blocks,
'conv_lambda': conv_lambda,
'growth_rate': growth_rate,
'atrous': atrous
}
layers_dict = get_layers_dict_dense(
**run_data,
pool=(2, 4, 4),
num_classes=num_classes)
things = [
'change_background{}'.format(
change_background),
'weighted_loss_{}'.format(weighted_loss),
'atrous_{}'.format(atrous),
'layers{}'.format(layer),
'max_conv_blocks_{}'.format(
max_conv_blocks),
'filters_{}'.format(filters),
'num_conv_blocks_{}'.format(
num_conv_blocks),
'conv_lambda_{}'.format(conv_lambda),
'growth_rate_{}'.format(growth_rate),
'{}_Iteration'.format(iteration)
]
out_path = os.path.join(
morfeus_drive, path_desc, 'DenseNet')
if records_add != '':
things = ['Gaussian'] + things
for thing in things:
out_path = os.path.join(out_path, thing)
if os.path.exists(out_path):
print('already done')
continue
os.makedirs(out_path)
print(out_path)
model = my_UNet(
layers_dict=layers_dict,
image_size=(None, None, None, 1),
mask_output=True,
out_classes=num_classes,
explictly_defined=True).created_model
k = TensorBoard(log_dir=out_path,
profile_batch=0,
write_graph=True)
k.set_model(model)
k.on_train_begin()
lr_opt = tf.keras.optimizers.Adam
base_path, morfeus_drive, train_generator, validation_generator = return_generators(
combined_lobes=combined_lobes,
records_add=records_add,
change_background=change_background)
loss = tf.keras.losses.CategoricalCrossentropy(
from_logits=False)
LearningRateFinder(
epochs=10,
model=model,
metrics=[
'categorical_accuracy',
MeanDSC(num_classes=num_classes)
],
out_path=out_path,
optimizer=lr_opt,
loss=loss,
steps_per_epoch=len(train_generator),
train_generator=train_generator.data_set,
lower_lr=min_lr,
high_lr=max_lr)
tf.keras.backend.clear_session()
return None # repeat!
