class DeepQ(object):
"""Constructs the desired deep q learning network"""
def __init__(self, action_size, observation_size, lr=LEARNING_RATE):
self.action_size = action_size
self.observation_size = observation_size
self.lr = lr
self.model = None
self.target_model = None
self.qvalue_evolution = []
self.construct_q_network()
def construct_q_network(self):
""" Construct both the actual Q-network and the target network with three hidden layers and ReLu activation
functions in between. The network uses an Adam optimizer with MSE loss."""
self.model = Sequential()
input_layer = Input(shape=(self.observation_size * NUM_FRAMES, ))
layer1 = Dense(self.observation_size * NUM_FRAMES)(input_layer)
layer1 = Activation('relu')(layer1)
layer3 = Dense(self.observation_size)(layer1)
layer3 = Activation('relu')(layer3)
layer4 = Dense(2 * self.action_size)(layer3)
layer4 = Activation('relu')(layer4)
output = Dense(self.action_size)(layer4)
self.model = Model(inputs=[input_layer], outputs=[output])
self.model.compile(loss='mse', optimizer=Adam(lr=self.lr))
self.target_model = Model(inputs=[input_layer], outputs=[output])
self.target_model.compile(loss='mse', optimizer=Adam(lr=self.lr))
self.target_model.set_weights(self.model.get_weights())
def predict_movement(self, data, epsilon):
""" Predict the next action from the network. Epsilon is the probability of making a random move.
Returns the optimal action and the predicted reward for that action. """
rand_val = np.random.random()
q_actions = self.model.predict(data.reshape(
1, self.observation_size * NUM_FRAMES),
batch_size=1)
if rand_val < epsilon:
opt_policy = np.random.randint(0, self.action_size)
else:
opt_policy = np.argmax(np.abs(q_actions[0]))
self.qvalue_evolution.append(q_actions[0][opt_policy])
return opt_policy, q_actions[0][opt_policy]
def predict_rewards(self, data):
""" Like predict_movement, only without a probability of a random move and returns only the predicted
q-values."""
q_actions = self.model.predict(np.array(data).reshape(
1, self.observation_size * NUM_FRAMES),
batch_size=1)
return q_actions[0]
def train(self, s_batch, a_batch, r_batch, d_batch, s2_batch):
""" Trains the network on a batch of input.
The parameters are batches of states, actions, rewards, done booleans and next states. """
batch_size = s_batch.shape[0]
# Train according to the Bellman Equation
targets = self.model.predict(s_batch, batch_size=batch_size)
fut_action = self.target_model.predict(s2_batch, batch_size=batch_size)
targets[:, a_batch.flatten()] = r_batch
targets[d_batch, a_batch[d_batch]] += DISCOUNT_RATE * np.max(
fut_action[d_batch], axis=-1)
targets_ts = tf.convert_to_tensor(targets, dtype=tf.float32)
loss = self.model.train_on_batch(s_batch, targets_ts)
return loss
def train_imitation(self, s_batch, t_batch):
""" Trains network on generated data: Imitation Learning. """
loss = self.model.train_on_batch(s_batch, t_batch)
return loss
def save_network(self, path):
if not os.path.exists(path):
os.mkdir(path)
self.model.save(os.path.join(path, 'network.h5'))
print("Successfully saved network.")
def load_network(self, path):
self.model = load_model(os.path.join(path, 'network.h5'))
print("Successfully loaded network.")
def target_train(self):
""" The target network is updated each step by 'merging' a small part of the actual network into it. """
model_weights = self.model.get_weights()
target_model_weights = self.target_model.get_weights()
for i in range(len(model_weights)):
target_model_weights[i] = TAU * model_weights[i] + (
1 - TAU) * target_model_weights[i]
self.target_model.set_weights(target_model_weights)
def layer_test(layer_cls,
kwargs={},
input_shape=None,
input_dtype=None,
input_data=None,
expected_output=None,
expected_output_dtype=None,
fixed_batch_size=False,
supports_masking=False):
# generate input data
if input_data is None:
if not input_shape:
raise AssertionError()
if not input_dtype:
input_dtype = K.floatx()
input_data_shape = list(input_shape)
for i, e in enumerate(input_data_shape):
if e is None:
input_data_shape[i] = np.random.randint(1, 4)
input_mask = []
if all(isinstance(e, tuple) for e in input_data_shape):
input_data = []
for e in input_data_shape:
input_data.append(
(10 * np.random.random(e)).astype(input_dtype))
if supports_masking:
a = np.full(e[:2], False)
a[:, :e[1] // 2] = True
input_mask.append(a)
else:
input_data = (10 * np.random.random(input_data_shape))
input_data = input_data.astype(input_dtype)
if supports_masking:
a = np.full(input_data_shape[:2], False)
a[:, :input_data_shape[1] // 2] = True
print(a)
print(a.shape)
input_mask.append(a)
else:
if input_shape is None:
input_shape = input_data.shape
if input_dtype is None:
input_dtype = input_data.dtype
if expected_output_dtype is None:
expected_output_dtype = input_dtype
# instantiation
layer = layer_cls(**kwargs)
# test get_weights , set_weights at layer level
weights = layer.get_weights()
layer.set_weights(weights)
try:
expected_output_shape = layer.compute_output_shape(input_shape)
except Exception:
expected_output_shape = layer._compute_output_shape(input_shape)
# test in functional API
if isinstance(input_shape, list):
if fixed_batch_size:
x = [Input(batch_shape=e, dtype=input_dtype) for e in input_shape]
if supports_masking:
mask = [
Input(batch_shape=e[0:2], dtype=bool) for e in input_shape
]
else:
x = [Input(shape=e[1:], dtype=input_dtype) for e in input_shape]
if supports_masking:
mask = [Input(shape=(e[1], ), dtype=bool) for e in input_shape]
else:
if fixed_batch_size:
x = Input(batch_shape=input_shape, dtype=input_dtype)
if supports_masking:
mask = Input(batch_shape=input_shape[0:2], dtype=bool)
else:
x = Input(shape=input_shape[1:], dtype=input_dtype)
if supports_masking:
mask = Input(shape=(input_shape[1], ), dtype=bool)
if supports_masking:
y = layer(Masking()(x), mask=mask)
else:
y = layer(x)
if not (K.dtype(y) == expected_output_dtype):
raise AssertionError()
# check with the functional API
if supports_masking:
model = Model([x, mask], y)
actual_output = model.predict([input_data, input_mask[0]])
else:
model = Model(x, y)
actual_output = model.predict(input_data)
actual_output_shape = actual_output.shape
for expected_dim, actual_dim in zip(expected_output_shape,
actual_output_shape):
if expected_dim is not None:
if not (expected_dim == actual_dim):
raise AssertionError("expected_shape", expected_output_shape,
"actual_shape", actual_output_shape)
if expected_output is not None:
assert_allclose(actual_output, expected_output, rtol=1e-3)
# test serialization, weight setting at model level
model_config = model.get_config()
recovered_model = model.__class__.from_config(model_config)
if model.weights:
weights = model.get_weights()
recovered_model.set_weights(weights)
_output = recovered_model.predict(input_data)
assert_allclose(_output, actual_output, rtol=1e-3)
# test training mode (e.g. useful when the layer has a
# different behavior at training and testing time).
if has_arg(layer.call, 'training'):
model.compile('rmsprop', 'mse')
model.train_on_batch(input_data, actual_output)
# test instantiation from layer config
layer_config = layer.get_config()
layer_config['batch_input_shape'] = input_shape
layer = layer.__class__.from_config(layer_config)
# for further checks in the caller function
return actual_output
def main():
data_dir = str(pathlib.Path.cwd()) + "/"
batch_size = 16
img_shape = (128, 128, 3)
print(data_dir)
gen = ImageDataGenerator(rescale=1 / 127.5, samplewise_center=True)
iters = gen.flow_from_directory(
directory=data_dir,
# classes=['doraemon'],
class_mode=None,
color_mode='rgb',
target_size=img_shape[:2],
batch_size=batch_size,
shuffle=True)
x_train_batch = next(iters)
for i in range(3):
plt.imshow(array_to_img(x_train_batch[i]))
optimizer = Adam(lr=0.00001, beta_1=0.5)
generator = generator_model(img_shape)
discriminator = discriminator_model(img_shape)
discriminator.trainable = True
generator.compile(loss='binary_crossentropy', optimizer=optimizer)
discriminator.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
z = Input(shape=(128, ))
img = generator(z)
discriminator.trainable = False
valid = discriminator(img)
generator_containing_discriminator = Model(z, valid)
generator_containing_discriminator.compile(loss='binary_crossentropy',
optimizer=optimizer)
#saveディレクトリ
model_save_dir = 'models'
img_save_dir = 'output_imgs'
#5×5生成画像生成
imgs_shape = (4, 4)
n_img_samples = np.prod(imgs_shape)
sample_seeds = np.random.uniform(-1, 1, (n_img_samples, 128))
hist = []
logs = []
#保存先フォルダ生成
os.makedirs(model_save_dir, exist_ok=True)
os.makedirs(img_save_dir, exist_ok=True)
Totalstep = 500000
for step, batch in enumerate(iters):
if len(batch) < batch_size:
continue
if step > Totalstep:
break
half_batch = int(batch_size / 2)
z_d = np.random.uniform(-1, 1, (half_batch, 128))
g_pred = generator.predict(z_d)
discriminator.trainable = True
real_loss = discriminator.train_on_batch(batch[:half_batch],
np.ones((half_batch, 1)))
fake_loss = discriminator.train_on_batch(g_pred,
np.zeros((half_batch, 1)))
d_loss = 0.5 * np.add(real_loss, fake_loss)
z_g = np.random.uniform(-1, 1, (batch_size, 128))
discriminator.trainable = False
g_loss = generator_containing_discriminator.train_on_batch(
z_g, np.ones((batch_size, 1)))
logs.append({
'step': step,
'd_loss': d_loss[0],
'acc[%]': 100 * d_loss[1],
'g_loss': g_loss
})
print(logs[-1])
if step % 200 == 0:
img_path = '{}/generate_{}.png'.format(img_save_dir, step)
save_imgs(img_path,
generator.predict(sample_seeds),
rows=imgs_shape[0],
cols=imgs_shape[1])
if step % 5000 == 0:
generator.save('{}/generator_{}.hd5'.format(model_save_dir, step))
discriminator.save('{}/discriminator_{}.hd5'.format(
model_save_dir, step))
'''
Totalstep=30
generator.load_weights('D:/python/jupyter_notebook/dcgan/models/generator_25000.hd5')
for step, batch in enumerate(iters):
if len(batch) < batch_size:
continue
if step > Totalstep:
break
z_d = np.random.uniform(-1, 1, (batch_size, 100))
g_pred = generator.predict(z_d)
img_path = '{}/pred_generate_{}.png'.format(img_save_dir, step)
save_imgs(img_path, generator.predict(z_d), rows=imgs_shape[0], cols=imgs_shape[1])
'''
return
def train_frcnn(options):
if options.parser == 'pascal_voc':
from utils import voc_parser as get_data
elif options.parser == 'simple':
from utils import simple_parser as get_data
else:
raise ValueError(
"Command line option parser must be one of 'pascal_voc' or 'simple'"
)
# pass the settings from the command line, and persist them in the config object
C = Config()
C.use_horizontal_flips = bool(options.horizontal_flips)
C.use_vertical_flips = bool(options.vertical_flips)
C.rot_90 = bool(options.rot_90)
C.model_path = options.output_weight_path.format(options.network)
C.num_rois = int(options.num_rois)
if options.network == 'resnet50':
C.network = 'resnet50'
from utils import rpn_res as rpn
from utils import classifier_res as classifier_func
from utils import get_img_output_length_res as get_img_output_length
from utils import nn_base_res as nn_base
elif options.network == 'vgg':
C.network = 'vgg'
from utils import rpn_vgg as rpn
from utils import classifier_vgg as classifier_func
from utils import get_img_output_length_vgg as get_img_output_length
from utils import nn_base_vgg as nn_base
else:
print('Not a valid model')
raise ValueError
# check if weight path was passed via command line
if options.input_weight_path:
C.base_net_weights = options.input_weight_path
else:
# set the path to weights based on backend and model
C.base_net_weights = get_weight_path(options.network)
all_imgs, classes_count, class_mapping = get_data(options.path)
if 'bg' not in classes_count:
classes_count['bg'] = 0
class_mapping['bg'] = len(class_mapping)
C.class_mapping = class_mapping
inv_map = {v: k for k, v in class_mapping.items()}
print('Training images per class:')
pprint.pprint(classes_count)
print('Num classes (including bg) = {}'.format(len(classes_count)))
config_output_filename = options.config_filename
with open(config_output_filename, 'wb') as config_f:
pickle.dump(C, config_f)
print(
'Config has been written to {}, and can be loaded when testing to ensure correct results'
.format(config_output_filename))
#
random.shuffle(all_imgs)
train_imgs = [s for s in all_imgs if s['imageset'] == 'trainval']
val_imgs = [s for s in all_imgs if s['imageset'] == 'test']
print('Num train samples {}'.format(len(train_imgs)))
print('Num val samples {}'.format(len(val_imgs)))
data_gen_train = get_anchor_gt(train_imgs,
classes_count,
C,
get_img_output_length,
K.backend(),
mode='train')
data_gen_val = get_anchor_gt(val_imgs,
classes_count,
C,
get_img_output_length,
K.backend(),
mode='val')
if K.backend() == "theano":
input_shape_img = (3, None, None)
else:
input_shape_img = (None, None, 3)
img_input = Input(shape=input_shape_img)
roi_input = Input(shape=(None, 4))
# define the base network (resnet here, can be VGG, Inception, etc)
shared_layers = nn_base(img_input, trainable=True)
# define the RPN, built on the base layers
num_anchors = len(C.anchor_box_scales) * len(C.anchor_box_ratios)
rpn = rpn(shared_layers, num_anchors)
classifier = classifier_func(shared_layers,
roi_input,
C.num_rois,
nb_classes=len(classes_count),
trainable=True)
model_rpn = Model(img_input, rpn[:2])
model_classifier = Model([img_input, roi_input], classifier)
# this is a model that holds both the RPN and the classifier, used to load/save weights for the models
model_all = Model([img_input, roi_input], rpn[:2] + classifier)
try:
print('loading weights from {}'.format(C.base_net_weights))
model_rpn.load_weights(C.base_net_weights + "rpn.h5", by_name=True)
model_classifier.load_weights(C.base_net_weights + "classifier.h5",
by_name=True)
except Exception as e:
model_rpn.load_weights(C.base_net_weights, by_name=True)
model_classifier.load_weights(C.base_net_weights, by_name=True)
print('Exception: {}'.format(e))
optimizer = Adam(lr=1e-5, decay=2e-7)
optimizer_classifier = Adam(lr=1e-5, decay=2e-7)
model_rpn.compile(
optimizer=optimizer,
loss=[rpn_loss_cls(num_anchors),
rpn_loss_regr(num_anchors)])
model_classifier.compile(
optimizer=optimizer_classifier,
loss=[class_loss_cls,
class_loss_regr(len(classes_count) - 1)],
metrics={'dense_class_{}'.format(len(classes_count)): 'accuracy'})
model_all.compile(optimizer='sgd', loss='mae')
epoch_length = options.epoch_length
num_epochs = int(options.num_epochs)
iter_num = 0
losses = np.zeros((epoch_length, 5))
rpn_accuracy_rpn_monitor = []
rpn_accuracy_for_epoch = []
start_time = time.time()
best_loss = np.Inf
print('Starting training')
for epoch_num in range(num_epochs):
progbar = generic_utils.Progbar(epoch_length)
print('Epoch {}/{}'.format(epoch_num + 1, num_epochs))
while True:
try:
if len(rpn_accuracy_rpn_monitor) == epoch_length and C.verbose:
mean_overlapping_bboxes = float(
sum(rpn_accuracy_rpn_monitor)) / len(
rpn_accuracy_rpn_monitor)
rpn_accuracy_rpn_monitor = []
print(
'Average number of overlapping bounding boxes from RPN = {} for {} previous iterations'
.format(mean_overlapping_bboxes, epoch_length))
if mean_overlapping_bboxes == 0:
print(
'RPN is not producing bounding boxes that overlap the ground truth boxes. '
'Check RPN settings or keep training.')
X, Y, img_data = next(data_gen_train)
loss_rpn = model_rpn.train_on_batch(X, Y)
P_rpn = model_rpn.predict_on_batch(X)
R = rpn_to_roi(P_rpn[0],
P_rpn[1],
C,
K.backend(),
use_regr=True,
overlap_thresh=0.7,
max_boxes=300)
# note: calc_iou converts from (x1,y1,x2,y2) to (x,y,w,h) format
X2, Y1, Y2, IouS = calc_iou(R, img_data, C, class_mapping)
if X2 is None:
rpn_accuracy_rpn_monitor.append(0)
rpn_accuracy_for_epoch.append(0)
continue
neg_samples = np.where(Y1[0, :, -1] == 1)
pos_samples = np.where(Y1[0, :, -1] == 0)
if len(neg_samples) > 0:
neg_samples = neg_samples[0]
else:
neg_samples = []
if len(pos_samples) > 0:
pos_samples = pos_samples[0]
else:
pos_samples = []
rpn_accuracy_rpn_monitor.append(len(pos_samples))
rpn_accuracy_for_epoch.append((len(pos_samples)))
if C.num_rois > 1:
if len(pos_samples) < C.num_rois // 2:
selected_pos_samples = pos_samples.tolist()
else:
selected_pos_samples = np.random.choice(
pos_samples, C.num_rois // 2,
replace=False).tolist()
try:
selected_neg_samples = np.random.choice(
neg_samples,
C.num_rois - len(selected_pos_samples),
replace=False).tolist()
except:
selected_neg_samples = np.random.choice(
neg_samples,
C.num_rois - len(selected_pos_samples),
replace=True).tolist()
sel_samples = selected_pos_samples + selected_neg_samples
else:
# in the extreme case where num_rois = 1, we pick a random pos or neg sample
selected_pos_samples = pos_samples.tolist()
selected_neg_samples = neg_samples.tolist()
if np.random.randint(0, 2):
sel_samples = random.choice(selected_neg_samples)
else:
sel_samples = random.choice(selected_pos_samples)
loss_class = model_classifier.train_on_batch(
[X, X2[:, sel_samples, :]],
[Y1[:, sel_samples, :], Y2[:, sel_samples, :]])
losses[iter_num, 0] = loss_rpn[1]
losses[iter_num, 1] = loss_rpn[2]
losses[iter_num, 2] = loss_class[1]
losses[iter_num, 3] = loss_class[2]
losses[iter_num, 4] = loss_class[3]
progbar.update(iter_num + 1,
[('rpn_cls', losses[iter_num, 0]),
('rpn_regr', losses[iter_num, 1]),
('detector_cls', losses[iter_num, 2]),
('detector_regr', losses[iter_num, 3])])
iter_num += 1
if iter_num == epoch_length:
loss_rpn_cls = np.mean(losses[:, 0])
loss_rpn_regr = np.mean(losses[:, 1])
loss_class_cls = np.mean(losses[:, 2])
loss_class_regr = np.mean(losses[:, 3])
class_acc = np.mean(losses[:, 4])
mean_overlapping_bboxes = float(sum(
rpn_accuracy_for_epoch)) / len(rpn_accuracy_for_epoch)
rpn_accuracy_for_epoch = []
if C.verbose:
print(
'Mean number of bounding boxes from RPN overlapping ground truth boxes: {}'
.format(mean_overlapping_bboxes))
print(
'Classifier accuracy for bounding boxes from RPN: {}'
.format(class_acc))
print('Loss RPN classifier: {}'.format(loss_rpn_cls))
print('Loss RPN regression: {}'.format(loss_rpn_regr))
print('Loss Detector classifier: {}'.format(
loss_class_cls))
print('Loss Detector regression: {}'.format(
loss_class_regr))
print('Elapsed time: {}'.format(time.time() -
start_time))
curr_loss = loss_rpn_cls + loss_rpn_regr + loss_class_cls + loss_class_regr
iter_num = 0
start_time = time.time()
if curr_loss < best_loss:
if C.verbose:
print(
f'Total loss decreased from {best_loss:.3f} to {curr_loss:.3f}, saving weights to '
f'{C.model_path}')
best_loss = curr_loss
model_classifier.save_weights(C.model_path +
"classifier.h5")
model_rpn.save_weights(C.model_path + "rpn.h5")
break
except Exception as e:
print('Exception: {}'.format(e))
continue
print('Training complete, exiting.')
class CartoonGAN():
def __init__(self, args):
self.model_name = 'CartoonGAN'
self.batch_size = args.batch_size
self.epochs = args.epochs
self.gpu = args.gpu_num
self.image_channels = args.image_channels
self.image_size = args.image_size
self.init_epoch = args.init_epoch
self.log_dir = args.log_dir
self.lr = args.lr
self.model_dir = args.model_dir
self.weight = args.weight
# method for generator
def generator(self):
input_shape = [self.image_size, self.image_size, self.image_channels]
input_img = Input(shape=input_shape, name="input")
# first block
x = ReflectionPadding2D(3)(input_img)
x = Conv2D(64, (7, 7),
strides=1,
use_bias=True,
padding='valid',
name="conv1")(x)
x = InstanceNormalization(name="norm1")(x)
x = Activation("relu")(x)
# down-convolution
channel = 128
for i in range(2):
x = Conv2D(channel, (3, 3),
strides=2,
use_bias=True,
padding='same',
name="conv{}_1".format(i + 2))(x)
x = Conv2D(channel, (3, 3),
strides=1,
use_bias=True,
padding='same',
name="conv{}_2".format(i + 2))(x)
x = InstanceNormalization(name="norm{}".format(i + 2))(x)
x = Activation("relu")(x)
channel = channel * 2
# residual blocks
x_res = x
for i in range(8):
x = ReflectionPadding2D(1)(x)
x = Conv2D(256, (3, 3),
strides=1,
use_bias=True,
padding='valid',
name="conv{}_1".format(i + 4))(x)
x = InstanceNormalization(name="norm{}_1".format(i + 4))(x)
x = Activation("relu")(x)
x = ReflectionPadding2D(1)(x)
x = Conv2D(256, (3, 3),
strides=1,
use_bias=True,
padding='valid',
name="conv{}_2".format(i + 4))(x)
x = InstanceNormalization(name="norm{}_2".format(i + 4))(x)
x = Add()([x, x_res])
x_res = x
# up-convolution
for i in range(2):
x = Conv2DTranspose(channel // 2,
3,
2,
padding="same",
output_padding=1,
name="deconv{}_1".format(i + 1))(x)
x = Conv2D(channel // 2, (3, 3),
strides=1,
use_bias=True,
padding="same",
name="deconv{}_2".format(i + 1))(x)
x = InstanceNormalization(name="norm_deconv" + str(i + 1))(x)
x = Activation("relu")(x)
channel = channel // 2
# last block
x = ReflectionPadding2D(3)(x)
x = Conv2D(3, (7, 7),
strides=1,
use_bias=True,
padding="valid",
name="deconv3")(x)
x = Activation("tanh")(x)
model = Model(input_img, x, name='Cartoon_Generator')
return model
# method for discriminator
def discriminator(self):
input_shape = [self.image_size, self.image_size, self.image_channels]
input_img = Input(shape=input_shape, name="input")
# first block
x = Conv2D(32, (3, 3),
strides=1,
use_bias=True,
padding='same',
name="conv1")(input_img)
x = LeakyReLU(alpha=0.2)(x)
# block loop
channel = 64
for i in range(2):
x = Conv2D(channel, (3, 3),
strides=2,
use_bias=True,
padding='same',
name="conv{}_1".format(i + 2))(x)
x = LeakyReLU(alpha=0.2)(x)
x = Conv2D(channel * 2, (3, 3),
strides=1,
use_bias=True,
padding='same',
name="conv{}_2".format(i + 2))(x)
x = InstanceNormalization()(x)
x = LeakyReLU(alpha=0.2)(x)
channel = channel * 2
# last block
x = Conv2D(256, (3, 3),
strides=1,
use_bias=True,
padding='same',
name="conv4")(x)
x = InstanceNormalization()(x)
x = LeakyReLU(alpha=0.2)(x)
x = Conv2D(1, (3, 3),
strides=1,
use_bias=True,
padding='same',
activation='sigmoid',
name="conv5")(x)
model = Model(input_img, x, name='Cartoon_Discriminator')
return model
# vgg loss function
def vgg_loss(self, y_true, y_pred):
# get vgg model
input_shape = [self.image_size, self.image_size, self.image_channels]
img_input = Input(shape=input_shape, name="vgg_input")
vgg19 = tf.keras.applications.vgg19.VGG19(weights='imagenet')
vggmodel = Model(inputs=vgg19.input,
outputs=vgg19.get_layer('block4_conv4').output)
x = vggmodel(img_input)
vgg = Model(img_input, x, name='VGG_for_Feature_Extraction')
# get l1 loss for the content loss
y_true = vgg(y_true)
y_pred = vgg(y_pred)
content_loss = tf.losses.absolute_difference(y_true, y_pred)
return content_loss
# compile each model
def compile_model(self):
# init summary writer for tensorboard
self.callback1 = TensorBoard(self.log_dir + '/discriminator')
self.callback2 = TensorBoard(self.log_dir + '/generator')
self.callback3 = TensorBoard(self.log_dir + '/generated_images')
# model stuff
input_shape = [self.image_size, self.image_size, self.image_channels]
adam1 = Adam(lr=self.lr)
adam2 = Adam(lr=self.lr * 2)
# init and add multi-gpu support
try:
self.discriminator = multi_gpu_model(self.discriminator(),
gpus=self.gpu)
except:
self.discriminator = self.discriminator()
try:
self.generator = multi_gpu_model(self.generator(), gpus=self.gpu)
except:
self.generator = self.generator()
# compile discriminator
self.discriminator.compile(loss='binary_crossentropy', optimizer=adam1)
# compile generator
input_tensor = Input(shape=input_shape)
generated_catroon_tensor = self.generator(input_tensor)
self.discriminator.trainable = False # for here we only train the generator
discriminator_output = self.discriminator(generated_catroon_tensor)
self.train_generator = Model(
input_tensor,
outputs=[generated_catroon_tensor, discriminator_output])
# add multi-gpu support
try:
self.train_generator = multi_gpu_model(self.train_generator,
gpus=self.gpu)
except:
pass
self.train_generator.compile(
loss=[self.vgg_loss, 'binary_crossentropy'],
loss_weights=[float(self.weight), 1.0],
optimizer=adam2)
# set callback model
self.callback1.set_model(self.discriminator)
self.callback2.set_model(self.train_generator)
self.callback3.set_model(self.train_generator)
# method for training process
def train(self):
# start training
flip = False
variance = 1 / 127.5
start_time = time.time()
for epoch in range(1, self.epochs + 1):
# create batch generator at each epoch
batch_generator = DataGenerator(image_size=self.image_size,
batch_size=self.batch_size)
batch_end = len(batch_generator)
print('Epoch {}'.format(epoch))
# start training for each batch
for idx, (photo, cartoon, smooth_cartoon,
index) in enumerate(batch_generator):
# these two tensors measure the output of generator and discriminator
real = np.ones((self.batch_size, ) + (64, 64, 1))
fake = np.zeros((self.batch_size, ) + (64, 64, 1))
# check if it is the end of an epoch
if index + 1 == batch_end:
break
# initial training or start training
if epoch < self.init_epoch:
g_loss = self.train_generator.train_on_batch(
photo, [photo, real])
generated_img = self.generator.predict(photo)
print(
"Batch %d (initial training for generator), g_loss: %.5f, with time: %4.4f"
% (idx, g_loss[2], time.time() - start_time))
start_time = time.time()
write_log(self.callback2, 'g_loss', g_loss[2],
idx + (epoch + 1) * len(batch_generator))
if idx % 20 == 0:
write_images(self.callback3, generated_img,
'generated_imgs',
idx + (epoch + 1) * len(batch_generator))
if epoch % 20 == 0 and K.eval(
self.train_generator.optimizer.lr) > 0.0001:
K.set_value(
self.train_generator.optimizer.lr,
K.eval(self.train_generator.optimizer.lr) * 0.99)
else:
# add noise to the input of discriminator
if variance > 0.00001:
variance = variance * 0.9999
gaussian = np.random.normal(
0, variance, (cartoon.shape[1], cartoon.shape[2]))
cartoon[:, :, :, 0] = cartoon[:, :, :, 0] + gaussian
cartoon[:, :, :, 1] = cartoon[:, :, :, 1] + gaussian
cartoon[:, :, :, 2] = cartoon[:, :, :, 2] + gaussian
gaussian = np.random.normal(
0, variance, (cartoon.shape[1], cartoon.shape[2]))
smooth_cartoon[:, :, :,
0] = smooth_cartoon[:, :, :,
0] + gaussian
smooth_cartoon[:, :, :,
1] = smooth_cartoon[:, :, :,
1] + gaussian
smooth_cartoon[:, :, :,
2] = smooth_cartoon[:, :, :,
2] + gaussian
# generate cartoonized images
generated_img = self.generator.predict(photo)
# to certain probability: flip the label of discriminator
if idx % 9 == 0 or np.random.uniform(0, 1) < 0.05:
real = fake
fake = fake + 1
flip = True
# train discriminator and adversarial loss
real_loss = self.discriminator.train_on_batch(
cartoon, real)
smooth_loss = self.discriminator.train_on_batch(
smooth_cartoon, fake)
fake_loss = self.discriminator.train_on_batch(
generated_img, fake)
d_loss = (real_loss + smooth_loss + fake_loss) / 3
# train generator
if flip:
real = fake
fake = fake - 1
flip = False
g_loss = self.train_generator.train_on_batch(
photo, [photo, real])
print(
"Batch %d, d_loss: %.5f, g_loss: %.5f, with time: %4.4f"
% (idx, d_loss, g_loss[2], time.time() - start_time))
start_time = time.time()
# add losses to writer
write_log(self.callback1, 'd_loss', d_loss,
idx + (epoch + 1) * len(batch_generator))
write_log(self.callback2, 'g_loss', g_loss[2],
idx + (epoch + 1) * len(batch_generator))
if idx % 20 == 0:
write_images(self.callback3, generated_img,
'generated_imgs',
idx + (epoch + 1) * len(batch_generator))
# change learning rate
if epoch % 20 == 0 and K.eval(
self.discriminator.optimizer.lr) > 0.0001:
K.set_value(
self.discriminator.optimizer.lr,
K.eval(self.discriminator.optimizer.lr) * 0.95)
if epoch % 20 == 0 and K.eval(
self.train_generator.optimizer.lr) > 0.0001:
K.set_value(
self.train_generator.optimizer.lr,
K.eval(self.train_generator.optimizer.lr) * 0.95)
# save model
if epoch % 50 == 0:
self.generator.save_weights(
self.model_dir + '/' +
'CartoonGan_generator_epoch_{}.h5'.format(epoch))
self.discriminator.save_weights(
self.model_dir + '/' +
'CartoonGan_discriminator_epoch_{}.h5'.format(epoch))
self.train_generator.save_weights(
self.model_dir + '/' +
'CartoonGan_train_generator_epoch_{}.h5'.format(epoch))
print('Done!')
self.generator.save('CartoonGan_generator.h5')
def layer_test(layer_cls, kwargs={}, input_shape=None, input_dtype=None,
input_data=None, expected_output=None,
expected_output_dtype=None, fixed_batch_size=False):
# generate input data
if input_data is None:
if not input_shape:
raise AssertionError()
if not input_dtype:
input_dtype = K.floatx()
input_data_shape = list(input_shape)
for i, e in enumerate(input_data_shape):
if e is None:
input_data_shape[i] = np.random.randint(1, 4)
if all(isinstance(e, tuple) for e in input_data_shape):
input_data = []
for e in input_data_shape:
input_data.append(
(10 * np.random.random(e)).astype(input_dtype))
else:
input_data = (10 * np.random.random(input_data_shape))
input_data = input_data.astype(input_dtype)
else:
if input_shape is None:
input_shape = input_data.shape
if input_dtype is None:
input_dtype = input_data.dtype
if expected_output_dtype is None:
expected_output_dtype = input_dtype
# instantiation
layer = layer_cls(**kwargs)
# test get_weights , set_weights at layer level
weights = layer.get_weights()
layer.set_weights(weights)
try:
expected_output_shape = layer.compute_output_shape(input_shape)
except Exception:
expected_output_shape = layer._compute_output_shape(input_shape)
# test in functional API
if isinstance(input_shape, list):
if fixed_batch_size:
x = [Input(batch_shape=e, dtype=input_dtype) for e in input_shape]
else:
x = [Input(shape=e[1:], dtype=input_dtype) for e in input_shape]
else:
if fixed_batch_size:
x = Input(batch_shape=input_shape, dtype=input_dtype)
else:
x = Input(shape=input_shape[1:], dtype=input_dtype)
y = layer(x)
if not (K.dtype(y) == expected_output_dtype):
raise AssertionError()
# check with the functional API
model = Model(x, y)
actual_output = model.predict(input_data)
actual_output_shape = actual_output.shape
for expected_dim, actual_dim in zip(expected_output_shape,
actual_output_shape):
if expected_dim is not None:
if not (expected_dim == actual_dim):
raise AssertionError()
if expected_output is not None:
assert_allclose(actual_output, expected_output, rtol=1e-3)
# test serialization, weight setting at model level
model_config = model.get_config()
recovered_model = model.__class__.from_config(model_config)
if model.weights:
weights = model.get_weights()
recovered_model.set_weights(weights)
_output = recovered_model.predict(input_data)
assert_allclose(_output, actual_output, rtol=1e-3)
# test training mode (e.g. useful when the layer has a
# different behavior at training and testing time).
if has_arg(layer.call, 'training'):
model.compile('rmsprop', 'mse')
model.train_on_batch(input_data, actual_output)
# test instantiation from layer config
layer_config = layer.get_config()
layer_config['batch_input_shape'] = input_shape
layer = layer.__class__.from_config(layer_config)
# for further checks in the caller function
return actual_output
class CGAN:
def __init__(self, config_options):
"""
:param config_options: config parser object
"""
# Images
self.img_dataset = config_options['Images']['dataset']
self.img_rows = int(config_options['Images']['rows'])
self.img_cols = int(config_options['Images']['cols'])
self.channels = int(config_options['Images']['channels'])
self.img_shape = (self.img_rows, self.img_cols, self.channels)
self.output_image_grid = [
int(el) for el in config_options['Images']['out_grid'].split(', ')
]
# Word embeddings
self.word_embeddings = load_embeddings(
config_options['Embeddings']['file'])
self.embeddings_dim = self.word_embeddings.vector_size
self.embeddings_vocab = len(self.word_embeddings.vocab)
self.train_labels = self.read_labels(
os.path.join(os.getcwd(), 'data',
config_options['Embeddings']['train_vocab']))
self.test_words = config_options['Embeddings']['test_vocab'].split(
', ')
_, self.index = self.build_index()
# GAN
optimizer = Adam(
0.0002, 0.5
) # the first value is the learning rate and the second is the beta1, right?
# Build and compile the discriminator
self.discriminator = self.build_discriminator(
config_options['Discriminator'])
self.discriminator.compile(
loss=config_options['Discriminator']['loss'],
optimizer=optimizer,
metrics=config_options['Discriminator']['metrics'].split(', '))
# Build the generator
self.generator = self.build_generator(config_options['Generator'])
# The generator takes noise and the target label as input
# and generates the corresponding digit of that label -> unclear what you mean by generate the digit
noise = Input(shape=(self.embeddings_dim, ))
label = Input(shape=(1, ))
img = self.generator([noise, label])
# For the combined model we will only train the generator
self.discriminator.trainable = False
# The discr takes generated image as input and determines validity
# and the label of that image
valid = self.discriminator([img, label])
# The combined model (stacked generator and discriminator)
# Trains generator to fool discriminator
self.gan = Model([noise, label], valid)
self.gan.compile(loss=config_options['GAN']['loss'],
optimizer=optimizer)
print('Discriminator summary:')
self.discriminator.summary()
print('\nGenerator summary:')
self.generator.summary()
print('\nGAN summary:')
self.gan.summary()
self.epochs = int(config_options['GAN']['epochs'])
self.batch_size = int(config_options['GAN']['batch_size'])
self.sample_interval = int(config_options['GAN']['sample_interval'])
self.image_folder = config_options['GAN']['image_folder']
if not os.path.exists(self.image_folder):
os.makedirs(self.image_folder)
self.outfolder = config_options['GAN']['outfolder']
if not os.path.exists(self.outfolder):
os.makedirs(self.outfolder)
@staticmethod
def read_labels(path):
with open(path, 'r') as fin:
labels = [w.strip() for w in fin if not w.startswith('#')]
return labels
def build_index(self, training=True):
target_words = deepcopy(self.train_labels)
if not training:
for w in self.test_words:
target_words.append(w)
indexed_words = []
for word in target_words:
indexed_words.append(self.word_embeddings.vocab[word].index)
return target_words, indexed_words
def build_generator(self, config_gener):
m = float(config_gener['momentum'])
a = float(config_gener['leaky_relu_alpha'])
model = Sequential()
model.add(
Dense(int(config_gener['dense1']), input_dim=self.embeddings_dim))
model.add(LeakyReLU(alpha=a))
model.add(BatchNormalization(momentum=m))
model.add(Dense(int(config_gener['dense2'])))
model.add(LeakyReLU(alpha=a))
model.add(BatchNormalization(momentum=m))
model.add(Dense(int(config_gener['dense3'])))
model.add(LeakyReLU(alpha=a))
model.add(BatchNormalization(momentum=m))
model.add(
Dense(np.prod(self.img_shape),
activation=config_gener['activation']))
model.add(Reshape(self.img_shape))
noise = Input(shape=(self.embeddings_dim, ))
label = Input(shape=(1, ), dtype='int32')
label_embedding = Flatten()(Embedding(
self.embeddings_vocab,
self.embeddings_dim,
weights=[self.word_embeddings.vectors],
trainable=False)(label))
model_input = multiply([noise, label_embedding])
img = model(model_input)
return Model([noise, label], img)
def build_discriminator(self, config_discr):
a = float(config_discr['leaky_relu_alpha'])
d = float(config_discr['dropout'])
model = Sequential()
model.add(
Dense(int(config_discr['dense1']),
input_dim=np.prod(self.img_shape)))
model.add(LeakyReLU(alpha=a))
model.add(Dense(int(config_discr['dense2'])))
model.add(LeakyReLU(alpha=a))
model.add(Dropout(d))
model.add(Dense(int(config_discr['dense3'])))
model.add(LeakyReLU(alpha=a))
model.add(Dropout(d))
model.add(Dense(1, activation=config_discr['activation']))
img = Input(shape=self.img_shape)
label = Input(shape=(1, ), dtype='int32')
label_embedding = Flatten()(Embedding(
self.embeddings_vocab,
self.embeddings_dim,
weights=[self.word_embeddings.vectors],
trainable=False)(label))
label_expansion = Dense(self.img_rows * self.img_cols *
self.channels)(label_embedding)
flat_img = Flatten()(img)
model_input = multiply([flat_img, label_expansion])
validity = model(model_input)
return Model([img, label], validity)
def train(self):
# Load the dataset
if 'mnist' in self.img_dataset:
(X_train, y_train), (_, _) = mnist.load_data()
elif 'cifar' in self.img_dataset:
(X_train, y_train), (_, _) = cifar.load_data('fine')
X_train = X_train / 255.0
classes2labels = {c: l for c, l in enumerate(self.train_labels)}
else:
raise ValueError(
"Unknown dataset {}! Please use either 'fasion_mnist' or 'cifar100'."
.format(self.img_dataset))
# Configure input
X_train = (X_train.astype(np.float32) - 127.5) / 127.5
# if self.channels == 1:
# X_train = np.expand_dims(X_train, axis=3)
for n, i in enumerate(self.index):
y_train = np.where(y_train == n, i, y_train)
y_train = y_train.reshape(-1, 1)
# Adversarial ground truths
true = np.ones((self.batch_size, 1))
fake = np.zeros((self.batch_size, 1))
for epoch in range(self.epochs):
# ---------------------
# Train Discriminator
# ---------------------
# Select a random half batch of images
idx = np.random.randint(0, X_train.shape[0], self.batch_size)
# with np.random.randint we can pick the same image more than once: is this a feature or a bug?
imgs, labels = X_train[idx], y_train[idx]
shuffled_labels = np.zeros_like(labels)
for i, label in enumerate(labels):
modified_index = np.delete(self.index,
np.where(self.index == label))
shuffled_labels[i, 0] = np.random.choice(modified_index, 1)
# Sample noise as generator input
noise = np.random.normal(0, 1,
(self.batch_size, self.embeddings_dim))
# Generate a half batch of new images
gen_imgs = self.generator.predict([noise, labels])
"""
Train the discriminator to do three tasks:
1. recognize real images for a concept as true images for that concept
2. recognize generated images for a concept as fake images for that concept
3. recognize real images for a concept as fake for a different concept
this way the generator cannot just fool the discriminator by generating sensible images in the abstract,
but has to generate credible images for a specific concept.
"""
d_loss_real = self.discriminator.train_on_batch([imgs, labels],
true)
d_loss_fake = self.discriminator.train_on_batch([gen_imgs, labels],
fake)
d_loss_unrelated = self.discriminator.train_on_batch(
[imgs, shuffled_labels], fake)
# this is the original loss, which could be extended to
# d_loss = np.add(d_loss_real, d_loss_fake, d_loss_unrelated) / 3
# to keep its spirit.
# d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
# this is the loss proposed in Reed et al 2018
d_loss = np.log(d_loss_real) + (np.log(1 - d_loss_unrelated) +
np.log(1 - d_loss_fake)) / 2
# ---------------------
# Train Generator
# ---------------------
# Condition on labels
sampled_labels = np.random.choice(self.index,
self.batch_size).reshape(-1, 1)
# Train the generator
g_loss = self.gan.train_on_batch([noise, sampled_labels], true)
# If at save interval => save generated image samples
if epoch % self.sample_interval == 0:
print("%d [D loss: %f, acc.: %.2f%%] [G loss: %f]" %
(epoch, d_loss[0], 100 * d_loss[1], g_loss))
self.sample_images(epoch)
self.save()
def save(self):
self.discriminator.trainable = False
self.gan.save(
os.path.join(self.outfolder, 'gan_{}_epochs'.format(self.epochs)))
self.discriminator.trainable = True
self.discriminator.save(
os.path.join(self.outfolder,
'discriminator_{}_epochs'.format(self.epochs)))
self.generator.save(os.path.join(
self.outfolder, 'generator_{}_epochs'.format(self.epochs)),
include_optimizer=False)
def sample_images(self, epoch):
r, c = self.output_image_grid
vocab, extended_index = self.build_index(training=False)
noise = np.random.normal(0, 1, (r * c, self.embeddings_dim))
sampled_labels = np.asarray(extended_index).reshape(-1, 1)
gen_imgs = self.generator.predict([noise, sampled_labels])
# Rescale images 0 - 1
gen_imgs = 0.5 * gen_imgs + 0.5
fig, axs = plt.subplots(r, c)
cnt = 0
for i in range(r):
for j in range(c):
if self.channels == 1:
axs[i, j].imshow(gen_imgs[cnt, :, :, 0], cmap='gray')
else:
axs[i, j].imshow(gen_imgs[cnt, :, :, :])
axs[i, j].set_title(vocab[cnt])
axs[i, j].axis('off')
cnt += 1
fig.savefig("{}/{}.png".format(self.image_folder, epoch))
plt.close()
class GAN:
def __init__(self):
self.img_rows = 28
self.img_cols = 28
self.channels = 1
self.img_shape = (self.img_rows, self.img_cols, self.channels)
self.latent_dim = 100
optimizer = Adam(0.0002, 0.5)
# Build and compile the discriminator
self.discriminator = self.build_discriminator()
# self.discriminator.summary()
self.discriminator.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
# Build the generator
self.generator = self.build_generator()
# self.generator.summary()
# The generator takes noise as input and generates imgs
z = Input(shape=(self.latent_dim, ))
img = self.generator(z)
# For the combined model we will only train the generator
# self.discriminator.trainable = False
# The discriminator takes generated images as input and determines validity
validity = self.discriminator(img)
# The combined model (stacked generator and discriminator)
# Trains the generator to fool the discriminator
self.combined = Model(z, validity)
# self.combined.summary()
self.combined.compile(loss='binary_crossentropy', optimizer=optimizer)
def build_generator(self):
model = Sequential()
model.add(Dense(256, input_dim=self.latent_dim))
model.add(LeakyReLU(alpha=0.2))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(512))
model.add(LeakyReLU(alpha=0.2))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(1024))
model.add(LeakyReLU(alpha=0.2))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(np.prod(self.img_shape), activation='tanh'))
model.add(Reshape(self.img_shape))
noise = Input(shape=(self.latent_dim, ))
img = model(noise)
return Model(noise, img)
def build_discriminator(self):
model = Sequential()
model.add(Flatten(input_shape=self.img_shape))
model.add(Dense(512))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(256))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(1, activation='sigmoid'))
model.summary()
img = Input(shape=self.img_shape)
validity = model(img)
return Model(img, validity)
def train(self, epochs, batch_size=128, sample_interval=50):
# from tensorflow.examples.tutorials.mnist import input_data
# mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
#
# x_train, y_train = mnist.train.images, mnist.train.labels
# x_test, y_test = mnist.test.images, mnist.test.labels
# X_train = x_train.reshape(-1, 28, 28, 1).astype('float32') / 127.5 - 1
# x_test = x_test.reshape(-1, 28, 28, 1).astype('float32')
# Rescale -1 to 1
from keras.datasets import mnist
(X_train, _), (_, _) = mnist.load_data()
X_train = X_train / 127.5 - 1.
X_train = np.expand_dims(X_train, axis=3)
# Adversarial ground truths
valid = np.ones((batch_size, 1))
fake = np.zeros((batch_size, 1))
for epoch in range(epochs):
# ---------------------
# Train Discriminator
# ---------------------
# Select a random batch of images
idx = np.random.randint(0, X_train.shape[0], batch_size)
imgs = X_train[idx]
noise = np.random.normal(0, 1, (batch_size, self.latent_dim))
# Generate a batch of new images
gen_imgs = self.generator.predict(noise)
if epoch > 80 and epoch % 40:
for i in range(8):
cv2.imwrite("imgs/%d_%d.jpg" % (epoch, i),
((gen_imgs[i] + 1) * 127.5).astype(np.uint8))
# Train the discriminator
for layer in self.discriminator.layers:
layer.trainable = True
self.discriminator.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
d_loss_real = self.discriminator.train_on_batch(imgs, valid)
d_loss_fake = self.discriminator.train_on_batch(gen_imgs, fake)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
for layer in self.discriminator.layers:
layer.trainable = False
# ---------------------
# Train Generator
# ---------------------
noise = np.random.normal(0, 1, (batch_size, self.latent_dim))
# Train the generator (to have the discriminator label samples as valid)
self.combined.compile(loss='binary_crossentropy', optimizer='adam')
g_loss = self.combined.train_on_batch(noise, valid)
# Plot the progress
print("%d [D loss: %f, acc.: %.2f%%] [G loss: %f]" %
(epoch, d_loss[0], 100 * d_loss[1], g_loss))
def test_cyclegan():
mnist_shape, mnist_images = load_mnist()
cats = load_input_images()
nb_epochs = 50
batch_size = 512
adam_lr = 0.0002
adam_beta_1 = 0.5
adam_decay = 0 # adam_lr/(nb_epochs*120)
cyc_multiplier = 10
history_size = int(batch_size * 7 / 3)
SAVEPATH = os.path.abspath(
os.path.join(os.path.dirname(__file__), 'output'))
generation_history_mnist = None
generation_history_cats = None
adam_optimizer = Adam(lr=adam_lr, beta_1=adam_beta_1, decay=adam_decay)
generator_cats = mnist_generator(mnist_shape)
generator_cats.compile(optimizer=adam_optimizer, loss='mean_squared_error')
generator_cats.summary()
discriminator_cats = mnist_discriminator(mnist_shape)
discriminator_cats.compile(optimizer=adam_optimizer,
loss=discriminator_loss)
discriminator_cats.summary()
generator_mnist = mnist_generator(mnist_shape)
generator_mnist.compile(optimizer=adam_optimizer,
loss='mean_squared_error')
generator_mnist.summary()
discriminator_mnist = mnist_discriminator(mnist_shape)
discriminator_mnist.compile(optimizer=adam_optimizer,
loss=discriminator_loss)
discriminator_mnist.summary()
mnist_input = Input(mnist_shape)
cat_input = Input(mnist_shape)
fake_cat = generator_cats(mnist_input)
fake_mnist = generator_mnist(cat_input)
# Only train discriminator during first phase
# (trainable only affects new models when they are compiled)
discriminator_cats.trainable = False
discriminator_mnist.trainable = False
mnist_gen_trainer = Model(cat_input, discriminator_mnist(fake_mnist))
mnist_gen_trainer.compile(optimizer=adam_optimizer,
loss='mean_squared_error')
mnist_gen_trainer.summary()
cats_gen_trainer = Model(mnist_input, discriminator_cats(fake_cat))
cats_gen_trainer.compile(optimizer=adam_optimizer,
loss='mean_squared_error')
cats_gen_trainer.summary()
mnist_cyc = Model(mnist_input, generator_mnist(fake_cat))
mnist_cyc.compile(optimizer=adam_optimizer,
loss=cycle_loss,
loss_weights=[cyc_multiplier])
mnist_cyc.summary()
cats_cyc = Model(cat_input, generator_cats(fake_mnist))
cats_cyc.compile(optimizer=adam_optimizer,
loss=cycle_loss,
loss_weights=[cyc_multiplier])
cats_cyc.summary()
# training time
mnist_discrim_loss = []
cats_discrim_loss = []
mnist_gen_loss = []
cats_gen_loss = []
mnist_cyc_loss = []
cats_cyc_loss = []
if not os.path.exists(SAVEPATH):
os.makedirs(os.path.join(SAVEPATH, 'images'))
for epoch in range(nb_epochs):
print("\n\n================================================")
print("Epoch", epoch, '\n')
if epoch == 0: # Initialize history for training the discriminator
mnist_indices = np.random.choice(mnist_images.shape[0],
history_size)
generation_history_mnist = mnist_images[mnist_indices]
cats_indices = np.random.choice(cats.shape[0], history_size)
generation_history_cats = cats[cats_indices]
# Make and save a test collage
choice = np.random.choice(mnist_images.shape[0])
if k.image_data_format() == 'channels_first':
mnist_in = mnist_images[choice].reshape((1, 1, 28, 28))
else:
mnist_in = mnist_images[choice].reshape((1, 28, 28, 1))
cat_out = generator_cats.predict(mnist_in)
mnist_cyc_out = generator_mnist.predict(cat_out)
choice = np.random.choice(cats.shape[0])
if k.image_data_format() == 'channels_first':
cat_in = cats[choice].reshape((1, 1, 28, 28))
else:
cat_in = cats[choice].reshape((1, 28, 28, 1))
mnist_out = generator_mnist.predict(cat_in)
cat_cyc_out = generator_cats.predict(mnist_out)
mnist_test_images = np.concatenate(
(prettify(mnist_in), prettify(cat_out), prettify(mnist_cyc_out)),
axis=1)
cat_test_images = np.concatenate(
(prettify(cat_in), prettify(mnist_out), prettify(cat_cyc_out)),
axis=1)
test_collage = np.concatenate((mnist_test_images, cat_test_images),
axis=0)
test_collage = Image.fromarray(test_collage, mode='L')
test_collage.save(os.path.join(SAVEPATH, 'images',
str(epoch) + '.png'))
for batch in range(int(mnist_images.shape[0] / batch_size)):
print("\nEpoch", epoch, "| Batch", batch)
# Get batch.
mnist_indices = np.random.choice(mnist_images.shape[0], batch_size)
mnist_batch_real = mnist_images[mnist_indices]
cats_indices = np.random.choice(cats.shape[0], batch_size)
cats_batch_real = cats[cats_indices]
# Update history with new generated images.
mnist_batch_gen = generator_mnist.predict_on_batch(cats_batch_real)
cats_batch_gen = generator_cats.predict_on_batch(mnist_batch_real)
generation_history_mnist = np.concatenate(
(generation_history_mnist[batch_size:], mnist_batch_gen))
generation_history_cats = np.concatenate(
(generation_history_cats[batch_size:], cats_batch_gen))
# Train discriminators.
real_label = np.ones(batch_size)
fake_label = np.zeros(batch_size)
mnist_discrim_loss.append(
discriminator_mnist.train_on_batch(
np.concatenate((generation_history_mnist[:batch_size],
mnist_batch_real)),
np.concatenate((fake_label, real_label))))
print("MNIST Discriminator Loss:", mnist_discrim_loss[-1])
cats_discrim_loss.append(
discriminator_cats.train_on_batch(
np.concatenate((generation_history_cats[:batch_size],
cats_batch_real)),
np.concatenate((fake_label, real_label))))
print("Cats Discriminator Loss:", cats_discrim_loss[-1])
# Train generators.
mnist_gen_loss.append(
mnist_gen_trainer.train_on_batch(cats_batch_real, real_label))
print("MNIST Generator Loss:", mnist_gen_loss[-1])
cats_gen_loss.append(
cats_gen_trainer.train_on_batch(mnist_batch_real, real_label))
print("Cats Generator Loss:", cats_gen_loss[-1])
mnist_cyc_loss.append(
mnist_cyc.train_on_batch(mnist_batch_real, mnist_batch_real))
print("MNIST Cyclic Loss:", mnist_cyc_loss[-1])
cats_cyc_loss.append(
cats_cyc.train_on_batch(cats_batch_real, cats_batch_real))
print("Cats Cyclic Loss:", cats_cyc_loss[-1])
# Save models.
generator_cats.save(os.path.join(SAVEPATH, 'generator_cats.h5'))
generator_mnist.save(os.path.join(SAVEPATH, 'generator_mnist.h5'))
discriminator_cats.save(os.path.join(SAVEPATH, 'discriminator_cats.h5'))
discriminator_mnist.save(os.path.join(SAVEPATH, 'discriminator_mnist.h5'))
# Save training history.
output_dict = {
'mnist_discrim_loss': [str(loss) for loss in mnist_discrim_loss],
'cats_discrim_loss': [str(loss) for loss in cats_discrim_loss],
'mnist_gen_loss': [str(loss) for loss in mnist_gen_loss],
'cats_gen_loss': [str(loss) for loss in cats_gen_loss],
'mnist_cyc_loss': [str(loss) for loss in mnist_cyc_loss],
'cats_cyc_loss': [str(loss) for loss in cats_cyc_loss]
}
with open(os.path.join(SAVEPATH, 'log.txt'), 'w') as f:
json.dump(output_dict, f, indent=4)
class merge_models:
def __init__(self, epochs, batch_size):
self.epochs = epochs
self.batch_size = batch_size
# Generator
filter_kernal = [[32, 128, 128, 128, 256],
[32, 128, 128, 128, 128, 256, 512],
[32, 128, 128, 128, 128, 128, 128, 128, 512]]
self.g_1 = generator(
3,
filter_kernal[2],
in_1_size,
'Adam',
'mse',
path='/home/mdo2/sid_codes/new_codes/gen_1.h5').generator_model()
self.g_2 = generator(
2,
filter_kernal[1],
in_2_size,
'Adam',
'mse',
path='/home/mdo2/sid_codes/new_codes/gen_2.h5').generator_model()
self.g_3 = generator(
1,
filter_kernal[0],
in_3_size,
'Adam',
'mse',
path='/home/mdo2/sid_codes/new_codes/gen_3.h5').generator_model()
# Discriminator
#self.D1 = discriminator(in_2_size).modified_vgg()
#self.D1.compile(loss='mse',optimizer='Adam',metrics=['accuracy'])
#self.D2 = discriminator(in_3_size).modified_vgg()
#self.D2.compile(loss='mse',optimizer='Adam',metrics=['accuracy'])
self.D = discriminator(
out, '/home/mdo2/sid_codes/new_codes/D.h5').modified_vgg()
self.D.compile(loss='mse', optimizer='Adam', metrics=['accuracy'])
self.g = 0
# Images
lr = Input(shape=in_1_size)
hr_1 = Input(shape=in_2_size)
hr_2 = Input(shape=in_3_size)
hr_3 = Input(shape=out)
fake_hr_image_1 = self.g_1(lr)
fake_hr_image_2 = self.g_2(fake_hr_image_1)
fake_hr_image_3 = self.g_3(fake_hr_image_2)
self.vgg_1 = vgg(in_2_size).vgg_loss_model()
self.vgg_2 = vgg(in_3_size).vgg_loss_model()
self.vgg_3 = vgg(out).vgg_loss_model()
fake_hr_feat_1 = self.vgg_1(fake_hr_image_1)
fake_hr_feat_2 = self.vgg_2(fake_hr_image_2)
fake_hr_feat_3 = self.vgg_3(fake_hr_image_3)
self.D.trainable = False
#self.D1.trainable = False
#self.D2.trainable = False
validity = self.D(fake_hr_image_3)
#self.D.load_weights('/media/arjun/119GB/attachments/D_best.h5')
self.merged_net = Model(
inputs=[lr, hr_1, hr_2, hr_3],
outputs=[validity, fake_hr_feat_1, fake_hr_feat_2, fake_hr_feat_3])
self.merged_net.compile(
loss=['binary_crossentropy', 'mse', 'mse', 'mse'],
loss_weights=[1e-3, .3, .3, .3],
optimizer='Adam')
self.merged_net.summary()
#self.merged_net.load_weights('/home/mdo2/sid_codes/newew/my_model.h5')
print('loaded')
def save(self):
#self.D.save_weights('D_best.h5')
#print('Discriminator saved')
#self.merged_net.save('my_model.h5')
self.merged_net.save('/home/mdo2/sid_codes/new_codes/model.h5')
self.g_1.save('/home/mdo2/sid_codes/new_codes/gen_1.h5')
self.g_2.save('/home/mdo2/sid_codes/new_codes/gen_2.h5')
self.g_3.save('/home/mdo2/sid_codes/new_codes/gen_3.h5')
self.D.save('/home/mdo2/sid_codes/new_codes/D.h5')
print('saved')
def write(self, g_loss, d_loss, flag):
with open('train.csv', mode='a') as csv_file:
if flag == True:
fieldnames = ['g_loss', 'd_loss']
writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
writer.writeheader()
fieldnames = ['g_loss', 'd_loss']
writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
writer.writerow({'g_loss': g_loss, 'd_loss': d_loss})
"""
def test(self):
test_path, test_path = ''
img_test_path = glob.glob(test_path)
random.shuffle(img_test_path)
d_sum = 0.
g_sum = np.array([0.,0.,0.,0.,0.,0.,0.,0.])
step_size = int(800/self.batch_size)
for steps in range(step_size):
# Load data
print('Loading Test Batch')
x_test, y_test_1, y_test_2, y_test_3 = load_data(img_test_path, start=steps*self.batch_size, batch_size=self.batch_size)
fake_hr_1 = self.g_1.predict(x_test)
fake_hr_2 = self.g_2.predict(fake_hr_1)
fake_hr_3 = self.g_3.predict(fake_hr_2)
valid = np.ones([self.batch_size,12,12,16])
fake = np.zeros([self.batch_size,12,12,16])
d_test_loss_real = self.D.test_on_batch(y_test_3, valid)
d_test_loss_fake = self.D.test_on_batch(fake_hr_3, fake)
d_test_loss = 0.5*np.add(d_test_loss_fake,d_test_loss_real)
8
real_feat_1 = self.vgg_1.predict(y_test_1)
real_feat_2 = self.vgg_2.predict(y_test_2)
real_feat_3 = self.vgg_3.predict(y_test_3)
g_test_loss = self.merged_net.test_on_batch([x_test,y_test_3], [valid,real_feat_3])
d_sum += d_test_loss[0]
g_sum = np.add(g_sum, g_test_loss)
save(d_sum/float(step_size),g_sum/float(step_size))
"""
def train(self):
try:
#self.D.load_weights('/media/arjun/119GB/attachments/D_best.h5')
#self.g_1.load_weights('/media/arjun/119GB/attachments/g_1_best.h5')
#self.g_2.load_weights('/media/arjun/119GB/attachments/g_2_best.h5')
#self.g_3.load_weights('/media/arjun/119GB/attachments/g_3_best.h5')
i = 0
flag = True
print("Discriminator loaded")
for epoch in range(self.epochs):
print("loading data")
train_path = dataset_dir
img_train_path = glob.glob(train_path)
#i=0
#if i==0:
#avg_loss=0
#i+=1
#avg_loss = avg_loss/int(25000/self.batch_size)
with tqdm(total=int(25000 / self.batch_size),
file=sys.stdout) as t:
for steps in range(int(25000 / self.batch_size)):
print('Loading Train Batch:', steps + 1)
x_train, y_train_1, y_train_2, y_train_3 = load_data(
train_path,
start=steps * self.batch_size,
batch_size=self.batch_size)
print('Training Discriminator')
t.update()
#dISCRIMINATOR
valid = np.ones([self.batch_size, 6, 6, 1])
fake = np.zeros([self.batch_size, 6, 6, 1])
fake_hr_1 = self.g_1.predict(x_train)
fake_hr_feat_1 = self.vgg_1(fake_hr_1)
fake_hr_2 = self.g_2.predict(fake_hr_1)
fake_hr_feat_2 = self.vgg_2(fake_hr_2)
fake_hr_3 = self.g_3.predict(fake_hr_2)
fake_hr_feat_3 = self.vgg_3(fake_hr_3)
d_train_loss_fake = self.D.train_on_batch(
fake_hr_3, fake)
d_train_loss_real = self.D.train_on_batch(
y_train_3, valid)
d_train_loss = 0.5 * np.add(d_train_loss_fake,
d_train_loss_real)
#GENERATOR
print('Training Generator')
valid = np.ones([self.batch_size, 6, 6, 1])
fake = np.zeros([self.batch_size, 6, 6, 1])
real_feat_1 = self.vgg_1.predict(y_train_1)
real_feat_2 = self.vgg_2.predict(y_train_2)
real_feat_3 = self.vgg_3.predict(y_train_3)
g_loss = self.merged_net.train_on_batch(
[x_train, y_train_1, y_train_2, y_train_3],
[valid, real_feat_1, real_feat_2, real_feat_3])
t.set_postfix_str(s='Epoch:' + str(epoch) +
' Generator_cost ' + str(g_loss) +
' Discriminator_cost ' +
str(d_train_loss))
if i > 0:
flag = False
self.write(g_loss, d_train_loss, flag)
i += 1
print(
'*******************************************************\n'
)
if self.g == 0:
try:
self.old_loss = np.load(
'/home/mdo2/sid_codes/newew/g_loss.npy')
print('latest gen')
self.g += 1
except:
self.old_loss = float('inf')
self.g += 1
if g_loss[0] < self.old_loss:
self.old_loss = g_loss[0]
print('saving best')
np.save('g_loss', g_loss[0])
self.g += 1
self.save()
#if (steps+1)%100==0:
# self.save()
#self.merged_net.inputs = [x_train,y_train_1,y_train_2,y_train_3]
#self.merged_net.outputs = [valid,fake_hr_feat_1,fake_hr_feat_2,fake_hr_feat_3]
#c_loss = K.sqrt(K.mean((fake_hr_feat_1 - real_feat_1)**2))
#log_loss = logcosh(fake_hr_feat_1,real_feat_1)
#grads = K.gradients([log_loss,c_loss],[fake_hr_feat_1,real_feat_1])[0]
#print(grads)
#iterate = K.function([(y_train_1)], [c_loss, grads])
#print(iterate,'c_loss')
#iterate = K.function([tf.convert_to_tensor(y_train_1)], [log_loss, grads])
#print(iterate,'log_loss_1')
#self.D.fit(y_train_3,valid, epochs=epoch, batch_size=self.batch_size, callbacks=[tbCallBack])
except KeyboardInterrupt:
print('Saving Weights')
#self.save()
def feed_forward(self):
in_1 = [64, 64, 3]
in_2 = [64, 64, 3]
in_3 = [64 * 4, 64 * 4, 3]
out_ = [128, 128, 3]
img = cv2.resize(
cv2.imread(
'/home/mdo2/sid_codes/datasets_big/qhd_no_pad/train/1.png', 1),
(64, 64))
img = img.reshape(1, 64, 64, 3)
filter_kernal = [[32, 128, 128, 128, 256],
[32, 128, 128, 128, 128, 256, 512],
[32, 128, 128, 128, 128, 128, 128, 128, 512]]
g3 = generator(
3,
filter_kernal[2],
in_1,
'Adam',
path='/home/mdo2/sid_codes/new_codes/gen_1.h5').generator_model()
#g2 = generator(2,filter_kernal[1],in_2,'Adam',path = '/home/mdo2/sid_codes/new_codes/gen_2.h5').generator_model()
#g1 = generator(1,filter_kernal[0],in_1,'Adam',path = '/home/mdo2/sid_codes/new_codes/gen_3.h5').generator_model()
img = g3.predict(img)
#img = g2.predict(img)
img = np.array(img).astype('uint8')
img = img.reshape(128, 128, 3)
cv2.imshow('img', img)
cv2.waitKey(0)
cv2.imwrite('ee.jpg', img)
class TmClassify:
def __init__(self):
self.input_shape = (224,224)
self.filter_count = 32
self.kernel_size = (3, 3)
self.leakrelu_alpha = 0.2
self.encoder = self.createEncoder()
Input1 = Input(shape=(224,224,3))
Input2 = Input(shape=(224,224,3))
# target = Dot(axes=1)([self.encoder(Input1), self.encoder(Input2)])
x = concatenate(inputs = [self.encoder(Input1), self.encoder(Input2)])
target = Dense(1)(x)
self.discriminator = self.createDiscriminator()
y = self.discriminator(x)
self.model = Model(inputs=[Input1,Input2],outputs=y)
self.model.summary()
self.pathlist = pathlist
self.train_data_count = [len(i) for i in self.pathlist]
op = Adam(lr=0.0001)
self.model.compile(optimizer=op,loss='mse',metrics=['accuracy'])
self.pad_param = 5
self.rotate_degree_param = 90
def createEncoder(self):
base_model=InceptionV3(input_shape=(224,224,3),weights=None,include_top=False)
x=base_model.output
x=GlobalAveragePooling2D()(x)
x = LayerNormalization()(x)
model=Model(inputs=base_model.input,outputs=x)
return model
def createDiscriminator(self):
x = Input(shape = 4096)
target = Dense(1)(x)
model = Model(inputs=x,outputs=target)
return model
def gen_data(self,path):
img_pil = Image.open(path).convert('RGB')
# img_pil.save("test.jpg")
pad_top = int(abs(np.random.uniform(0,self.pad_param)))
pad_bottom = int(abs(np.random.uniform(0,self.pad_param)))
pad_left = int(abs(np.random.uniform(0,self.pad_param)))
pad_right = int(abs(np.random.uniform(0,self.pad_param)))
rotate_param = np.random.uniform(0,self.rotate_degree_param)
flip_flag = np.random.randint(0,1)
mirror_flag = np.random.randint(0,1)
if(flip_flag):
img_pil = ImageOps.flip(img_pil)
if(mirror_flag):
img_pil = ImageOps.mirror(img_pil)
blur_rad = np.random.normal(loc=0.0, scale=1, size=None)
img_pil = img_pil.filter(ImageFilter.GaussianBlur(blur_rad))
enhancer_contrat = ImageEnhance.Contrast(img_pil)
enhancer_brightness = ImageEnhance.Brightness(img_pil)
enhancer_color = ImageEnhance.Color(img_pil)
contrast_factor = np.random.normal(loc=1.0, scale=0.25, size=None)
color_factor = np.max([0,1-abs(np.random.normal(loc=0, scale=0.5, size=None))])
translate_factor_hor = np.random.normal(loc=0, scale=5, size=None)
translate_factor_ver = np.random.normal(loc=0, scale=5, size=None)
brightness_factor = np.random.normal(loc=1.0, scale=0.5, size=None)
img_pil = enhancer_contrat.enhance(contrast_factor)
img_pil = enhancer_brightness.enhance(brightness_factor)
img_pil = enhancer_color.enhance(color_factor)
img_pil = ImageChops.offset(img_pil, int(translate_factor_hor), int(translate_factor_ver))
img_pil = img_pil.rotate(rotate_param,resample = Image.BILINEAR,expand = True, fillcolor = (255))
img = np.asarray(img_pil)
img = cv2.copyMakeBorder(img, pad_top, pad_bottom, pad_left, pad_right, cv2.BORDER_CONSTANT,value=(255,255,255))
img= cv2.resize(img, dsize=(self.input_shape))
img = img/127.5 - 1
return img
def train(self,start_epoch, max_epoch, batch_size, viz_interval):
max_step = sum([len(i) for i in self.pathlist]) // batch_size
# permu_ind = list(range(len(self.pathlist)))
step = 0
for epoch in range(start_epoch,max_epoch):
# permu_ind = np.random.permutation(permu_ind)
real_index = 0
for step_index in range(max_step):
batch_img1 = np.zeros((batch_size,self.input_shape[0],self.input_shape[1],3))
batch_img2 = np.zeros((batch_size,self.input_shape[0],self.input_shape[1],3))
batch_target = np.zeros(batch_size)
for batch_index in range(batch_size//2+1):
random_permu = np.random.permutation(len(sdir))
filelist = glob2.glob(sdir[batch_index]+'/*')
file_permu = np.random.permutation(len(filelist))
img1 = self.gen_data(filelist[file_permu[0]])
img2 = self.gen_data(filelist[file_permu[1]])
batch_target[batch_index] = 1
batch_img1[batch_index] = img1
batch_img2[batch_index] = img2
real_index = real_index+1
for batch_index in range(batch_size//2,batch_size):
random_permu = np.random.permutation(len(sdir))
filelist1 = glob2.glob(sdir[batch_index]+'/*')
filelist2 = glob2.glob(sdir[-1-(-1*batch_index)]+'/*')
file_permu1 = np.random.permutation(len(filelist1))
file_permu2 = np.random.permutation(len(filelist2))
img1 = self.gen_data(filelist1[file_permu1[0]])
img2 = self.gen_data(filelist2[file_permu2[0]])
batch_target[batch_index] = 0
batch_img1[batch_index] = img1
batch_img2[batch_index] = img2
real_index = real_index+1
train_loss = self.model.train_on_batch([batch_img1,batch_img2],batch_target)
with train_summary_writer.as_default():
tf.summary.scalar('loss', train_loss[0], step=step)
tf.summary.scalar('accuracy', train_loss[1], step=step)
step = step + 1
# train_summary_writer = tf.summary.create_file_writer(train_log_dir)
print('\r epoch ' + str(epoch) + ' / ' + str(max_epoch) + ' ' + 'step ' + str(step_index) + ' / ' + str(max_step) + ' loss = ' + str(train_loss))
if(step_index%viz_interval==0):
self.encoder.save('DIPencoder.h5')
self.discriminator.save('DIPdiscriminator.h5')
self.model.save('DIPMatch.h5')
class StyleBank(object):
def __init__(self):
######################### Tunable Parameters ################################
# General
self.img_shape = (None, 128, 128, 3
) # (None, 512, 512, 3) ## Image Shape
self.n_styles = 4 # 50 ## Number of styles in the bank
self.n_content = 1000 ## Number of content images
self.N_steps = 300000 ## Total number of training steps
self.T = 2 ## Number of consecutive steps for training styles before training the AutoEncoder
self.print_iter = 100 ## Log output
self.Batch_Size = 4 ## Batch size
self.Use_Batch_Norm = True ## Use batch normalization instead of instance normalization
# LR
self.LR_Initial = 0.01 ## Initial ADAM learning rate
self.LR_Current = self.LR_Initial ## For logging
self.LR_Decay = 0.8 ## LR decay
self.LR_Update_Every = self.N_steps / 10 ## LR decay period
# Loss
self.Optimizer = optimizers.Adam(
lr=self.LR_Initial) ## Optimizer for both branches
self.LossAlpha = 0.025 # Content weight
self.LossBeta = 1.2 # Style weight
self.LossGamma = 1.0 # Total Variation weight
######################### \Tunable Parameters ################################
self.StyleNetLoss = {k: None for k in range(self.n_styles)}
self.StyleNetContentLoss = {k: None for k in range(self.n_styles)}
self.StyleNetStyleLoss = {k: None for k in range(self.n_styles)}
# Data
self.Content_DB = None
self.Style_DB = None
self.Content_DB_path = './DB/content/'
self.Style_DB_path = './DB/style/'
self.Content_DB_list = glob(self.Content_DB_path + '*')
self.Style_DB_list = glob(self.Style_DB_path + '*')
# VGG
self.VGG16 = None
# auto-encoder
self.encoder = None
self.decoder = None
# style bank
self.style_bank = {k: None for k in range(self.n_styles)}
self.StyleNet = {k: None for k in range(self.n_styles)}
self.AutoEncoderNet = None
# inputs - content and one for style
self.KinputContent = None
self.KinputStyle = None
self.tfStyleIndices = None
self.TensorBoardStyleNet = {k: None for k in range(self.n_styles)}
self.TensorBoardAutoEncoder = None
def initialize_placeholders(self):
# initialize the content and style image tensors
self.KinputContent = Input(shape=self.img_shape[1:],
name="InputContent")
self.KinputDecoded = None
def build_models(self):
###########
# Encoder #
###########
print("Building Encoder")
input_layer = Input(shape=self.img_shape[1:])
t_encoder = Conv2D(32, (9, 9),
strides=(1, 1),
padding='same',
use_bias=False)(input_layer)
if self.Use_Batch_Norm:
t_encoder = BatchNormalization()(t_encoder)
else:
t_encoder = InstanceNormalization()(t_encoder)
t_encoder = Activation('relu')(t_encoder)
t_encoder = Conv2D(64, (3, 3),
strides=(2, 2),
padding='same',
use_bias=False)(t_encoder)
if self.Use_Batch_Norm:
t_encoder = BatchNormalization()(t_encoder)
else:
t_encoder = InstanceNormalization()(t_encoder)
t_encoder = Activation('relu')(t_encoder)
t_encoder = Conv2D(128, (3, 3),
strides=(2, 2),
padding='same',
use_bias=False)(t_encoder)
if self.Use_Batch_Norm:
t_encoder = BatchNormalization()(t_encoder)
else:
t_encoder = InstanceNormalization()(t_encoder)
t_encoder = Activation('relu')(t_encoder)
self.encoder = Model(input_layer, t_encoder, name='Encoder')
print(self.encoder.summary())
###########
# Decoder #
###########
print("Building Decoder")
input_layer = Input(shape=self.encoder.layers[-1].output_shape[1:])
t_decoder = Conv2DTranspose(64, (3, 3),
strides=(2, 2),
padding='same',
use_bias=False)(input_layer)
if self.Use_Batch_Norm:
t_decoder = BatchNormalization()(t_decoder)
else:
t_decoder = InstanceNormalization()(t_decoder)
t_decoder = Activation('relu')(t_decoder)
t_decoder = Conv2DTranspose(32, (3, 3),
strides=(2, 2),
padding='same',
use_bias=False)(t_decoder)
if self.Use_Batch_Norm:
t_decoder = BatchNormalization()(t_decoder)
else:
t_decoder = InstanceNormalization()(t_decoder)
t_decoder = Activation('relu')(t_decoder)
t_decoder = Conv2DTranspose(3, (9, 9),
strides=(1, 1),
padding='same',
use_bias=False)(t_decoder)
self.decoder = Model(input_layer, t_decoder, name='Decoder')
print(self.decoder.summary())
#############
# StyleBank #
#############
for i in self.style_bank:
print("Building Style {}".format(i))
bank_name = "StyleBank{}".format(i)
stylenet_name = "StyleNet{}".format(i)
input_layer = Input(shape=self.encoder.layers[-1].output_shape[1:])
t_style = Conv2D(256, (3, 3),
strides=(1, 1),
padding='same',
use_bias=False)(input_layer)
if self.Use_Batch_Norm:
t_style = BatchNormalization()(t_style)
else:
t_style = InstanceNormalization()(t_style)
t_style = Activation('relu')(t_style)
t_style = Conv2D(256, (3, 3),
strides=(1, 1),
padding='same',
use_bias=False)(t_style)
if self.Use_Batch_Norm:
t_style = BatchNormalization()(t_style)
else:
t_style = InstanceNormalization()(t_style)
t_style = Activation('relu')(t_style)
t_style = Conv2D(128, (3, 3),
strides=(1, 1),
padding='same',
use_bias=False)(t_style)
if self.Use_Batch_Norm:
t_style = BatchNormalization()(t_style)
else:
t_style = InstanceNormalization()(t_style)
t_style = Activation('relu')(t_style)
self.style_bank[i] = Model(input_layer, t_style, name=bank_name)
#########################
# StyleBank Full Models #
#########################
input_layer = self.encoder.layers[
0].output # layers.Input(batch_shape=self.encoder.layers[0].input_shape)
prev_layer = input_layer
for layer in self.encoder.layers[1:]:
prev_layer = layer(prev_layer)
for layer in self.style_bank[i].layers[1:]:
prev_layer = layer(prev_layer)
for layer in self.decoder.layers[1:]:
prev_layer = layer(prev_layer)
self.StyleNet[i] = Model([input_layer], [prev_layer],
name=stylenet_name)
print(self.StyleNet[i].summary())
##########################
# AutoEncoder Full Model #
##########################
print("Building AutoEncoder")
input_layer = self.encoder.layers[
0].output # layers.Input(batch_shape=self.encoder.layers[0].input_shape)
prev_layer = input_layer
for layer in self.encoder.layers[1:]:
prev_layer = layer(prev_layer)
for layer in self.decoder.layers[1:]:
prev_layer = layer(prev_layer)
self.AutoEncoderNet = Model([input_layer], [prev_layer],
name='AutoEncoder')
print(self.AutoEncoderNet.summary())
### VGG
print("Importing VGG")
self.VGG16 = VGG16(include_top=False,
weights='imagenet',
input_shape=self.img_shape[1:])
print("Plotting Models")
plot_model(self.AutoEncoderNet,
to_file='Model_AutoEncoderNet.png',
show_shapes=True)
plot_model(self.VGG16, to_file='Model_VGG16.png', show_shapes=True)
for i in self.style_bank:
stylenet_model_file = "Model_StyleNet{}.png".format(i)
plot_model(self.StyleNet[i],
to_file=stylenet_model_file,
show_shapes=True)
def compile_models(self):
print("Compiling models")
def total_variation_loss(x):
img_nrows = self.img_shape[1]
img_ncols = self.img_shape[2]
assert K.ndim(x) == 4
if K.image_data_format() == 'channels_first':
a = K.square(x[:, :, :img_nrows - 1, :img_ncols - 1] -
x[:, :, 1:, :img_ncols - 1])
b = K.square(x[:, :, :img_nrows - 1, :img_ncols - 1] -
x[:, :, :img_nrows - 1, 1:])
else:
a = K.square(x[:, :img_nrows - 1, :img_ncols - 1, :] -
x[:, 1:, :img_ncols - 1, :])
b = K.square(x[:, :img_nrows - 1, :img_ncols - 1, :] -
x[:, :img_nrows - 1, 1:, :])
return K.sum(K.pow(a + b, 1.25))
def gram_matrix(x):
assert K.ndim(x) == 4
grams = list()
for i in range(self.Batch_Size):
img = x[i, :, :, :]
if K.image_data_format() == 'channels_first':
features = K.batch_flatten(img)
else:
features = K.batch_flatten(
K.permute_dimensions(img, (2, 0, 1)))
grams.append(K.dot(features, K.transpose(features)))
gram = tf.keras.backend.stack(grams)
return gram
def stylenet_loss_wrapper(input_tensor):
def stylenet_loss(S, O):
style_loss = K.variable(0.0)
content_loss = K.variable(0.0)
vgg16_layers = [l for l in self.VGG16.layers]
vgg16_layers = vgg16_layers[1:]
FlI = input_tensor #self.encoder.layers[0].output
FlS = S
FlO = O
for i in range(len(vgg16_layers)):
FlI = vgg16_layers[i](FlI)
FlS = vgg16_layers[i](FlS)
FlO = vgg16_layers[i](FlO)
if vgg16_layers[i] == self.VGG16.get_layer(
'block4_conv2') or self.VGG16.get_layer(
'block3_conv2') or self.VGG16.get_layer(
'block2_conv2') or self.VGG16.get_layer(
'block1_conv2'):
gram_mse = K.mean(
K.square(gram_matrix(FlO) - gram_matrix(FlS)))
layer_channels = vgg16_layers[i].output_shape[3]
layer_size = vgg16_layers[i].output_shape[
1] * vgg16_layers[i].output_shape[2]
gram_mse_norm = gram_mse / (2.0**2 *
(layer_channels**2) *
(layer_size**2))
style_loss = style_loss + gram_mse_norm
if vgg16_layers[i] == self.VGG16.get_layer('block4_conv2'):
content_loss = K.mean(K.square(FlO - FlI))
break
tv_loss = total_variation_loss(O)
return self.LossAlpha * content_loss + self.LossBeta * style_loss + self.LossGamma * tv_loss
return stylenet_loss
# Compile Models
for i in self.style_bank:
print("Compiling StyleBank {}".format(i))
self.StyleNet[i].compile(optimizer=self.Optimizer,
loss=stylenet_loss_wrapper(
self.StyleNet[i].layers[0].output))
self.AutoEncoderNet.compile(optimizer=self.Optimizer, loss=mse)
print("Initial learning rates: StyleNet={}, AutoEncoder={}".format(
K.eval(self.StyleNet[0].optimizer.lr),
K.eval(self.AutoEncoderNet.optimizer.lr)))
def get_batch_ids(self, batch_size, data_size):
return np.random.choice(np.arange(0, data_size),
size=batch_size,
replace=False)
def train_models(self):
style_id = 0
new_lr = self.LR_Initial
for step in range(self.N_steps):
style_ids = [style_id for i in range(self.Batch_Size)]
batch_ids = self.get_batch_ids(self.Batch_Size, self.n_content)
# Load the DB
print("Loading DB, step {}...".format(step), end='')
self.Content_DB = np.array([
resize(imread(self.Content_DB_list[batch_id]),
self.img_shape[1:]) for batch_id in batch_ids
])
style_im = resize(imread(self.Style_DB_list[style_id]),
self.img_shape[1:])
self.Style_DB = np.array([style_im for style_id in style_ids])
print("Finished Loading DB")
if step % (self.T + 1) != self.T: # Train Style
loss_style = self.StyleNet[style_id].train_on_batch(
self.Content_DB, self.Style_DB)
self.TensorBoardStyleNet[style_id].on_epoch_end(
step, self.named_logs(self.StyleNet[style_id], loss_style))
else: # Train AE
loss_autoencoder = self.AutoEncoderNet.train_on_batch(
self.Content_DB, self.Content_DB)
self.TensorBoardAutoEncoder.on_epoch_end(
step, self.named_logs(self.AutoEncoderNet,
loss_autoencoder))
style_id += 1
style_id = style_id % self.n_styles
if step % self.print_iter == 0 and step != 0:
print(
"step {0}, loss_style={1}, loss_autoencoder={2}, timestamp={3}"
.format(step, loss_style, loss_autoencoder,
datetime.now()))
if step % self.LR_Update_Every == 0 and step != 0:
new_lr = new_lr * self.LR_Decay
self.LR_Current = new_lr
for i in self.style_bank:
K.set_value(self.StyleNet[i].optimizer.lr, new_lr)
K.set_value(self.AutoEncoderNet.optimizer.lr, new_lr)
print("Updating LR to: StyleNet={}, AutoEncoder={}".format(
K.eval(self.StyleNet[0].optimizer.lr),
K.eval(self.AutoEncoderNet.optimizer.lr)))
for i in self.style_bank:
self.TensorBoardStyleNet[i].on_train_end(None)
self.TensorBoardAutoEncoder.on_train_end(None)
def prepare_tensorboard(self):
for i in self.style_bank:
self.TensorBoardStyleNet[i] = keras.callbacks.TensorBoard(
log_dir="tb_logs/stylenet_{}".format(i),
histogram_freq=0,
batch_size=self.Batch_Size,
write_graph=True,
write_grads=True)
self.TensorBoardStyleNet[i].set_model(self.StyleNet[i])
self.TensorBoardAutoEncoder = keras.callbacks.TensorBoard(
log_dir="tb_logs/autoencoder",
histogram_freq=0,
batch_size=self.Batch_Size,
write_graph=True,
write_grads=True)
self.TensorBoardAutoEncoder.set_model(self.AutoEncoderNet)
def named_logs(self, model, logs):
result = {}
for l in zip(model.metrics_names, [logs]):
result[l[0]] = l[1]
return result
def save_models(self):
# serialize model to JSON
if self.Use_Batch_Norm:
out_mod_dir = 'Save_BatchNorm'
else:
out_mod_dir = 'Save_InstNorm'
os.makedirs(out_mod_dir)
ae_json = self.AutoEncoderNet.to_json()
with open(os.path.join(out_mod_dir, "autoencoder.json"),
"w") as json_file:
json_file.write(ae_json)
# serialize weights to HDF5
self.AutoEncoderNet.save_weights(
os.path.join(out_mod_dir, "autoencoder.h5"))
for i in self.style_bank:
ae_json = self.StyleNet[i].to_json()
with open(os.path.join(out_mod_dir, "stylenet_{}.json".format(i)),
"w") as json_file:
json_file.write(ae_json)
# serialize weights to HDF5
self.StyleNet[i].save_weights(
os.path.join(out_mod_dir, "stylenet_{}.h5".format(i)))
print("Saved model to disk")
def load_models(self):
# load json and create model
if self.Use_Batch_Norm:
out_mod_dir = 'Save_BatchNorm'
else:
out_mod_dir = 'Save_InstNorm'
ae_json = open(os.path.join(out_mod_dir, 'autoencoder.json'), 'r')
ae_model_json = ae_json.read()
ae_json.close()
ae_model = models.model_from_json(
ae_model_json,
custom_objects={'InstanceNormalization': InstanceNormalization})
# load weights into new model
ae_model.load_weights(os.path.join(out_mod_dir, "autoencoder.h5"))
self.AutoEncoderNet = ae_model
for i in self.style_bank:
ae_json = open(
os.path.join(out_mod_dir, "stylenet_{}.json".format(i)), 'r')
ae_model_json = ae_json.read()
ae_json.close()
ae_model = models.model_from_json(ae_model_json,
custom_objects={
'InstanceNormalization':
InstanceNormalization
})
# load weights into new model
ae_model.load_weights(
os.path.join(out_mod_dir, "stylenet_{}.h5".format(i)))
self.StyleNet[i] = ae_model
print("Loaded models from disk")
class CycleGAN:
""" Do OOP for this GAN due to number of generators and discriminators """
def __init__(self):
# Input shape
self.img_rows = 128
self.img_cols = 128
self.channels = 3
# Configure dataloader
self.dataset_name = 'apple2orange'
self.data_loader = DataLoader(dataset_name=self.dataset_name,
img_res=(self.img_rows, self.img_cols))
# Calculate output shape of D (PatchGAN)
patch = int(self.img_rows / 2**4)
self.disc_patch = (patch, patch, 1)
# No. of filters in the 1st layer of G and D
self.generator_filters = 32
self.discriminator_filters = 64
# Loss weights
self.lambda_cycle = 10. # Cycle-consistency loss
self.lambda_id = 0.9 * self.lambda_cycle # Identity loss
optimizer = Adam(0.0002, 0.5)
# Build and compile the discriminators
self.discriminator_a = self.build_discriminator()
self.discriminator_b = self.build_discriminator()
self.discriminator_a.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
self.discriminator_b.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
# Construct computational graph of generators
# Build the generators
self.generator_ab = self.build_generator()
self.generator_ba = self.build_generator()
# Input images from both domains
img_a = layers.Input(shape=(self.img_rows, self.img_cols,
self.channels))
img_b = layers.Input(shape=(self.img_rows, self.img_cols,
self.channels))
# Translate images to other domains
fake_b = self.generator_ab(img_a)
fake_a = self.generator_ba(img_b)
# Translate images back to original domain
reconstr_a = self.generator_ba(fake_b)
reconstr_b = self.generator_ab(fake_a)
# Identity mapping of images
img_a_id = self.generator_ba(img_a)
img_b_id = self.generator_ab(img_b)
# For the combined model, only train the generators
self.discriminator_a.trainable = False
self.discriminator_b.trainable = False
# Discriminators determine validity of translated images
valid_a = self.discriminator_a(fake_a)
valid_b = self.discriminator_b(fake_b)
# Combined model trains generators to fool discriminators
self.combined = Model(inputs=[img_a, img_b],
outputs=[
valid_a, valid_b, reconstr_a, reconstr_b,
img_a_id, img_b_id
])
self.combined.compile(loss=['mse', 'mse', 'mae', 'mae', 'mae', 'mae'],
loss_weights=[
1, 1, self.lambda_cycle, self.lambda_cycle,
self.lambda_id, self.lambda_id
],
optimizer=optimizer)
@staticmethod
def conv2d(layer_input, filters, f_size=4, normalization=True):
"""Discriminator layer"""
output = Conv2D(filters, kernel_size=f_size, strides=2,
padding='same')(layer_input)
output = LeakyReLU(alpha=0.2)(output)
if normalization:
output = InstanceNormalization()(output)
return output
@staticmethod
def deconv2d(layer_input, skip_input, filters, f_size=4, dropout_rate=0):
"""Layer used during upsampling"""
output = UpSampling2D(size=2)(layer_input)
output = Conv2D(filters,
kernel_size=f_size,
strides=1,
padding='same',
activation='relu')(output)
if dropout_rate:
output = layers.Dropout(dropout_rate)(output)
output = InstanceNormalization()(output)
output = layers.Concatenate()([output, skip_input])
return output
def build_generator(self):
"""U-Net Generator"""
d_0 = layers.Input(shape=(self.img_rows, self.img_cols, self.channels))
# Downsampling
d_1 = self.conv2d(d_0, self.generator_filters)
d_2 = self.conv2d(d_1, self.generator_filters * 2)
d_3 = self.conv2d(d_2, self.generator_filters * 4)
d_4 = self.conv2d(d_3, self.generator_filters * 8)
# Upsampling
u_1 = self.deconv2d(d_4, d_3, self.generator_filters * 4)
u_2 = self.deconv2d(u_1, d_2, self.generator_filters * 2)
u_3 = self.deconv2d(u_2, d_1, self.generator_filters)
u_4 = UpSampling2D(size=2)(u_3)
output_img = Conv2D(self.channels,
kernel_size=4,
strides=1,
padding='same',
activation='tanh')(u_4)
return Model(d_0, output_img)
def build_discriminator(self):
""" Simple CNN discriminator but outputting patches instead of a scalar """
img = layers.Input(shape=(self.img_rows, self.img_cols, self.channels))
d_1 = self.conv2d(img, self.discriminator_filters, normalization=False)
d_2 = self.conv2d(d_1, self.discriminator_filters * 2)
d_3 = self.conv2d(d_2, self.discriminator_filters * 4)
d_4 = self.conv2d(d_3, self.discriminator_filters * 8)
validity = Conv2D(1, kernel_size=4, strides=1, padding='same')(d_4)
return Model(img, validity)
def sample_images(self, epoch, batch_i):
"""
generate 2x3 grid images
where each row represents a generator e.g., row 1 is A
1st column is the original image drawn from the dataset
2nd column is the generated image in the other domain
3rd column is the reconstructed image back in the original domain
"""
row, column = 2, 3
imgs_a = self.data_loader.load_data(domain='A',
batch_size=1,
is_testing=True)
imgs_b = self.data_loader.load_data(domain='B',
batch_size=1,
is_testing=True)
# Translate images to other domain
fake_b = self.generator_ab.predict(imgs_a)
fake_a = self.generator_ba.predict(imgs_b)
# Translate back to original domain
reconstr_a = self.generator_ba.predict(fake_b)
reconstr_b = self.generator_ab.predict(fake_a)
gen_imgs = np.concatenate(
[imgs_a, fake_b, reconstr_a, imgs_b, fake_a, reconstr_b])
gen_imgs = 0.5 * gen_imgs + 0.5
titles = ['Original', 'Translated', 'Reconstructed']
fig, axs = plt.subplots(row, column)
cnt = 0
for i in range(row):
for j in range(column):
axs[i, j].imshow(gen_imgs[cnt])
axs[i, j].set_title(titles[j])
axs[i, j].axis('off')
cnt += 1
fig.savefig('data/img{}-{}.png'.format(epoch, batch_i))
def train(self, epochs, batch_size=1, sample_interval=50):
"""
alternately train Discriminator and Generator for specified
epochs and batch_size, also generate sample images at specified intervals
"""
valid = np.ones((batch_size, ) + self.disc_patch)
fake = np.zeros((batch_size, ) + self.disc_patch)
for epoch in range(epochs):
for batch_i, (imgs_a, imgs_b) in enumerate(
self.data_loader.load_batch(batch_size)):
# Train Discriminators
# -------------------
# Translate images to opposite domains
fake_b = self.generator_ab.predict(imgs_a)
fake_a = self.generator_ba.predict(imgs_b)
# Train discriminators (original images = real/translated = Fake)
self.discriminator_a.train_on_batch(imgs_a, valid)
self.discriminator_a.train_on_batch(fake_a, fake)
self.discriminator_b.train_on_batch(imgs_b, valid)
self.discriminator_b.train_on_batch(fake_b, fake)
# Train Generators
# -------------------
self.combined.train_on_batch(
[imgs_a, imgs_b],
[valid, valid, imgs_a, imgs_b, imgs_a, imgs_b])
# If at save interval, plot
if batch_i % sample_interval == 0:
self.sample_images(epoch, batch_i)
class merge_models:
def __init__(self,epochs,batch_size):
self.epochs = epochs
self.batch_size = batch_size
# Generator
filter_kernal = [[32,128,128,128,256],[32,128,128,128,128,256,512],[32,128,128,128,128,128,128,128,512]]
#self.g_1 = generator(3,filter_kernal[2],in_1_size,'Adam','mse',path = '/home/mdo2/sid_codes/new_codes/gen_1.h5').generator_model()
#self.g_2 = generator(2,filter_kernal[1],in_2_size,'Adam','mse',path = '/home/mdo2/sid_codes/new_codes/gen_2.h5').generator_model()
self.g_3 = generator(1,filter_kernal[0],in_3_size,'Adam','mse',path = '/home/mdo2/sid_codes/new_codes/gen_3.h5').generator_model()
# Discriminator
#self.D1 = discriminator(in_2_size).modified_vgg()
#self.D1.compile(loss='mse',optimizer='Adam',metrics=['accuracy'])
#self.D2 = discriminator(in_3_size).modified_vgg()
#self.D2.compile(loss='mse',optimizer='Adam',metrics=['accuracy'])
self.D = discriminator(out,'/home/mdo2/sid_codes/new_codes/D.h5').modified_vgg()
self.D.compile(loss='mse',optimizer='Adam',metrics=['accuracy'])
self.g = 0
# Images
lr = Input(shape=in_1_size)
hr_1 = Input(shape=in_2_size)
hr_2 = Input(shape=in_2_size)
hr_3 = Input(shape=out)
#fake_hr_image_1 = self.g_1(lr)
#fake_hr_image_2 = self.g_2(lr)
fake_hr_image_3 = self.g_3(lr)
#self.vgg_1 = vgg(in_2_size).vgg_loss_model()
#self.vgg_2 = vgg(in_2_size).vgg_loss_model()
self.vgg_3 = vgg(in_2_size).vgg_loss_model()
#fake_hr_feat_1 = self.vgg_1(fake_hr_image_1)
#fake_hr_feat_2 = self.vgg_2(fake_hr_image_2)
fake_hr_feat_3 = self.vgg_3(fake_hr_image_3)
self.D.trainable = False
#self.D1.trainable = False
#self.D2.trainable = False
validity = self.D(fake_hr_image_3)
#self.D.load_weights('/media/arjun/119GB/attachments/D_best.h5')
self.merged_net = Model(inputs=[lr,hr_2],outputs=[validity,fake_hr_feat_3])
self.merged_net.compile(loss=['binary_crossentropy','mse'],loss_weights=[1e-3,.3],optimizer='Adam')
self.merged_net.summary()
#self.merged_net.load_weights('/home/mdo2/sid_codes/newew/my_model.h5')
print('loaded')
def save(self):
#self.D.save_weights('D_best.h5')
#print('Discriminator saved')
#self.merged_net.save('my_model.h5')
self.merged_net.save('/home/mdo2/sid_codes/new_codes/model.h5')
#self.g_2.save('/home/mdo2/sid_codes/new_codes/gen_2.h5')
#self.g_2.save('/home/mdo2/sid_codes/new_codes/gen_2.h5')
self.g_3.save('/home/mdo2/sid_codes/new_codes/gen_3.h5')
print('saved')
"""
def test(self):
test_path, test_path = ''
img_test_path = glob.glob(test_path)
random.shuffle(img_test_path)
d_sum = 0.
g_sum = np.array([0.,0.,0.,0.,0.,0.,0.,0.])
step_size = int(800/self.batch_size)
for steps in range(step_size):
# Load data
print('Loading Test Batch')
x_test, y_test_1, y_test_2, y_test_3 = load_data(img_test_path, start=steps*self.batch_size, batch_size=self.batch_size)
fake_hr_1 = self.g_1.predict(x_test)
fake_hr_2 = self.g_2.predict(fake_hr_1)
fake_hr_3 = self.g_3.predict(fake_hr_2)
valid = np.ones([self.batch_size,12,12,16])
fake = np.zeros([self.batch_size,12,12,16])
d_test_loss_real = self.D.test_on_batch(y_test_3, valid)
d_test_loss_fake = self.D.test_on_batch(fake_hr_3, fake)
d_test_loss = 0.5*np.add(d_test_loss_fake,d_test_loss_real)
8
real_feat_1 = self.vgg_1.predict(y_test_1)
real_feat_2 = self.vgg_2.predict(y_test_2)
real_feat_3 = self.vgg_3.predict(y_test_3)
g_test_loss = self.merged_net.test_on_batch([x_test,y_test_3], [valid,real_feat_3])
d_sum += d_test_loss[0]
g_sum = np.add(g_sum, g_test_loss)
save(d_sum/float(step_size),g_sum/float(step_size))
"""
def train(self):
try:
#self.D.load_weights('/media/arjun/119GB/attachments/D_best.h5')
#self.g_1.load_weights('/media/arjun/119GB/attachments/g_1_best.h5')
#self.g_2.load_weights('/media/arjun/119GB/attachments/g_2_best.h5')
#self.g_3.load_weights('/media/arjun/119GB/attachments/g_3_best.h5')
print("Discriminator loaded")
for epoch in range(self.epochs):
print("loading data")
train_path = dataset_dir
img_train_path = glob.glob(train_path)
i=0
if i==0:
avg_loss=0
i+=1
avg_loss = avg_loss/int(25000/self.batch_size)
for steps in range(int(25000/self.batch_size)):
# Load data
print(epoch)
print('Loading Train Batch:',steps+1)
x_train, y_train_1, y_train_2, y_train_3 = load_data(train_path, start=steps*self.batch_size, batch_size=self.batch_size)
tbCallBack = keras.callbacks.TensorBoard(log_dir='./Graph', histogram_freq=0, write_graph=True, write_images=True)
# Discriminator
print('Training Discriminator')
'''
valid1 = np.ones([self.batch_size,1,1,1])
fake1 = np.zeros([self.batch_size,1,1,1])
valid2 = np.ones([self.batch_size,3,3,1])
fake2 = np.zeros([self.batch_size,3,3,1])
fake_hr_1 = self.g_1.predict(x_train)
fake_hr_feat_1 = self.vgg_1(fake_hr_1)
#d_train_loss_real = self.D1.train_on_batch(y_train_1, valid1)
#d_train_loss_fake = self.D1.train_on_batch(fake_hr_1, fake1)
fake_hr_2 = self.g_2.predict(fake_hr_1)
fake_hr_feat_2 = self.vgg_2(fake_hr_2)
#d_train_loss_real = self.D2.train_on_batch(y_train_2, valid2)
#d_train_loss_fake = self.D2.train_on_batch(fake_hr_2, fake2)
fake_hr_3 = self.g_3.predict(fake_hr_2)
fake_hr_feat_3 = self.vgg_3(fake_hr_3)
d_train_loss_fake = self.D.train_on_batch(fake_hr_3, fake)
d_train_loss_real = self.D.train_on_batch(y_train_3, valid)
#d_train_loss_real = self.D.train_on_batch(y_train_3, valid)
#d_train_loss_fake = self.D.train_on_batch(fake_hr_3, fake)
#print(d_train_loss_fake,' fake loss ')
#print(d_train_loss_real,' real loss ')
d_train_loss = 0.5*np.add(d_train_loss_fake,d_train_loss_real)
# Generator
#valid = np.ones([self.batch_size,192,192,3])
'''
print('Training Generator')
valid = np.ones([self.batch_size,1,1,1])
fake = np.zeros([self.batch_size,1,1,1])
#real_feat_1 = self.vgg_1.predict(y_train_1)
#real_feat_2 = self.vgg_2.predict(y_train_1)
real_feat_3 = self.vgg_3.predict(y_train_1)
d_train_loss=0
#cv2.imshow('image',real_feat_3)
g_loss = self.merged_net.train_on_batch([x_train,y_train_1], [valid,real_feat_3])
#g_loss=0
#d_train_loss = 0
print(g_loss,d_train_loss)
avg_loss+=g_loss[0]
#self.low = g_loss
#print("aaaaaa",K.eval(K.sum(log_loss_1)))
print('*******************************************************\n')
if self.g == 0:
try:
self.old_loss = np.load('/home/mdo2/sid_codes/newew/g_loss_3.npy')
print('latest gen')
self.g+=1
except:
self.old_loss = float('inf')
self.g+=1
if g_loss[0]< self.old_loss:
self.old_loss = g_loss[0]
print('saving best')
np.save ('g_loss_3',g_loss[0])
self.g+=1
self.save()
#if (steps+1)%100==0:
# self.save()
#self.merged_net.inputs = [x_train,y_train_1,y_train_2,y_train_3]
#self.merged_net.outputs = [valid,fake_hr_feat_1,fake_hr_feat_2,fake_hr_feat_3]
#c_loss = K.sqrt(K.mean((fake_hr_feat_1 - real_feat_1)**2))
#log_loss = logcosh(fake_hr_feat_1,real_feat_1)
#grads = K.gradients([log_loss,c_loss],[fake_hr_feat_1,real_feat_1])[0]
#print(grads)
#iterate = K.function([(y_train_1)], [c_loss, grads])
#print(iterate,'c_loss')
#iterate = K.function([tf.convert_to_tensor(y_train_1)], [log_loss, grads])
#print(iterate,'log_loss_1')
#self.D.fit(y_train_3,valid, epochs=epoch, batch_size=self.batch_size, callbacks=[tbCallBack])
except KeyboardInterrupt:
print('Saving Weights')
#self.save()
def feed_forward(self):
in_1 = [256,256,3]
in_2 = [512,512,3]
in_3 = [1024,1024,3]
out_ = [128,128,3]
#/home/mdo2/sid_codes/newew/ee2.jpg
#/home/mdo2/sid_codes/datasets_big/qhd_no_pad/train/1.png
img = cv2.resize(cv2.imread('/home/mdo2/sid_codes/datasets_big/qhd_no_pad/train/1.png',1),(512*2,512*2))
img = img.reshape(1,512*2,512*2,3)
filter_kernal = [[32,128,128,128,256],[32,128,128,128,128,256,512],[32,128,128,128,128,128,128,128,512]]
g3 = generator(3,filter_kernal[2],in_3,'Adam',path = '/home/mdo2/sid_codes/new_codes/gen_1.h5').generator_model()
#g2 = generator(2,filter_kernal[1],in_2,'Adam',path = '/home/mdo2/sid_codes/new_codes/new_save/gen_2.h5').generator_model()
#g1 = generator(1,filter_kernal[0],in_1,'Adam',path = '/home/mdo2/sid_codes/new_codes/new_save/gen_3.h5').generator_model()
img = g3.predict(img)
#img = g2.predict(img)
img = np.array(img).astype('uint8')
img = img.reshape(1024*2,1024*2,3)
cv2.imshow('img',img)
cv2.waitKey(0)
cv2.imwrite('ee3.png',img)
im1 = tf.io.decode_image('/home/mdo2/sid_codes/newew/ee3.png')
#im1 = tf.image.convert_image_dtype(im1, tf.float32)
im2 = tf.io.decode_image('/home/mdo2/sid_codes/datasets_big/qhd_no_pad/train/1.png')
#im2 = tf.image.convert_image_dtype(im2, tf.float32)
psnr1 = tf.image.psnr(im2, im2, max_val=1.0)
print(psnr1)
def fit(self,
X_train,
epochs=50,
batch_size=50,
loss=tf.keras.losses.BinaryCrossentropy(),
optimizer=tf.keras.optimizers.Adam(lr=1e-5, beta_1=0.5),
test=tuple(),
early_stop_num=50,
verbose=1):
#Convert training-data to numpy format
X_train = np.array(X_train)
#If "input_dim" is not greater than 1, it is set to the number of features in the training-data
if not self.input_dim >= 1: self.input_dim = X_train.shape[1]
#Define model for Discriminator
self.dis = self.get_discriminator()
self.dis.compile(loss=loss, optimizer=optimizer)
#Define model to train Encoder
self.enc = self.get_encoder()
x = Input(shape=(self.input_dim, ))
z_gen = self.enc(x)
valid = self.dis([x, z_gen])
enc_dis = Model(inputs=x, outputs=valid, name='enc_to_dis')
enc_dis.compile(loss=loss, optimizer=optimizer)
#Define model to train Generator
self.gen = self.get_generator()
z = Input(shape=(self.latent_dim, ))
x_gen = self.gen(z)
valid = self.dis([x_gen, z])
gen_dis = Model(inputs=z, outputs=valid, name='gen_to_dis')
gen_dis.compile(loss=loss, optimizer=optimizer)
#Training
min_val_loss = float('inf')
stop_count = 0
for i in range(epochs):
#Unfreeze Discriminator
self.dis.trainable = True
#From the training data, randomly choice half of the "batch_size" as "real_data"
idx = np.random.randint(0, X_train.shape[0], batch_size // 2)
real_data = X_train[idx]
#Generates noise and get the data generated by that noise
noise = np.random.normal(0, 1, (len(idx), self.latent_dim))
gen_data = self.gen.predict(noise)
#Get noise predicted from "real_data"
enc_noise = self.enc.predict(real_data)
#Train Discriminator
d_enc_loss = self.dis.train_on_batch([real_data, enc_noise],
np.ones((len(real_data), 1)))
d_gen_loss = self.dis.train_on_batch([gen_data, noise],
np.zeros((len(gen_data), 1)))
d_loss = d_enc_loss + d_gen_loss
#Freeze Discriminator to train Encoder and Generator
self.dis.trainable = False
#Train Encoder
e_loss = enc_dis.train_on_batch(real_data,
np.zeros((len(real_data), 1)))
#Train Generator
g_loss = gen_dis.train_on_batch(noise, np.ones((len(noise), 1)))
#Calculate test loss
if len(test) > 0:
#Get testing-data
X_test = test[0]
y_true = test[1]
#Predict testing-data
proba = self.predict(X_test)
proba = minmax_scale(proba)
#As "val_loss", calculate binary cross entropy
val_loss = tf.keras.losses.binary_crossentropy(y_true,
proba).numpy()
#If "val_loss" is less than "min_val_loss"
if min_val_loss > val_loss:
min_val_loss = val_loss #Update "min_val_loss" to "val_loss"
stop_count = 0 #Change "stop_count" to 0
#If "stop_count" is equal or more than "early_stop_num", training is end
elif stop_count >= early_stop_num:
break
#Else, "stop_count" is added 1
else:
stop_count += 1
#Display learning progress
if verbose == 1 and i % 100 == 0:
if len(test) == 0:
print(
f'epoch{i}-> d_loss:{d_loss} e_loss:{e_loss} g_loss:{g_loss}'
)
else:
print(
f'epoch{i}-> d_loss:{d_loss} e_loss:{e_loss} g_loss:{g_loss} val_loss:{val_loss}'
)
def layer_test(layer_cls,
kwargs={},
input_shape=None,
input_dtype=None,
input_data=None,
expected_output=None,
expected_output_dtype=None,
fixed_batch_size=False):
"""Test routine for a layer with a single input tensor
and single output tensor.
"""
# generate input data
if input_data is None:
assert input_shape
if not input_dtype:
input_dtype = K.floatx()
input_data_shape = list(input_shape)
for i, e in enumerate(input_data_shape):
if e is None:
input_data_shape[i] = np.random.randint(1, 4)
input_data = (10 * np.random.random(input_data_shape))
input_data = input_data.astype(input_dtype)
else:
if input_shape is None:
input_shape = input_data.shape
if input_dtype is None:
input_dtype = input_data.dtype
if expected_output_dtype is None:
expected_output_dtype = input_dtype
# instantiation
layer = layer_cls(**kwargs)
# test get_weights , set_weights at layer level
weights = layer.get_weights()
layer.set_weights(weights)
try:
expected_output_shape = layer.compute_output_shape(input_shape)
except:
expected_output_shape = layer._compute_output_shape(input_shape)
# test in functional API
if fixed_batch_size:
x = Input(batch_shape=input_shape, dtype=input_dtype)
else:
x = Input(shape=input_shape[1:], dtype=input_dtype)
y = layer(x)
assert K.dtype(y) == expected_output_dtype
# check with the functional API
model = Model(x, y)
actual_output = model.predict(input_data)
actual_output_shape = actual_output.shape
for expected_dim, actual_dim in zip(expected_output_shape,
actual_output_shape):
if expected_dim is not None:
assert expected_dim == actual_dim
if expected_output is not None:
assert_allclose(actual_output, expected_output, rtol=1e-3)
# test serialization, weight setting at model level
model_config = model.get_config()
recovered_model = model.__class__.from_config(model_config)
if model.weights:
weights = model.get_weights()
recovered_model.set_weights(weights)
_output = recovered_model.predict(input_data)
assert_allclose(_output, actual_output, rtol=1e-3)
# test training mode (e.g. useful when the layer has a
# different behavior at training and testing time).
if has_arg(layer.call, 'training'):
model.compile('rmsprop', 'mse')
model.train_on_batch(input_data, actual_output)
# test instantiation from layer config
layer_config = layer.get_config()
layer_config['batch_input_shape'] = input_shape
layer = layer.__class__.from_config(layer_config)
# for further checks in the caller function
return actual_output
size=batch_size)]
#Construct different batches of real and fake data
X = np.concatenate([image_batch, generated_images])
# Labels for generated and real data
y_dis = np.zeros(2 * batch_size)
y_dis[:batch_size] = 0.9
#Pre train discriminator on fake and real data before starting the gan.
discriminator.trainable = True
discriminator.train_on_batch(X, y_dis)
#Tricking the noised input of the Generator as real data
noise = np.random.normal(0, 1, [batch_size, 100])
y_gen = np.ones(batch_size)
# During the training of gan,
# the weights of discriminator should be fixed.
#We can enforce that by setting the trainable flag
discriminator.trainable = False
#training the GAN by alternating the training of the Discriminator
#and training the chained GAN model with Discriminator’s weights freezed.
gan.train_on_batch(noise, y_gen)
if e == 1 or e % 20 == 0:
plot_generated_images(e, generator)
#%%
class merge_models:
def __init__(self, epochs, batch_size):
self.epochs = epochs
self.batch_size = batch_size
# Generator
filter_kernal = [[32, 128, 128, 128, 256],
[32, 128, 128, 128, 128, 256, 512],
[32, 128, 128, 128, 128, 128, 128, 512, 512]]
self.g_1 = generator(3, filter_kernal[2], in_1_size, 'Adam',
'mse').generator_model()
self.g_2 = generator(2, filter_kernal[1], in_2_size, 'Adam',
'mse').generator_model()
self.g_3 = generator(1, filter_kernal[0], in_3_size, 'Adam',
'mse').generator_model()
# Discriminator
self.D = discriminator(out).modified_vgg()
self.D.compile(loss='mse', optimizer='Adam', metrics=['accuracy'])
# Images
lr = Input(shape=in_1_size)
hr_1 = Input(shape=in_2_size)
hr_2 = Input(shape=in_3_size)
hr_3 = Input(shape=out)
fake_hr_image_1 = self.g_1(lr)
fake_hr_image_2 = self.g_2(fake_hr_image_1)
fake_hr_image_3 = self.g_3(fake_hr_image_2)
self.vgg_1 = vgg(in_2_size).vgg_loss_model()
self.vgg_2 = vgg(in_3_size).vgg_loss_model()
self.vgg_3 = vgg(out).vgg_loss_model()
fake_hr_feat_1 = self.vgg_1(fake_hr_image_1)
fake_hr_feat_2 = self.vgg_2(fake_hr_image_2)
fake_hr_feat_3 = self.vgg_3(fake_hr_image_3)
self.D.trainable = False
valid = self.D(fake_hr_image_3)
self.merged_net = Model(
inputs=[lr, hr_1, hr_2, hr_3],
outputs=[valid, fake_hr_feat_1, fake_hr_feat_2, fake_hr_feat_3])
self.merged_net.compile(
loss=['binary_crossentropy', 'mse', 'mse', 'mse'],
loss_weights=[1e-3, .3, .3, .3],
optimizer='Adam')
self.merged_net.summary()
def save(self):
self.D.save_weights('D_best.h5')
print('Discriminator saved')
self.g_1.save_weights('g_1_best.h5')
self.g_2.save_weights('g_2_best.h5')
self.g_3.save_weights('g_3_best.h5')
print('All generators saved')
"""
def test(self):
test_path, test_path = ''
img_test_path = glob.glob(test_path)
random.shuffle(img_test_path)
d_sum = 0.
g_sum = np.array([0.,0.,0.,0.,0.,0.,0.,0.])
step_size = int(800/self.batch_size)
for steps in range(step_size):
# Load data
print('Loading Test Batch')
x_test, y_test_1, y_test_2, y_test_3 = load_data(img_test_path, start=steps*self.batch_size, batch_size=self.batch_size)
fake_hr_1 = self.g_1.predict(x_test)
fake_hr_2 = self.g_2.predict(fake_hr_1)
fake_hr_3 = self.g_3.predict(fake_hr_2)
valid = np.ones([self.batch_size,12,12,16])
fake = np.zeros([self.batch_size,12,12,16])
d_test_loss_real = self.D.test_on_batch(y_test_3, valid)
d_test_loss_fake = self.D.test_on_batch(fake_hr_3, fake)
d_test_loss = 0.5*np.add(d_test_loss_fake,d_test_loss_real)
8
real_feat_1 = self.vgg_1.predict(y_test_1)
real_feat_2 = self.vgg_2.predict(y_test_2)
real_feat_3 = self.vgg_3.predict(y_test_3)
g_test_loss = self.merged_net.test_on_batch([x_test,y_test_3], [valid,real_feat_3])
d_sum += d_test_loss[0]
g_sum = np.add(g_sum, g_test_loss)
save(d_sum/float(step_size),g_sum/float(step_size))
"""
def train(self):
try:
for epoch in range(self.epochs):
train_path = dataset_dir + '/*.jpg'
img_train_path = glob.glob(train_path)
random.shuffle(img_train_path)
for steps in range(int(25000 / self.batch_size)):
# Load data
print('Loading Train Batch:', steps + 1)
x_train, y_train_1, y_train_2, y_train_3 = load_data(
train_path,
start=steps * self.batch_size,
batch_size=self.batch_size)
# Discriminator
print('Training Discriminator')
fake_hr_1 = self.g_1.predict(x_train)
fake_hr_2 = self.g_2.predict(fake_hr_1)
fake_hr_3 = self.g_3.predict(fake_hr_2)
valid = np.ones([self.batch_size, 6, 6, 1])
fake = np.zeros([self.batch_size, 6, 6, 1])
#d_train_loss_real = self.D.train_on_batch(y_train_3, valid)
#d_train_loss_fake = self.D.train_on_batch(fake_hr_3, fake)
#d_train_loss = 0.5*np.add(d_train_loss_fake,d_train_loss_real)
d_train_loss = 0
# Generator
print('Training Generator')
real_feat_1 = self.vgg_1.predict(y_train_1)
real_feat_2 = self.vgg_2.predict(y_train_2)
real_feat_3 = self.vgg_3.predict(y_train_3)
g_loss = self.merged_net.train_on_batch(
[x_train, y_train_1, y_train_2, y_train_3],
[valid, real_feat_1, real_feat_2, real_feat_3])
print(g_loss, d_train_loss)
self.low = g_loss
print("aaaaaa", K.eval(K.sum(log_loss_1)))
print(
'*******************************************************\n'
)
if (steps + 1) % 100 == 0:
self.save()
except KeyboardInterrupt:
print('Saving Weights')
#self.save()
def loss(self):
print("0000......................................")
class A2C(Agent):
"""Advantage Actor-Critic (A2C)
A2C is a synchronous version of A3C which gives equal or better performance.
For more information on A2C refer to the OpenAI blog post: https://blog.openai.com/baselines-acktr-a2c/.
The A3C algorithm is described in "Asynchronous Methods for Deep Reinforcement Learning" (Mnih et al., 2016)
Since this algorithm is on-policy, it can and should be trained with multiple simultaneous environment instances.
The parallelism decorrelates the agents' data into a more stationary process which aids learning.
"""
def __init__(self,
model,
actions,
optimizer=None,
policy=None,
test_policy=None,
gamma=0.99,
instances=8,
nsteps=1,
value_loss=0.5,
entropy_loss=0.01):
"""
TODO: Describe parameters
"""
self.actions = actions
self.optimizer = Adam(lr=3e-3) if optimizer is None else optimizer
self.memory = memory.OnPolicy(steps=nsteps, instances=instances)
if policy is None:
# Create one policy per instance, with varying exploration parameters
self.policy = [Greedy()] + [
GaussianEpsGreedy(eps, 0.1)
for eps in np.arange(0, 1, 1 / (instances - 1))
]
else:
self.policy = policy
self.test_policy = Greedy() if test_policy is None else test_policy
self.gamma = gamma
self.instances = instances
self.nsteps = nsteps
self.value_loss = value_loss
self.entropy_loss = entropy_loss
self.training = True
# Create output model layers based on number of actions
raw_output = model.layers[-1].output
actor = Dense(actions, activation='softmax')(
raw_output) # Actor (Policy Network)
critic = Dense(1, activation='linear')(
raw_output) # Critic (Value Network)
output_layer = Concatenate()([actor, critic])
self.model = Model(inputs=model.input, outputs=output_layer)
def a2c_loss(targets_actions, y_pred):
# Unpack input
targets, actions = targets_actions[:,
0], targets_actions[:,
1:] # Unpack
probs, values = y_pred[:, :-1], y_pred[:, -1]
# Compute advantages and logprobabilities
adv = targets - values
logprob = tf.math.log(
tf.reduce_sum(probs * actions, axis=1, keepdims=False) + 1e-10)
# Compute composite loss
loss_policy = -adv * logprob
loss_value = self.value_loss * tf.square(adv)
entropy = self.entropy_loss * tf.reduce_sum(
probs * tf.math.log(probs + 1e-10), axis=1, keepdims=False)
return tf.reduce_mean(loss_policy + loss_value + entropy)
self.model.compile(optimizer=self.optimizer, loss=a2c_loss)
def save(self, filename, overwrite=False):
"""Saves the model parameters to the specified file."""
self.model.save_weights(filename, overwrite=overwrite)
def act(self, state, instance=0):
"""Returns the action to be taken given a state."""
qvals = self.model.predict(np.array([state]))[0][:-1]
if self.training:
return self.policy[instance].act(qvals) if isinstance(
self.policy, list) else self.policy.act(qvals)
else:
return self.test_policy[instance].act(qvals) if isinstance(
self.test_policy, list) else self.test_policy.act(qvals)
def push(self, transition, instance=0):
"""Stores the transition in memory."""
self.memory.put(transition, instance)
def train(self, step):
"""Trains the agent for one step."""
if len(self.memory) < self.instances:
return
state_batch, action_batch, reward_batches, end_state_batch, not_done_mask = self.memory.get(
)
# Compute the value of the last next states
target_qvals = np.zeros(self.instances)
non_final_last_next_states = [
es for es in end_state_batch if es is not None
]
if len(non_final_last_next_states) > 0:
non_final_mask = list(map(lambda s: s is not None,
end_state_batch))
target_qvals[non_final_mask] = self.model.predict_on_batch(
np.array(non_final_last_next_states))[:, -1].squeeze()
# Compute n-step discounted return
# If episode ended within any sampled nstep trace - zero out remaining rewards
for n in reversed(range(self.nsteps)):
rewards = np.array([b[n] for b in reward_batches])
target_qvals *= np.array([t[n] for t in not_done_mask])
target_qvals = rewards + (self.gamma * target_qvals)
# Prepare loss data: target Q-values and actions taken (as a mask)
ran = np.arange(self.instances)
targets_actions = np.zeros((self.instances, self.actions + 1))
targets_actions[ran, 0] = target_qvals
targets_actions[ran, np.array(action_batch) + 1] = 1
self.model.train_on_batch(np.array(state_batch), targets_actions)
class CycleGAN():
def __init__(self):
# Input shape
self.class_num = config.class_num
self.img_shape = config.img_shape
self.img_width = config.img_width
self.img_height = config.img_height
self.img_channel = config.img_channel
self.gf = 32
self.df = 64
self.patch = int(config.img_height // (2**4))
self.patch_size = (self.patch, self.patch, 1)
self.s1 = Dataset('./t1ce', './tice_label')
self.s2 = Dataset('./t2', './t2_label')
self.target = Dataset('./OpenBayes',
'./Openbayes_label',
need_resize=1)
optimizer = RMSprop(0.0002)
self.D = self.build_discriminator()
self.G = self.build_generator()
self.G.trainable = False
real_img = Input(shape=config.img_shape)
real_src, real_cls = self.D(real_img)
fake_cls = Input(shape=(self.class_num, ))
fake_img = self.G([real_img, fake_cls])
fake_src, fake_output = self.D(fake_img)
self.Train_D = Model([real_img, fake_cls],
[real_src, real_cls, fake_src, fake_output])
self.Train_D.compile(loss=[
'mse', self.classification_loss, 'mse', self.classification_loss
],
optimizer=optimizer,
loss_weights=[1.0, 1.0, 1.0, 1.0])
self.G.trainable = True
self.D.trainable = False
real_x = Input(shape=self.img_shape)
now_label = Input(shape=(self.class_num, ))
target_label = Input(shape=(self.class_num, ))
fake_x = self.G([real_x, target_label])
fake_out_src, fake_out_cls = self.D(fake_x)
x_rec = self.G([fake_x, now_label])
self.train_G = Model([real_x, now_label, target_label],
[fake_out_src, fake_out_cls, x_rec])
self.train_G.compile(loss=['mse', self.classification_loss, 'mae'],
optimizer=optimizer,
loss_weights=[1.0, 1.0, 1.0])
'''
self.AdataSet=Dataset('./done', './done_label',need_resize=1)
to get different model
change image path
for example t1ce to t1
self.BdataSet=Dataset('./Data/2/t1','./Data/2/label')
# Number of filters in the first layer of G and D
self.patch = int(config.img_height //(2**4))
self.patch_size = (self.patch,self.patch,1)
# Loss weights
self.lambda_cycle = 10.0 # Cycle-consistency loss
self.lambda_id = 0.1 * self.lambda_cycle # Identity loss
optimizer = RMSprop(0.0002)
# Build and compile the discriminators
self.d_B = self.build_discriminator()
self.d_A = self.build_discriminator()
self.conv_init = RandomNormal(0, 0.02) # for convolution kernel
self.gamma_init = RandomNormal(1., 0.02)
self.d_A.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
self.d_B.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
#-------------------------
# Construct Computational
# Graph of Generators
#-------------------------
# Build the generators
self.g_AB = self.resnet_6blocks()
self.g_BA = self.resnet_6blocks()
# Input images from both domains
img_A = Input(shape=self.img_shape)
img_B = Input(shape=self.img_shape)
# Translate images to the other domain
fake_B = self.g_AB(img_A)
fake_A = self.g_BA(img_B)
# Translate images back to original domain
reconstr_A = self.g_BA(fake_B)
reconstr_B = self.g_AB(fake_A)
# Identity mapping of images
img_A_id = self.g_BA(img_A)
img_B_id = self.g_AB(img_B)
# For the combined model we will only train the generators
self.d_A.trainable = False
self.d_B.trainable = False
# Discriminators determines validity of translated images
valid_A = self.d_A(fake_A)
valid_B = self.d_B(fake_B)
# Combined model trains generators to fool discriminators
self.combined = Model(inputs=[img_A, img_B],
outputs=[ valid_A, valid_B,
reconstr_A, reconstr_B,
img_A_id, img_B_id ])
self.combined.compile(loss=['mse', 'mse',
'mae', 'mae',
'mae', 'mae'],
loss_weights=[ 1, 1,
self.lambda_cycle, self.lambda_cycle,
self.lambda_id, self.lambda_id ],
optimizer=optimizer)
'''
def classification_loss(self, Y_true, Y_pred):
return tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(labels=Y_true,
logits=Y_pred))
def normalize(self, **kwargs):
# return batchnorm()#axis=get_filter_dim()
return InstanceNormalization()
def get_onehot_label(self, label):
now_label = [0.] * self.class_num
now_label[label] = 1.
now_label = np.array(now_label).reshape([1, self.class_num])
return now_label
def resnet_block(self, input, dim, ks=(3, 3), strides=(1, 1)):
x = ZeroPadding2D((1, 1))(input)
x = Conv2D(dim, ks, strides=strides,
kernel_initializer=self.conv_init)(x)
x = InstanceNormalization()(x)
x = Activation('relu')(x)
x = ZeroPadding2D((1, 1))(x)
x = Conv2D(dim, ks, strides=strides,
kernel_initializer=self.conv_init)(x)
x = InstanceNormalization()(x)
res = Add()([input, x])
return res
def resnet_6blocks(self, ngf=64, **kwargs):
input = Input(config.img_shape)
x = ZeroPadding2D((3, 3))(input)
x = Conv2D(ngf, (7, 7), kernel_initializer=self.conv_init)(x)
x = InstanceNormalization()(x)
x = Activation('relu')(x)
n_downsampling = 2
for i in range(n_downsampling):
mult = 2**i
x = Conv2D(ngf * mult * 2, (3, 3),
padding='same',
strides=(2, 2),
kernel_initializer=self.conv_init)(x)
x = InstanceNormalization()(x)
x = Activation('relu')(x)
mult = 2**n_downsampling
for i in range(6):
x = self.resnet_block(x, ngf * mult)
for i in range(n_downsampling):
mult = 2**(n_downsampling - i)
f = int(ngf * mult / 2)
x = UpSampling2D()(x)
x = InstanceNormalization()(x)
x = Activation('relu')(x)
x = ZeroPadding2D((3, 3))(x)
x = Conv2D(config.img_channel, (7, 7),
kernel_initializer=self.conv_init)(x)
x = Activation('tanh')(x)
model = Model(inputs=input, outputs=[x])
print('Model resnet 6blocks:')
model.summary()
return model
def build_generator(self):
"""U-Net Generator"""
def conv2d(layer_input, filters, f_size=4):
"""Layers used during downsampling"""
d = Conv2D(filters, kernel_size=f_size, strides=2,
padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
d = InstanceNormalization()(d)
return d
def deconv2d(layer_input,
skip_input,
filters,
f_size=4,
dropout_rate=0):
"""Layers used during upsampling"""
u = UpSampling2D(size=2)(layer_input)
u = Conv2D(filters,
kernel_size=f_size,
strides=1,
padding='same',
activation='relu')(u)
if dropout_rate:
u = Dropout(dropout_rate)(u)
u = InstanceNormalization()(u)
u = Concatenate()([u, skip_input])
return u
# Image input
input_img = Input(shape=self.img_shape)
inp_c = Input(shape=(self.class_num, ))
c = Lambda(lambda x: K.repeat(x, self.img_width * self.img_height))(
inp_c)
c = Reshape((self.img_width, self.img_height, self.class_num))(c)
d0 = Concatenate()([input_img, c])
# Downsampling
d1 = conv2d(d0, self.gf)
d2 = conv2d(d1, self.gf * 2)
d3 = conv2d(d2, self.gf * 4)
d4 = conv2d(d3, self.gf * 8)
# Upsampling
u1 = deconv2d(d4, d3, self.gf * 4)
u2 = deconv2d(u1, d2, self.gf * 2)
u3 = deconv2d(u2, d1, self.gf)
u4 = UpSampling2D(size=2)(u3)
output_img = Conv2D(self.img_channel,
kernel_size=4,
strides=1,
padding='same',
activation='tanh')(u4)
return Model([input_img, inp_c], output_img)
def build_discriminator(self):
def d_layer(layer_input, filters, f_size=4, normalization=True):
"""Discriminator layer"""
d = Conv2D(filters, kernel_size=f_size, strides=2,
padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
if normalization:
d = InstanceNormalization()(d)
return d
img = Input(shape=self.img_shape)
d1 = d_layer(img, self.df, normalization=False)
d2 = d_layer(d1, self.df * 2)
d3 = d_layer(d2, self.df * 4)
d4 = d_layer(d3, self.df * 8)
validity = Conv2D(1, kernel_size=4, strides=1, padding='same')(d4)
class_cls = Conv2D(self.class_num,
kernel_size=15,
strides=1,
padding='valid')(d4)
class_cls = Reshape((self.class_num, ))(class_cls)
model = Model(img, [validity, class_cls])
return model
def train(self, epochs, batch_size=1, sample_interval=50):
#start_time = datetime.datetime.now()
# Adversarial loss ground truths
valid = np.ones((batch_size, ) + self.patch_size)
fake = np.zeros((batch_size, ) + self.patch_size)
self.train_G.summary()
self.Train_D.summary()
for epoch in range(epochs):
for batch_i in range(0, 1000):
rand_number = random.randint(0, 1)
target_label = 2
target_label = self.get_onehot_label(target_label)
now_img = None
now_label = None
if (rand_number == 0):
now_img, _ = self.s1.get_one_data()
now_label = self.get_onehot_label(0)
if (rand_number == 1):
now_img, _ = self.s2.get_one_data()
now_label = self.get_onehot_label(1)
if (rand_number == 2):
now_img, _ = self.target.get_one_data()
now_label = self.get_onehot_label(2)
#print(now_label.shape)
#print(target_label.shape)
#print(self.D.output)
d_loss = self.Train_D.train_on_batch(
[now_img, target_label],
[valid, now_label, fake, target_label],
class_weight=[1.0, 1.0, 1.0, 1.0])
g_loss = self.train_G.train_on_batch(
[now_img, now_label, target_label],
[valid, target_label, now_img],
class_weight=[1.0, 1.0, 1.0])
print("[Epoch %d/%d][Batch %d/%d]" %
(epoch, epochs, batch_i, 1000))
print(
"[d loss: %.3f][real dloss: %.3f][real cl: %.6f][fake dloss: %.3f][fake cl:%.6f]"
% (d_loss[0], d_loss[1], d_loss[2], d_loss[3], d_loss[4]))
print(
"[g loss: %.3f][g dloss: %.3f][g cl loss: %.6f][rec loss: %.3f]"
% (g_loss[0], g_loss[1], g_loss[2], g_loss[3]))
# If at save interval => save generated image samples
if batch_i % sample_interval == 0:
'''
change parameter path to save image to different dir
'''
self.sample_images(epoch, batch_i, path='sample_t1ce')
'''
remember to save model with different name
'''
self.G.save_weights('g_300.h5')
def sample_images(self, epoch, batch_i, path='sample'):
os.makedirs(path + '/', exist_ok=True)
r, c = 2, 2
imgs_A, label_A = self.s1.get_one_data()
imgs_B, label_B = self.s2.get_one_data()
target_label = self.get_onehot_label(2)
fake_A = self.G.predict([imgs_A, target_label])
fake_B = self.G.predict([imgs_B, target_label])
gen_imgs = np.concatenate([imgs_A, fake_A, imgs_B, fake_B])
gen_imgs = np.reshape(gen_imgs,
(r * c, config.img_width, config.img_height))
# Rescale images 0 - 1
gen_imgs = 0.5 * gen_imgs + 0.5
for x in range(r * c):
now_image = gen_imgs[x, :, :]
now_image = now_image[:, :, np.newaxis]
now_image = image.array_to_img(now_image, scale=True)
now_image.save(os.path.join(path, 'generated_' \
+ str(epoch) + '_' + str(batch_i)+'_'+str(x) + '_.png'))
'''
class Pix2Pix():
def __init__(self):
# Input shape
self.img_rows = 256
self.img_cols = 256
self.channels = 3
self.img_shape = (self.img_rows, self.img_cols, self.channels)
# Configure data loader
self.dataset_name = 'facades'
self.data_loader = DataLoader(dataset_name=self.dataset_name,
img_res=(self.img_rows, self.img_cols))
# Calculate output shape of D (PatchGAN)
patch = int(self.img_rows / 2**4)
self.disc_patch = (patch, patch, 1)
# Number of filters in the first layer of G and D
self.gf = 64
self.df = 64
optimizer = Adam(0.0002, 0.5)
# Build and compile the discriminator
self.discriminator = self.build_discriminator()
self.discriminator.compile(loss='mse',
optimizer=optimizer,
metrics=['accuracy'])
#-------------------------
# Construct Computational
# Graph of Generator
#-------------------------
# Build the generator
self.generator = self.build_generator()
# Input images and their conditioning images
img_A = Input(shape=self.img_shape)
img_B = Input(shape=self.img_shape)
# By conditioning on B generate a fake version of A
fake_A = self.generator(img_B)
# For the combined model we will only train the generator
#self.discriminator.trainable = False
# Discriminators determines validity of translated images / condition pairs
valid = self.discriminator([fake_A, img_B])
self.combined = Model(inputs=[img_A, img_B], outputs=[valid, fake_A])
self.combined.compile(loss=['mse', 'mae'],
loss_weights=[1, 100],
optimizer=optimizer)
valid.trainable = False
#self.combined.load_weights("Weights/199.h5")
def build_generator(self):
layer_per_block = [4, 4, 4, 4, 4, 15, 4, 4, 4, 4, 4]
tiramisu = Tiramisu(layer_per_block)
tiramisu.summary()
#d0 = Input(shape=self.img_shape)
return tiramisu
def build_discriminator(self):
def d_layer(layer_input, filters, f_size=4, bn=True):
"""Discriminator layer"""
d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
if bn:
d = BatchNormalization(momentum=0.8)(d)
return d
img_A = Input(shape=self.img_shape)
img_B = Input(shape=self.img_shape)
# Concatenate image and conditioning image by channels to produce input
combined_imgs = Concatenate(axis=-1)([img_A, img_B])
d1 = d_layer(combined_imgs, self.df, bn=False)
d2 = d_layer(d1, self.df*2)
d3 = d_layer(d2, self.df*4)
d4 = d_layer(d3, self.df*8)
validity = Conv2D(1, kernel_size=4, strides=1, padding='same')(d4)
return Model([img_A, img_B], validity)
def train(self, epochs, batch_size=1, sample_interval=50):
start_time = datetime.datetime.now()
# Adversarial loss ground truths
valid = np.ones((batch_size,) + self.disc_patch)
fake = np.zeros((batch_size,) + self.disc_patch)
for epoch in range(epochs):
for batch_i, (imgs_A, imgs_B) in enumerate(self.data_loader.load_batch(batch_size)):
# ---------------------
# Train Discriminator
# ---------------------
print(imgs_A.shape)
# Condition on B and generate a translated version
fake_A = self.generator.predict(imgs_B)
# Train the discriminators (original images = real / generated = Fake)
d_loss_real = self.discriminator.train_on_batch([imgs_A, imgs_B], valid)
d_loss_fake = self.discriminator.train_on_batch([fake_A, imgs_B], fake)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
# -----------------
# Train Generator
# -----------------
# Train the generators
g_loss = self.combined.train_on_batch([imgs_A, imgs_B], [valid, imgs_A])
elapsed_time = datetime.datetime.now() - start_time
# Plot the progress
print ("[Epoch %d/%d] [Batch %d/%d] [D loss: %f, acc: %3d%%] [G loss: %f] time: %s" % (epoch, epochs,
batch_i, self.data_loader.n_batches,
d_loss[0], 100*d_loss[1],
g_loss[0],
elapsed_time))
# If at save interval => save generated image samples
if batch_i % sample_interval == 0:
self.sample_images(epoch, batch_i)
self.combined.save_weights("Weights/"+str(epoch)+".h5")
def img_to_frame(self,imgA,imgB,fakeA):
no_images = imgA.shape[0]
img_height = imgA.shape[1]
img_width = imgA.shape[2]
pad = 20
title_pad=20
pad_top = pad+title_pad
frame=np.zeros((no_images*(img_height+pad_top),no_images*(img_width+pad),3))
count=0
gen_imgs = np.concatenate([imgB, fakeA, imgA])
gen_imgs = 0.5 * gen_imgs + 0.5
titles = ['Condition', 'Generated', 'Original']
for r in range(no_images):
for c in range(no_images):
im = gen_imgs[count]
count=count+1
y0 = r*(img_height+pad_top) + pad//2
x0 = c*(img_width+pad) + pad//2
# print(frame[y0:y0+img_height,x0:x0+img_width,:].shape)
frame[y0:y0+img_height,x0:x0+img_width,:] = im*255
frame = cv2.putText(frame, titles[r], (x0, y0-title_pad//4), cv2.FONT_HERSHEY_COMPLEX, .5, (255,255,255))
return frame
def sample_images(self, epoch, batch_i):
os.makedirs('images/%s' % self.dataset_name, exist_ok=True)
os.makedirs('images/dehazed', exist_ok=True)
os.makedirs('images/haze', exist_ok=True)
os.makedirs('images/original',exist_ok=True)
r, c = 3, 3
imgs_A, imgs_B, or_A, or_B = self.data_loader.load_data(batch_size=3, is_testing=True)
fake_A = self.generator.predict(imgs_B)
cv2.imwrite("images/dehazed"+"/"+"Img:"+str(epoch)+"_"+str(batch_i)+".jpg",(fake_A[0]*0.5+0.5)*255)
cv2.imwrite("images/haze"+"/"+"Img:"+str(epoch)+"_"+str(batch_i)+".jpg",(or_B[0]*0.5+0.5)*255)
cv2.imwrite("images/original"+"/"+"Img:"+str(epoch)+"_"+str(batch_i)+".jpg",(or_A[0]*0.5+0.5)*255)
frame=self.img_to_frame(imgs_A,imgs_B,fake_A)
cv2.imwrite("images/"+self.dataset_name+"/"+"Img:"+str(epoch)+"_"+str(batch_i)+".png",frame)
class NS_MODEL:
def __init__(self):
input_shape = (224, 224, 3)
# 変換ネットワークの構築
self.model_gen = self.build_encoder_decoder(
input_shape=input_shape
)
# 学習済みモデルVGG16の呼び出し
self.vgg16 = VGG16()
# 重みパラメータを学習させない設定をする
for layer in self.vgg16.layers:
layer.trainable = False
# 特徴量を抽出する層の名前を定義
self.style_layer_names = (
'block1_conv2',
'block2_conv2',
'block3_conv3',
'block4_conv3'
)
self.contents_layer_names = ('block3_conv3',)
# 中間層の出力を保持するためのリスト
self.style_outputs_gen = []
self.contents_outputs_gen = []
self.input_gen = self.model_gen.output # 変換ネットワークの出力を入力とする
z = Lambda(self.norm_vgg16)(self.input_gen) # 入力値の正規化
for layer in self.vgg16.layers:
z = layer(z) # VGG16の層を積み上げてネットワークを再構築
if layer.name in self.style_layer_names:
# スタイル特徴量抽出用の中間層の出力を追加
self.style_outputs_gen.append(z)
if layer.name in self.contents_layer_names:
# コンテンツ特徴量抽出用の中間層の出力を追加
self.contents_outputs_gen.append(z)
# モデルを定義
self.model = Model(
inputs=self.model_gen.input,
outputs=self.style_outputs_gen + self.contents_outputs_gen
)
input_sty = Input(shape=input_shape, name='input_sty')
style_outputs = []
x = Lambda(self.norm_vgg16)(input_sty)
for layer in self.vgg16.layers:
x = layer(x)
if layer.name in self.style_layer_names:
style_outputs.append(x)
self.model_sty = Model(
inputs=input_sty,
outputs=style_outputs
)
input_con = Input(shape=input_shape, name='input_con')
contents_outputs = []
y = Lambda(self.norm_vgg16)(input_con)
for layer in self.vgg16.layers:
y = layer(y)
if layer.name in self.contents_layer_names:
contents_outputs.append(y)
self.model_con = Model(
inputs = input_con,
outputs = contents_outputs
)
def residual_block(self,input_ts):
"""ResidualBlockの構築する関数"""
x = Conv2D(
128, (3, 3), strides=1, padding='same'
)(input_ts)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(128, (3, 3), strides=1, padding='same')(x)
x = BatchNormalization()(x)
return Add()([x, input_ts])
def build_encoder_decoder(self,input_shape=(224, 224, 3)):
"""変換用ネットワークの構築"""
# Encoder部分
input_ts = Input(shape=input_shape, name='input')
# 入力を[0, 1]の範囲に正規化
x = Lambda(lambda a: a/255.)(input_ts)
x = Conv2D(32, (9, 9), strides=1, padding='same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(64, (3, 3), strides=2, padding='same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(128, (3, 3), strides=2, padding='same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# ResidualBlockを5ブロック追加
for _ in range(5):
x = self.residual_block(x)
# Decoder部分
x = Conv2DTranspose(
64, (3, 3), strides=2, padding='same'
)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2DTranspose(32, (3, 3), strides=2, padding='same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2DTranspose(3, (9, 9), strides=1, padding='same')(x)
x = BatchNormalization()(x)
x = Activation('tanh')(x)
# 出力値が[0, 255]になるようにスケール変換
gen_out = Lambda(lambda a: (a + 1)*127.5)(x)
model_gen = Model(
inputs=[input_ts],
outputs=[gen_out]
)
return model_gen
# VGG16のための入力値を正規化する関数
def norm_vgg16(self,x):
"""RGB->BGR変換と近似的に中心化をおこなう関数"""
return (x[:, :, :, ::-1] - 120) / 255.
# 画像ファイル読み込み用のラッパー関数定義
def load_imgs(self,img_paths, target_size=(224, 224)):
"""画像ファイルのパスのリストから、配列のバッチを返す"""
_load_img = lambda x: img_to_array(
load_img(x, target_size=target_size)
)
img_list = [
np.expand_dims(_load_img(img_path), axis=0)
for img_path in img_paths
]
return np.concatenate(img_list, axis=0)
def train_generator(self,img_paths, batch_size, model, y_true_sty, shuffle=True, epochs=None):
"""学習データを生成するジェネレータ"""
n_samples = len(img_paths)
indices = list(range(n_samples))
steps_per_epoch = math.ceil(n_samples / batch_size)
img_paths = np.array(img_paths)
cnt_epoch = 0
while True:
cnt_epoch += 1
if shuffle:
np.random.shuffle(indices)
for i in range(steps_per_epoch):
start = batch_size*i
end = batch_size*(i + 1)
X = self.load_imgs(img_paths[indices[start:end]])
batch_size_act = X.shape[0]
y_true_sty_t = [
np.repeat(feat, batch_size_act, axis=0)
for feat in y_true_sty
]
# コンテンツ特徴量の抽出
y_true_con = model.predict(X)
yield (X, y_true_sty_t + [y_true_con])
if epochs is not None:
if cnt_epoch >= epochs:
raise StopIteration
def feature_loss(self, y_true, y_pred):
"""コンテンツ特徴量の損失関数"""
norm = K.prod(K.cast(K.shape(y_true)[1:], 'float32'))
return K.sum(
K.square(y_pred - y_true), axis=(1, 2, 3)
)/norm
def gram_matrix(self, X):
"""グラム行列の算出"""
X_sw = K.permute_dimensions(
X, (0, 3, 2, 1)
) # 軸の入れ替え
s = K.shape(X_sw)
new_shape = (s[0], s[1], s[2]*s[3])
X_rs = K.reshape(X_sw, new_shape)
X_rs_t = K.permute_dimensions(
X_rs, (0, 2, 1)
) # 行列の転置
dot = K.batch_dot(X_rs, X_rs_t) # 内積の計算
norm = K.prod(K.cast(s[1:], 'float32'))
return dot/norm
def style_loss(self, y_true, y_pred):
"""スタイル用の損失関数定義"""
return K.sum(
K.square(
self.gram_matrix(y_pred) - self.gram_matrix(y_true)
),
axis=(1, 2)
)
# Total Variation Regularizerの定義
def TVRegularizer(self, x, weight=1e-6, beta=1.0, input_size=(224, 224)):
delta = 1e-8
h, w = input_size
d_h = K.square(x[:, :h - 1, :w - 1, :] - x[:, 1:, :w - 1, :])
d_w = K.square(x[:, :h - 1, :w - 1, :] - x[:, :h - 1, 1:, :])
return weight * K.mean(K.sum(K.pow(d_h + d_w + delta, beta/2.)))
def learn(self, style_name):
img_paths = glob.glob("transfer/ns_model/train_img/*")
batch_size = 2
epochs = 5
input_shape = (224, 224, 3)
input_size = input_shape[:2]
style = glob.glob("media/style/*")[0].split("\\")[-1]
img_sty = load_img(
'media/style/'+style,
target_size=input_size
)
img_arr_sty = np.expand_dims(img_to_array(img_sty), axis=0)
self.y_true_sty = self.model_sty.predict(img_arr_sty)
shutil.rmtree("./media/style")
os.mkdir("./media/style")
self.gen = self.train_generator(
img_paths,
batch_size,
self.model_con,
self.y_true_sty,
epochs=epochs
)
gen_output_layer = self.model_gen.layers[-1]
tv_loss = self.TVRegularizer(gen_output_layer.output)
gen_output_layer.add_loss(tv_loss)
self.model.compile(
optimizer = Adadelta(),
loss = [
self.style_loss,
self.style_loss,
self.style_loss,
self.style_loss,
self.feature_loss
],
loss_weights = [1.0, 1.0, 1.0, 1.0, 3.0]
)
now_epoch = 0
min_loss = np.inf
steps_per_epoch = math.ceil(len(img_paths)/batch_size)
# 学習
for i , (X_train, y_train) in enumerate(self.gen):
if i % steps_per_epoch == 0:
now_epoch += 1
loss = self.model.train_on_batch(X_train, y_train)
if loss[0]<min_loss:
min_loss = loss[0]
self.model.save("transfer/ns_model/pretrained_model/" + style_name + ".h5")
print("epoch: {}, iters: {}, loss: {:.3f}".format(now_epoch, i, loss[0]))
class GAN():
def __init__(self, rows):
self.seq_length = rows
self.seq_shape = (self.seq_length, 1)
self.latent_dim = 1000
self.disc_loss = []
self.gen_loss = []
optimizer = Adam(0.0002, 0.5)
# Build and compile the discriminator
self.discriminator = self.build_discriminator()
self.discriminator.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
# Build the generator
self.generator = self.build_generator()
# The generator takes noise as input and generates note sequences
z = Input(shape=(self.latent_dim, ))
generated_seq = self.generator(z)
# For the combined model we will only train the generator
self.discriminator.trainable = False
# The discriminator takes generated images as input and determines validity
validity = self.discriminator(generated_seq)
# The combined model (stacked generator and discriminator)
# Trains the generator to fool the discriminator
self.combined = Model(z, validity)
self.combined.compile(loss='binary_crossentropy', optimizer=optimizer)
def build_discriminator(self):
model = Sequential()
model.add(
CuDNNLSTM(512, input_shape=self.seq_shape, return_sequences=True))
model.add(Bidirectional(CuDNNLSTM(512)))
model.add(Dense(512))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(256))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(1, activation='sigmoid'))
model.summary()
seq = Input(shape=self.seq_shape)
validity = model(seq)
return Model(seq, validity)
def build_generator(self):
model = Sequential()
model.add(Dense(256, input_dim=self.latent_dim))
model.add(LeakyReLU(alpha=0.2))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(512))
model.add(LeakyReLU(alpha=0.2))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(1024))
model.add(LeakyReLU(alpha=0.2))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(np.prod(self.seq_shape), activation='tanh'))
model.add(Reshape(self.seq_shape))
model.summary()
noise = Input(shape=(self.latent_dim, ))
seq = model(noise)
return Model(noise, seq)
def train(self, epochs, batch_size=128, sample_interval=50):
# Load and convert the data
notes = get_notes()
n_vocab = len(set(notes))
X_train, y_train = prepare_sequences(notes, n_vocab)
# Adversarial ground truths
real = np.ones((batch_size, 1))
fake = np.zeros((batch_size, 1))
# Training the model
for epoch in range(epochs):
# Training the discriminator
# Select a random batch of note sequences
idx = np.random.randint(0, X_train.shape[0], batch_size)
real_seqs = X_train[idx]
#noise = np.random.choice(range(484), (batch_size, self.latent_dim))
#noise = (noise-242)/242
noise = np.random.normal(0, 1, (batch_size, self.latent_dim))
# Generate a batch of new note sequences
gen_seqs = self.generator.predict(noise)
# Train the discriminator
d_loss_real = self.discriminator.train_on_batch(real_seqs, real)
d_loss_fake = self.discriminator.train_on_batch(gen_seqs, fake)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
# Training the Generator
noise = np.random.normal(0, 1, (batch_size, self.latent_dim))
# Train the generator (to have the discriminator label samples as real)
g_loss = self.combined.train_on_batch(noise, real)
# Print the progress and save into loss lists
if epoch % sample_interval == 0:
print("%d [D loss: %f, acc.: %.2f%%] [G loss: %f]" %
(epoch, d_loss[0], 100 * d_loss[1], g_loss))
self.disc_loss.append(d_loss[0])
self.gen_loss.append(g_loss)
self.generate(notes)
self.plot_loss()
def generate(self, input_notes):
# Get pitch names and store in a dictionary
notes = input_notes
pitchnames = sorted(set(item for item in notes))
int_to_note = dict(
(number, note) for number, note in enumerate(pitchnames))
# Use random noise to generate sequences
noise = np.random.normal(0, 1, (1, self.latent_dim))
predictions = self.generator.predict(noise)
pred_notes = [((x + 1) * (len(set(notes))) / 2)
for x in predictions[0]]
pred_notes = [int_to_note[int(x)] for x in pred_notes]
create_midi(pred_notes, 'gan_final')
def plot_loss(self):
plt.plot(self.disc_loss, c='red')
plt.plot(self.gen_loss, c='blue')
plt.title("GAN Loss per Epoch")
plt.legend(['Discriminator', 'Generator'])
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.savefig('GAN_Loss_per_Epoch_final.png', transparent=True)
plt.close()
class WGANGP():
def __init__(self):
self.img_rows = 2
self.img_cols = 235
self.channels = 1
self.img_shape = (self.img_rows, self.img_cols, self.channels)
self.latent_dim = 470
# Following parameter and optimizer set as recommended in paper
self.n_critic = 5
optimizer = RMSprop(lr=0.00002)
# Build the generator and critic
self.generator = self.build_generator()
self.critic = self.build_critic()
#-------------------------------
# Construct Computational Graph
# for the Critic
#-------------------------------
# Freeze generator's layers while training critic
self.generator.trainable = False
# Image input (real sample)
real_img = Input(shape=self.img_shape)
# Noise input
z_disc = Input(shape=(self.latent_dim, ))
# Generate image based of noise (fake sample)
fake_img = self.generator(z_disc)
# Discriminator determines validity of the real and fake images
fake = self.critic(fake_img)
valid = self.critic(real_img)
# Construct weighted average between real and fake images
interpolated_img = RandomWeightedAverage()([real_img, fake_img])
# Determine validity of weighted sample
validity_interpolated = self.critic(interpolated_img)
# Use Python partial to provide loss function with additional
# 'averaged_samples' argument
partial_gp_loss = partial(self.gradient_penalty_loss,
averaged_samples=interpolated_img)
partial_gp_loss.__name__ = 'gradient_penalty' # Keras requires function names
self.critic_model = Model(inputs=[real_img, z_disc],
outputs=[valid, fake, validity_interpolated])
self.critic_model.compile(loss=[
self.wasserstein_loss, self.wasserstein_loss, partial_gp_loss
],
optimizer=optimizer,
loss_weights=[2, 2, 10])
#-------------------------------
# Construct Computational Graph
# for Generator
#-------------------------------
# For the generator we freeze the critic's layers
self.critic.trainable = False
self.generator.trainable = True
# Sampled noise for input to generator
z_gen = Input(shape=(470, ))
# Generate images based of noise
img = self.generator(z_gen)
# Discriminator determines validity
valid = self.critic(img)
# Defines generator model
self.generator_model = Model(z_gen, valid)
self.generator_model.compile(loss=self.wasserstein_loss,
optimizer=optimizer)
def gradient_penalty_loss(self, y_true, y_pred, averaged_samples):
"""
Computes gradient penalty based on prediction and weighted real / fake samples
"""
gradients = K.gradients(y_pred, averaged_samples)[0]
# compute the euclidean norm by squaring ...
gradients_sqr = K.square(gradients)
# ... summing over the rows ...
gradients_sqr_sum = K.sum(gradients_sqr,
axis=np.arange(1, len(gradients_sqr.shape)))
# ... and sqrt
gradient_l2_norm = K.sqrt(gradients_sqr_sum)
# compute lambda * (1 - ||grad||)^2 still for each single sample
gradient_penalty = K.square(1 - gradient_l2_norm)
# return the mean as loss over all the batch samples
return K.mean(gradient_penalty)
def wasserstein_loss(self, y_true, y_pred):
return K.mean(y_true * y_pred)
def build_generator(self):
model = Sequential()
model.add(
Dense(235 * 2 * 1, activation="relu", input_dim=self.latent_dim))
model.add(Reshape((2, 235, 1)))
model.add(UpSampling2D())
model.add(Conv2D(128, kernel_size=4, padding="same"))
model.add(BatchNormalization(momentum=0.8))
model.add(Activation("relu"))
model.add(UpSampling2D())
model.add(Conv2D(64, kernel_size=4, padding="same"))
model.add(BatchNormalization(momentum=0.8))
model.add(Activation("relu"))
model.add(Conv2D(self.channels, 2, strides=4))
model.add(Activation("relu"))
model.summary()
noise = Input(shape=(self.latent_dim, ))
img = model(noise)
return Model(noise, img)
def build_critic(self):
model = Sequential()
model.add(
Conv2D(16,
kernel_size=3,
strides=2,
input_shape=self.img_shape,
padding="same"))
model.add(LeakyReLU(alpha=0.2))
model.add(Dropout(0.25))
model.add(Conv2D(32, kernel_size=3, strides=2, padding="same"))
model.add(ZeroPadding2D(padding=((0, 1), (0, 1))))
model.add(BatchNormalization(momentum=0.8))
model.add(LeakyReLU(alpha=0.2))
model.add(Dropout(0.25))
model.add(Conv2D(64, kernel_size=3, strides=2, padding="same"))
model.add(BatchNormalization(momentum=0.8))
model.add(LeakyReLU(alpha=0.2))
model.add(Dropout(0.25))
model.add(Conv2D(128, kernel_size=3, strides=1, padding="same"))
model.add(BatchNormalization(momentum=0.8))
model.add(LeakyReLU(alpha=0.2))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(1))
model.summary()
img = Input(shape=self.img_shape)
validity = model(img)
return Model(img, validity)
def train(self, epochs, batch_size, sample_interval=50):
# Load the dataset
X_train = trains
# Adversarial ground truths
valid = -np.ones((batch_size, 1))
fake = np.ones((batch_size, 1))
dummy = np.zeros((batch_size, 1)) # Dummy gt for gradient penalty
for epoch in range(epochs):
for _ in range(self.n_critic):
# ---------------------
# Train Discriminator
# ---------------------
# Select a random batch of images
idx = np.random.randint(0, X_train.shape[0], batch_size)
imgs = X_train[idx]
# Sample generator input
noise = np.random.normal(0, 1, (batch_size, self.latent_dim))
# Train the critic
d_loss = self.critic_model.train_on_batch([imgs, noise],
[valid, fake, dummy])
# ---------------------
# Train Generator
# ---------------------
g_loss = self.generator_model.train_on_batch(noise, valid)
# Plot the progress
# If at save interval => save generated image samples
if epoch % 200 == 0:
self.sample_images(epoch)
print("%d [D loss: %f] [G loss: %f]" %
(epoch, d_loss[0], g_loss))
def sample_images(self, epoch):
r, c = 1, 1
noise = np.random.normal(0, 1, (r * c, self.latent_dim))
gen_imgs = self.generator.predict(noise)
generated_images = gen_imgs[0].transpose(2, 0, 1)
plt.plot(generated_images[0][0], generated_images[0][1], "-")
plt.pause(4)
class GAN():
def __init__(self):
self.img_rows = 128
self.img_cols = 128
self.channels = 4
self.img_shape = (self.img_rows, self.img_cols, self.channels)
self.latent_dim = 100
optimizer = Adam(0.0002, 0.5)
# Build and compile the discriminator
self.discriminator = self.build_discriminator()
self.discriminator.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
# Build the generator
self.generator = self.build_generator()
# The generator takes noise as input and generates imgs
z = Input(shape=(self.latent_dim, ))
img = self.generator(z)
# For the combined model we will only train the generator
self.discriminator.trainable = False
# The discriminator takes generated images as input and determines validity
validity = self.discriminator(img)
# The combined model (stacked generator and discriminator)
# Trains the generator to fool the discriminator
self.combined = Model(z, validity)
self.combined.compile(loss='binary_crossentropy', optimizer=optimizer)
def build_generator(self):
model = Sequential()
model.add(Dense(256, input_dim=self.latent_dim))
model.add(LeakyReLU(alpha=0.2))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(512))
model.add(LeakyReLU(alpha=0.2))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(1024))
model.add(LeakyReLU(alpha=0.2))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(np.prod(self.img_shape), activation='tanh'))
model.add(Reshape(self.img_shape))
model.summary()
noise = Input(shape=(self.latent_dim, ))
img = model(noise)
return Model(noise, img)
def build_discriminator(self):
model = Sequential()
model.add(Flatten(input_shape=self.img_shape))
model.add(Dense(512))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(256))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(1, activation='sigmoid'))
model.summary()
img = Input(shape=self.img_shape)
validity = model(img)
return Model(img, validity)
def train(self, epochs, batch_size=128, sample_interval=50):
mystr = ""
# Load the dataset
X_train = np.load(os.getcwd() + "/pickles/train_data.npy")
# Rescale -1 to 1
X_train = X_train / 127.5 - 1.
#X_train = np.expand_dims(X_train, axis=4)
# Adversarial ground truths
valid = np.ones((batch_size, 1))
fake = np.zeros((batch_size, 1))
for epoch in range(epochs):
# ---------------------
# Train Discriminator
# ---------------------
# Select a random batch of images
idx = np.random.randint(0, X_train.shape[0], batch_size)
imgs = X_train[idx]
noise = np.random.normal(0, 1, (batch_size, self.latent_dim))
# Generate a batch of new images
gen_imgs = self.generator.predict(noise)
# Train the discriminator
d_loss_real = self.discriminator.train_on_batch(imgs, valid)
d_loss_fake = self.discriminator.train_on_batch(gen_imgs, fake)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
# ---------------------
# Train Generator
# ---------------------
noise = np.random.normal(0, 1, (batch_size, self.latent_dim))
# Train the generator (to have the discriminator label samples as valid)
g_loss = self.combined.train_on_batch(noise, valid)
# Plot the progress
print("%d [D loss: %f, acc.: %.2f%%] [G loss: %f]" %
(epoch, d_loss[0], 100 * d_loss[1], g_loss))
mystr += ("%d [D loss: %f, acc.: %.2f%%] [G loss: %f]\n" %
(epoch, d_loss[0], 100 * d_loss[1], g_loss))
# If at save interval => save generated image samples
if epoch % sample_interval == 0:
self.sample_images(epoch)
with open("analytics.txt", "w") as f:
f.write(mystr)
def sample_images(self, epoch):
r, c = 5, 5
noise = np.random.normal(0, 1, (r * c, self.latent_dim))
gen_imgs = self.generator.predict(noise)
# Rescale images 0 - 1
gen_imgs = 0.5 * gen_imgs + 0.5
fig, axs = plt.subplots(r, c)
cnt = 0
for i in range(r):
for j in range(c):
axs[i, j].imshow(gen_imgs[cnt, :, :, :])
axs[i, j].axis('off')
cnt += 1
fig.savefig("images/%d.png" % epoch)
plt.close()
class GAN():
def __init__(self, model_yaml, train_yaml):
"""
Args:
model_yaml: dictionnary with the model parameters
train_yaml: dictionnary the tran parameters
"""
self.sigma_val = 0
self.model_yaml = model_yaml
self.img_rows = 28
self.img_cols = 28
self.channels = 1
self.img_shape = (self.img_rows, self.img_cols, self.channels)
if "dict_band_x" not in train_yaml:
self.dict_band_X = None
self.dict_band_label = None
self.dict_rescale_type = None
else:
self.dict_band_X = train_yaml["dict_band_x"]
self.dict_band_label = train_yaml["dict_band_label"]
self.dict_rescale_type = train_yaml["dict_rescale_type"]
self.s1bands = train_yaml["s1bands"]
self.s2bands = train_yaml["s2bands"]
# self.latent_dim = 100
# PATH
self.model_name = model_yaml["model_name"]
self.model_dir = train_yaml["training_dir"] + self.model_name + "/"
self.this_training_dir = self.model_dir + "training_{}/".format(
train_yaml["training_number"])
self.saving_image_path = self.this_training_dir + "saved_training_images/"
self.saving_logs_path = self.this_training_dir + "logs/"
self.checkpoint_dir = self.this_training_dir + "checkpoints/"
self.previous_checkpoint = train_yaml["load_model"]
# TRAIN PARAMETER
self.normalization = train_yaml["normalization"]
self.epoch = train_yaml["epoch"]
self.batch_size = train_yaml["batch_size"]
# self.sess = sess
self.learning_rate = train_yaml["lr"]
self.fact_g_lr = train_yaml["fact_g_lr"]
self.beta1 = train_yaml["beta1"]
self.val_directory = train_yaml["val_directory"]
self.fact_s2 = train_yaml["s2_scale"]
self.fact_s1 = train_yaml["s1_scale"]
self.data_X, self.data_y, self.scale_dict_train = load_data(
train_yaml["train_directory"],
x_shape=model_yaml["input_shape"],
label_shape=model_yaml["dim_gt_image"],
normalization=self.normalization,
dict_band_X=self.dict_band_X,
dict_band_label=self.dict_band_label,
dict_rescale_type=self.dict_rescale_type,
fact_s2=self.fact_s2,
fact_s1=self.fact_s1,
s2_bands=self.s2bands,
s1_bands=self.s1bands,
lim=train_yaml["lim_train_tile"])
self.val_X, self.val_Y, scale_dict_val = load_data(
self.val_directory,
x_shape=model_yaml["input_shape"],
label_shape=model_yaml["dim_gt_image"],
normalization=self.normalization,
dict_band_X=self.dict_band_X,
dict_band_label=self.dict_band_label,
dict_rescale_type=self.dict_rescale_type,
dict_scale=self.scale_dict_train,
fact_s2=self.fact_s2,
fact_s1=self.fact_s1,
s2_bands=self.s2bands,
s1_bands=self.s1bands,
lim=train_yaml["lim_val_tile"])
print("Loading the data done dataX {} dataY {}".format(
self.data_X.shape, self.data_y.shape))
self.gpu = train_yaml["n_gpu"]
self.num_batches = self.data_X.shape[0] // self.batch_size
self.model_yaml = model_yaml
self.im_saving_step = train_yaml["im_saving_step"]
self.w_saving_step = train_yaml["weights_saving_step"]
self.val_metric_step = train_yaml["metric_step"]
# REDUCE THE DISCRIMINATOR PERFORMANCE
self.val_lambda = train_yaml["lambda"]
self.real_label_smoothing = tuple(train_yaml["real_label_smoothing"])
self.fake_label_smoothing = tuple(train_yaml["fake_label_smoothing"])
self.sigma_init = train_yaml["sigma_init"]
self.sigma_step = train_yaml['sigma_step']
self.sigma_decay = train_yaml["sigma_decay"]
self.ite_train_g = train_yaml["train_g_multiple_time"]
self.max_im = 10
self.strategy = tf.distribute.MirroredStrategy()
print('Number of devices: {}'.format(
self.strategy.num_replicas_in_sync))
self.buffer_size = self.data_X.shape[0]
self.global_batch_size = self.batch_size * self.strategy.num_replicas_in_sync
with self.strategy.scope():
self.d_optimizer = Adam(self.learning_rate, self.beta1)
self.g_optimizer = Adam(self.learning_rate * self.fact_g_lr,
self.beta1)
self.build_model()
self.model_writer = tf.summary.create_file_writer(
self.saving_logs_path)
#self.strategy = tf.distribute.MirroredStrategy()
def build_model(self):
# strategy = tf.distribute.MirroredStrategy()
# print('Number of devices: {}'.format(strategy.num_replicas_in_sync))
# We use the discriminator
self.discriminator = self.build_discriminator(self.model_yaml)
self.discriminator.compile(loss='binary_crossentropy',
optimizer=self.d_optimizer,
metrics=['accuracy'])
self.generator = self.build_generator(self.model_yaml,
is_training=True)
print("Input G")
g_input = Input(shape=(self.data_X.shape[1], self.data_X.shape[2],
self.data_X.shape[3]),
name="g_build_model_input_data")
G = self.generator(g_input)
print("G", G)
# For the combined model we will only train the generator
self.discriminator.trainable = False
D_input = tf.concat([G, g_input], axis=-1)
print("INPUT DISCRI ", D_input)
# The discriminator takes generated images as input and determines validity
D_output_fake = self.discriminator(D_input)
# print(D_output)
# The combined model (stacked generator and discriminator)
# TO TRAIN WITH MULTIPLE GPU
self.combined = Model(g_input, [D_output_fake, G],
name="Combined_model")
self.combined.compile(loss=['binary_crossentropy', L1_loss],
loss_weights=[1, self.val_lambda],
optimizer=self.g_optimizer)
print("[INFO] combined model loss are : ".format(
self.combined.metrics_names))
def build_generator(self, model_yaml, is_training=True):
def build_resnet_block(input, id=0):
"""Define the ResNet block"""
x = Conv2D(model_yaml["dim_resnet"],
model_yaml["k_resnet"],
padding=model_yaml["padding"],
strides=tuple(model_yaml["stride"]),
name="g_block_{}_conv1".format(id))(input)
x = BatchNormalization(momentum=model_yaml["bn_momentum"],
trainable=is_training,
name="g_block_{}_bn1".format(id))(x)
x = ReLU(name="g_block_{}_relu1".format(id))(x)
x = Dropout(rate=model_yaml["do_rate"],
name="g_block_{}_do".format(id))(x)
x = Conv2D(model_yaml["dim_resnet"],
model_yaml["k_resnet"],
padding=model_yaml["padding"],
strides=tuple(model_yaml["stride"]),
name="g_block_{}_conv2".format(id))(x)
x = BatchNormalization(momentum=model_yaml["bn_momentum"],
trainable=is_training,
name="g_block_{}_bn2".format(id))(x)
x = Add(name="g_block_{}_add".format(id))([x, input])
x = ReLU(name="g_block_{}_relu2".format(id))(x)
return x
img_input = Input(shape=(self.data_X.shape[1], self.data_X.shape[2],
self.data_X.shape[3]),
name="g_input_data")
if model_yaml["last_activation"] == "tanh":
print("use tanh keras")
last_activ = lambda x: tf.keras.activations.tanh(x)
else:
last_activ = model_yaml["last_activation"]
x = img_input
for i, param_lay in enumerate(
model_yaml["param_before_resnet"]
): # build the blocks before the Resnet Blocks
x = Conv2D(param_lay[0],
param_lay[1],
strides=tuple(model_yaml["stride"]),
padding=model_yaml["padding"],
name="g_conv{}".format(i))(x)
x = BatchNormalization(momentum=model_yaml["bn_momentum"],
trainable=is_training,
name="g_{}_bn".format(i))(x)
x = ReLU(name="g_{}_lay_relu".format(i))(x)
for j in range(model_yaml["nb_resnet_blocs"]): # add the Resnet blocks
x = build_resnet_block(x, id=j)
for i, param_lay in enumerate(model_yaml["param_after_resnet"]):
x = Conv2D(param_lay[0],
param_lay[1],
strides=tuple(model_yaml["stride"]),
padding=model_yaml["padding"],
name="g_conv_after_resnetblock{}".format(i))(x)
x = BatchNormalization(
momentum=model_yaml["bn_momentum"],
trainable=is_training,
name="g_after_resnetblock{}_bn2".format(i))(x)
x = ReLU(name="g_after_resnetblock_relu_{}".format(i))(x)
# The last layer
x = Conv2D(model_yaml["last_layer"][0],
model_yaml["last_layer"][1],
strides=tuple(model_yaml["stride"]),
padding=model_yaml["padding"],
name="g_final_conv",
activation=last_activ)(x)
model_gene = Model(img_input, x, name="Generator")
model_gene.summary()
return model_gene
def build_discriminator(self, model_yaml, is_training=True):
discri_input = Input(shape=tuple([256, 256, 12]), name="d_input")
if model_yaml["d_activation"] == "lrelu":
d_activation = lambda x: tf.nn.leaky_relu(
x, alpha=model_yaml["lrelu_alpha"])
else:
d_activation = model_yaml["d_activation"]
if model_yaml["add_discri_noise"]:
x = GaussianNoise(self.sigma_val,
input_shape=self.model_yaml["dim_gt_image"],
name="d_GaussianNoise")(discri_input)
else:
x = discri_input
for i, layer_index in enumerate(model_yaml["dict_discri_archi"]):
layer_val = model_yaml["dict_discri_archi"][layer_index]
layer_key = model_yaml["layer_key"]
layer_param = dict(zip(layer_key, layer_val))
pad = layer_param["padding"]
vpadding = tf.constant([[0, 0], [pad, pad], [pad, pad],
[0, 0]]) # the last dimension is 12
x = tf.pad(
x,
vpadding,
model_yaml["discri_opt_padding"],
name="{}_padding_{}".format(
model_yaml["discri_opt_padding"],
layer_index)) # the type of padding is defined the yaml,
# more infomration in https://www.tensorflow.org/api_docs/python/tf/pad
#
# x = ZeroPadding2D(
# padding=(layer_param["padding"], layer_param["padding"]), name="d_pad_{}".format(layer_index))(x)
x = Conv2D(layer_param["nfilter"],
layer_param["kernel"],
padding="valid",
activation=d_activation,
strides=(layer_param["stride"], layer_param["stride"]),
name="d_conv{}".format(layer_index))(x)
if i > 0:
x = BatchNormalization(momentum=model_yaml["bn_momentum"],
trainable=is_training,
name="d_bn{}".format(layer_index))(x)
# x = Flatten(name="flatten")(x)
# for i, dlayer_idx in enumerate(model_yaml["discri_dense_archi"]):
# dense_layer = model_yaml["discri_dense_archi"][dlayer_idx]
# x = Dense(dense_layer, activation=d_activation, name="dense_{}".format(dlayer_idx))(x)
if model_yaml["d_last_activ"] == "sigmoid":
x_final = tf.keras.layers.Activation('sigmoid',
name="d_last_activ")(x)
else:
x_final = x
model_discri = Model(discri_input, x_final, name="discriminator")
model_discri.summary()
return model_discri
def produce_noisy_input(self, input, sigma_val):
if self.model_yaml["add_discri_white_noise"]:
# print("[INFO] On each batch GT label we add Gaussian Noise before training discri on labelled image")
new_gt = GaussianNoise(sigma_val,
input_shape=self.model_yaml["dim_gt_image"],
name="d_inputGN")(input)
if self.model_yaml["add_relu_after_noise"]:
new_gt = tf.keras.layers.Activation(
lambda x: tf.keras.activations.tanh(x),
name="d_before_activ")(new_gt)
else:
new_gt = input
return new_gt
def define_callback(self):
# Define Tensorboard callbacks
self.g_tensorboard_callback = TensorBoard(
log_dir=self.saving_logs_path,
histogram_freq=0,
batch_size=self.batch_size,
write_graph=True,
write_grads=True)
self.g_tensorboard_callback.set_model(self.combined)
def train_gpu(self):
valid = np.ones(
(self.batch_size, 30, 30, 1)) # because of the shape of the discri
fake = np.zeros((self.batch_size, 30, 30, 1))
print("valid shape {}".format(valid.shape))
if self.previous_checkpoint is not None:
print("LOADING the model from step {}".format(
self.previous_checkpoint))
start_epoch = int(self.previous_checkpoint) + 1
self.load_from_checkpoint(self.previous_checkpoint)
else:
# create_safe_directory(self.saving_logs_path)
create_safe_directory(self.saving_image_path)
train_dataset = tf.data.Dataset.from_tensor_slices(
(self.data_X, self.data_y)).shuffle(self.batch_size).batch(
self.global_batch_size)
train_dist_dataset = self.strategy.experimental_distribute_dataset(
train_dataset)
def train(self):
# Adversarial ground truths
valid = np.ones((self.global_batch_size, 30, 30,
1)) # because of the shape of the discri
fake = np.zeros((self.global_batch_size, 30, 30, 1))
#print("valid shape {}".format(valid.shape))
if self.previous_checkpoint is not None:
print("LOADING the model from step {}".format(
self.previous_checkpoint))
start_epoch = int(self.previous_checkpoint) + 1
self.load_from_checkpoint(self.previous_checkpoint)
else:
# create_safe_directory(self.saving_logs_path)
create_safe_directory(self.saving_image_path)
start_epoch = 0
train_dataset = tf.data.Dataset.from_tensor_slices(
(self.data_X, self.data_y)).shuffle(self.batch_size).batch(
self.global_batch_size)
# loop for epoch
sigma_val = self.sigma_init
# dict_metric={"epoch":[],"d_loss_real":[],"d_loss_fake":[],"d_loss":[],"g_loss":[]}
d_loss_real = [100, 100] # init losses
d_loss_fake = [100, 100]
d_loss = [100, 100]
l_val_name_metrics, l_val_value_metrics = [], []
start_time = time.time()
for epoch in range(start_epoch, self.epoch):
# print("starting epoch {}".format(epoch))
for idx, (batch_input, batch_gt) in enumerate(train_dataset):
#print(batch_input)
## TRAIN THE DISCRIMINATOR
d_noise_real = random.uniform(
self.real_label_smoothing[0],
self.real_label_smoothing[1]) # Add noise on the loss
d_noise_fake = random.uniform(
self.fake_label_smoothing[0],
self.fake_label_smoothing[1]) # Add noise on the loss
# Create a noisy gt images
batch_new_gt = self.produce_noisy_input(batch_gt, sigma_val)
# Generate a batch of new images
# print("Make a prediction")
gen_imgs = self.generator.predict(
batch_input) # .astype(np.float32)
D_input_real = tf.concat([batch_new_gt, batch_input], axis=-1)
D_input_fake = tf.concat([gen_imgs, batch_input], axis=-1)
print("shape d train")
print(valid.shape, D_input_fake.shape)
d_loss_real = self.discriminator.train_on_batch(
D_input_real, d_noise_real * valid)
d_loss_fake = self.discriminator.train_on_batch(
D_input_fake, d_noise_fake * fake)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
g_loss = self.combined.train_on_batch(batch_input,
[valid, batch_gt])
# Plot the progress
print("%d iter %d [D loss: %f, acc.: %.2f%%] [G loss: %f %f]" %
(epoch, self.num_batches * epoch + idx, d_loss[0],
100 * d_loss[1], g_loss[0], g_loss[1]))
if epoch % self.im_saving_step == 0 and idx < self.max_im: # to save some generated_images
gen_imgs = self.generator.predict(batch_input)
save_images(gen_imgs, self.saving_image_path, ite=idx)
# LOGS to print in Tensorboard
if epoch % self.val_metric_step == 0:
l_val_name_metrics, l_val_value_metrics = self.val_metric()
name_val_metric = [
"val_{}".format(name) for name in l_val_name_metrics
]
name_logs = self.combined.metrics_names + [
"g_loss_tot", "d_loss_real", "d_loss_fake",
"d_loss_tot", "d_acc_real", "d_acc_fake", "d_acc_tot"
]
val_logs = g_loss + [
g_loss[0] + 100 * g_loss[1], d_loss_real[0],
d_loss_fake[0], d_loss[0], d_loss_real[1],
d_loss_fake[1], d_loss[1]
]
# The metrics
#print(type(batch_gt),type(gen_imgs))
l_name_metrics, l_value_metrics = compute_metric(
batch_gt.numpy(), gen_imgs)
assert len(val_logs) == len(
name_logs
), "The name and value list of logs does not have the same lenght {} vs {}".format(
name_logs, val_logs)
write_log_tf2(
self.model_writer, name_logs + l_name_metrics +
name_val_metric + ["time_in_sec"],
val_logs + l_value_metrics + l_val_value_metrics +
[start_time - time.time()], epoch)
if epoch % self.sigma_step == 0: # update simga
sigma_val = sigma_val * self.sigma_decay
# save the models
if epoch % self.w_saving_step == 0:
self.save_model(epoch)
def save_model(self, step):
print("Saving model at {} step {}".format(self.checkpoint_dir, step))
checkpoint_dir = self.checkpoint_dir
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
if not os.path.isfile("{}model_generator.yaml".format(
self.checkpoint_dir)):
gene_yaml = self.generator.to_yaml()
with open("{}model_generator.yaml".format(self.checkpoint_dir),
"w") as yaml_file:
yaml_file.write(gene_yaml)
if not os.path.isfile("{}model_combined.yaml".format(
self.checkpoint_dir)):
comb_yaml = self.combined.to_yaml()
with open("{}model_combined.yaml".format(self.checkpoint_dir),
"w") as yaml_file:
yaml_file.write(comb_yaml)
if not os.path.isfile("{}model_discri.yaml".format(
self.checkpoint_dir)):
discri_yaml = self.discriminator.to_yaml()
with open("{}model_discri.yaml".format(self.checkpoint_dir),
"w") as yaml_file:
yaml_file.write(discri_yaml)
self.generator.save_weights("{}model_gene_i{}.h5".format(
self.checkpoint_dir, step))
self.discriminator.save_weights("{}model_discri_i{}.h5".format(
self.checkpoint_dir, step))
self.combined.save_weights("{}model_combined_i{}.h5".format(
self.checkpoint_dir, step))
def load_from_checkpoint(self, step):
assert os.path.isfile("{}model_discri_i{}.h5".format(
self.checkpoint_dir,
step)), "No file at {}".format("{}model_discri_i{}.h5".format(
self.checkpoint_dir, step))
self.discriminator.load_weights("{}model_discri_i{}.h5".format(
self.checkpoint_dir, step))
self.generator.load_weights("{}model_gene_i{}.h5".format(
self.checkpoint_dir, step))
self.combined.load_weights("{}model_combined_i{}.h5".format(
self.checkpoint_dir, step))
def load_generator(self, path_yaml, path_weight):
# load YAML and create model
yaml_file = open(path_yaml, 'r')
loaded_model_yaml = yaml_file.read()
yaml_file.close()
loaded_model = model_from_yaml(loaded_model_yaml)
# load weights into new model
loaded_model.load_weights(path_weight)
print("Loaded model from disk")
return loaded_model
def val_metric(self):
test_dataset = tf.data.Dataset.from_tensor_slices(
(self.val_X, self.val_Y)).batch(self.val_X.shape[0])
#test_dist_dataset = self.strategy.experimental_distribute_dataset(test_dataset)
for i, (x, y) in enumerate(test_dataset):
#print("eval on {}".format(i))
val_pred = self.generator.predict(x)
#print("type {} {}".format(type(y),type(val_pred)))
label = y
return compute_metric(label.numpy(), val_pred)
def predict_on_iter(self,
batch,
path_save,
l_image_id=None,
un_rescale=True):
"""given an iter load the model at this iteration, returns the a predicted_batch but check if image have been saved at this directory
:param dataset:
:param batch could be a string : path to the dataset or an array corresponding to the batch we are going to predict on
"""
if type(batch) == type(
"u"
): # the param is an string we load the bathc from this directory
#print("We load our data from {}".format(batch))
l_image_id = find_image_indir(batch + XDIR, "npy")
batch, _ = load_data(batch,
x_shape=self.model_yaml["input_shape"],
label_shape=self.model_yaml["dim_gt_image"],
normalization=self.normalization,
dict_band_X=self.dict_band_X,
dict_band_label=self.dict_band_label,
dict_rescale_type=self.dict_rescale_type,
dict_scale=self.scale_dict_train,
fact_s2=self.fact_s2,
fact_s1=self.fact_s1,
s2_bands=self.s2bands,
s1_bands=self.s1bands,
clip_s2=False)
else:
if l_image_id is None:
print("We defined our own index for image name")
l_image_id = [i for i in range(batch.shape[0])]
assert len(l_image_id) == batch.shape[
0], "Wrong size of the name of the images is {} should be {} ".format(
len(l_image_id), batch.shape[0])
if os.path.isdir(path_save):
print(
"[INFO] the directory where to store the image already exists")
data_array, path_tile, _ = load_from_dir(
path_save, self.model_yaml["dim_gt_image"])
return data_array
else:
create_safe_directory(path_save)
batch_res = self.generator.predict(batch)
# if un_rescale: # remove the normalization made on the data
# _, batch_res, _ = rescale_array(batch, batch_res, dict_group_band_X=self.dict_band_X,
# dict_group_band_label=self.dict_band_label,
# dict_rescale_type=self.dict_rescale_type,
# dict_scale=self.scale_dict_train, invert=True, fact_scale2=self.fact_s2,
# fact_scale1=self.fact_s1,clip_s2=False)
assert batch_res.shape[0] == batch.shape[
0], "Wrong prediction should have shape {} but has shape {}".format(
batch_res.shape, batch.shape)
if path_save is not None:
# we store the data at path_save
for i in range(batch_res.shape[0]):
np.save(
"{}_image_{}".format(path_save,
l_image_id[i].split("/")[-1]),
batch_res[i, :, :, :])
return batch_res
continue
if step > Totalstep:
break
half_batch = int(batch_size / 2)
z_d = np.random.uniform(-1, 1, (half_batch, 100))
g_pred = generator.predict(z_d)
discriminator.trainable = True
real_loss = discriminator.train_on_batch(batch[:half_batch],
np.ones((half_batch, 1)))
fake_loss = discriminator.train_on_batch(g_pred, np.zeros((half_batch, 1)))
d_loss = 0.5 * np.add(real_loss, fake_loss)
z_g = np.random.uniform(-1, 1, (batch_size, 100))
discriminator.trainable = False
g_loss = generator_containing_discriminator.train_on_batch(
z_g, np.ones((batch_size, 1)))
logs.append({
'step': step,
'd_loss': d_loss[0],
'acc[%]': 100 * d_loss[1],
'g_loss': g_loss
})
print(logs[-1])
if step % 200 == 0:
img_path = '{}/generate_{}.png'.format(img_save_dir, step)
save_imgs(img_path,
generator.predict(sample_seeds),
rows=imgs_shape[0],
cols=imgs_shape[1])
x = y = np.random.randn(32, 12) # dummy data
ipt = Input((12, ))
out = Dense(12)(ipt)
model = Model(ipt, out)
# starter_learning_rate = 0.1
# end_learning_rate = 0.01
# decay_steps = 10000
# learning_rate_fn = tf.keras.optimizers.schedules.PolynomialDecay(
# starter_learning_rate,
# decay_steps,
# end_learning_rate,
# power=0.5)
# model.compile(SGD(learning_rate = 1e-4, decay=1e-2), loss='mse')
model.compile(SGD(learning_rate=0.1), loss='mse')
lr_list = []
for iteration in range(10):
model.train_on_batch(x, y)
model.optimizer.lr = model.optimizer.lr - 0.001
# print(model.optimizer.get_update())
print("lr at iteration {}: {}".format(
iteration + 1,
model.optimizer._decayed_lr('float32').numpy()))
lr_list.append(model.optimizer._decayed_lr("float32").numpy())
print("model.optimizer.lr", model.optimizer.lr)
plt.plot(range(10), lr_list)
plt.show()
class DEC(object):
def __init__(self, dims, n_clusters=10, alpha=1.0, init='glorot_uniform'):
super(DEC, self).__init__()
self.dims = dims
self.input_dim = dims[0]
self.n_stacks = len(self.dims) - 1
self.n_clusters = n_clusters
self.alpha = alpha
self.encoder = autoencoder(self.dims, init=init)
# prepare DEC model
clustering_layer = ClusteringLayer(self.n_clusters, name='clustering')(
self.encoder.output)
self.model = Model(inputs=self.encoder.input, outputs=clustering_layer)
def load_weights(self, weights): # load weights of DEC model
self.model.load_weights(weights)
def extract_features(self, x):
return self.encoder.predict(x)
def predict(
self,
x): # predict cluster labels using the output of clustering layer
q = self.model.predict(x, verbose=0)
return q.argmax(1)
@staticmethod
def target_distribution(q):
weight = q**2 / q.sum(0)
return (weight.T / weight.sum(1)).T
def compile(self, optimizer='sgd', loss='kld'):
self.model.compile(optimizer=optimizer, loss=loss)
def fit(self,
x,
y=None,
maxiter=2e4,
batch_size=256,
tol=1e-3,
update_interval=140,
save_dir='./results/temp'):
print('Update interval', update_interval)
save_interval = int(x.shape[0] / batch_size) * 5 # 5 epochs
print('Save interval', save_interval)
# Step 1: initialize cluster centers using k-means
t1 = time()
print('Initializing cluster centers with k-means.')
kmeans = KMeans(n_clusters=self.n_clusters, n_init=20)
y_pred = kmeans.fit_predict(self.encoder.predict(x))
y_pred_last = np.copy(y_pred)
self.model.get_layer(name='clustering').set_weights(
[kmeans.cluster_centers_])
# Step 2: deep clustering
# logging file
import csv
logfile = open(save_dir + '/dec_log.csv', 'w')
logwriter = csv.DictWriter(
logfile, fieldnames=['iter', 'acc', 'nmi', 'ari', 'loss'])
logwriter.writeheader()
loss = 0
index = 0
index_array = np.arange(x.shape[0])
for ite in range(int(maxiter)):
if ite % update_interval == 0:
q = self.model.predict(x, verbose=0)
p = self.target_distribution(
q) # update the auxiliary target distribution p
# evaluate the clustering performance
y_pred = q.argmax(1)
if y is not None:
acc = np.round(metrics.acc(y, y_pred), 5)
nmi = np.round(metrics.nmi(y, y_pred), 5)
ari = np.round(metrics.ari(y, y_pred), 5)
loss = np.round(loss, 5)
logdict = dict(iter=ite,
acc=acc,
nmi=nmi,
ari=ari,
loss=loss)
logwriter.writerow(logdict)
print(
'Iter %d: acc = %.5f, nmi = %.5f, ari = %.5f' %
(ite, acc, nmi, ari), ' ; loss=', loss)
# check stop criterion
delta_label = np.sum(y_pred != y_pred_last).astype(
np.float32) / y_pred.shape[0]
y_pred_last = np.copy(y_pred)
if ite > 0 and delta_label < tol:
print('delta_label ', delta_label, '< tol ', tol)
print('Reached tolerance threshold. Stopping training.')
logfile.close()
break
# train on batch
# if index == 0:
# np.random.shuffle(index_array)
idx = index_array[index * batch_size:min((index + 1) *
batch_size, x.shape[0])]
loss = self.model.train_on_batch(x=x[idx], y=p[idx])
index = index + 1 if (index + 1) * batch_size <= x.shape[0] else 0
# save intermediate model
if ite % save_interval == 0:
print('saving model to:',
save_dir + '/DEC_model_' + str(ite) + '.h5')
self.model.save_weights(save_dir + '/DEC_model_' + str(ite) +
'.h5')
ite += 1
# save the trained model
logfile.close()
print('saving model to:', save_dir + '/DEC_model_final.h5')
self.model.save_weights(save_dir + '/DEC_model_final.h5')
return y_pred
class GAN():
def __init__(self):
self.img_rows = 28
self.img_cols = 28
self.channels = 1
self.img_shape = (self.img_rows, self.img_cols, self.channels)
self.latent_dim = 100
# latent_dim is dimension inputted to generator, (rows, column, 1) is resolution returned by generator and inputted to
# discriminator, then discriminator returns 0 or 1 - wether image is real or fake
optimizer = Adam(0.0002, 0.5)
self.discriminator = self.build_discriminator()
self.discriminator.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
# for training discriminator
self.generator = self.build_generator()
z = Input(shape=(self.latent_dim,))
img = self.generator(z)
self.discriminator.trainable = False
# so only generator weights are trained while training entire GAN
validity = self.discriminator(img)
self.combined = Model(z, validity)
self.combined.compile(loss='binary_crossentropy', optimizer=optimizer)
# combined is the GAN - generator + discriminator(but only gen weights are trained)
def build_generator(self):
model = Sequential()
model.add(Dense(256, input_dim=self.latent_dim))
model.add(LeakyReLU(alpha=0.2))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(512))
model.add(LeakyReLU(alpha=0.2))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(1024))
model.add(LeakyReLU(alpha=0.2))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(np.prod(self.img_shape), activation='tanh'))
model.add(Reshape(self.img_shape))
model.summary()
noise = Input(shape=(self.latent_dim,))
img = model(noise)
return Model(noise, img)
def build_discriminator(self):
model = Sequential()
model.add(Flatten(input_shape=self.img_shape))
model.add(Dense(512))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(256))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(1, activation='sigmoid'))
model.summary()
img = Input(shape=self.img_shape)
validity = model(img)
return Model(img, validity)
def train(self, epochs, batch_size=128, sample_interval=50):
(X_train, _), (_, _) = mnist.load_data()
X_train = X_train / 127.5 - 1.
X_train = np.expand_dims(X_train, axis=3)
valid = np.ones((batch_size, 1))
fake = np.zeros((batch_size, 1))
# we will first train GAN(generator + discriminator) by inputting noise and give output as valid(1) and only generator weights
# will be trained(the purpose is therefore to fool the discriminator into thinking these generated images are valid).
# next we will train discriminator by inputting images obtained from generator with output - fake(0) and also by images from
# mnist with output - valid(1)(basically train discriminator to label images generated by generator as fake).
# therefore we set up a rivalry between generator and discriminator where purpose of generator is to fool the discriminator
# and purpose of discriminator is to recognise images by generator and not get fooled.
for epoch in range(epochs+1):
idx = np.random.randint(0, X_train.shape[0], batch_size)
imgs = X_train[idx]
# real images - inputted to discriminator for training
noise = np.random.normal(0, 1, (batch_size, self.latent_dim))
# random noise - inputted to gan(generator part)
g_loss = self.combined.train_on_batch(noise, valid)
# training gan
gen_imgs = self.generator.predict(noise)
# output from generator
d_loss_real = self.discriminator.train_on_batch(imgs, valid)
# discriminator trained on real images from mnist
d_loss_fake = self.discriminator.train_on_batch(gen_imgs, fake)
# discriminator trained on fake images from generator
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
print("%d [Discriminator loss: %f, acc.: %.2f%%] [GAN loss: %f]" % (epoch, d_loss[0], 100 * d_loss[1], g_loss))
if epoch % sample_interval == 0:
self.sample_images(epoch)
def sample_images(self, epoch):
r, c = 5, 5
noise = np.random.normal(0, 1, (r * c, self.latent_dim))
gen_imgs = self.generator.predict(noise)
gen_imgs = 0.5 * gen_imgs + 0.5
fig, axs = plt.subplots(r, c)
cnt = 0
for i in range(r):
for j in range(c):
axs[i, j].imshow(gen_imgs[cnt, :, :, 0], cmap='gray')
axs[i, j].axis('off')
cnt += 1
plt.show()
