def DocNetWork(input_shape=(224, 224, 3)):
vgg16 = VGG16(include_top=True,
input_tensor=None,
input_shape=input_shape,
weights='imagenet')
vgg16.layers.pop()
vgg16.layers.pop()
vgg16.layers.pop()
# Fine tuning
for layer in vgg16.layers:
if layer.name == "block5_conv1":
break
else:
layer.trainable = False
# model_t : Secondary Model
model_t = Model(inputs=vgg16.input, outputs=vgg16.layers[-1].output)
# model_r : Reference Model
model_r = Network(inputs=model_t.input,
outputs=model_t.output,
name="shared_layer")
# add prediction layer
prediction = Dense(classes, activation='softmax')(model_t.output)
model_r = Model(inputs=model_r.input, outputs=prediction)
optimizer = SGD(lr=5e-5, decay=0.00005)
model_r.compile(optimizer=optimizer, loss="categorical_crossentropy")
model_t.compile(optimizer=optimizer, loss=original_loss)
return model_r, model_t
def build_model():
frames = None
s_size = None
# prepare shared layers
dummy_frame_inputs = Input(shape=((9, s_size, s_size, 3)))
shared_layer_h = Network(dummy_frame_inputs,
conv3D_branch(dummy_frame_inputs))
shared_layer_v = Network(dummy_frame_inputs,
conv3D_branch(dummy_frame_inputs))
# build model
inputs_h = Input(shape=((frames, 9, s_size, s_size, 3)), name='inputs_h')
processed_h = TimeDistributed(shared_layer_h,
name='shared_3Dconv_branch_h')(inputs_h)
inputs_v = Input(shape=((frames, 9, s_size, s_size, 3)), name='inputs_v')
processed_v = TimeDistributed(shared_layer_v,
name='shared_3Dconv_branch_v')(inputs_v)
x = Concatenate()([processed_h, processed_v])
for n_filters in [64, 32, 32, 16]:
x = TimeDistributed(
Conv2D(n_filters,
kernel_size=3,
padding='same',
activation='relu',
kernel_initializer='glorot_uniform'))(x)
x = TimeDistributed(
Conv2D(1,
kernel_size=3,
padding='same',
kernel_initializer='glorot_uniform'))(x)
x = TimeDistributed(Lambda(lambda x: K.squeeze(x, axis=3)))(x)
return Model(inputs=[inputs_h, inputs_v], outputs=x)
def build(self):
self.input_tgt = Input(self.input_shape) # 検出対象画像
self.input_search = Input(self.input_shape) # 探索対象画像
# 同じ特徴量抽出器を共有
x_in = Input(self.input_shape)
feature_net = MobileNetV2(input_tensor=x_in,
alpha=1.0,
include_top=False)
for temp in feature_net.layers:
temp.trainable = False
feature_net = Network(x_in, feature_net.output, name='feature')
self.feature_tgt = feature_net(self.input_tgt)
self.feature_search = feature_net(self.input_search)
# 出力結果を連結
self.concat = concatenate([self.feature_tgt, self.feature_search],
axis=3)
self.fc0 = Flatten()(self.concat)
# 全結合(オリジナルは(4096,) x 3 から最後に(4,)だがメモリに乗らなかったので小さくしてある
x = Dense(1024)(self.fc0)
x = BatchNormalization()(x)
self.fc1 = Activation('relu')(x)
x = Dense(1024)(self.fc1)
x = BatchNormalization()(x)
self.fc2 = Activation('relu')(x)
x = Dense(1024)(self.fc2)
x = BatchNormalization()(x)
self.fc3 = Activation('relu')(x)
self.output = Dense(4)(self.fc3)
# モデル出力
self.model = Model(inputs=[self.input_tgt, self.input_search],
outputs=self.output)
return self.model
def make_q_network(input_shape, name=None):
inputs = Input(shape=input_shape)
h1 = Dense(24, activation='relu')(inputs)
h2 = Dense(48, activation='relu')(h1)
h3 = Dense(24, activation='relu')(h2)
output = Dense(1, activation='linear')(h3)
return Network(inputs=inputs, outputs=output, name=name)
def __init__(self):
# Input shape
self.img_rows = 32
self.img_cols = 32
self.channels = 3
self.img_shape = (self.img_rows, self.img_cols, self.channels)
self.latent_dim = 32
optimizer = Adam(0.0002, 0.5)
# Build and compile the discriminator
inp_discriminator, out_discriminator = self.build_discriminator()
self.discriminator = Model(inp_discriminator, out_discriminator, name='discriminator')
self.discriminator.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
# Build the frozen discriminator copy
self.frozen_discriminator = Network(inp_discriminator, out_discriminator, name='frozen_discriminator')
self.frozen_discriminator.trainable = False
# Build the generator
inp_generator, out_generator = self.build_generator()
self.generator = Model(inp_generator, out_generator, name='generator')
# The combined model (stacked generator and discriminator)
# Trains the generator to fool the discriminator
optimizer = Adam(0.0002, 0.5)
self.combined = Sequential()
self.combined.add(self.generator)
self.combined.add(self.frozen_discriminator)
self.combined.compile(loss='binary_crossentropy', optimizer=optimizer)
self.discriminator.summary()
self.generator.summary()
self.combined.summary()
def create_model(optimizer, target_size=(224, 224, 3)):
"""
モデルを作って返す。
諸々の値がハードコーディングされている。きたない。
keras applicationsで定義されたモデルを特徴抽出器として利用するモデル。
最後にconcat, fully connectedに渡して結果を計算する。
"""
base_model = InceptionV3(include_top=False, weights='imagenet')
shared_layers = Network(base_model.inputs,
base_model.output,
name='shared_layers')
shared_layers.trainable = False
input_x1 = Input(shape=target_size, name='input_x1')
x1 = shared_layers(input_x1)
x1 = GlobalAveragePooling2D(name='x1_gap')(x1)
x1 = BatchNormalization()(x1)
input_x2 = Input(shape=target_size, name='input_x2')
x2 = shared_layers(input_x2)
x2 = GlobalAveragePooling2D(name='x2_gap')(x2)
x2 = BatchNormalization()(x2)
input_x3 = Input(shape=target_size, name='input_x3')
x3 = shared_layers(input_x3)
x3 = GlobalAveragePooling2D(name='x3_gap')(x3)
x3 = BatchNormalization()(x3)
x = Concatenate(name='concat_triple_image')([x1, x2, x3])
x = Dense(1024, activation='relu')(x)
# x = Dense(256, activation='relu', kernel_regularizer=regularizers.l2(0.001))(x)
x = Dropout(0.5)(x)
predictions = Dense(2, activation='softmax')(x)
model = Model([input_x1, input_x2, input_x3], predictions)
model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def __init__(self):
self.img_rows = 28
self.img_cols = 28
self.channels = 1
self.img_shape = (self.img_rows, self.img_cols, self.channels)
self.latent_dim = 100
optimizer = Adam(0.0002, 0.5)
print('Discriminator Model:')
model = Sequential()
model.add(Flatten(input_shape=self.img_shape))
model.add(Dense(512))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(256))
model.add(LeakyReLU(alpha=0.2))
model.add(Dense(1, activation='sigmoid'))
model.summary()
img = Input(shape=self.img_shape)
validity = model(img)
# Build and Compile the Discriminator Model
self.discriminator = Model(img, validity)
self.discriminator.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
self.discriminator_fixed = Network(
img, validity) # -> I Add THIS CONTAINER!
# Build & Compile the Generator Model
self.generator = self.build_generator()
#Creates the NOISE (z) to be input for generator and produce the image
z = Input(shape=(self.latent_dim, ))
# Produce the image
img = self.generator(z)
# This steps ensure when we train our Netowrks we ONLY train the Generator Net
# self.discriminator.trainable = False
self.discriminator_fixed.trainable = False
# specifies that our Discriminator will take the images generated by our Generator + true dataset and set its output
# to a parameter called validity, which will indicate whether the input is real or not.
# validity = self.discriminator(img)
validity = self.discriminator_fixed(img)
# combined the models and also set our loss function and optimizer. The ultimate goal here is for the Generator to fool the Discriminator.
self.combined = Model(z, validity)
# self.combined = Model(z, self.discriminator_fixed(validity)) # Use container
self.combined.compile(loss='binary_crossentropy', optimizer=optimizer)
def build_vgg(self, vgg_path, loss_layer=12):
vgg = load_model(self.vgg_path)
self.shared_vgg = Network(inputs=vgg.input,
outputs=vgg.get_layer(
vgg.layers[loss_layer].name).output,
name='vgg')
input_layer = Input(shape=self.tgt_shape)
out = self.shared_vgg(input_layer)
model = Model(input_layer, out)
for l in model.layers:
l.trainable = False
return model
def build_discriminators(self, filters=64):
"""
Build the discriminator network according to description in the paper.
:param optimizer: Keras optimizer to use for network
:param int filters: How many filters to use in first conv layer
:return: the compiled model
"""
def conv2d_block(input, filters, strides=1, bn=True):
d = Conv2D(filters, kernel_size=3, strides=strides,
padding='same')(input)
d = LeakyReLU(alpha=0.2)(d)
if bn:
d = BatchNormalization(momentum=0.8)(d)
return d
# Input high resolution image
img = Input(shape=self.shape_hr)
x = conv2d_block(img, filters, bn=False)
x = conv2d_block(x, filters, strides=2)
x = conv2d_block(x, filters * 2)
x = conv2d_block(x, filters * 2, strides=2)
x = conv2d_block(x, filters * 4)
x = conv2d_block(x, filters * 4, strides=2)
x = conv2d_block(x, filters * 8)
x = conv2d_block(x, filters * 8, strides=2)
x = Dense(filters * 16)(x)
x = LeakyReLU(alpha=0.2)(x)
x = Dense(1, activation='sigmoid')(x)
# Create model and compile
model = Model(inputs=img, outputs=x)
# Build "frozen discriminator"
frozen_discriminator = Network(inputs=img,
outputs=x,
name='frozen_discriminator')
frozen_discriminator.trainable = False
return model, frozen_discriminator
def gradient_penalty(D, input_merged, base_name):
gradients = K.gradients(D, [input_merged])[0]
# compute the euclidean norm by squaring ...
gradients_sqr = K.square(gradients)
# ... summing over the rows ...
gradients_sqr_sum = K.sum(gradients_sqr,
axis=np.arange(1, len(gradients_sqr.shape)))
# ... and sqrt
gradient_l2_norm = K.sqrt(gradients_sqr_sum)
# compute lambda * (1 - ||grad||)^2 still for each single sample
gradient_penalty = K.square(1 - gradient_l2_norm)
# return the mean as loss over all the batch samples
out = Network(input=[D, input_merged],
output=[gradient_penalty],
name=base_name + "_gp")
return out
def __init__(self):
# Input shape
self.img_rows = 64
self.img_cols = 64
self.channels = 3
self.img_shape = (self.img_rows, self.img_cols, self.channels)
self.latent_dim = 16
# Following parameter and optimizer set as recommended in paper
self.n_critic = 5
self.clip_value = 0.01
optimizer = Adam(0.0002, 0.5)
# Build and compile the discriminator
inp_discriminator, out_discriminator = self.build_discriminator()
self.discriminator = Model(inp_discriminator,
out_discriminator,
name='discriminator')
self.discriminator.compile(loss=self.wasserstein_loss,
optimizer=optimizer,
metrics=['accuracy'])
# Build the frozen discriminator copy
self.frozen_discriminator = Network(inp_discriminator,
out_discriminator,
name='frozen_discriminator')
self.frozen_discriminator.trainable = False
# Build the generator
inp_generator, out_generator = self.build_generator()
self.generator = Model(inp_generator, out_generator, name='generator')
# The combined model (stacked generator and discriminator)
# Trains the generator to fool the discriminator
optimizer = Adam(0.0002, 0.5)
self.combined = Sequential()
self.combined.add(self.generator)
self.combined.add(self.frozen_discriminator)
self.combined.compile(loss=self.wasserstein_loss, optimizer=optimizer)
self.discriminator.summary()
self.generator.summary()
self.combined.summary()
def build_discriminator(self, filters=8):
input_layer = Input(shape=self.tgt_shape)
layers = [input_layer]
# Down 0
layer_name = 'DD0' #0~7
self.__add_conv_disc(layer_name, filters)
self.dis_net.append(MaxPooling2D(name=layer_name + '_pool'))
filters *= 2
# Down 1
layer_name = 'DD1' #8~13
self.__add_conv_disc(layer_name, filters)
self.dis_net.append(MaxPooling2D(name=layer_name + '_pool'))
filters *= 2
# Down 2
layer_name = 'DD2' #8~13
self.__add_conv_disc(layer_name, filters)
self.dis_net.append(MaxPooling2D(name=layer_name + '_pool'))
filters *= 2
# Down 3
layer_name = 'DD3' #8~13
self.__add_conv_disc(layer_name, filters)
self.dis_net.append(MaxPooling2D(name=layer_name + '_pool'))
self.dis_net.append(Dropout(0.25))
self.dis_net.append(BatchNormalization(momentum=0.8))
self.dis_net.append(Flatten())
self.dis_net.append(Dense(1024))
self.dis_net.append(BatchNormalization(momentum=0.8))
self.dis_net.append(LeakyReLU())
self.dis_net.append(Dense(1, activation='sigmoid'))
# body first up
for i, operator in enumerate(self.dis_net):
layers.append(operator(layers[i]))
self.shared_discriminator = Network(input=layers[0],
output=layers[-1],
name='discriminator')
model = Model(layers[0], layers[-1])
return model
def __init__(self):
# Input shape
self.img_rows = 64
self.img_cols = 64
self.channels = 3
self.img_shape = (self.img_rows, self.img_cols, self.channels)
self.latent_dim = 16
self.num_classes = 17
optimizer = Adam(0.0002, 0.5)
losses = ['binary_crossentropy', 'sparse_categorical_crossentropy']
# Build and compile the discriminator
inp_discr, out_discr, lbl_discr = self.build_discriminator()
self.discriminator = Model(inp_discr, [out_discr, lbl_discr],
name='discriminator')
self.discriminator.compile(loss=losses,
optimizer=optimizer,
metrics=['accuracy'])
# Build the frozen discriminator copy
self.frozen_discriminator = Network(inp_discr, [out_discr, lbl_discr],
name='frozen_discriminator')
self.frozen_discriminator.trainable = False
# Build the generator
inp_gen, lbl_gen, out_gen = self.build_generator()
self.generator = Model([inp_gen, lbl_gen], out_gen, name='generator')
# The combined model (stacked generator and discriminator)
# Trains the generator to fool the discriminator
optimizer = Adam(0.0002, 0.5)
comb_valid, comb_label = self.frozen_discriminator(
self.generator([inp_gen, lbl_gen]))
self.combined = Model(inputs=[inp_gen, lbl_gen],
outputs=[comb_valid, comb_label])
self.combined.compile(loss=losses, optimizer=optimizer)
self.discriminator.summary()
self.generator.summary()
self.combined.summary()
def mapping_function_Unet(input_shape, base_name, num_res_blocks):
initializer = TruncatedNormal(mean=0, stddev=0.2, seed=42)
x = in_x = Input(shape=input_shape)
# size→size//2→size//4→size//8
x = Conv2D(32,
kernel_size=7,
strides=1,
padding="same",
kernel_initializer=initializer,
use_bias=False,
name=base_name + "_conv1")(x)
x = BatchNormalization(momentum=0.9, epsilon=1e-5,
name=base_name + "_bn1")(x)
conv1 = LeakyReLU(0.2)(x)
x = Conv2D(64,
kernel_size=3,
strides=2,
padding="same",
kernel_initializer=initializer,
use_bias=False,
name=base_name + "_conv2")(conv1)
x = BatchNormalization(momentum=0.9, epsilon=1e-5,
name=base_name + "_bn2")(x)
conv2 = LeakyReLU(0.2)(x)
x = Conv2D(128,
kernel_size=3,
strides=2,
padding="same",
kernel_initializer=initializer,
use_bias=False,
name=base_name + "_conv3")(conv2)
x = BatchNormalization(momentum=0.9, epsilon=1e-5,
name=base_name + "_bn3")(x)
conv3 = LeakyReLU(0.2)(x)
x = conv3
for i in range(num_res_blocks):
x = residual_block(x,
base_name=base_name,
block_num=i,
initializer=initializer)
x = Concatenate(axis=3)([x, conv3])
# size//8→size//4→size//2→size
x = Conv2DTranspose(64,
kernel_size=3,
strides=2,
padding='same',
kernel_initializer=initializer,
name=base_name + "_deconv2")(x)
x = BatchNormalization(momentum=0.9, epsilon=1e-5,
name=base_name + "_bn6")(x)
x = Activation("relu")(x)
x = Concatenate(axis=3)([x, conv2])
x = Conv2DTranspose(32,
kernel_size=3,
strides=2,
padding='same',
kernel_initializer=initializer,
name=base_name + "_deconv3")(x)
x = BatchNormalization(momentum=0.9, epsilon=1e-5,
name=base_name + "_bn7")(x)
x = Activation("relu")(x)
x = Concatenate(axis=3)([x, conv1])
out = Conv2DTranspose(3,
kernel_size=7,
strides=1,
padding='same',
activation="tanh",
kernel_initializer=initializer,
name=base_name + "_out")(x)
network = Network(in_x, out, name=base_name)
return network
def train(x_target, x_ref, y_ref, epoch_num):
# VGG16読み込み, S network用
print("Model build...")
#mobile = VGG16(include_top=False, input_shape=input_shape, weights='imagenet')
# mobile net読み込み, S network用
mobile = MobileNetV2(include_top=True, input_shape=input_shape, alpha=alpha,
, weights='imagenet')
#最終層削除
mobile.layers.pop()
# 重みを固定
for layer in mobile.layers:
if layer.name == "block_13_expand": # "block5_conv1": for VGG16
break
else:
layer.trainable = False
model_t = Model(inputs=mobile.input,outputs=mobile.layers[-1].output)
# R network用 Sと重み共有
model_r = Network(inputs=model_t.input,
outputs=model_t.output,
name="shared_layer")
#Rに全結合層を付ける
prediction = Dense(classes, activation='softmax')(model_t.output)
model_r = Model(inputs=model_r.input,outputs=prediction)
#コンパイル
optimizer = SGD(lr=5e-5, decay=0.00005)
model_r.compile(optimizer=optimizer, loss="categorical_crossentropy")
model_t.compile(optimizer=optimizer, loss=original_loss)
model_t.summary()
model_r.summary()
print("x_target is",x_target.shape[0],'samples')
print("x_ref is",x_ref.shape[0],'samples')
ref_samples = np.arange(x_ref.shape[0])
loss, loss_c = [], []
print("training...")
#学習
for epochnumber in range(epoch_num):
x_r, y_r, lc, ld = [], [], [], []
#ターゲットデータシャッフル
np.random.shuffle(x_target)
#リファレンスデータシャッフル
np.random.shuffle(ref_samples)
for i in range(len(x_target)):
x_r.append(x_ref[ref_samples[i]])
y_r.append(y_ref[ref_samples[i]])
x_r = np.array(x_r)
y_r = np.array(y_r)
for i in range(int(len(x_target) / batchsize)):
#batchsize分のデータロード
batch_target = x_target[i*batchsize:i*batchsize+batchsize]
batch_ref = x_r[i*batchsize:i*batchsize+batchsize]
batch_y = y_r[i*batchsize:i*batchsize+batchsize]
#target data
#学習しながら、損失を取得
lc.append(model_t.train_on_batch(batch_target, np.zeros((batchsize, feature_out))))
#reference data
#学習しながら、損失を取得
ld.append(model_r.train_on_batch(batch_ref, batch_y))
loss.append(np.mean(ld))
loss_c.append(np.mean(lc))
if (epochnumber+1) % 5 == 0:
print("epoch:",epochnumber+1)
print("Descriptive loss:", loss[-1])
print("Compact loss", loss_c[-1])
#結果グラフ
plt.plot(loss,label="Descriptive loss")
plt.xlabel("epoch")
plt.legend()
plt.show()
plt.plot(loss_c,label="Compact loss")
plt.xlabel("epoch")
plt.legend()
plt.show()
return model_t
def __init__(self,
dataset_path: str,
gen_mod_name: str,
disc_mod_name: str,
latent_dim: int,
training_progress_save_path: str,
testing_dataset_path: str = None,
generator_optimizer: Optimizer = Adam(0.0002, 0.5),
discriminator_optimizer: Optimizer = Adam(0.0002, 0.5),
discriminator_label_noise: float = None,
discriminator_label_noise_decay: float = None,
discriminator_label_noise_min: float = 0.001,
batch_size: int = 32,
buffered_batches: int = 20,
generator_weights: Union[str, None] = None,
discriminator_weights: Union[str, None] = None,
start_episode: int = 0,
load_from_checkpoint: bool = False,
check_dataset: bool = True,
num_of_loading_workers: int = 8):
self.disc_mod_name = disc_mod_name
self.gen_mod_name = gen_mod_name
self.generator_optimizer = generator_optimizer
self.latent_dim = latent_dim
assert self.latent_dim > 0, Fore.RED + "Invalid latent dim" + Fore.RESET
self.batch_size = batch_size
assert self.batch_size > 0, Fore.RED + "Invalid batch size" + Fore.RESET
self.discriminator_label_noise = discriminator_label_noise
self.discriminator_label_noise_decay = discriminator_label_noise_decay
self.discriminator_label_noise_min = discriminator_label_noise_min
self.progress_image_dim = (16, 9)
if start_episode < 0: start_episode = 0
self.episode_counter = start_episode
# Initialize training data folder and logging
self.training_progress_save_path = training_progress_save_path
self.training_progress_save_path = os.path.join(
self.training_progress_save_path,
f"{self.gen_mod_name}__{self.disc_mod_name}")
self.tensorboard = TensorBoardCustom(
log_dir=os.path.join(self.training_progress_save_path, "logs"))
# Create array of input image paths
self.train_data = get_paths_of_files_from_path(dataset_path,
only_files=True)
assert self.train_data, Fore.RED + "Training dataset is not loaded" + Fore.RESET
self.testing_data = None
if testing_dataset_path:
self.testing_data = get_paths_of_files_from_path(
testing_dataset_path)
assert self.testing_data, Fore.RED + "Testing dataset is not loaded" + Fore.RESET
# Load one image to get shape of it
tmp_image = cv.imread(self.train_data[0])
self.image_shape = tmp_image.shape
self.image_channels = self.image_shape[2]
# Check image size validity
if self.image_shape[0] < 4 or self.image_shape[1] < 4:
raise Exception("Images too small, min size (4, 4)")
# Check validity of whole datasets
if check_dataset:
self.validate_dataset()
# Define static vars
if os.path.exists(
f"{self.training_progress_save_path}/static_noise.npy"):
self.static_noise = np.load(
f"{self.training_progress_save_path}/static_noise.npy")
if self.static_noise.shape[0] != (self.progress_image_dim[0] *
self.progress_image_dim[1]):
print(Fore.YELLOW +
"Progress image dim changed, restarting static noise!" +
Fore.RESET)
os.remove(
f"{self.training_progress_save_path}/static_noise.npy")
self.static_noise = np.random.normal(
0.0,
1.0,
size=(self.progress_image_dim[0] *
self.progress_image_dim[1], self.latent_dim))
else:
self.static_noise = np.random.normal(
0.0,
1.0,
size=(self.progress_image_dim[0] * self.progress_image_dim[1],
self.latent_dim))
self.kernel_initializer = RandomNormal(stddev=0.02)
# Load checkpoint
self.initiated = False
loaded_gen_weights_path = None
loaded_disc_weights_path = None
if load_from_checkpoint:
loaded_gen_weights_path, loaded_disc_weights_path = self.load_checkpoint(
)
# Create batchmaker and start it
self.batch_maker = BatchMaker(
self.train_data,
self.batch_size,
buffered_batches=buffered_batches,
num_of_loading_workers=num_of_loading_workers)
self.testing_batchmaker = None
if self.testing_data:
self.testing_batchmaker = BatchMaker(
self.testing_data,
self.batch_size,
buffered_batches=buffered_batches,
num_of_loading_workers=num_of_loading_workers)
self.testing_batchmaker.start()
#################################
### Create discriminator ###
#################################
self.discriminator = self.build_discriminator(disc_mod_name)
self.discriminator.compile(loss="binary_crossentropy",
optimizer=discriminator_optimizer)
#################################
### Create generator ###
#################################
self.generator = self.build_generator(gen_mod_name)
if self.generator.output_shape[1:] != self.image_shape:
raise Exception(
"Invalid image input size for this generator model")
#################################
### Create combined generator ###
#################################
noise_input = Input(shape=(self.latent_dim, ), name="noise_input")
gen_images = self.generator(noise_input)
# Create frozen version of discriminator
frozen_discriminator = Network(self.discriminator.inputs,
self.discriminator.outputs,
name="frozen_discriminator")
frozen_discriminator.trainable = False
# Discriminator takes images and determinates validity
valid = frozen_discriminator(gen_images)
# Combine models
# Train generator to fool discriminator
self.combined_generator_model = Model(noise_input,
valid,
name="dcgan_model")
self.combined_generator_model.compile(
loss="binary_crossentropy", optimizer=self.generator_optimizer)
# Print all summaries
print("\nDiscriminator Summary:")
self.discriminator.summary()
print("\nGenerator Summary:")
self.generator.summary()
print("\nGAN Summary")
self.combined_generator_model.summary()
# Load weights from checkpoint
try:
if loaded_gen_weights_path:
self.generator.load_weights(loaded_gen_weights_path)
except:
print(Fore.YELLOW +
"Failed to load generator weights from checkpoint" +
Fore.RESET)
try:
if loaded_disc_weights_path:
self.discriminator.load_weights(loaded_disc_weights_path)
except:
print(Fore.YELLOW +
"Failed to load discriminator weights from checkpoint" +
Fore.RESET)
# Load weights from param and override checkpoint weights
if generator_weights: self.generator.load_weights(generator_weights)
if discriminator_weights:
self.discriminator.load_weights(discriminator_weights)
def I3D(input_shape=(20, 224, 224, 3), nb_class=1, name='I3D'):
# Determine proper input shape
input_shape = _obtain_input_shape(input_shape,
default_frame_size=224,
min_frame_size=32,
default_num_frames=64,
min_num_frames=8,
data_format=K.image_data_format(),
require_flatten=False,
weights=None)
img_input = Input(shape=input_shape)
if K.image_data_format() == 'channels_first':
channel_axis = 1
else:
channel_axis = 4
# Downsampling via convolution (spatial and temporal)
x = conv3d_bn(img_input,
64,
7,
7,
7,
strides=(2, 2, 2),
padding='same',
name='Conv3d_1a_7x7')
# Downsampling (spatial only)
x = MaxPooling3D((1, 3, 3),
strides=(1, 2, 2),
padding='same',
name='MaxPool2d_2a_3x3')(x)
x = conv3d_bn(x,
64,
1,
1,
1,
strides=(1, 1, 1),
padding='same',
name='Conv3d_2b_1x1')
x = conv3d_bn(x,
192,
3,
3,
3,
strides=(1, 1, 1),
padding='same',
name='Conv3d_2c_3x3')
# Downsampling (spatial only)
x = MaxPooling3D((1, 3, 3),
strides=(1, 2, 2),
padding='same',
name='MaxPool2d_3a_3x3')(x)
# Mixed 3b
branch_0 = conv3d_bn(x,
64,
1,
1,
1,
padding='same',
name='Conv3d_3b_0a_1x1')
branch_1 = conv3d_bn(x,
96,
1,
1,
1,
padding='same',
name='Conv3d_3b_1a_1x1')
branch_1 = conv3d_bn(branch_1,
128,
3,
3,
3,
padding='same',
name='Conv3d_3b_1b_3x3')
branch_2 = conv3d_bn(x,
16,
1,
1,
1,
padding='same',
name='Conv3d_3b_2a_1x1')
branch_2 = conv3d_bn(branch_2,
32,
3,
3,
3,
padding='same',
name='Conv3d_3b_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3),
strides=(1, 1, 1),
padding='same',
name='MaxPool2d_3b_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3,
32,
1,
1,
1,
padding='same',
name='Conv3d_3b_3b_1x1')
x = layers.concatenate([branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_3b')
# Mixed 3c
branch_0 = conv3d_bn(x,
128,
1,
1,
1,
padding='same',
name='Conv3d_3c_0a_1x1')
branch_1 = conv3d_bn(x,
128,
1,
1,
1,
padding='same',
name='Conv3d_3c_1a_1x1')
branch_1 = conv3d_bn(branch_1,
192,
3,
3,
3,
padding='same',
name='Conv3d_3c_1b_3x3')
branch_2 = conv3d_bn(x,
32,
1,
1,
1,
padding='same',
name='Conv3d_3c_2a_1x1')
branch_2 = conv3d_bn(branch_2,
96,
3,
3,
3,
padding='same',
name='Conv3d_3c_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3),
strides=(1, 1, 1),
padding='same',
name='MaxPool2d_3c_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3,
64,
1,
1,
1,
padding='same',
name='Conv3d_3c_3b_1x1')
x = layers.concatenate([branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_3c')
# Downsampling (spatial and temporal)
x = MaxPooling3D((3, 3, 3),
strides=(2, 2, 2),
padding='same',
name='MaxPool2d_4a_3x3')(x)
# Mixed 4b
branch_0 = conv3d_bn(x,
192,
1,
1,
1,
padding='same',
name='Conv3d_4b_0a_1x1')
branch_1 = conv3d_bn(x,
96,
1,
1,
1,
padding='same',
name='Conv3d_4b_1a_1x1')
branch_1 = conv3d_bn(branch_1,
208,
3,
3,
3,
padding='same',
name='Conv3d_4b_1b_3x3')
branch_2 = conv3d_bn(x,
16,
1,
1,
1,
padding='same',
name='Conv3d_4b_2a_1x1')
branch_2 = conv3d_bn(branch_2,
48,
3,
3,
3,
padding='same',
name='Conv3d_4b_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3),
strides=(1, 1, 1),
padding='same',
name='MaxPool2d_4b_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3,
64,
1,
1,
1,
padding='same',
name='Conv3d_4b_3b_1x1')
x = layers.concatenate([branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_4b')
# Mixed 4c
branch_0 = conv3d_bn(x,
160,
1,
1,
1,
padding='same',
name='Conv3d_4c_0a_1x1')
branch_1 = conv3d_bn(x,
112,
1,
1,
1,
padding='same',
name='Conv3d_4c_1a_1x1')
branch_1 = conv3d_bn(branch_1,
224,
3,
3,
3,
padding='same',
name='Conv3d_4c_1b_3x3')
branch_2 = conv3d_bn(x,
24,
1,
1,
1,
padding='same',
name='Conv3d_4c_2a_1x1')
branch_2 = conv3d_bn(branch_2,
64,
3,
3,
3,
padding='same',
name='Conv3d_4c_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3),
strides=(1, 1, 1),
padding='same',
name='MaxPool2d_4c_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3,
64,
1,
1,
1,
padding='same',
name='Conv3d_4c_3b_1x1')
x = layers.concatenate([branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_4c')
# Mixed 4d
branch_0 = conv3d_bn(x,
128,
1,
1,
1,
padding='same',
name='Conv3d_4d_0a_1x1')
branch_1 = conv3d_bn(x,
128,
1,
1,
1,
padding='same',
name='Conv3d_4d_1a_1x1')
branch_1 = conv3d_bn(branch_1,
256,
3,
3,
3,
padding='same',
name='Conv3d_4d_1b_3x3')
branch_2 = conv3d_bn(x,
24,
1,
1,
1,
padding='same',
name='Conv3d_4d_2a_1x1')
branch_2 = conv3d_bn(branch_2,
64,
3,
3,
3,
padding='same',
name='Conv3d_4d_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3),
strides=(1, 1, 1),
padding='same',
name='MaxPool2d_4d_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3,
64,
1,
1,
1,
padding='same',
name='Conv3d_4d_3b_1x1')
x = layers.concatenate([branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_4d')
# Mixed 4e
branch_0 = conv3d_bn(x,
112,
1,
1,
1,
padding='same',
name='Conv3d_4e_0a_1x1')
branch_1 = conv3d_bn(x,
144,
1,
1,
1,
padding='same',
name='Conv3d_4e_1a_1x1')
branch_1 = conv3d_bn(branch_1,
288,
3,
3,
3,
padding='same',
name='Conv3d_4e_1b_3x3')
branch_2 = conv3d_bn(x,
32,
1,
1,
1,
padding='same',
name='Conv3d_4e_2a_1x1')
branch_2 = conv3d_bn(branch_2,
64,
3,
3,
3,
padding='same',
name='Conv3d_4e_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3),
strides=(1, 1, 1),
padding='same',
name='MaxPool2d_4e_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3,
64,
1,
1,
1,
padding='same',
name='Conv3d_4e_3b_1x1')
x = layers.concatenate([branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_4e')
# Mixed 4f
branch_0 = conv3d_bn(x,
256,
1,
1,
1,
padding='same',
name='Conv3d_4f_0a_1x1')
branch_1 = conv3d_bn(x,
160,
1,
1,
1,
padding='same',
name='Conv3d_4f_1a_1x1')
branch_1 = conv3d_bn(branch_1,
320,
3,
3,
3,
padding='same',
name='Conv3d_4f_1b_3x3')
branch_2 = conv3d_bn(x,
32,
1,
1,
1,
padding='same',
name='Conv3d_4f_2a_1x1')
branch_2 = conv3d_bn(branch_2,
128,
3,
3,
3,
padding='same',
name='Conv3d_4f_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3),
strides=(1, 1, 1),
padding='same',
name='MaxPool2d_4f_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3,
128,
1,
1,
1,
padding='same',
name='Conv3d_4f_3b_1x1')
x = layers.concatenate([branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_4f')
# Downsampling (spatial and temporal)
x = MaxPooling3D((2, 2, 2),
strides=(2, 2, 2),
padding='same',
name='MaxPool2d_5a_2x2')(x)
# Mixed 5b
branch_0 = conv3d_bn(x,
256,
1,
1,
1,
padding='same',
name='Conv3d_5b_0a_1x1')
branch_1 = conv3d_bn(x,
160,
1,
1,
1,
padding='same',
name='Conv3d_5b_1a_1x1')
branch_1 = conv3d_bn(branch_1,
320,
3,
3,
3,
padding='same',
name='Conv3d_5b_1b_3x3')
branch_2 = conv3d_bn(x,
32,
1,
1,
1,
padding='same',
name='Conv3d_5b_2a_1x1')
branch_2 = conv3d_bn(branch_2,
128,
3,
3,
3,
padding='same',
name='Conv3d_5b_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3),
strides=(1, 1, 1),
padding='same',
name='MaxPool2d_5b_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3,
128,
1,
1,
1,
padding='same',
name='Conv3d_5b_3b_1x1')
x = layers.concatenate([branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_5b')
# Mixed 5c
branch_0 = conv3d_bn(x,
384,
1,
1,
1,
padding='same',
name='Conv3d_5c_0a_1x1')
branch_1 = conv3d_bn(x,
192,
1,
1,
1,
padding='same',
name='Conv3d_5c_1a_1x1')
branch_1 = conv3d_bn(branch_1,
384,
3,
3,
3,
padding='same',
name='Conv3d_5c_1b_3x3')
branch_2 = conv3d_bn(x,
48,
1,
1,
1,
padding='same',
name='Conv3d_5c_2a_1x1')
branch_2 = conv3d_bn(branch_2,
128,
3,
3,
3,
padding='same',
name='Conv3d_5c_2b_3x3')
branch_3 = MaxPooling3D((3, 3, 3),
strides=(1, 1, 1),
padding='same',
name='MaxPool2d_5c_3a_3x3')(x)
branch_3 = conv3d_bn(branch_3,
128,
1,
1,
1,
padding='same',
name='Conv3d_5c_3b_1x1')
x = layers.concatenate([branch_0, branch_1, branch_2, branch_3],
axis=channel_axis,
name='Mixed_5c')
h = int(x.shape[2])
w = int(x.shape[3])
x = AveragePooling3D((2, h, w),
strides=(1, 1, 1),
padding='valid',
name='global_avg_pool')(x)
#x = conv3d_bn(x, classes, 1, 1, 1, padding='same', name='Conv3d_6a_1x1_1')
# create model
#model = Model([input1, input2], [output1, output2], name='i3d_inception')
return Network(img_input, x, name=name)
def build_generator(self, filters=128):
input_layer = Input(shape=self.input_shape)
layers = [input_layer]
# define operators
layer_name = 'G_Head' # 0~2
self.gen_net.append(
Conv2D(filters, (3, 3),
padding='same',
name=layer_name + '_conv0',
activation='relu'))
self.gen_net.append(
BatchNormalization(momentum=0.8, name=layer_name + '_norm0'))
self.gen_net.append(LeakyReLU(name=layer_name + '_act0'))
layer_name = 'G_Body_0' # 3~8
self.__add_conv_gen(layer_name, filters)
layer_name = 'G_Body_1' # 9~14
self.__add_conv_gen(layer_name, filters)
layer_name = "G_Up_1" # 15
self.gen_net.append(
UpSampling2D(size=(2, 2), name=layer_name + '_upsamp'))
layer_name = 'G_Body_3' # 16~21
self.__add_conv_gen(layer_name, filters)
layer_name = "G_Tail" # 22~23
self.gen_net.append(
BatchNormalization(momentum=0.8, name=layer_name + '_norm0'))
self.gen_net.append(
Conv2D(filters=3,
kernel_size=(3, 3),
padding='same',
name='outRGB',
activation="sigmoid"))
# build network
# head
num = len(layers)
for i, operator in enumerate(self.gen_net[:3]):
layers.append(operator(layers[i + num - 1]))
head_out = layers[-1]
# body first 0
num = len(layers)
for i, operator in enumerate(self.gen_net[3:9]):
layers.append(operator(layers[i + num - 1]))
layers.append(Add()([layers[-1], head_out]))
body_0_out = layers[-1]
# body first 1
num = len(layers)
for i, operator in enumerate(self.gen_net[9:15]):
layers.append(operator(layers[i + num - 1]))
layers.append(Add()([layers[-1], body_0_out]))
body_1_out = layers[-1]
layers.append(Add()([head_out, body_1_out]))
# up
layers.append(self.gen_net[15](layers[-1]))
# tail
prior_tail = layers[-1]
num = len(layers)
for i, operator in enumerate(self.gen_net[16:22]):
layers.append(operator(layers[i + num - 1]))
layers.append(Add()([layers[-1], prior_tail]))
layers.append(self.gen_net[-2](layers[-1])) # batch_norm
layers.append(self.gen_net[-1](layers[-1])) # rgb
self.shared_generator = Network(input=layers[0],
output=layers[-1],
name='generator')
return Model(layers[0], layers[-1])
def train(x_target, x_ref, y_ref, epoch_num):
#Read VGG16, S network
print("Model build...")
#mobile = VGG16(include_top=False, input_shape=input_shape, weights='imagenet')
#Read mobile net, S network
mobile = MobileNetV2(include_top=False,
input_shape=input_shape,
alpha=alpha,
depth_multiplier=1,
weights='imagenet')
#Delete last layer
mobile.layers.pop()
#Fixed weight
for layer in mobile.layers:
if layer.name == "block_13_expand": # "block5_conv1": for VGG16
break
else:
layer.trainable = False
model_t = Model(inputs=mobile.input, outputs=mobile.layers[-1].output)
#R network S and Weight sharing
model_r = Network(inputs=model_t.input,
outputs=model_t.output,
name="shared_layer")
#Apply a Fully Connected Layer to R
prediction = Dense(classes, activation='softmax')(model_t.output)
model_r = Model(inputs=model_r.input, outputs=prediction)
#Compile
optimizer = SGD(lr=5e-5, decay=0.00005)
model_r.compile(optimizer=optimizer, loss="categorical_crossentropy")
model_t.compile(optimizer=optimizer, loss=original_loss)
model_t.summary()
model_r.summary()
print("x_target is", x_target.shape[0], 'samples')
print("x_ref is", x_ref.shape[0], 'samples')
ref_samples = np.arange(x_ref.shape[0])
loss, loss_c = [], []
print("training...")
#Learning
for epochnumber in range(epoch_num):
x_r, y_r, lc, ld = [], [], [], []
#Shuffle target data
np.random.shuffle(x_target)
#Shuffle reference data
np.random.shuffle(ref_samples)
for i in range(len(x_target)):
x_r.append(x_ref[ref_samples[i]])
y_r.append(y_ref[ref_samples[i]])
x_r = np.array(x_r)
y_r = np.array(y_r)
for i in range(int(len(x_target) / batchsize)):
#Load data for batch size
batch_target = x_target[i * batchsize:i * batchsize + batchsize]
batch_ref = x_r[i * batchsize:i * batchsize + batchsize]
batch_y = y_r[i * batchsize:i * batchsize + batchsize]
#target data
#Get loss while learning
lc.append(
model_t.train_on_batch(batch_target,
np.zeros((batchsize, feature_out))))
#reference data
#Get loss while learning
ld.append(model_r.train_on_batch(batch_ref, batch_y))
loss.append(np.mean(ld))
loss_c.append(np.mean(lc))
if (epochnumber + 1) % 5 == 0:
print("epoch:", epochnumber + 1)
print("Descriptive loss:", loss[-1])
print("Compact loss", loss_c[-1])
#Result graph
plt.plot(loss, label="Descriptive loss")
plt.xlabel("epoch")
plt.legend()
plt.show()
plt.plot(loss_c, label="Compact loss")
plt.xlabel("epoch")
plt.legend()
plt.show()
return model_t
def C3D(input_shape=(20, 224, 224, 3), nb_class=1, activation='tanh'):
input_ = Input(shape=input_shape)
x = Conv3D(64, (3, 3, 3),
activation='relu',
border_mode='same',
subsample=(1, 1, 1),
input_shape=input_shape)(input_)
x = MaxPooling3D(pool_size=(1, 2, 2),
strides=(1, 2, 2),
border_mode='valid')(x)
# 2nd layer group
x = Conv3D(128, (3, 3, 3),
activation='relu',
border_mode='same',
subsample=(1, 1, 1))(x)
x = MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
border_mode='valid')(x)
# 3rd layer group
x = Conv3D(256, (3, 3, 3),
activation='relu',
border_mode='same',
subsample=(1, 1, 1))(x)
x = Conv3D(256, (3, 3, 3),
activation='relu',
border_mode='same',
subsample=(1, 1, 1))(x)
x = MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
border_mode='valid')(x)
# 4th layer group
x = Conv3D(512, (3, 3, 3),
activation='relu',
border_mode='same',
subsample=(1, 1, 1))(x)
x = Conv3D(512, (3, 3, 3),
activation='relu',
border_mode='same',
subsample=(1, 1, 1))(x)
x = MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
border_mode='valid')(x)
# 5th layer group
x = Conv3D(512, (3, 3, 3),
activation='relu',
border_mode='same',
subsample=(1, 1, 1))(x)
x = Conv3D(512, (3, 3, 3),
activation='relu',
border_mode='same',
subsample=(1, 1, 1))(x)
x = ZeroPadding3D(padding=(0, 1, 1))(x)
x = MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
border_mode='valid')(x)
x = Flatten()(x)
# FC layers group
x = Dense(4096, activation='relu')(x)
x = Dropout(0.5)(x)
x = Dense(4096, activation='relu')(x)
x = Dropout(0.5)(x)
output_ = Dense(nb_class, activation=activation)(x)
return Network(input_, output_, name='C3D')
def main(overwrite=False):
# convert input images into an hdf5 file
if overwrite or not os.path.exists(config["data_file"]):
create_data_file(config)
data_file_opened = open_data_file(config["data_file"])
seg_loss_func = getattr(fetal_net.metrics, config['loss'])
dis_loss_func = getattr(fetal_net.metrics, config['dis_loss'])
# instantiate new model
seg_model_func = getattr(fetal_net.model, config['model_name'])
gen_model = seg_model_func(
input_shape=config["input_shape"],
initial_learning_rate=config["initial_learning_rate"],
**{
'dropout_rate':
config['dropout_rate'],
'loss_function':
seg_loss_func,
'mask_shape':
None if config["weight_mask"] is None else config["input_shape"],
'old_model_path':
config['old_model']
})
dis_model_func = getattr(fetal_net.model, config['dis_model_name'])
dis_model = dis_model_func(
input_shape=[config["input_shape"][0] + config["n_labels"]] +
config["input_shape"][1:],
initial_learning_rate=config["initial_learning_rate"],
**{
'dropout_rate': config['dropout_rate'],
'loss_function': dis_loss_func
})
if not overwrite \
and len(glob.glob(config["model_file"] + 'g_*.h5')) > 0:
# dis_model_path = get_last_model_path(config["model_file"] + 'dis_')
gen_model_path = get_last_model_path(config["model_file"] + 'g_')
# print('Loading dis model from: {}'.format(dis_model_path))
print('Loading gen model from: {}'.format(gen_model_path))
# dis_model = load_old_model(dis_model_path)
# gen_model = load_old_model(gen_model_path)
# dis_model.load_weights(dis_model_path)
gen_model.load_weights(gen_model_path)
gen_model.summary()
dis_model.summary()
# Build "frozen discriminator"
frozen_dis_model = Network(dis_model.inputs,
dis_model.outputs,
name='frozen_discriminator')
frozen_dis_model.trainable = False
inputs_real = Input(shape=config["input_shape"])
inputs_fake = Input(shape=config["input_shape"])
segs_real = Activation(None, name='seg_real')(gen_model(inputs_real))
segs_fake = Activation(None, name='seg_fake')(gen_model(inputs_fake))
valid = Activation(None, name='dis')(frozen_dis_model(
Concatenate(axis=1)([segs_fake, inputs_fake])))
combined_model = Model(inputs=[inputs_real, inputs_fake],
outputs=[segs_real, valid])
combined_model.compile(loss=[seg_loss_func, 'binary_crossentropy'],
loss_weights=[1, config["gd_loss_ratio"]],
optimizer=Adam(config["initial_learning_rate"]))
combined_model.summary()
# get training and testing generators
train_generator, validation_generator, n_train_steps, n_validation_steps = get_training_and_validation_generators(
data_file_opened,
batch_size=config["batch_size"],
data_split=config["validation_split"],
overwrite=overwrite,
validation_keys_file=config["validation_file"],
training_keys_file=config["training_file"],
test_keys_file=config["test_file"],
n_labels=config["n_labels"],
labels=config["labels"],
patch_shape=(*config["patch_shape"], config["patch_depth"]),
validation_batch_size=config["validation_batch_size"],
augment=config["augment"],
skip_blank_train=config["skip_blank_train"],
skip_blank_val=config["skip_blank_val"],
truth_index=config["truth_index"],
truth_size=config["truth_size"],
prev_truth_index=config["prev_truth_index"],
prev_truth_size=config["prev_truth_size"],
truth_downsample=config["truth_downsample"],
truth_crop=config["truth_crop"],
patches_per_epoch=config["patches_per_epoch"],
categorical=config["categorical"],
is3d=config["3D"],
drop_easy_patches_train=config["drop_easy_patches_train"],
drop_easy_patches_val=config["drop_easy_patches_val"])
# get training and testing generators
_, semi_generator, _, _ = get_training_and_validation_generators(
data_file_opened,
batch_size=config["batch_size"],
data_split=config["validation_split"],
overwrite=overwrite,
validation_keys_file=config["validation_file"],
training_keys_file=config["training_file"],
test_keys_file=config["test_file"],
n_labels=config["n_labels"],
labels=config["labels"],
patch_shape=(*config["patch_shape"], config["patch_depth"]),
validation_batch_size=config["validation_batch_size"],
val_augment=config["augment"],
skip_blank_train=config["skip_blank_train"],
skip_blank_val=config["skip_blank_val"],
truth_index=config["truth_index"],
truth_size=config["truth_size"],
prev_truth_index=config["prev_truth_index"],
prev_truth_size=config["prev_truth_size"],
truth_downsample=config["truth_downsample"],
truth_crop=config["truth_crop"],
patches_per_epoch=config["patches_per_epoch"],
categorical=config["categorical"],
is3d=config["3D"],
drop_easy_patches_train=config["drop_easy_patches_train"],
drop_easy_patches_val=config["drop_easy_patches_val"])
# start training
scheduler = Scheduler(config["dis_steps"],
config["gen_steps"],
init_lr=config["initial_learning_rate"],
lr_patience=config["patience"],
lr_decay=config["learning_rate_drop"])
best_loss = np.inf
for epoch in range(config["n_epochs"]):
postfix = {'g': None, 'd': None} # , 'val_g': None, 'val_d': None}
with tqdm(range(n_train_steps // config["gen_steps"]),
dynamic_ncols=True,
postfix={
'gen': None,
'dis': None,
'val_gen': None,
'val_dis': None,
None: None
}) as pbar:
for n_round in pbar:
# train D
outputs = np.zeros(dis_model.metrics_names.__len__())
for i in range(scheduler.get_dsteps()):
real_patches, real_segs = next(train_generator)
semi_patches, _ = next(semi_generator)
d_x_batch, d_y_batch = input2discriminator(
real_patches, real_segs, semi_patches,
gen_model.predict(semi_patches,
batch_size=config["batch_size"]),
dis_model.output_shape)
outputs += dis_model.train_on_batch(d_x_batch, d_y_batch)
if scheduler.get_dsteps():
outputs /= scheduler.get_dsteps()
postfix['d'] = build_dsc(dis_model.metrics_names, outputs)
pbar.set_postfix(**postfix)
# train G (freeze discriminator)
outputs = np.zeros(combined_model.metrics_names.__len__())
for i in range(scheduler.get_gsteps()):
real_patches, real_segs = next(train_generator)
semi_patches, _ = next(validation_generator)
g_x_batch, g_y_batch = input2gan(real_patches, real_segs,
semi_patches,
dis_model.output_shape)
outputs += combined_model.train_on_batch(
g_x_batch, g_y_batch)
outputs /= scheduler.get_gsteps()
postfix['g'] = build_dsc(combined_model.metrics_names, outputs)
pbar.set_postfix(**postfix)
# evaluate on validation set
dis_metrics = np.zeros(dis_model.metrics_names.__len__(),
dtype=float)
gen_metrics = np.zeros(gen_model.metrics_names.__len__(),
dtype=float)
evaluation_rounds = n_validation_steps
for n_round in range(evaluation_rounds): # rounds_for_evaluation:
val_patches, val_segs = next(validation_generator)
# D
if scheduler.get_dsteps() > 0:
d_x_test, d_y_test = input2discriminator(
val_patches, val_segs, val_patches,
gen_model.predict(
val_patches,
batch_size=config["validation_batch_size"]),
dis_model.output_shape)
dis_metrics += dis_model.evaluate(
d_x_test,
d_y_test,
batch_size=config["validation_batch_size"],
verbose=0)
# G
# gen_x_test, gen_y_test = input2gan(val_patches, val_segs, dis_model.output_shape)
gen_metrics += gen_model.evaluate(
val_patches,
val_segs,
batch_size=config["validation_batch_size"],
verbose=0)
dis_metrics /= float(evaluation_rounds)
gen_metrics /= float(evaluation_rounds)
# save the model and weights with the best validation loss
if gen_metrics[0] < best_loss:
best_loss = gen_metrics[0]
print('Saving Model...')
with open(
os.path.join(
config["base_dir"],
"g_{}_{:.3f}.json".format(epoch, gen_metrics[0])),
'w') as f:
f.write(gen_model.to_json())
gen_model.save_weights(
os.path.join(
config["base_dir"],
"g_{}_{:.3f}.h5".format(epoch, gen_metrics[0])))
postfix['val_d'] = build_dsc(dis_model.metrics_names, dis_metrics)
postfix['val_g'] = build_dsc(gen_model.metrics_names, gen_metrics)
# pbar.set_postfix(**postfix)
print('val_d: ' + postfix['val_d'], end=' | ')
print('val_g: ' + postfix['val_g'])
# pbar.refresh()
# update step sizes, learning rates
scheduler.update_steps(epoch, gen_metrics[0])
K.set_value(dis_model.optimizer.lr, scheduler.get_lr())
K.set_value(combined_model.optimizer.lr, scheduler.get_lr())
data_file_opened.close()
def train(x_target, x_ref, y_ref, epoch_num):
print("Model build...")
# mobile = VGG16(include_top=False, input_shape=input_shape, weights='imagenet', pooling= 'avg')
mobile = InceptionResNetV2(include_top=False,
input_shape=input_shape,
weights='imagenet')
# mobile = MobileNetV2(include_top=True, input_shape=input_shape, alpha=alpha,
# weights='imagenet')
mobile.layers.pop()
'''
Last layer :
- block_13_expand : Mobilenet v2
- block5_conv1 : VGG16
- mixed_7a : Inception resnetv2
'''
# for layer in mobile.layers:
# print(layer.name)
# if layer.name == "block_13_expand": # "block5_conv1": for VGG16
# if layer.name == "block5_conv1":
# if layer.name == "mixed_7a":
# break
# else:
# layer.trainable = False
# exit()
flat = GlobalAveragePooling2D()(mobile.layers[-1].output)
model_t = Model(inputs=mobile.input, outputs=flat)
model_r = Network(inputs=model_t.input,
outputs=model_t.output,
name="shared_layer")
prediction = Dense(classes, activation='softmax')(model_t.output)
model_r = Model(inputs=model_r.input, outputs=prediction)
optimizer = SGD(lr=5e-5, decay=0.00005)
model_r.compile(optimizer=optimizer, loss="categorical_crossentropy")
model_t.compile(optimizer=optimizer, loss=original_loss)
model_t.summary()
ref_samples = np.arange(x_ref.shape[0])
loss, loss_c = [], []
epochs = []
print("training...")
for epochnumber in range(epoch_num):
x_r, y_r, lc, ld = [], [], [], []
np.random.shuffle(x_target)
np.random.shuffle(ref_samples)
for i in range(len(x_target)):
x_r.append(x_ref[ref_samples[i]])
y_r.append(y_ref[ref_samples[i]])
x_r = np.array(x_r)
y_r = np.array(y_r)
for i in range(int(len(x_target) / batchsize)):
batch_target = x_target[i * batchsize:i * batchsize + batchsize]
batch_ref = x_r[i * batchsize:i * batchsize + batchsize]
batch_y = y_r[i * batchsize:i * batchsize + batchsize]
#target data
lc.append(
model_t.train_on_batch(batch_target,
np.zeros((batchsize, feature_out))))
#reference data
ld.append(model_r.train_on_batch(batch_ref, batch_y))
loss.append(np.mean(ld))
loss_c.append(np.mean(lc))
epochs.append(epochnumber)
print("epoch : {} ,Descriptive loss : {}, Compact loss : {}".format(
epochnumber + 1, loss[-1], loss_c[-1]))
if epochnumber % 10 == 0:
model_t.save_weights('model/model_t_smd_{}.h5'.format(epochnumber))
model_r.save_weights('model/model_r_smd_{}.h5'.format(epochnumber))
discriminator_input, discriminator_output = build_discriminator(
image_shape, dropout_rate)
discriminator = Model(discriminator_input,
discriminator_output,
name='discriminator')
discriminator.compile(optimizer,
loss='binary_crossentropy',
metrics=['accuracy'])
assert (len(discriminator._collected_trainable_weights) > 0)
frozen_discriminator = Network(discriminator_input,
discriminator_output,
name='frozen_discriminator')
frozen_discriminator.trainable = False
generator_input, generator_output = build_generator()
generator = Model(generator_input, generator_output, name='generator')
adversarial_model = Model(generator_input,
frozen_discriminator(generator_output),
name='adversarial_model')
adversarial_model.compile(optimizer, loss='binary_crossentropy')
assert (len(adversarial_model._collected_trainable_weights) == len(
generator.trainable_weights))
batch_size = 64
def train(s_data, s_out, nuc, bestwight, samplesize, epoch_num):
batch_size = 1024
num_classes_org = 1024
num_classes = 63
shape1 = (None, DATA_LENGTH, 2)
model = cnn_network_keras.build_network(shape=shape1,
num_classes=num_classes_org)
model.load_weights(bestwight)
model.layers.pop() # remove last layer
model.layers.pop() # remove last layer
model.layers.pop() # remove last layer
for layer in model.layers:
if layer.name == "conv1d_20":
break
else:
layer.trainable = False
flat = GlobalAveragePooling1D()(model.layers[-1].output)
model_t = Model(inputs=model.input, outputs=flat)
model_r = Network(inputs=model_t.input,
outputs=model_t.output,
name="shared_layer")
prediction = Dense(num_classes, activation='softmax')(model_t.output)
model_r = Model(inputs=model_r.input, outputs=prediction)
optimizer = SGD(lr=5e-5, decay=0.00005)
model_r.compile(optimizer=optimizer, loss="categorical_crossentropy")
model_t.compile(optimizer=optimizer, loss=original_loss)
x_ref, test_x_r, y_ref, test_y_r, num_classes_r = prepDataNear(
s_data, samplesize, nuc)
x_target, test_x, train_y, test_y, num_classes = prepData(
s_data, samplesize, nuc)
ref_samples = np.arange(x_ref.shape[0])
loss, loss_c = [], []
epochs = []
print("training...")
for epochnumber in range(epoch_num):
x_r, y_r, lc, ld = [], [], [], []
np.random.shuffle(x_target)
np.random.shuffle(ref_samples)
for i in range(len(x_target)):
x_r.append(x_ref[ref_samples[i]])
y_r.append(y_ref[ref_samples[i]])
x_r = np.array(x_r)
y_r = np.array(y_r)
for i in range(int(len(x_target) / batchsize)):
batch_target = x_target[i * batchsize:i * batchsize + batchsize]
batch_ref = x_r[i * batchsize:i * batchsize + batchsize]
batch_y = y_r[i * batchsize:i * batchsize + batchsize]
# target data
lc.append(
model_t.train_on_batch(batch_target,
np.zeros((batchsize, feature_out))))
# reference data
ld.append(model_r.train_on_batch(batch_ref, batch_y))
loss.append(np.mean(ld))
loss_c.append(np.mean(lc))
epochs.append(epochnumber)
print("epoch : {} ,Descriptive loss : {}, Compact loss : {}".format(
epochnumber + 1, loss[-1], loss_c[-1]))
if epochnumber % 1 == 0:
model_t.save_weights(s_out + '/' + nuc +
'/model_t_ep_{}.h5'.format(epochnumber))
model_r.save_weights(s_out + '/' + nuc +
'/model_r_ep_{}.h5'.format(epochnumber))
def train_test(result_df, df_t, df_r, df_test):
# data
x_target = np.array(df_t.vecs.to_list())
x_ref = np.array(df_r.vecs.to_list())
y_ref = np.array(df_r.target.to_list())
y_ref = to_categorical(y_ref)
test_vecs = np.array(df_test.vecs.to_list())
n_sup = 10000
n_per_targ = 1000
df_r_temp = df_r.groupby('target', group_keys=False).apply(
lambda df: df.sample(n=min(df.shape[0], n_per_targ), random_state=42))
x_tr = np.array(df_t.head(n_sup).append(df_r_temp).vecs.to_list())
y_tr = np.array(df_t.head(n_sup).append(df_r_temp).label.to_list())
#y_tr = to_categorical(y_tr)
#print(f"{df.where(df.label == 0).dropna().target.value_counts()}")
#print(f"x_target: {x_target.shape}\nx_ref: {x_ref.shape}\ny_ref: {y_ref.shape}\n")
res_path = "/home/philipp/projects/dad4td/reports/one_class/all.tsv"
classes = df_r.target.unique().shape[0]
print(f"classes: {classes}")
batchsize = 128
epoch_num = 15
epoch_report = 5
feature_out = 64
pred_mode = "nn"
# get the loss for compactness
original_loss = create_loss(classes, batchsize)
# model creation
model = create_model(loss="binary_crossentropy", n_in=x_target[0].shape[0])
model_t = Model(inputs=model.input, outputs=model.output)
model_r = Network(inputs=model_t.input,
outputs=model_t.output,
name="shared_layer")
prediction = Dense(classes, activation='softmax')(model_t.output)
model_r = Model(inputs=model_r.input, outputs=prediction)
#latent_t = Dense(2, activation='relu')(model_t.output)
#model_t = Model(inputs=model_t.input,outputs=latent_t)
prediction_t = Dense(feature_out, activation='softmax')(model_t.output)
model_t = Model(inputs=model_t.input, outputs=prediction_t)
#optimizer = SGD(lr=5e-5, decay=0.00005)
optimizer = Adam(learning_rate=5e-5)
model_r.compile(optimizer=optimizer, loss="categorical_crossentropy")
model_t.compile(optimizer=optimizer, loss=original_loss)
model_t.summary()
model_r.summary()
ref_samples = np.arange(x_ref.shape[0])
loss, loss_c = [], []
epochs = []
best_acc = 0
print("training...")
for epochnumber in range(epoch_num):
x_r, y_r, lc, ld = [], [], [], []
np.random.shuffle(x_target)
np.random.shuffle(ref_samples)
for i in range(len(x_ref)):
x_r.append(x_ref[ref_samples[i]])
y_r.append(y_ref[ref_samples[i]])
x_r = np.array(x_r)
y_r = np.array(y_r)
for i in range(int(len(x_target) / batchsize)):
batch_target = x_target[i * batchsize:i * batchsize + batchsize]
batch_ref = x_r[i * batchsize:i * batchsize + batchsize]
batch_y = y_r[i * batchsize:i * batchsize + batchsize]
# target data
lc.append(
model_t.train_on_batch(batch_target,
np.zeros((batchsize, feature_out))))
# reference data
ld.append(model_r.train_on_batch(batch_ref, batch_y))
loss.append(np.mean(ld))
loss_c.append(np.mean(lc))
epochs.append(epochnumber)
if epochnumber % epoch_report == 0 or epochnumber == epoch_num - 1:
print(
f"-----\n\nepoch : {epochnumber+1} ,Descriptive loss : {loss[-1]}, Compact loss : {loss_c[-1]}"
)
model_t.save_weights(
'/home/philipp/projects/dad4td/models/one_class/model_t_smd_{}.h5'
.format(epochnumber))
model_r.save_weights(
'/home/philipp/projects/dad4td/models/one_class/model_r_smd_{}.h5'
.format(epochnumber))
#test_b = model_t.predict(test_vecs)
#od = OCSVM()
# od.fit(test_b)
#decision_scores = od.labels_
# decision_scores = decision_scores.astype(float)
labels = df_test["label"].astype(int).values
# threshold = 0.5
# scores = get_scores(dict(),labels, np.where(decision_scores > threshold, 0, 1), outlabel=0)
if pred_mode == "svm":
x_tr_pred = model_t.predict(x_tr)
clf = SVC()
clf.fit(x_tr_pred, y_tr)
preds = model_t.predict(test_vecs)
preds = clf.predict(preds)
elif pred_mode == "nn":
y_tr = y_tr.astype(int)
print(y_tr)
x_tr_pred = model_t.predict(x_tr)
clf = create_sup_model(n_in=feature_out)
clf.summary()
clf.fit(x_tr_pred,
y=y_tr,
epochs=15,
batch_size=64,
verbose=True)
decision_scores = model_t.predict(test_vecs)
decision_scores = clf.predict(decision_scores)
preds = decision_scores.astype(float)
_ = plt.hist(preds, bins=10)
plt.show()
else:
raise Exception(f"{pred_mode} must be one of svm, nn, osvm")
scores = get_scores(dict(), labels, preds, outlabel=0)
print(f"\n\nTest scores:\n{pd.DataFrame([scores], index=[0])}")
if scores["accuracy"] > best_acc:
best_acc = scores["accuracy"]
print(f"best_acc updated to: {best_acc}")
normalize = "true"
print(f"{confusion_matrix(labels, preds, normalize=normalize)}")
result_df = result_df.append(dict(cclass=list(df_test.target.unique()),
accuracy=best_acc),
ignore_index=True)
result_df.to_csv(res_path, sep="\t")
return result_df
def build_model():
frames = None
s_size = None
# prepare shared layers
dummy_frame_inputs = Input(shape=((9, s_size, s_size, 3)))
shared_layer_h = Network(dummy_frame_inputs,
conv3D_branch(dummy_frame_inputs))
shared_layer_v = Network(dummy_frame_inputs,
conv3D_branch(dummy_frame_inputs))
# build model
inputs_h = Input(shape=((frames, 9, s_size, s_size, 3)), name='inputs_h')
processed_h = TimeDistributed(shared_layer_h,
name='shared_3Dconv_branch_h')(inputs_h)
inputs_v = Input(shape=((frames, 9, s_size, s_size, 3)), name='inputs_v')
processed_v = TimeDistributed(shared_layer_v,
name='shared_3Dconv_branch_v')(inputs_v)
x = Concatenate()([processed_h, processed_v])
for n_filters in [64, 32, 32, 16]:
x = TimeDistributed(
Conv2D(n_filters,
kernel_size=3,
padding='same',
activation='relu',
kernel_initializer='glorot_uniform'))(x)
x = ConvRNN2D(STConvLSTM2DCell(8,
kernel_size=3,
padding='same',
activation='tanh',
recurrent_activation='hard_sigmoid',
kernel_initializer='glorot_uniform',
recurrent_initializer='orthogonal'),
return_sequences=True,
name='STConvLSTM2D')(x)
x = TimeDistributed(Lambda(lambda x: K.squeeze(x, axis=-1)),
name='squeeze')(x)
return Model(inputs=[inputs_h, inputs_v], outputs=x)
# ________________________________________________________________________________________________________________
# Layer (type) Output Shape Param # Connected to
# ================================================================================================================
# inputs_h (InputLayer) [(None, None, 9, None, None, 3)] 0
# ________________________________________________________________________________________________________________
# inputs_v (InputLayer) [(None, None, 9, None, None, 3)] 0
# ________________________________________________________________________________________________________________
# shared_3Dconv_branch_h (TimeDistributed) (None, None, None, None, 64) 279296 inputs_h[0][0]
# ________________________________________________________________________________________________________________
# shared_3Dconv_branch_v (TimeDistributed) (None, None, None, None, 64) 279296 inputs_v[0][0]
# ________________________________________________________________________________________________________________
# concatenate (Concatenate) (None, None, None, None, 128) 0 shared_3Dconv_branch_h[0][0]
# shared_3Dconv_branch_v[0][0]
# ________________________________________________________________________________________________________________
# time_distributed (TimeDistributed) (None, None, None, None, 64) 73792 concatenate[0][0]
# ________________________________________________________________________________________________________________
# time_distributed_1 (TimeDistributed) (None, None, None, None, 32) 18464 time_distributed[0][0]
# ________________________________________________________________________________________________________________
# time_distributed_2 (TimeDistributed) (None, None, None, None, 32) 9248 time_distributed_1[0][0]
# ________________________________________________________________________________________________________________
# time_distributed_3 (TimeDistributed) (None, None, None, None, 16) 4624 time_distributed_2[0][0]
# ________________________________________________________________________________________________________________
# STConvLSTM2D (ConvRNN2D) (None, None, None, None, 1) 40009 time_distributed_3[0][0]
# ________________________________________________________________________________________________________________
# squeeze (TimeDistributed) (None, None, None, None) 0 conv_rn_n2d[0][0]
# ================================================================================================================
# Total params: 704,729
# Trainable params: 704,633
# Non-trainable params: 96
# ________________________________________________________________________________________________________________
def __init__(self,
dataset_path: str,
num_of_upscales: int,
gen_mod_name: str,
disc_mod_name: str,
training_progress_save_path: str,
dataset_augmentation_settings: Union[AugmentationSettings,
None] = None,
generator_optimizer: Optimizer = Adam(0.0001, 0.9),
discriminator_optimizer: Optimizer = Adam(0.0001, 0.9),
gen_loss="mae",
disc_loss="binary_crossentropy",
feature_loss="mae",
gen_loss_weight: float = 1.0,
disc_loss_weight: float = 0.003,
feature_loss_weights: Union[list, float, None] = None,
feature_extractor_layers: Union[list, None] = None,
generator_lr_decay_interval: Union[int, None] = None,
discriminator_lr_decay_interval: Union[int, None] = None,
generator_lr_decay_factor: Union[float, None] = None,
discriminator_lr_decay_factor: Union[float, None] = None,
generator_min_lr: Union[float, None] = None,
discriminator_min_lr: Union[float, None] = None,
discriminator_label_noise: Union[float, None] = None,
discriminator_label_noise_decay: Union[float, None] = None,
discriminator_label_noise_min: Union[float, None] = 0.001,
batch_size: int = 4,
buffered_batches: int = 20,
generator_weights: Union[str, None] = None,
discriminator_weights: Union[str, None] = None,
load_from_checkpoint: bool = False,
custom_hr_test_images_paths: Union[list, None] = None,
check_dataset: bool = True,
num_of_loading_workers: int = 8):
# Save params to inner variables
self.__disc_mod_name = disc_mod_name
self.__gen_mod_name = gen_mod_name
self.__num_of_upscales = num_of_upscales
assert self.__num_of_upscales >= 0, Fore.RED + "Invalid number of upscales" + Fore.RESET
self.__discriminator_label_noise = discriminator_label_noise
self.__discriminator_label_noise_decay = discriminator_label_noise_decay
self.__discriminator_label_noise_min = discriminator_label_noise_min
if self.__discriminator_label_noise_min is None:
self.__discriminator_label_noise_min = 0
self.__batch_size = batch_size
assert self.__batch_size > 0, Fore.RED + "Invalid batch size" + Fore.RESET
self.__episode_counter = 0
# Insert empty lists if feature extractor settings are empty
if feature_extractor_layers is None:
feature_extractor_layers = []
if feature_loss_weights is None:
feature_loss_weights = []
# If feature_loss_weights is float then create list of the weights from it
if isinstance(feature_loss_weights,
float) and len(feature_extractor_layers) > 0:
feature_loss_weights = [
feature_loss_weights / len(feature_extractor_layers)
] * len(feature_extractor_layers)
assert len(feature_extractor_layers) == len(
feature_loss_weights
), Fore.RED + "Number of extractor layers and feature loss weights must match!" + Fore.RESET
# Create array of input image paths
self.__train_data = get_paths_of_files_from_path(dataset_path,
only_files=True)
assert self.__train_data, Fore.RED + "Training dataset is not loaded" + Fore.RESET
# Load one image to get shape of it
self.__target_image_shape = cv.imread(self.__train_data[0]).shape
# Check image size validity
if self.__target_image_shape[0] < 4 or self.__target_image_shape[1] < 4:
raise Exception("Images too small, min size (4, 4)")
# Starting image size calculate
self.__start_image_shape = count_upscaling_start_size(
self.__target_image_shape, self.__num_of_upscales)
# Check validity of whole datasets
if check_dataset:
self.__validate_dataset()
# Initialize training data folder and logging
self.__training_progress_save_path = training_progress_save_path
self.__training_progress_save_path = os.path.join(
self.__training_progress_save_path,
f"{self.__gen_mod_name}__{self.__disc_mod_name}__{self.__start_image_shape}_to_{self.__target_image_shape}"
)
self.__tensorboard = TensorBoardCustom(
log_dir=os.path.join(self.__training_progress_save_path, "logs"))
self.__stat_logger = StatLogger(self.__tensorboard)
# Define static vars
self.kernel_initializer = RandomNormal(stddev=0.02)
self.__custom_loading_failed = False
self.__custom_test_images = True if custom_hr_test_images_paths else False
if custom_hr_test_images_paths:
self.__progress_test_images_paths = custom_hr_test_images_paths
for idx, image_path in enumerate(
self.__progress_test_images_paths):
if not os.path.exists(image_path):
self.__custom_loading_failed = True
self.__progress_test_images_paths[idx] = random.choice(
self.__train_data)
else:
self.__progress_test_images_paths = [
random.choice(self.__train_data)
]
# Create batchmaker and start it
self.__batch_maker = BatchMaker(
self.__train_data,
self.__batch_size,
buffered_batches=buffered_batches,
secondary_size=self.__start_image_shape,
num_of_loading_workers=num_of_loading_workers,
augmentation_settings=dataset_augmentation_settings)
# Create LR Schedulers for both "Optimizer"
self.__gen_lr_scheduler = LearningRateScheduler(
start_lr=float(K.get_value(generator_optimizer.lr)),
lr_decay_factor=generator_lr_decay_factor,
lr_decay_interval=generator_lr_decay_interval,
min_lr=generator_min_lr)
self.__disc_lr_scheduler = LearningRateScheduler(
start_lr=float(K.get_value(discriminator_optimizer.lr)),
lr_decay_factor=discriminator_lr_decay_factor,
lr_decay_interval=discriminator_lr_decay_interval,
min_lr=discriminator_min_lr)
#####################################
### Create discriminator ###
#####################################
self.__discriminator = self.__build_discriminator(disc_mod_name)
self.__discriminator.compile(loss=disc_loss,
optimizer=discriminator_optimizer)
#####################################
### Create generator ###
#####################################
self.__generator = self.__build_generator(gen_mod_name)
if self.__generator.output_shape[1:] != self.__target_image_shape:
raise Exception(
f"Invalid image input size for this generator model\nGenerator shape: {self.__generator.output_shape[1:]}, Target shape: {self.__target_image_shape}"
)
self.__generator.compile(loss=gen_loss,
optimizer=generator_optimizer,
metrics=[PSNR_Y, PSNR, SSIM])
#####################################
### Create vgg network ###
#####################################
self.__vgg = create_feature_extractor(self.__target_image_shape,
feature_extractor_layers)
#####################################
### Create combined generator ###
#####################################
small_image_input_generator = Input(shape=self.__start_image_shape,
name="small_image_input")
# Images upscaled by generator
gen_images = self.__generator(small_image_input_generator)
# Discriminator takes images and determinates validity
frozen_discriminator = Network(self.__discriminator.inputs,
self.__discriminator.outputs,
name="frozen_discriminator")
frozen_discriminator.trainable = False
validity = frozen_discriminator(gen_images)
# Extracts features from generated images
generated_features = self.__vgg(preprocess_vgg(gen_images))
# Combine models
# Train generator to fool discriminator
self.__combined_generator_model = Model(
inputs=small_image_input_generator,
outputs=[gen_images, validity] + [*generated_features],
name="srgan")
self.__combined_generator_model.compile(
loss=[gen_loss, disc_loss] +
([feature_loss] * len(generated_features)),
loss_weights=[gen_loss_weight, disc_loss_weight] +
feature_loss_weights,
optimizer=generator_optimizer,
metrics={"generator": [PSNR_Y, PSNR, SSIM]})
# Print all summaries
print("\nDiscriminator Summary:")
self.__discriminator.summary()
print("\nGenerator Summary:")
self.__generator.summary()
# Load checkpoint
self.__initiated = False
if load_from_checkpoint: self.__load_checkpoint()
# Load weights from param and override checkpoint weights
if generator_weights: self.__generator.load_weights(generator_weights)
if discriminator_weights:
self.__discriminator.load_weights(discriminator_weights)
# Set LR
self.__gen_lr_scheduler.set_lr(self.__combined_generator_model,
self.__episode_counter)
self.__disc_lr_scheduler.set_lr(self.__discriminator,
self.__episode_counter)
def GAN_Main(result_dir="output", data_dir="data"):
alpha = 0.0004
input_shape = (256, 256, 3)
local_shape = (128, 128, 3)
batchSize = 4
Epochs = 150
l1 = int(Epochs * 0.18)
l2 = int(Epochs * 0.02)
train_datagen = Data(data_dir, input_shape[:2], local_shape[:2])
Gen = Generative(input_shape)
Dis = Discriminative(input_shape, local_shape)
optimizer = Adadelta()
#######
orgVal = Input(shape=input_shape)
mask = Input(shape=(input_shape[0], input_shape[1], 1))
imgContent = Lambda(lambda x: x[0] * (1 - x[1]),
output_shape=input_shape)([orgVal, mask])
mimic = Gen(imgContent)
completion = Lambda(lambda x: x[0] * x[2] + x[1] * (1 - x[2]),
output_shape=input_shape)([mimic, orgVal, mask])
Gen_container = Network([orgVal, mask], completion)
Gen_out = Gen_container([orgVal, mask])
Gen_model = Model([orgVal, mask], Gen_out)
Gen_model.compile(loss='mse', optimizer=optimizer)
inputLayer = Input(shape=(4, ), dtype='int32')
Dis_container = Network([orgVal, inputLayer], Dis([orgVal, inputLayer]))
Dis_model = Model([orgVal, inputLayer], Dis_container([orgVal,
inputLayer]))
Dis_model.compile(loss='binary_crossentropy', optimizer=optimizer)
Dis_container.trainable = False
totalModel = Model([orgVal, mask, inputLayer],
[Gen_out, Dis_container([Gen_out, inputLayer])])
totalModel.compile(loss=['mse', 'binary_crossentropy'],
loss_weights=[1.0, alpha],
optimizer=optimizer)
for n in range(Epochs):
progress = generic_utils.Progbar(len(train_datagen))
for inputs, points, masks in train_datagen.flow(batchSize):
Gen_image = Gen_model.predict([inputs, masks])
real = np.ones((batchSize, 1))
unreal = np.zeros((batchSize, 1))
generatorLoss = 0.0
discriminatorLoss = 0.0
if n < l1:
generatorLoss = Gen_model.train_on_batch([inputs, masks],
inputs)
else:
discriminatorLoss_real = Dis_model.train_on_batch(
[inputs, points], real)
discriminatorLoss_unreal = Dis_model.train_on_batch(
[Gen_image, points], unreal)
discriminatorLoss = 0.5 * np.add(discriminatorLoss_real,
discriminatorLoss_unreal)
if n >= l1 + l2:
generatorLoss = totalModel.train_on_batch(
[inputs, masks, points], [inputs, real])
generatorLoss = generatorLoss[0] + alpha * generatorLoss[1]
progress.add(inputs.shape[0])
imgs = min(5, batchSize)
Display, Axis = mplt.subplots(imgs, 3)
Axis[0, 0].set_title('Input Image')
Axis[0, 1].set_title('Output Image')
Axis[0, 2].set_title('Original Image')
for i in range(imgs):
Axis[i, 0].imshow(inputs[i] * (1 - masks[i]))
Axis[i, 0].axis('off')
Axis[i, 1].imshow(Gen_image[i])
Axis[i, 1].axis('off')
Axis[i, 2].imshow(inputs[i])
Axis[i, 2].axis('off')
Display.savefig(os.path.join(result_dir, "Batch_%d.png" % n))
mplt.close()
#Trained Model Files...
Gen.save(os.path.join(result_dir, "generator.h5"))
Dis.save(os.path.join(result_dir, "discriminator.h5"))
def discriminator(input_shape,
base_name,
num_res_blocks=0,
is_wgangp=False,
use_res=False):
initializer_d = TruncatedNormal(mean=0, stddev=0.1, seed=42)
D = in_D = Input(shape=input_shape)
D = Conv2D(64,
kernel_size=4,
strides=2,
padding="same",
kernel_initializer=initializer_d,
use_bias=False,
name=base_name + "_conv1")(D)
D = LeakyReLU(0.2)(D)
D = Conv2D(128,
kernel_size=4,
strides=2,
padding="same",
kernel_initializer=initializer_d,
use_bias=False,
name=base_name + "_conv2")(D)
D = BatchNormalization(momentum=0.9, epsilon=1e-5,
name=base_name + "_bn1")(D)
D = LeakyReLU(0.2)(D)
D = Conv2D(256,
kernel_size=4,
strides=2,
padding="same",
kernel_initializer=initializer_d,
use_bias=False,
name=base_name + "_conv3")(D)
D = BatchNormalization(momentum=0.9, epsilon=1e-5,
name=base_name + "_bn2")(D)
D = LeakyReLU(0.2)(D)
if use_res:
for i in range(5):
D = residual_block(D,
base_name=base_name,
block_num=i,
initializer=initializer_d,
num_channels=256,
is_wgangp=is_wgangp)
D = Conv2D(512,
kernel_size=4,
strides=2,
padding="same",
kernel_initializer=initializer_d,
use_bias=False,
name=base_name + "_conv4")(D)
D = BatchNormalization(momentum=0.9, epsilon=1e-5,
name=base_name + "_bn3")(D)
D = LeakyReLU(0.2)(D)
if not is_wgangp:
D = Flatten()(D)
D = Dense(units=128, name=base_name + "_dense1")(D)
D = LeakyReLU(0.2)(D)
out = Dense(units=1, activation="sigmoid", name=base_name + "_out")(D)
else:
#D = GlobalAveragePooling2D()(D)
D = Flatten()(D)
D = Dense(units=128, name=base_name + "_dense1")(D)
D = LeakyReLU(0.2)(D)
out = Dense(units=1, activation=None, name=base_name + "_out")(D)
network = Network(in_D, out, name=base_name)
return network
