def DocNetWork(input_shape=(224, 224, 3)): vgg16 = VGG16(include_top=True, input_tensor=None, input_shape=input_shape, weights='imagenet') vgg16.layers.pop() vgg16.layers.pop() vgg16.layers.pop() # Fine tuning for layer in vgg16.layers: if layer.name == "block5_conv1": break else: layer.trainable = False # model_t : Secondary Model model_t = Model(inputs=vgg16.input, outputs=vgg16.layers[-1].output) # model_r : Reference Model model_r = Network(inputs=model_t.input, outputs=model_t.output, name="shared_layer") # add prediction layer prediction = Dense(classes, activation='softmax')(model_t.output) model_r = Model(inputs=model_r.input, outputs=prediction) optimizer = SGD(lr=5e-5, decay=0.00005) model_r.compile(optimizer=optimizer, loss="categorical_crossentropy") model_t.compile(optimizer=optimizer, loss=original_loss) return model_r, model_t
def build_model(): frames = None s_size = None # prepare shared layers dummy_frame_inputs = Input(shape=((9, s_size, s_size, 3))) shared_layer_h = Network(dummy_frame_inputs, conv3D_branch(dummy_frame_inputs)) shared_layer_v = Network(dummy_frame_inputs, conv3D_branch(dummy_frame_inputs)) # build model inputs_h = Input(shape=((frames, 9, s_size, s_size, 3)), name='inputs_h') processed_h = TimeDistributed(shared_layer_h, name='shared_3Dconv_branch_h')(inputs_h) inputs_v = Input(shape=((frames, 9, s_size, s_size, 3)), name='inputs_v') processed_v = TimeDistributed(shared_layer_v, name='shared_3Dconv_branch_v')(inputs_v) x = Concatenate()([processed_h, processed_v]) for n_filters in [64, 32, 32, 16]: x = TimeDistributed( Conv2D(n_filters, kernel_size=3, padding='same', activation='relu', kernel_initializer='glorot_uniform'))(x) x = TimeDistributed( Conv2D(1, kernel_size=3, padding='same', kernel_initializer='glorot_uniform'))(x) x = TimeDistributed(Lambda(lambda x: K.squeeze(x, axis=3)))(x) return Model(inputs=[inputs_h, inputs_v], outputs=x)
def build(self): self.input_tgt = Input(self.input_shape) # 検出対象画像 self.input_search = Input(self.input_shape) # 探索対象画像 # 同じ特徴量抽出器を共有 x_in = Input(self.input_shape) feature_net = MobileNetV2(input_tensor=x_in, alpha=1.0, include_top=False) for temp in feature_net.layers: temp.trainable = False feature_net = Network(x_in, feature_net.output, name='feature') self.feature_tgt = feature_net(self.input_tgt) self.feature_search = feature_net(self.input_search) # 出力結果を連結 self.concat = concatenate([self.feature_tgt, self.feature_search], axis=3) self.fc0 = Flatten()(self.concat) # 全結合(オリジナルは(4096,) x 3 から最後に(4,)だがメモリに乗らなかったので小さくしてある x = Dense(1024)(self.fc0) x = BatchNormalization()(x) self.fc1 = Activation('relu')(x) x = Dense(1024)(self.fc1) x = BatchNormalization()(x) self.fc2 = Activation('relu')(x) x = Dense(1024)(self.fc2) x = BatchNormalization()(x) self.fc3 = Activation('relu')(x) self.output = Dense(4)(self.fc3) # モデル出力 self.model = Model(inputs=[self.input_tgt, self.input_search], outputs=self.output) return self.model
def make_q_network(input_shape, name=None): inputs = Input(shape=input_shape) h1 = Dense(24, activation='relu')(inputs) h2 = Dense(48, activation='relu')(h1) h3 = Dense(24, activation='relu')(h2) output = Dense(1, activation='linear')(h3) return Network(inputs=inputs, outputs=output, name=name)
def __init__(self): # Input shape self.img_rows = 32 self.img_cols = 32 self.channels = 3 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.latent_dim = 32 optimizer = Adam(0.0002, 0.5) # Build and compile the discriminator inp_discriminator, out_discriminator = self.build_discriminator() self.discriminator = Model(inp_discriminator, out_discriminator, name='discriminator') self.discriminator.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) # Build the frozen discriminator copy self.frozen_discriminator = Network(inp_discriminator, out_discriminator, name='frozen_discriminator') self.frozen_discriminator.trainable = False # Build the generator inp_generator, out_generator = self.build_generator() self.generator = Model(inp_generator, out_generator, name='generator') # The combined model (stacked generator and discriminator) # Trains the generator to fool the discriminator optimizer = Adam(0.0002, 0.5) self.combined = Sequential() self.combined.add(self.generator) self.combined.add(self.frozen_discriminator) self.combined.compile(loss='binary_crossentropy', optimizer=optimizer) self.discriminator.summary() self.generator.summary() self.combined.summary()
def create_model(optimizer, target_size=(224, 224, 3)): """ モデルを作って返す。 諸々の値がハードコーディングされている。きたない。 keras applicationsで定義されたモデルを特徴抽出器として利用するモデル。 最後にconcat, fully connectedに渡して結果を計算する。 """ base_model = InceptionV3(include_top=False, weights='imagenet') shared_layers = Network(base_model.inputs, base_model.output, name='shared_layers') shared_layers.trainable = False input_x1 = Input(shape=target_size, name='input_x1') x1 = shared_layers(input_x1) x1 = GlobalAveragePooling2D(name='x1_gap')(x1) x1 = BatchNormalization()(x1) input_x2 = Input(shape=target_size, name='input_x2') x2 = shared_layers(input_x2) x2 = GlobalAveragePooling2D(name='x2_gap')(x2) x2 = BatchNormalization()(x2) input_x3 = Input(shape=target_size, name='input_x3') x3 = shared_layers(input_x3) x3 = GlobalAveragePooling2D(name='x3_gap')(x3) x3 = BatchNormalization()(x3) x = Concatenate(name='concat_triple_image')([x1, x2, x3]) x = Dense(1024, activation='relu')(x) # x = Dense(256, activation='relu', kernel_regularizer=regularizers.l2(0.001))(x) x = Dropout(0.5)(x) predictions = Dense(2, activation='softmax')(x) model = Model([input_x1, input_x2, input_x3], predictions) model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy']) return model
def __init__(self): self.img_rows = 28 self.img_cols = 28 self.channels = 1 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.latent_dim = 100 optimizer = Adam(0.0002, 0.5) print('Discriminator Model:') model = Sequential() model.add(Flatten(input_shape=self.img_shape)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(256)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(1, activation='sigmoid')) model.summary() img = Input(shape=self.img_shape) validity = model(img) # Build and Compile the Discriminator Model self.discriminator = Model(img, validity) self.discriminator.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) self.discriminator_fixed = Network( img, validity) # -> I Add THIS CONTAINER! # Build & Compile the Generator Model self.generator = self.build_generator() #Creates the NOISE (z) to be input for generator and produce the image z = Input(shape=(self.latent_dim, )) # Produce the image img = self.generator(z) # This steps ensure when we train our Netowrks we ONLY train the Generator Net # self.discriminator.trainable = False self.discriminator_fixed.trainable = False # specifies that our Discriminator will take the images generated by our Generator + true dataset and set its output # to a parameter called validity, which will indicate whether the input is real or not. # validity = self.discriminator(img) validity = self.discriminator_fixed(img) # combined the models and also set our loss function and optimizer. The ultimate goal here is for the Generator to fool the Discriminator. self.combined = Model(z, validity) # self.combined = Model(z, self.discriminator_fixed(validity)) # Use container self.combined.compile(loss='binary_crossentropy', optimizer=optimizer)
def build_vgg(self, vgg_path, loss_layer=12): vgg = load_model(self.vgg_path) self.shared_vgg = Network(inputs=vgg.input, outputs=vgg.get_layer( vgg.layers[loss_layer].name).output, name='vgg') input_layer = Input(shape=self.tgt_shape) out = self.shared_vgg(input_layer) model = Model(input_layer, out) for l in model.layers: l.trainable = False return model
def build_discriminators(self, filters=64): """ Build the discriminator network according to description in the paper. :param optimizer: Keras optimizer to use for network :param int filters: How many filters to use in first conv layer :return: the compiled model """ def conv2d_block(input, filters, strides=1, bn=True): d = Conv2D(filters, kernel_size=3, strides=strides, padding='same')(input) d = LeakyReLU(alpha=0.2)(d) if bn: d = BatchNormalization(momentum=0.8)(d) return d # Input high resolution image img = Input(shape=self.shape_hr) x = conv2d_block(img, filters, bn=False) x = conv2d_block(x, filters, strides=2) x = conv2d_block(x, filters * 2) x = conv2d_block(x, filters * 2, strides=2) x = conv2d_block(x, filters * 4) x = conv2d_block(x, filters * 4, strides=2) x = conv2d_block(x, filters * 8) x = conv2d_block(x, filters * 8, strides=2) x = Dense(filters * 16)(x) x = LeakyReLU(alpha=0.2)(x) x = Dense(1, activation='sigmoid')(x) # Create model and compile model = Model(inputs=img, outputs=x) # Build "frozen discriminator" frozen_discriminator = Network(inputs=img, outputs=x, name='frozen_discriminator') frozen_discriminator.trainable = False return model, frozen_discriminator
def gradient_penalty(D, input_merged, base_name): gradients = K.gradients(D, [input_merged])[0] # compute the euclidean norm by squaring ... gradients_sqr = K.square(gradients) # ... summing over the rows ... gradients_sqr_sum = K.sum(gradients_sqr, axis=np.arange(1, len(gradients_sqr.shape))) # ... and sqrt gradient_l2_norm = K.sqrt(gradients_sqr_sum) # compute lambda * (1 - ||grad||)^2 still for each single sample gradient_penalty = K.square(1 - gradient_l2_norm) # return the mean as loss over all the batch samples out = Network(input=[D, input_merged], output=[gradient_penalty], name=base_name + "_gp") return out
def __init__(self): # Input shape self.img_rows = 64 self.img_cols = 64 self.channels = 3 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.latent_dim = 16 # Following parameter and optimizer set as recommended in paper self.n_critic = 5 self.clip_value = 0.01 optimizer = Adam(0.0002, 0.5) # Build and compile the discriminator inp_discriminator, out_discriminator = self.build_discriminator() self.discriminator = Model(inp_discriminator, out_discriminator, name='discriminator') self.discriminator.compile(loss=self.wasserstein_loss, optimizer=optimizer, metrics=['accuracy']) # Build the frozen discriminator copy self.frozen_discriminator = Network(inp_discriminator, out_discriminator, name='frozen_discriminator') self.frozen_discriminator.trainable = False # Build the generator inp_generator, out_generator = self.build_generator() self.generator = Model(inp_generator, out_generator, name='generator') # The combined model (stacked generator and discriminator) # Trains the generator to fool the discriminator optimizer = Adam(0.0002, 0.5) self.combined = Sequential() self.combined.add(self.generator) self.combined.add(self.frozen_discriminator) self.combined.compile(loss=self.wasserstein_loss, optimizer=optimizer) self.discriminator.summary() self.generator.summary() self.combined.summary()
def build_discriminator(self, filters=8): input_layer = Input(shape=self.tgt_shape) layers = [input_layer] # Down 0 layer_name = 'DD0' #0~7 self.__add_conv_disc(layer_name, filters) self.dis_net.append(MaxPooling2D(name=layer_name + '_pool')) filters *= 2 # Down 1 layer_name = 'DD1' #8~13 self.__add_conv_disc(layer_name, filters) self.dis_net.append(MaxPooling2D(name=layer_name + '_pool')) filters *= 2 # Down 2 layer_name = 'DD2' #8~13 self.__add_conv_disc(layer_name, filters) self.dis_net.append(MaxPooling2D(name=layer_name + '_pool')) filters *= 2 # Down 3 layer_name = 'DD3' #8~13 self.__add_conv_disc(layer_name, filters) self.dis_net.append(MaxPooling2D(name=layer_name + '_pool')) self.dis_net.append(Dropout(0.25)) self.dis_net.append(BatchNormalization(momentum=0.8)) self.dis_net.append(Flatten()) self.dis_net.append(Dense(1024)) self.dis_net.append(BatchNormalization(momentum=0.8)) self.dis_net.append(LeakyReLU()) self.dis_net.append(Dense(1, activation='sigmoid')) # body first up for i, operator in enumerate(self.dis_net): layers.append(operator(layers[i])) self.shared_discriminator = Network(input=layers[0], output=layers[-1], name='discriminator') model = Model(layers[0], layers[-1]) return model
def __init__(self): # Input shape self.img_rows = 64 self.img_cols = 64 self.channels = 3 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.latent_dim = 16 self.num_classes = 17 optimizer = Adam(0.0002, 0.5) losses = ['binary_crossentropy', 'sparse_categorical_crossentropy'] # Build and compile the discriminator inp_discr, out_discr, lbl_discr = self.build_discriminator() self.discriminator = Model(inp_discr, [out_discr, lbl_discr], name='discriminator') self.discriminator.compile(loss=losses, optimizer=optimizer, metrics=['accuracy']) # Build the frozen discriminator copy self.frozen_discriminator = Network(inp_discr, [out_discr, lbl_discr], name='frozen_discriminator') self.frozen_discriminator.trainable = False # Build the generator inp_gen, lbl_gen, out_gen = self.build_generator() self.generator = Model([inp_gen, lbl_gen], out_gen, name='generator') # The combined model (stacked generator and discriminator) # Trains the generator to fool the discriminator optimizer = Adam(0.0002, 0.5) comb_valid, comb_label = self.frozen_discriminator( self.generator([inp_gen, lbl_gen])) self.combined = Model(inputs=[inp_gen, lbl_gen], outputs=[comb_valid, comb_label]) self.combined.compile(loss=losses, optimizer=optimizer) self.discriminator.summary() self.generator.summary() self.combined.summary()
def mapping_function_Unet(input_shape, base_name, num_res_blocks): initializer = TruncatedNormal(mean=0, stddev=0.2, seed=42) x = in_x = Input(shape=input_shape) # size→size//2→size//4→size//8 x = Conv2D(32, kernel_size=7, strides=1, padding="same", kernel_initializer=initializer, use_bias=False, name=base_name + "_conv1")(x) x = BatchNormalization(momentum=0.9, epsilon=1e-5, name=base_name + "_bn1")(x) conv1 = LeakyReLU(0.2)(x) x = Conv2D(64, kernel_size=3, strides=2, padding="same", kernel_initializer=initializer, use_bias=False, name=base_name + "_conv2")(conv1) x = BatchNormalization(momentum=0.9, epsilon=1e-5, name=base_name + "_bn2")(x) conv2 = LeakyReLU(0.2)(x) x = Conv2D(128, kernel_size=3, strides=2, padding="same", kernel_initializer=initializer, use_bias=False, name=base_name + "_conv3")(conv2) x = BatchNormalization(momentum=0.9, epsilon=1e-5, name=base_name + "_bn3")(x) conv3 = LeakyReLU(0.2)(x) x = conv3 for i in range(num_res_blocks): x = residual_block(x, base_name=base_name, block_num=i, initializer=initializer) x = Concatenate(axis=3)([x, conv3]) # size//8→size//4→size//2→size x = Conv2DTranspose(64, kernel_size=3, strides=2, padding='same', kernel_initializer=initializer, name=base_name + "_deconv2")(x) x = BatchNormalization(momentum=0.9, epsilon=1e-5, name=base_name + "_bn6")(x) x = Activation("relu")(x) x = Concatenate(axis=3)([x, conv2]) x = Conv2DTranspose(32, kernel_size=3, strides=2, padding='same', kernel_initializer=initializer, name=base_name + "_deconv3")(x) x = BatchNormalization(momentum=0.9, epsilon=1e-5, name=base_name + "_bn7")(x) x = Activation("relu")(x) x = Concatenate(axis=3)([x, conv1]) out = Conv2DTranspose(3, kernel_size=7, strides=1, padding='same', activation="tanh", kernel_initializer=initializer, name=base_name + "_out")(x) network = Network(in_x, out, name=base_name) return network
def train(x_target, x_ref, y_ref, epoch_num): # VGG16読み込み, S network用 print("Model build...") #mobile = VGG16(include_top=False, input_shape=input_shape, weights='imagenet') # mobile net読み込み, S network用 mobile = MobileNetV2(include_top=True, input_shape=input_shape, alpha=alpha, , weights='imagenet') #最終層削除 mobile.layers.pop() # 重みを固定 for layer in mobile.layers: if layer.name == "block_13_expand": # "block5_conv1": for VGG16 break else: layer.trainable = False model_t = Model(inputs=mobile.input,outputs=mobile.layers[-1].output) # R network用 Sと重み共有 model_r = Network(inputs=model_t.input, outputs=model_t.output, name="shared_layer") #Rに全結合層を付ける prediction = Dense(classes, activation='softmax')(model_t.output) model_r = Model(inputs=model_r.input,outputs=prediction) #コンパイル optimizer = SGD(lr=5e-5, decay=0.00005) model_r.compile(optimizer=optimizer, loss="categorical_crossentropy") model_t.compile(optimizer=optimizer, loss=original_loss) model_t.summary() model_r.summary() print("x_target is",x_target.shape[0],'samples') print("x_ref is",x_ref.shape[0],'samples') ref_samples = np.arange(x_ref.shape[0]) loss, loss_c = [], [] print("training...") #学習 for epochnumber in range(epoch_num): x_r, y_r, lc, ld = [], [], [], [] #ターゲットデータシャッフル np.random.shuffle(x_target) #リファレンスデータシャッフル np.random.shuffle(ref_samples) for i in range(len(x_target)): x_r.append(x_ref[ref_samples[i]]) y_r.append(y_ref[ref_samples[i]]) x_r = np.array(x_r) y_r = np.array(y_r) for i in range(int(len(x_target) / batchsize)): #batchsize分のデータロード batch_target = x_target[i*batchsize:i*batchsize+batchsize] batch_ref = x_r[i*batchsize:i*batchsize+batchsize] batch_y = y_r[i*batchsize:i*batchsize+batchsize] #target data #学習しながら、損失を取得 lc.append(model_t.train_on_batch(batch_target, np.zeros((batchsize, feature_out)))) #reference data #学習しながら、損失を取得 ld.append(model_r.train_on_batch(batch_ref, batch_y)) loss.append(np.mean(ld)) loss_c.append(np.mean(lc)) if (epochnumber+1) % 5 == 0: print("epoch:",epochnumber+1) print("Descriptive loss:", loss[-1]) print("Compact loss", loss_c[-1]) #結果グラフ plt.plot(loss,label="Descriptive loss") plt.xlabel("epoch") plt.legend() plt.show() plt.plot(loss_c,label="Compact loss") plt.xlabel("epoch") plt.legend() plt.show() return model_t
def __init__(self, dataset_path: str, gen_mod_name: str, disc_mod_name: str, latent_dim: int, training_progress_save_path: str, testing_dataset_path: str = None, generator_optimizer: Optimizer = Adam(0.0002, 0.5), discriminator_optimizer: Optimizer = Adam(0.0002, 0.5), discriminator_label_noise: float = None, discriminator_label_noise_decay: float = None, discriminator_label_noise_min: float = 0.001, batch_size: int = 32, buffered_batches: int = 20, generator_weights: Union[str, None] = None, discriminator_weights: Union[str, None] = None, start_episode: int = 0, load_from_checkpoint: bool = False, check_dataset: bool = True, num_of_loading_workers: int = 8): self.disc_mod_name = disc_mod_name self.gen_mod_name = gen_mod_name self.generator_optimizer = generator_optimizer self.latent_dim = latent_dim assert self.latent_dim > 0, Fore.RED + "Invalid latent dim" + Fore.RESET self.batch_size = batch_size assert self.batch_size > 0, Fore.RED + "Invalid batch size" + Fore.RESET self.discriminator_label_noise = discriminator_label_noise self.discriminator_label_noise_decay = discriminator_label_noise_decay self.discriminator_label_noise_min = discriminator_label_noise_min self.progress_image_dim = (16, 9) if start_episode < 0: start_episode = 0 self.episode_counter = start_episode # Initialize training data folder and logging self.training_progress_save_path = training_progress_save_path self.training_progress_save_path = os.path.join( self.training_progress_save_path, f"{self.gen_mod_name}__{self.disc_mod_name}") self.tensorboard = TensorBoardCustom( log_dir=os.path.join(self.training_progress_save_path, "logs")) # Create array of input image paths self.train_data = get_paths_of_files_from_path(dataset_path, only_files=True) assert self.train_data, Fore.RED + "Training dataset is not loaded" + Fore.RESET self.testing_data = None if testing_dataset_path: self.testing_data = get_paths_of_files_from_path( testing_dataset_path) assert self.testing_data, Fore.RED + "Testing dataset is not loaded" + Fore.RESET # Load one image to get shape of it tmp_image = cv.imread(self.train_data[0]) self.image_shape = tmp_image.shape self.image_channels = self.image_shape[2] # Check image size validity if self.image_shape[0] < 4 or self.image_shape[1] < 4: raise Exception("Images too small, min size (4, 4)") # Check validity of whole datasets if check_dataset: self.validate_dataset() # Define static vars if os.path.exists( f"{self.training_progress_save_path}/static_noise.npy"): self.static_noise = np.load( f"{self.training_progress_save_path}/static_noise.npy") if self.static_noise.shape[0] != (self.progress_image_dim[0] * self.progress_image_dim[1]): print(Fore.YELLOW + "Progress image dim changed, restarting static noise!" + Fore.RESET) os.remove( f"{self.training_progress_save_path}/static_noise.npy") self.static_noise = np.random.normal( 0.0, 1.0, size=(self.progress_image_dim[0] * self.progress_image_dim[1], self.latent_dim)) else: self.static_noise = np.random.normal( 0.0, 1.0, size=(self.progress_image_dim[0] * self.progress_image_dim[1], self.latent_dim)) self.kernel_initializer = RandomNormal(stddev=0.02) # Load checkpoint self.initiated = False loaded_gen_weights_path = None loaded_disc_weights_path = None if load_from_checkpoint: loaded_gen_weights_path, loaded_disc_weights_path = self.load_checkpoint( ) # Create batchmaker and start it self.batch_maker = BatchMaker( self.train_data, self.batch_size, buffered_batches=buffered_batches, num_of_loading_workers=num_of_loading_workers) self.testing_batchmaker = None if self.testing_data: self.testing_batchmaker = BatchMaker( self.testing_data, self.batch_size, buffered_batches=buffered_batches, num_of_loading_workers=num_of_loading_workers) self.testing_batchmaker.start() ################################# ### Create discriminator ### ################################# self.discriminator = self.build_discriminator(disc_mod_name) self.discriminator.compile(loss="binary_crossentropy", optimizer=discriminator_optimizer) ################################# ### Create generator ### ################################# self.generator = self.build_generator(gen_mod_name) if self.generator.output_shape[1:] != self.image_shape: raise Exception( "Invalid image input size for this generator model") ################################# ### Create combined generator ### ################################# noise_input = Input(shape=(self.latent_dim, ), name="noise_input") gen_images = self.generator(noise_input) # Create frozen version of discriminator frozen_discriminator = Network(self.discriminator.inputs, self.discriminator.outputs, name="frozen_discriminator") frozen_discriminator.trainable = False # Discriminator takes images and determinates validity valid = frozen_discriminator(gen_images) # Combine models # Train generator to fool discriminator self.combined_generator_model = Model(noise_input, valid, name="dcgan_model") self.combined_generator_model.compile( loss="binary_crossentropy", optimizer=self.generator_optimizer) # Print all summaries print("\nDiscriminator Summary:") self.discriminator.summary() print("\nGenerator Summary:") self.generator.summary() print("\nGAN Summary") self.combined_generator_model.summary() # Load weights from checkpoint try: if loaded_gen_weights_path: self.generator.load_weights(loaded_gen_weights_path) except: print(Fore.YELLOW + "Failed to load generator weights from checkpoint" + Fore.RESET) try: if loaded_disc_weights_path: self.discriminator.load_weights(loaded_disc_weights_path) except: print(Fore.YELLOW + "Failed to load discriminator weights from checkpoint" + Fore.RESET) # Load weights from param and override checkpoint weights if generator_weights: self.generator.load_weights(generator_weights) if discriminator_weights: self.discriminator.load_weights(discriminator_weights)
def I3D(input_shape=(20, 224, 224, 3), nb_class=1, name='I3D'): # Determine proper input shape input_shape = _obtain_input_shape(input_shape, default_frame_size=224, min_frame_size=32, default_num_frames=64, min_num_frames=8, data_format=K.image_data_format(), require_flatten=False, weights=None) img_input = Input(shape=input_shape) if K.image_data_format() == 'channels_first': channel_axis = 1 else: channel_axis = 4 # Downsampling via convolution (spatial and temporal) x = conv3d_bn(img_input, 64, 7, 7, 7, strides=(2, 2, 2), padding='same', name='Conv3d_1a_7x7') # Downsampling (spatial only) x = MaxPooling3D((1, 3, 3), strides=(1, 2, 2), padding='same', name='MaxPool2d_2a_3x3')(x) x = conv3d_bn(x, 64, 1, 1, 1, strides=(1, 1, 1), padding='same', name='Conv3d_2b_1x1') x = conv3d_bn(x, 192, 3, 3, 3, strides=(1, 1, 1), padding='same', name='Conv3d_2c_3x3') # Downsampling (spatial only) x = MaxPooling3D((1, 3, 3), strides=(1, 2, 2), padding='same', name='MaxPool2d_3a_3x3')(x) # Mixed 3b branch_0 = conv3d_bn(x, 64, 1, 1, 1, padding='same', name='Conv3d_3b_0a_1x1') branch_1 = conv3d_bn(x, 96, 1, 1, 1, padding='same', name='Conv3d_3b_1a_1x1') branch_1 = conv3d_bn(branch_1, 128, 3, 3, 3, padding='same', name='Conv3d_3b_1b_3x3') branch_2 = conv3d_bn(x, 16, 1, 1, 1, padding='same', name='Conv3d_3b_2a_1x1') branch_2 = conv3d_bn(branch_2, 32, 3, 3, 3, padding='same', name='Conv3d_3b_2b_3x3') branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_3b_3a_3x3')(x) branch_3 = conv3d_bn(branch_3, 32, 1, 1, 1, padding='same', name='Conv3d_3b_3b_1x1') x = layers.concatenate([branch_0, branch_1, branch_2, branch_3], axis=channel_axis, name='Mixed_3b') # Mixed 3c branch_0 = conv3d_bn(x, 128, 1, 1, 1, padding='same', name='Conv3d_3c_0a_1x1') branch_1 = conv3d_bn(x, 128, 1, 1, 1, padding='same', name='Conv3d_3c_1a_1x1') branch_1 = conv3d_bn(branch_1, 192, 3, 3, 3, padding='same', name='Conv3d_3c_1b_3x3') branch_2 = conv3d_bn(x, 32, 1, 1, 1, padding='same', name='Conv3d_3c_2a_1x1') branch_2 = conv3d_bn(branch_2, 96, 3, 3, 3, padding='same', name='Conv3d_3c_2b_3x3') branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_3c_3a_3x3')(x) branch_3 = conv3d_bn(branch_3, 64, 1, 1, 1, padding='same', name='Conv3d_3c_3b_1x1') x = layers.concatenate([branch_0, branch_1, branch_2, branch_3], axis=channel_axis, name='Mixed_3c') # Downsampling (spatial and temporal) x = MaxPooling3D((3, 3, 3), strides=(2, 2, 2), padding='same', name='MaxPool2d_4a_3x3')(x) # Mixed 4b branch_0 = conv3d_bn(x, 192, 1, 1, 1, padding='same', name='Conv3d_4b_0a_1x1') branch_1 = conv3d_bn(x, 96, 1, 1, 1, padding='same', name='Conv3d_4b_1a_1x1') branch_1 = conv3d_bn(branch_1, 208, 3, 3, 3, padding='same', name='Conv3d_4b_1b_3x3') branch_2 = conv3d_bn(x, 16, 1, 1, 1, padding='same', name='Conv3d_4b_2a_1x1') branch_2 = conv3d_bn(branch_2, 48, 3, 3, 3, padding='same', name='Conv3d_4b_2b_3x3') branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_4b_3a_3x3')(x) branch_3 = conv3d_bn(branch_3, 64, 1, 1, 1, padding='same', name='Conv3d_4b_3b_1x1') x = layers.concatenate([branch_0, branch_1, branch_2, branch_3], axis=channel_axis, name='Mixed_4b') # Mixed 4c branch_0 = conv3d_bn(x, 160, 1, 1, 1, padding='same', name='Conv3d_4c_0a_1x1') branch_1 = conv3d_bn(x, 112, 1, 1, 1, padding='same', name='Conv3d_4c_1a_1x1') branch_1 = conv3d_bn(branch_1, 224, 3, 3, 3, padding='same', name='Conv3d_4c_1b_3x3') branch_2 = conv3d_bn(x, 24, 1, 1, 1, padding='same', name='Conv3d_4c_2a_1x1') branch_2 = conv3d_bn(branch_2, 64, 3, 3, 3, padding='same', name='Conv3d_4c_2b_3x3') branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_4c_3a_3x3')(x) branch_3 = conv3d_bn(branch_3, 64, 1, 1, 1, padding='same', name='Conv3d_4c_3b_1x1') x = layers.concatenate([branch_0, branch_1, branch_2, branch_3], axis=channel_axis, name='Mixed_4c') # Mixed 4d branch_0 = conv3d_bn(x, 128, 1, 1, 1, padding='same', name='Conv3d_4d_0a_1x1') branch_1 = conv3d_bn(x, 128, 1, 1, 1, padding='same', name='Conv3d_4d_1a_1x1') branch_1 = conv3d_bn(branch_1, 256, 3, 3, 3, padding='same', name='Conv3d_4d_1b_3x3') branch_2 = conv3d_bn(x, 24, 1, 1, 1, padding='same', name='Conv3d_4d_2a_1x1') branch_2 = conv3d_bn(branch_2, 64, 3, 3, 3, padding='same', name='Conv3d_4d_2b_3x3') branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_4d_3a_3x3')(x) branch_3 = conv3d_bn(branch_3, 64, 1, 1, 1, padding='same', name='Conv3d_4d_3b_1x1') x = layers.concatenate([branch_0, branch_1, branch_2, branch_3], axis=channel_axis, name='Mixed_4d') # Mixed 4e branch_0 = conv3d_bn(x, 112, 1, 1, 1, padding='same', name='Conv3d_4e_0a_1x1') branch_1 = conv3d_bn(x, 144, 1, 1, 1, padding='same', name='Conv3d_4e_1a_1x1') branch_1 = conv3d_bn(branch_1, 288, 3, 3, 3, padding='same', name='Conv3d_4e_1b_3x3') branch_2 = conv3d_bn(x, 32, 1, 1, 1, padding='same', name='Conv3d_4e_2a_1x1') branch_2 = conv3d_bn(branch_2, 64, 3, 3, 3, padding='same', name='Conv3d_4e_2b_3x3') branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_4e_3a_3x3')(x) branch_3 = conv3d_bn(branch_3, 64, 1, 1, 1, padding='same', name='Conv3d_4e_3b_1x1') x = layers.concatenate([branch_0, branch_1, branch_2, branch_3], axis=channel_axis, name='Mixed_4e') # Mixed 4f branch_0 = conv3d_bn(x, 256, 1, 1, 1, padding='same', name='Conv3d_4f_0a_1x1') branch_1 = conv3d_bn(x, 160, 1, 1, 1, padding='same', name='Conv3d_4f_1a_1x1') branch_1 = conv3d_bn(branch_1, 320, 3, 3, 3, padding='same', name='Conv3d_4f_1b_3x3') branch_2 = conv3d_bn(x, 32, 1, 1, 1, padding='same', name='Conv3d_4f_2a_1x1') branch_2 = conv3d_bn(branch_2, 128, 3, 3, 3, padding='same', name='Conv3d_4f_2b_3x3') branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_4f_3a_3x3')(x) branch_3 = conv3d_bn(branch_3, 128, 1, 1, 1, padding='same', name='Conv3d_4f_3b_1x1') x = layers.concatenate([branch_0, branch_1, branch_2, branch_3], axis=channel_axis, name='Mixed_4f') # Downsampling (spatial and temporal) x = MaxPooling3D((2, 2, 2), strides=(2, 2, 2), padding='same', name='MaxPool2d_5a_2x2')(x) # Mixed 5b branch_0 = conv3d_bn(x, 256, 1, 1, 1, padding='same', name='Conv3d_5b_0a_1x1') branch_1 = conv3d_bn(x, 160, 1, 1, 1, padding='same', name='Conv3d_5b_1a_1x1') branch_1 = conv3d_bn(branch_1, 320, 3, 3, 3, padding='same', name='Conv3d_5b_1b_3x3') branch_2 = conv3d_bn(x, 32, 1, 1, 1, padding='same', name='Conv3d_5b_2a_1x1') branch_2 = conv3d_bn(branch_2, 128, 3, 3, 3, padding='same', name='Conv3d_5b_2b_3x3') branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_5b_3a_3x3')(x) branch_3 = conv3d_bn(branch_3, 128, 1, 1, 1, padding='same', name='Conv3d_5b_3b_1x1') x = layers.concatenate([branch_0, branch_1, branch_2, branch_3], axis=channel_axis, name='Mixed_5b') # Mixed 5c branch_0 = conv3d_bn(x, 384, 1, 1, 1, padding='same', name='Conv3d_5c_0a_1x1') branch_1 = conv3d_bn(x, 192, 1, 1, 1, padding='same', name='Conv3d_5c_1a_1x1') branch_1 = conv3d_bn(branch_1, 384, 3, 3, 3, padding='same', name='Conv3d_5c_1b_3x3') branch_2 = conv3d_bn(x, 48, 1, 1, 1, padding='same', name='Conv3d_5c_2a_1x1') branch_2 = conv3d_bn(branch_2, 128, 3, 3, 3, padding='same', name='Conv3d_5c_2b_3x3') branch_3 = MaxPooling3D((3, 3, 3), strides=(1, 1, 1), padding='same', name='MaxPool2d_5c_3a_3x3')(x) branch_3 = conv3d_bn(branch_3, 128, 1, 1, 1, padding='same', name='Conv3d_5c_3b_1x1') x = layers.concatenate([branch_0, branch_1, branch_2, branch_3], axis=channel_axis, name='Mixed_5c') h = int(x.shape[2]) w = int(x.shape[3]) x = AveragePooling3D((2, h, w), strides=(1, 1, 1), padding='valid', name='global_avg_pool')(x) #x = conv3d_bn(x, classes, 1, 1, 1, padding='same', name='Conv3d_6a_1x1_1') # create model #model = Model([input1, input2], [output1, output2], name='i3d_inception') return Network(img_input, x, name=name)
def build_generator(self, filters=128): input_layer = Input(shape=self.input_shape) layers = [input_layer] # define operators layer_name = 'G_Head' # 0~2 self.gen_net.append( Conv2D(filters, (3, 3), padding='same', name=layer_name + '_conv0', activation='relu')) self.gen_net.append( BatchNormalization(momentum=0.8, name=layer_name + '_norm0')) self.gen_net.append(LeakyReLU(name=layer_name + '_act0')) layer_name = 'G_Body_0' # 3~8 self.__add_conv_gen(layer_name, filters) layer_name = 'G_Body_1' # 9~14 self.__add_conv_gen(layer_name, filters) layer_name = "G_Up_1" # 15 self.gen_net.append( UpSampling2D(size=(2, 2), name=layer_name + '_upsamp')) layer_name = 'G_Body_3' # 16~21 self.__add_conv_gen(layer_name, filters) layer_name = "G_Tail" # 22~23 self.gen_net.append( BatchNormalization(momentum=0.8, name=layer_name + '_norm0')) self.gen_net.append( Conv2D(filters=3, kernel_size=(3, 3), padding='same', name='outRGB', activation="sigmoid")) # build network # head num = len(layers) for i, operator in enumerate(self.gen_net[:3]): layers.append(operator(layers[i + num - 1])) head_out = layers[-1] # body first 0 num = len(layers) for i, operator in enumerate(self.gen_net[3:9]): layers.append(operator(layers[i + num - 1])) layers.append(Add()([layers[-1], head_out])) body_0_out = layers[-1] # body first 1 num = len(layers) for i, operator in enumerate(self.gen_net[9:15]): layers.append(operator(layers[i + num - 1])) layers.append(Add()([layers[-1], body_0_out])) body_1_out = layers[-1] layers.append(Add()([head_out, body_1_out])) # up layers.append(self.gen_net[15](layers[-1])) # tail prior_tail = layers[-1] num = len(layers) for i, operator in enumerate(self.gen_net[16:22]): layers.append(operator(layers[i + num - 1])) layers.append(Add()([layers[-1], prior_tail])) layers.append(self.gen_net[-2](layers[-1])) # batch_norm layers.append(self.gen_net[-1](layers[-1])) # rgb self.shared_generator = Network(input=layers[0], output=layers[-1], name='generator') return Model(layers[0], layers[-1])
def train(x_target, x_ref, y_ref, epoch_num): #Read VGG16, S network print("Model build...") #mobile = VGG16(include_top=False, input_shape=input_shape, weights='imagenet') #Read mobile net, S network mobile = MobileNetV2(include_top=False, input_shape=input_shape, alpha=alpha, depth_multiplier=1, weights='imagenet') #Delete last layer mobile.layers.pop() #Fixed weight for layer in mobile.layers: if layer.name == "block_13_expand": # "block5_conv1": for VGG16 break else: layer.trainable = False model_t = Model(inputs=mobile.input, outputs=mobile.layers[-1].output) #R network S and Weight sharing model_r = Network(inputs=model_t.input, outputs=model_t.output, name="shared_layer") #Apply a Fully Connected Layer to R prediction = Dense(classes, activation='softmax')(model_t.output) model_r = Model(inputs=model_r.input, outputs=prediction) #Compile optimizer = SGD(lr=5e-5, decay=0.00005) model_r.compile(optimizer=optimizer, loss="categorical_crossentropy") model_t.compile(optimizer=optimizer, loss=original_loss) model_t.summary() model_r.summary() print("x_target is", x_target.shape[0], 'samples') print("x_ref is", x_ref.shape[0], 'samples') ref_samples = np.arange(x_ref.shape[0]) loss, loss_c = [], [] print("training...") #Learning for epochnumber in range(epoch_num): x_r, y_r, lc, ld = [], [], [], [] #Shuffle target data np.random.shuffle(x_target) #Shuffle reference data np.random.shuffle(ref_samples) for i in range(len(x_target)): x_r.append(x_ref[ref_samples[i]]) y_r.append(y_ref[ref_samples[i]]) x_r = np.array(x_r) y_r = np.array(y_r) for i in range(int(len(x_target) / batchsize)): #Load data for batch size batch_target = x_target[i * batchsize:i * batchsize + batchsize] batch_ref = x_r[i * batchsize:i * batchsize + batchsize] batch_y = y_r[i * batchsize:i * batchsize + batchsize] #target data #Get loss while learning lc.append( model_t.train_on_batch(batch_target, np.zeros((batchsize, feature_out)))) #reference data #Get loss while learning ld.append(model_r.train_on_batch(batch_ref, batch_y)) loss.append(np.mean(ld)) loss_c.append(np.mean(lc)) if (epochnumber + 1) % 5 == 0: print("epoch:", epochnumber + 1) print("Descriptive loss:", loss[-1]) print("Compact loss", loss_c[-1]) #Result graph plt.plot(loss, label="Descriptive loss") plt.xlabel("epoch") plt.legend() plt.show() plt.plot(loss_c, label="Compact loss") plt.xlabel("epoch") plt.legend() plt.show() return model_t
def C3D(input_shape=(20, 224, 224, 3), nb_class=1, activation='tanh'): input_ = Input(shape=input_shape) x = Conv3D(64, (3, 3, 3), activation='relu', border_mode='same', subsample=(1, 1, 1), input_shape=input_shape)(input_) x = MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2), border_mode='valid')(x) # 2nd layer group x = Conv3D(128, (3, 3, 3), activation='relu', border_mode='same', subsample=(1, 1, 1))(x) x = MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), border_mode='valid')(x) # 3rd layer group x = Conv3D(256, (3, 3, 3), activation='relu', border_mode='same', subsample=(1, 1, 1))(x) x = Conv3D(256, (3, 3, 3), activation='relu', border_mode='same', subsample=(1, 1, 1))(x) x = MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), border_mode='valid')(x) # 4th layer group x = Conv3D(512, (3, 3, 3), activation='relu', border_mode='same', subsample=(1, 1, 1))(x) x = Conv3D(512, (3, 3, 3), activation='relu', border_mode='same', subsample=(1, 1, 1))(x) x = MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), border_mode='valid')(x) # 5th layer group x = Conv3D(512, (3, 3, 3), activation='relu', border_mode='same', subsample=(1, 1, 1))(x) x = Conv3D(512, (3, 3, 3), activation='relu', border_mode='same', subsample=(1, 1, 1))(x) x = ZeroPadding3D(padding=(0, 1, 1))(x) x = MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), border_mode='valid')(x) x = Flatten()(x) # FC layers group x = Dense(4096, activation='relu')(x) x = Dropout(0.5)(x) x = Dense(4096, activation='relu')(x) x = Dropout(0.5)(x) output_ = Dense(nb_class, activation=activation)(x) return Network(input_, output_, name='C3D')
def main(overwrite=False): # convert input images into an hdf5 file if overwrite or not os.path.exists(config["data_file"]): create_data_file(config) data_file_opened = open_data_file(config["data_file"]) seg_loss_func = getattr(fetal_net.metrics, config['loss']) dis_loss_func = getattr(fetal_net.metrics, config['dis_loss']) # instantiate new model seg_model_func = getattr(fetal_net.model, config['model_name']) gen_model = seg_model_func( input_shape=config["input_shape"], initial_learning_rate=config["initial_learning_rate"], **{ 'dropout_rate': config['dropout_rate'], 'loss_function': seg_loss_func, 'mask_shape': None if config["weight_mask"] is None else config["input_shape"], 'old_model_path': config['old_model'] }) dis_model_func = getattr(fetal_net.model, config['dis_model_name']) dis_model = dis_model_func( input_shape=[config["input_shape"][0] + config["n_labels"]] + config["input_shape"][1:], initial_learning_rate=config["initial_learning_rate"], **{ 'dropout_rate': config['dropout_rate'], 'loss_function': dis_loss_func }) if not overwrite \ and len(glob.glob(config["model_file"] + 'g_*.h5')) > 0: # dis_model_path = get_last_model_path(config["model_file"] + 'dis_') gen_model_path = get_last_model_path(config["model_file"] + 'g_') # print('Loading dis model from: {}'.format(dis_model_path)) print('Loading gen model from: {}'.format(gen_model_path)) # dis_model = load_old_model(dis_model_path) # gen_model = load_old_model(gen_model_path) # dis_model.load_weights(dis_model_path) gen_model.load_weights(gen_model_path) gen_model.summary() dis_model.summary() # Build "frozen discriminator" frozen_dis_model = Network(dis_model.inputs, dis_model.outputs, name='frozen_discriminator') frozen_dis_model.trainable = False inputs_real = Input(shape=config["input_shape"]) inputs_fake = Input(shape=config["input_shape"]) segs_real = Activation(None, name='seg_real')(gen_model(inputs_real)) segs_fake = Activation(None, name='seg_fake')(gen_model(inputs_fake)) valid = Activation(None, name='dis')(frozen_dis_model( Concatenate(axis=1)([segs_fake, inputs_fake]))) combined_model = Model(inputs=[inputs_real, inputs_fake], outputs=[segs_real, valid]) combined_model.compile(loss=[seg_loss_func, 'binary_crossentropy'], loss_weights=[1, config["gd_loss_ratio"]], optimizer=Adam(config["initial_learning_rate"])) combined_model.summary() # get training and testing generators train_generator, validation_generator, n_train_steps, n_validation_steps = get_training_and_validation_generators( data_file_opened, batch_size=config["batch_size"], data_split=config["validation_split"], overwrite=overwrite, validation_keys_file=config["validation_file"], training_keys_file=config["training_file"], test_keys_file=config["test_file"], n_labels=config["n_labels"], labels=config["labels"], patch_shape=(*config["patch_shape"], config["patch_depth"]), validation_batch_size=config["validation_batch_size"], augment=config["augment"], skip_blank_train=config["skip_blank_train"], skip_blank_val=config["skip_blank_val"], truth_index=config["truth_index"], truth_size=config["truth_size"], prev_truth_index=config["prev_truth_index"], prev_truth_size=config["prev_truth_size"], truth_downsample=config["truth_downsample"], truth_crop=config["truth_crop"], patches_per_epoch=config["patches_per_epoch"], categorical=config["categorical"], is3d=config["3D"], drop_easy_patches_train=config["drop_easy_patches_train"], drop_easy_patches_val=config["drop_easy_patches_val"]) # get training and testing generators _, semi_generator, _, _ = get_training_and_validation_generators( data_file_opened, batch_size=config["batch_size"], data_split=config["validation_split"], overwrite=overwrite, validation_keys_file=config["validation_file"], training_keys_file=config["training_file"], test_keys_file=config["test_file"], n_labels=config["n_labels"], labels=config["labels"], patch_shape=(*config["patch_shape"], config["patch_depth"]), validation_batch_size=config["validation_batch_size"], val_augment=config["augment"], skip_blank_train=config["skip_blank_train"], skip_blank_val=config["skip_blank_val"], truth_index=config["truth_index"], truth_size=config["truth_size"], prev_truth_index=config["prev_truth_index"], prev_truth_size=config["prev_truth_size"], truth_downsample=config["truth_downsample"], truth_crop=config["truth_crop"], patches_per_epoch=config["patches_per_epoch"], categorical=config["categorical"], is3d=config["3D"], drop_easy_patches_train=config["drop_easy_patches_train"], drop_easy_patches_val=config["drop_easy_patches_val"]) # start training scheduler = Scheduler(config["dis_steps"], config["gen_steps"], init_lr=config["initial_learning_rate"], lr_patience=config["patience"], lr_decay=config["learning_rate_drop"]) best_loss = np.inf for epoch in range(config["n_epochs"]): postfix = {'g': None, 'd': None} # , 'val_g': None, 'val_d': None} with tqdm(range(n_train_steps // config["gen_steps"]), dynamic_ncols=True, postfix={ 'gen': None, 'dis': None, 'val_gen': None, 'val_dis': None, None: None }) as pbar: for n_round in pbar: # train D outputs = np.zeros(dis_model.metrics_names.__len__()) for i in range(scheduler.get_dsteps()): real_patches, real_segs = next(train_generator) semi_patches, _ = next(semi_generator) d_x_batch, d_y_batch = input2discriminator( real_patches, real_segs, semi_patches, gen_model.predict(semi_patches, batch_size=config["batch_size"]), dis_model.output_shape) outputs += dis_model.train_on_batch(d_x_batch, d_y_batch) if scheduler.get_dsteps(): outputs /= scheduler.get_dsteps() postfix['d'] = build_dsc(dis_model.metrics_names, outputs) pbar.set_postfix(**postfix) # train G (freeze discriminator) outputs = np.zeros(combined_model.metrics_names.__len__()) for i in range(scheduler.get_gsteps()): real_patches, real_segs = next(train_generator) semi_patches, _ = next(validation_generator) g_x_batch, g_y_batch = input2gan(real_patches, real_segs, semi_patches, dis_model.output_shape) outputs += combined_model.train_on_batch( g_x_batch, g_y_batch) outputs /= scheduler.get_gsteps() postfix['g'] = build_dsc(combined_model.metrics_names, outputs) pbar.set_postfix(**postfix) # evaluate on validation set dis_metrics = np.zeros(dis_model.metrics_names.__len__(), dtype=float) gen_metrics = np.zeros(gen_model.metrics_names.__len__(), dtype=float) evaluation_rounds = n_validation_steps for n_round in range(evaluation_rounds): # rounds_for_evaluation: val_patches, val_segs = next(validation_generator) # D if scheduler.get_dsteps() > 0: d_x_test, d_y_test = input2discriminator( val_patches, val_segs, val_patches, gen_model.predict( val_patches, batch_size=config["validation_batch_size"]), dis_model.output_shape) dis_metrics += dis_model.evaluate( d_x_test, d_y_test, batch_size=config["validation_batch_size"], verbose=0) # G # gen_x_test, gen_y_test = input2gan(val_patches, val_segs, dis_model.output_shape) gen_metrics += gen_model.evaluate( val_patches, val_segs, batch_size=config["validation_batch_size"], verbose=0) dis_metrics /= float(evaluation_rounds) gen_metrics /= float(evaluation_rounds) # save the model and weights with the best validation loss if gen_metrics[0] < best_loss: best_loss = gen_metrics[0] print('Saving Model...') with open( os.path.join( config["base_dir"], "g_{}_{:.3f}.json".format(epoch, gen_metrics[0])), 'w') as f: f.write(gen_model.to_json()) gen_model.save_weights( os.path.join( config["base_dir"], "g_{}_{:.3f}.h5".format(epoch, gen_metrics[0]))) postfix['val_d'] = build_dsc(dis_model.metrics_names, dis_metrics) postfix['val_g'] = build_dsc(gen_model.metrics_names, gen_metrics) # pbar.set_postfix(**postfix) print('val_d: ' + postfix['val_d'], end=' | ') print('val_g: ' + postfix['val_g']) # pbar.refresh() # update step sizes, learning rates scheduler.update_steps(epoch, gen_metrics[0]) K.set_value(dis_model.optimizer.lr, scheduler.get_lr()) K.set_value(combined_model.optimizer.lr, scheduler.get_lr()) data_file_opened.close()
def train(x_target, x_ref, y_ref, epoch_num): print("Model build...") # mobile = VGG16(include_top=False, input_shape=input_shape, weights='imagenet', pooling= 'avg') mobile = InceptionResNetV2(include_top=False, input_shape=input_shape, weights='imagenet') # mobile = MobileNetV2(include_top=True, input_shape=input_shape, alpha=alpha, # weights='imagenet') mobile.layers.pop() ''' Last layer : - block_13_expand : Mobilenet v2 - block5_conv1 : VGG16 - mixed_7a : Inception resnetv2 ''' # for layer in mobile.layers: # print(layer.name) # if layer.name == "block_13_expand": # "block5_conv1": for VGG16 # if layer.name == "block5_conv1": # if layer.name == "mixed_7a": # break # else: # layer.trainable = False # exit() flat = GlobalAveragePooling2D()(mobile.layers[-1].output) model_t = Model(inputs=mobile.input, outputs=flat) model_r = Network(inputs=model_t.input, outputs=model_t.output, name="shared_layer") prediction = Dense(classes, activation='softmax')(model_t.output) model_r = Model(inputs=model_r.input, outputs=prediction) optimizer = SGD(lr=5e-5, decay=0.00005) model_r.compile(optimizer=optimizer, loss="categorical_crossentropy") model_t.compile(optimizer=optimizer, loss=original_loss) model_t.summary() ref_samples = np.arange(x_ref.shape[0]) loss, loss_c = [], [] epochs = [] print("training...") for epochnumber in range(epoch_num): x_r, y_r, lc, ld = [], [], [], [] np.random.shuffle(x_target) np.random.shuffle(ref_samples) for i in range(len(x_target)): x_r.append(x_ref[ref_samples[i]]) y_r.append(y_ref[ref_samples[i]]) x_r = np.array(x_r) y_r = np.array(y_r) for i in range(int(len(x_target) / batchsize)): batch_target = x_target[i * batchsize:i * batchsize + batchsize] batch_ref = x_r[i * batchsize:i * batchsize + batchsize] batch_y = y_r[i * batchsize:i * batchsize + batchsize] #target data lc.append( model_t.train_on_batch(batch_target, np.zeros((batchsize, feature_out)))) #reference data ld.append(model_r.train_on_batch(batch_ref, batch_y)) loss.append(np.mean(ld)) loss_c.append(np.mean(lc)) epochs.append(epochnumber) print("epoch : {} ,Descriptive loss : {}, Compact loss : {}".format( epochnumber + 1, loss[-1], loss_c[-1])) if epochnumber % 10 == 0: model_t.save_weights('model/model_t_smd_{}.h5'.format(epochnumber)) model_r.save_weights('model/model_r_smd_{}.h5'.format(epochnumber))
discriminator_input, discriminator_output = build_discriminator( image_shape, dropout_rate) discriminator = Model(discriminator_input, discriminator_output, name='discriminator') discriminator.compile(optimizer, loss='binary_crossentropy', metrics=['accuracy']) assert (len(discriminator._collected_trainable_weights) > 0) frozen_discriminator = Network(discriminator_input, discriminator_output, name='frozen_discriminator') frozen_discriminator.trainable = False generator_input, generator_output = build_generator() generator = Model(generator_input, generator_output, name='generator') adversarial_model = Model(generator_input, frozen_discriminator(generator_output), name='adversarial_model') adversarial_model.compile(optimizer, loss='binary_crossentropy') assert (len(adversarial_model._collected_trainable_weights) == len( generator.trainable_weights)) batch_size = 64
def train(s_data, s_out, nuc, bestwight, samplesize, epoch_num): batch_size = 1024 num_classes_org = 1024 num_classes = 63 shape1 = (None, DATA_LENGTH, 2) model = cnn_network_keras.build_network(shape=shape1, num_classes=num_classes_org) model.load_weights(bestwight) model.layers.pop() # remove last layer model.layers.pop() # remove last layer model.layers.pop() # remove last layer for layer in model.layers: if layer.name == "conv1d_20": break else: layer.trainable = False flat = GlobalAveragePooling1D()(model.layers[-1].output) model_t = Model(inputs=model.input, outputs=flat) model_r = Network(inputs=model_t.input, outputs=model_t.output, name="shared_layer") prediction = Dense(num_classes, activation='softmax')(model_t.output) model_r = Model(inputs=model_r.input, outputs=prediction) optimizer = SGD(lr=5e-5, decay=0.00005) model_r.compile(optimizer=optimizer, loss="categorical_crossentropy") model_t.compile(optimizer=optimizer, loss=original_loss) x_ref, test_x_r, y_ref, test_y_r, num_classes_r = prepDataNear( s_data, samplesize, nuc) x_target, test_x, train_y, test_y, num_classes = prepData( s_data, samplesize, nuc) ref_samples = np.arange(x_ref.shape[0]) loss, loss_c = [], [] epochs = [] print("training...") for epochnumber in range(epoch_num): x_r, y_r, lc, ld = [], [], [], [] np.random.shuffle(x_target) np.random.shuffle(ref_samples) for i in range(len(x_target)): x_r.append(x_ref[ref_samples[i]]) y_r.append(y_ref[ref_samples[i]]) x_r = np.array(x_r) y_r = np.array(y_r) for i in range(int(len(x_target) / batchsize)): batch_target = x_target[i * batchsize:i * batchsize + batchsize] batch_ref = x_r[i * batchsize:i * batchsize + batchsize] batch_y = y_r[i * batchsize:i * batchsize + batchsize] # target data lc.append( model_t.train_on_batch(batch_target, np.zeros((batchsize, feature_out)))) # reference data ld.append(model_r.train_on_batch(batch_ref, batch_y)) loss.append(np.mean(ld)) loss_c.append(np.mean(lc)) epochs.append(epochnumber) print("epoch : {} ,Descriptive loss : {}, Compact loss : {}".format( epochnumber + 1, loss[-1], loss_c[-1])) if epochnumber % 1 == 0: model_t.save_weights(s_out + '/' + nuc + '/model_t_ep_{}.h5'.format(epochnumber)) model_r.save_weights(s_out + '/' + nuc + '/model_r_ep_{}.h5'.format(epochnumber))
def train_test(result_df, df_t, df_r, df_test): # data x_target = np.array(df_t.vecs.to_list()) x_ref = np.array(df_r.vecs.to_list()) y_ref = np.array(df_r.target.to_list()) y_ref = to_categorical(y_ref) test_vecs = np.array(df_test.vecs.to_list()) n_sup = 10000 n_per_targ = 1000 df_r_temp = df_r.groupby('target', group_keys=False).apply( lambda df: df.sample(n=min(df.shape[0], n_per_targ), random_state=42)) x_tr = np.array(df_t.head(n_sup).append(df_r_temp).vecs.to_list()) y_tr = np.array(df_t.head(n_sup).append(df_r_temp).label.to_list()) #y_tr = to_categorical(y_tr) #print(f"{df.where(df.label == 0).dropna().target.value_counts()}") #print(f"x_target: {x_target.shape}\nx_ref: {x_ref.shape}\ny_ref: {y_ref.shape}\n") res_path = "/home/philipp/projects/dad4td/reports/one_class/all.tsv" classes = df_r.target.unique().shape[0] print(f"classes: {classes}") batchsize = 128 epoch_num = 15 epoch_report = 5 feature_out = 64 pred_mode = "nn" # get the loss for compactness original_loss = create_loss(classes, batchsize) # model creation model = create_model(loss="binary_crossentropy", n_in=x_target[0].shape[0]) model_t = Model(inputs=model.input, outputs=model.output) model_r = Network(inputs=model_t.input, outputs=model_t.output, name="shared_layer") prediction = Dense(classes, activation='softmax')(model_t.output) model_r = Model(inputs=model_r.input, outputs=prediction) #latent_t = Dense(2, activation='relu')(model_t.output) #model_t = Model(inputs=model_t.input,outputs=latent_t) prediction_t = Dense(feature_out, activation='softmax')(model_t.output) model_t = Model(inputs=model_t.input, outputs=prediction_t) #optimizer = SGD(lr=5e-5, decay=0.00005) optimizer = Adam(learning_rate=5e-5) model_r.compile(optimizer=optimizer, loss="categorical_crossentropy") model_t.compile(optimizer=optimizer, loss=original_loss) model_t.summary() model_r.summary() ref_samples = np.arange(x_ref.shape[0]) loss, loss_c = [], [] epochs = [] best_acc = 0 print("training...") for epochnumber in range(epoch_num): x_r, y_r, lc, ld = [], [], [], [] np.random.shuffle(x_target) np.random.shuffle(ref_samples) for i in range(len(x_ref)): x_r.append(x_ref[ref_samples[i]]) y_r.append(y_ref[ref_samples[i]]) x_r = np.array(x_r) y_r = np.array(y_r) for i in range(int(len(x_target) / batchsize)): batch_target = x_target[i * batchsize:i * batchsize + batchsize] batch_ref = x_r[i * batchsize:i * batchsize + batchsize] batch_y = y_r[i * batchsize:i * batchsize + batchsize] # target data lc.append( model_t.train_on_batch(batch_target, np.zeros((batchsize, feature_out)))) # reference data ld.append(model_r.train_on_batch(batch_ref, batch_y)) loss.append(np.mean(ld)) loss_c.append(np.mean(lc)) epochs.append(epochnumber) if epochnumber % epoch_report == 0 or epochnumber == epoch_num - 1: print( f"-----\n\nepoch : {epochnumber+1} ,Descriptive loss : {loss[-1]}, Compact loss : {loss_c[-1]}" ) model_t.save_weights( '/home/philipp/projects/dad4td/models/one_class/model_t_smd_{}.h5' .format(epochnumber)) model_r.save_weights( '/home/philipp/projects/dad4td/models/one_class/model_r_smd_{}.h5' .format(epochnumber)) #test_b = model_t.predict(test_vecs) #od = OCSVM() # od.fit(test_b) #decision_scores = od.labels_ # decision_scores = decision_scores.astype(float) labels = df_test["label"].astype(int).values # threshold = 0.5 # scores = get_scores(dict(),labels, np.where(decision_scores > threshold, 0, 1), outlabel=0) if pred_mode == "svm": x_tr_pred = model_t.predict(x_tr) clf = SVC() clf.fit(x_tr_pred, y_tr) preds = model_t.predict(test_vecs) preds = clf.predict(preds) elif pred_mode == "nn": y_tr = y_tr.astype(int) print(y_tr) x_tr_pred = model_t.predict(x_tr) clf = create_sup_model(n_in=feature_out) clf.summary() clf.fit(x_tr_pred, y=y_tr, epochs=15, batch_size=64, verbose=True) decision_scores = model_t.predict(test_vecs) decision_scores = clf.predict(decision_scores) preds = decision_scores.astype(float) _ = plt.hist(preds, bins=10) plt.show() else: raise Exception(f"{pred_mode} must be one of svm, nn, osvm") scores = get_scores(dict(), labels, preds, outlabel=0) print(f"\n\nTest scores:\n{pd.DataFrame([scores], index=[0])}") if scores["accuracy"] > best_acc: best_acc = scores["accuracy"] print(f"best_acc updated to: {best_acc}") normalize = "true" print(f"{confusion_matrix(labels, preds, normalize=normalize)}") result_df = result_df.append(dict(cclass=list(df_test.target.unique()), accuracy=best_acc), ignore_index=True) result_df.to_csv(res_path, sep="\t") return result_df
def build_model(): frames = None s_size = None # prepare shared layers dummy_frame_inputs = Input(shape=((9, s_size, s_size, 3))) shared_layer_h = Network(dummy_frame_inputs, conv3D_branch(dummy_frame_inputs)) shared_layer_v = Network(dummy_frame_inputs, conv3D_branch(dummy_frame_inputs)) # build model inputs_h = Input(shape=((frames, 9, s_size, s_size, 3)), name='inputs_h') processed_h = TimeDistributed(shared_layer_h, name='shared_3Dconv_branch_h')(inputs_h) inputs_v = Input(shape=((frames, 9, s_size, s_size, 3)), name='inputs_v') processed_v = TimeDistributed(shared_layer_v, name='shared_3Dconv_branch_v')(inputs_v) x = Concatenate()([processed_h, processed_v]) for n_filters in [64, 32, 32, 16]: x = TimeDistributed( Conv2D(n_filters, kernel_size=3, padding='same', activation='relu', kernel_initializer='glorot_uniform'))(x) x = ConvRNN2D(STConvLSTM2DCell(8, kernel_size=3, padding='same', activation='tanh', recurrent_activation='hard_sigmoid', kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal'), return_sequences=True, name='STConvLSTM2D')(x) x = TimeDistributed(Lambda(lambda x: K.squeeze(x, axis=-1)), name='squeeze')(x) return Model(inputs=[inputs_h, inputs_v], outputs=x) # ________________________________________________________________________________________________________________ # Layer (type) Output Shape Param # Connected to # ================================================================================================================ # inputs_h (InputLayer) [(None, None, 9, None, None, 3)] 0 # ________________________________________________________________________________________________________________ # inputs_v (InputLayer) [(None, None, 9, None, None, 3)] 0 # ________________________________________________________________________________________________________________ # shared_3Dconv_branch_h (TimeDistributed) (None, None, None, None, 64) 279296 inputs_h[0][0] # ________________________________________________________________________________________________________________ # shared_3Dconv_branch_v (TimeDistributed) (None, None, None, None, 64) 279296 inputs_v[0][0] # ________________________________________________________________________________________________________________ # concatenate (Concatenate) (None, None, None, None, 128) 0 shared_3Dconv_branch_h[0][0] # shared_3Dconv_branch_v[0][0] # ________________________________________________________________________________________________________________ # time_distributed (TimeDistributed) (None, None, None, None, 64) 73792 concatenate[0][0] # ________________________________________________________________________________________________________________ # time_distributed_1 (TimeDistributed) (None, None, None, None, 32) 18464 time_distributed[0][0] # ________________________________________________________________________________________________________________ # time_distributed_2 (TimeDistributed) (None, None, None, None, 32) 9248 time_distributed_1[0][0] # ________________________________________________________________________________________________________________ # time_distributed_3 (TimeDistributed) (None, None, None, None, 16) 4624 time_distributed_2[0][0] # ________________________________________________________________________________________________________________ # STConvLSTM2D (ConvRNN2D) (None, None, None, None, 1) 40009 time_distributed_3[0][0] # ________________________________________________________________________________________________________________ # squeeze (TimeDistributed) (None, None, None, None) 0 conv_rn_n2d[0][0] # ================================================================================================================ # Total params: 704,729 # Trainable params: 704,633 # Non-trainable params: 96 # ________________________________________________________________________________________________________________
def __init__(self, dataset_path: str, num_of_upscales: int, gen_mod_name: str, disc_mod_name: str, training_progress_save_path: str, dataset_augmentation_settings: Union[AugmentationSettings, None] = None, generator_optimizer: Optimizer = Adam(0.0001, 0.9), discriminator_optimizer: Optimizer = Adam(0.0001, 0.9), gen_loss="mae", disc_loss="binary_crossentropy", feature_loss="mae", gen_loss_weight: float = 1.0, disc_loss_weight: float = 0.003, feature_loss_weights: Union[list, float, None] = None, feature_extractor_layers: Union[list, None] = None, generator_lr_decay_interval: Union[int, None] = None, discriminator_lr_decay_interval: Union[int, None] = None, generator_lr_decay_factor: Union[float, None] = None, discriminator_lr_decay_factor: Union[float, None] = None, generator_min_lr: Union[float, None] = None, discriminator_min_lr: Union[float, None] = None, discriminator_label_noise: Union[float, None] = None, discriminator_label_noise_decay: Union[float, None] = None, discriminator_label_noise_min: Union[float, None] = 0.001, batch_size: int = 4, buffered_batches: int = 20, generator_weights: Union[str, None] = None, discriminator_weights: Union[str, None] = None, load_from_checkpoint: bool = False, custom_hr_test_images_paths: Union[list, None] = None, check_dataset: bool = True, num_of_loading_workers: int = 8): # Save params to inner variables self.__disc_mod_name = disc_mod_name self.__gen_mod_name = gen_mod_name self.__num_of_upscales = num_of_upscales assert self.__num_of_upscales >= 0, Fore.RED + "Invalid number of upscales" + Fore.RESET self.__discriminator_label_noise = discriminator_label_noise self.__discriminator_label_noise_decay = discriminator_label_noise_decay self.__discriminator_label_noise_min = discriminator_label_noise_min if self.__discriminator_label_noise_min is None: self.__discriminator_label_noise_min = 0 self.__batch_size = batch_size assert self.__batch_size > 0, Fore.RED + "Invalid batch size" + Fore.RESET self.__episode_counter = 0 # Insert empty lists if feature extractor settings are empty if feature_extractor_layers is None: feature_extractor_layers = [] if feature_loss_weights is None: feature_loss_weights = [] # If feature_loss_weights is float then create list of the weights from it if isinstance(feature_loss_weights, float) and len(feature_extractor_layers) > 0: feature_loss_weights = [ feature_loss_weights / len(feature_extractor_layers) ] * len(feature_extractor_layers) assert len(feature_extractor_layers) == len( feature_loss_weights ), Fore.RED + "Number of extractor layers and feature loss weights must match!" + Fore.RESET # Create array of input image paths self.__train_data = get_paths_of_files_from_path(dataset_path, only_files=True) assert self.__train_data, Fore.RED + "Training dataset is not loaded" + Fore.RESET # Load one image to get shape of it self.__target_image_shape = cv.imread(self.__train_data[0]).shape # Check image size validity if self.__target_image_shape[0] < 4 or self.__target_image_shape[1] < 4: raise Exception("Images too small, min size (4, 4)") # Starting image size calculate self.__start_image_shape = count_upscaling_start_size( self.__target_image_shape, self.__num_of_upscales) # Check validity of whole datasets if check_dataset: self.__validate_dataset() # Initialize training data folder and logging self.__training_progress_save_path = training_progress_save_path self.__training_progress_save_path = os.path.join( self.__training_progress_save_path, f"{self.__gen_mod_name}__{self.__disc_mod_name}__{self.__start_image_shape}_to_{self.__target_image_shape}" ) self.__tensorboard = TensorBoardCustom( log_dir=os.path.join(self.__training_progress_save_path, "logs")) self.__stat_logger = StatLogger(self.__tensorboard) # Define static vars self.kernel_initializer = RandomNormal(stddev=0.02) self.__custom_loading_failed = False self.__custom_test_images = True if custom_hr_test_images_paths else False if custom_hr_test_images_paths: self.__progress_test_images_paths = custom_hr_test_images_paths for idx, image_path in enumerate( self.__progress_test_images_paths): if not os.path.exists(image_path): self.__custom_loading_failed = True self.__progress_test_images_paths[idx] = random.choice( self.__train_data) else: self.__progress_test_images_paths = [ random.choice(self.__train_data) ] # Create batchmaker and start it self.__batch_maker = BatchMaker( self.__train_data, self.__batch_size, buffered_batches=buffered_batches, secondary_size=self.__start_image_shape, num_of_loading_workers=num_of_loading_workers, augmentation_settings=dataset_augmentation_settings) # Create LR Schedulers for both "Optimizer" self.__gen_lr_scheduler = LearningRateScheduler( start_lr=float(K.get_value(generator_optimizer.lr)), lr_decay_factor=generator_lr_decay_factor, lr_decay_interval=generator_lr_decay_interval, min_lr=generator_min_lr) self.__disc_lr_scheduler = LearningRateScheduler( start_lr=float(K.get_value(discriminator_optimizer.lr)), lr_decay_factor=discriminator_lr_decay_factor, lr_decay_interval=discriminator_lr_decay_interval, min_lr=discriminator_min_lr) ##################################### ### Create discriminator ### ##################################### self.__discriminator = self.__build_discriminator(disc_mod_name) self.__discriminator.compile(loss=disc_loss, optimizer=discriminator_optimizer) ##################################### ### Create generator ### ##################################### self.__generator = self.__build_generator(gen_mod_name) if self.__generator.output_shape[1:] != self.__target_image_shape: raise Exception( f"Invalid image input size for this generator model\nGenerator shape: {self.__generator.output_shape[1:]}, Target shape: {self.__target_image_shape}" ) self.__generator.compile(loss=gen_loss, optimizer=generator_optimizer, metrics=[PSNR_Y, PSNR, SSIM]) ##################################### ### Create vgg network ### ##################################### self.__vgg = create_feature_extractor(self.__target_image_shape, feature_extractor_layers) ##################################### ### Create combined generator ### ##################################### small_image_input_generator = Input(shape=self.__start_image_shape, name="small_image_input") # Images upscaled by generator gen_images = self.__generator(small_image_input_generator) # Discriminator takes images and determinates validity frozen_discriminator = Network(self.__discriminator.inputs, self.__discriminator.outputs, name="frozen_discriminator") frozen_discriminator.trainable = False validity = frozen_discriminator(gen_images) # Extracts features from generated images generated_features = self.__vgg(preprocess_vgg(gen_images)) # Combine models # Train generator to fool discriminator self.__combined_generator_model = Model( inputs=small_image_input_generator, outputs=[gen_images, validity] + [*generated_features], name="srgan") self.__combined_generator_model.compile( loss=[gen_loss, disc_loss] + ([feature_loss] * len(generated_features)), loss_weights=[gen_loss_weight, disc_loss_weight] + feature_loss_weights, optimizer=generator_optimizer, metrics={"generator": [PSNR_Y, PSNR, SSIM]}) # Print all summaries print("\nDiscriminator Summary:") self.__discriminator.summary() print("\nGenerator Summary:") self.__generator.summary() # Load checkpoint self.__initiated = False if load_from_checkpoint: self.__load_checkpoint() # Load weights from param and override checkpoint weights if generator_weights: self.__generator.load_weights(generator_weights) if discriminator_weights: self.__discriminator.load_weights(discriminator_weights) # Set LR self.__gen_lr_scheduler.set_lr(self.__combined_generator_model, self.__episode_counter) self.__disc_lr_scheduler.set_lr(self.__discriminator, self.__episode_counter)
def GAN_Main(result_dir="output", data_dir="data"): alpha = 0.0004 input_shape = (256, 256, 3) local_shape = (128, 128, 3) batchSize = 4 Epochs = 150 l1 = int(Epochs * 0.18) l2 = int(Epochs * 0.02) train_datagen = Data(data_dir, input_shape[:2], local_shape[:2]) Gen = Generative(input_shape) Dis = Discriminative(input_shape, local_shape) optimizer = Adadelta() ####### orgVal = Input(shape=input_shape) mask = Input(shape=(input_shape[0], input_shape[1], 1)) imgContent = Lambda(lambda x: x[0] * (1 - x[1]), output_shape=input_shape)([orgVal, mask]) mimic = Gen(imgContent) completion = Lambda(lambda x: x[0] * x[2] + x[1] * (1 - x[2]), output_shape=input_shape)([mimic, orgVal, mask]) Gen_container = Network([orgVal, mask], completion) Gen_out = Gen_container([orgVal, mask]) Gen_model = Model([orgVal, mask], Gen_out) Gen_model.compile(loss='mse', optimizer=optimizer) inputLayer = Input(shape=(4, ), dtype='int32') Dis_container = Network([orgVal, inputLayer], Dis([orgVal, inputLayer])) Dis_model = Model([orgVal, inputLayer], Dis_container([orgVal, inputLayer])) Dis_model.compile(loss='binary_crossentropy', optimizer=optimizer) Dis_container.trainable = False totalModel = Model([orgVal, mask, inputLayer], [Gen_out, Dis_container([Gen_out, inputLayer])]) totalModel.compile(loss=['mse', 'binary_crossentropy'], loss_weights=[1.0, alpha], optimizer=optimizer) for n in range(Epochs): progress = generic_utils.Progbar(len(train_datagen)) for inputs, points, masks in train_datagen.flow(batchSize): Gen_image = Gen_model.predict([inputs, masks]) real = np.ones((batchSize, 1)) unreal = np.zeros((batchSize, 1)) generatorLoss = 0.0 discriminatorLoss = 0.0 if n < l1: generatorLoss = Gen_model.train_on_batch([inputs, masks], inputs) else: discriminatorLoss_real = Dis_model.train_on_batch( [inputs, points], real) discriminatorLoss_unreal = Dis_model.train_on_batch( [Gen_image, points], unreal) discriminatorLoss = 0.5 * np.add(discriminatorLoss_real, discriminatorLoss_unreal) if n >= l1 + l2: generatorLoss = totalModel.train_on_batch( [inputs, masks, points], [inputs, real]) generatorLoss = generatorLoss[0] + alpha * generatorLoss[1] progress.add(inputs.shape[0]) imgs = min(5, batchSize) Display, Axis = mplt.subplots(imgs, 3) Axis[0, 0].set_title('Input Image') Axis[0, 1].set_title('Output Image') Axis[0, 2].set_title('Original Image') for i in range(imgs): Axis[i, 0].imshow(inputs[i] * (1 - masks[i])) Axis[i, 0].axis('off') Axis[i, 1].imshow(Gen_image[i]) Axis[i, 1].axis('off') Axis[i, 2].imshow(inputs[i]) Axis[i, 2].axis('off') Display.savefig(os.path.join(result_dir, "Batch_%d.png" % n)) mplt.close() #Trained Model Files... Gen.save(os.path.join(result_dir, "generator.h5")) Dis.save(os.path.join(result_dir, "discriminator.h5"))
def discriminator(input_shape, base_name, num_res_blocks=0, is_wgangp=False, use_res=False): initializer_d = TruncatedNormal(mean=0, stddev=0.1, seed=42) D = in_D = Input(shape=input_shape) D = Conv2D(64, kernel_size=4, strides=2, padding="same", kernel_initializer=initializer_d, use_bias=False, name=base_name + "_conv1")(D) D = LeakyReLU(0.2)(D) D = Conv2D(128, kernel_size=4, strides=2, padding="same", kernel_initializer=initializer_d, use_bias=False, name=base_name + "_conv2")(D) D = BatchNormalization(momentum=0.9, epsilon=1e-5, name=base_name + "_bn1")(D) D = LeakyReLU(0.2)(D) D = Conv2D(256, kernel_size=4, strides=2, padding="same", kernel_initializer=initializer_d, use_bias=False, name=base_name + "_conv3")(D) D = BatchNormalization(momentum=0.9, epsilon=1e-5, name=base_name + "_bn2")(D) D = LeakyReLU(0.2)(D) if use_res: for i in range(5): D = residual_block(D, base_name=base_name, block_num=i, initializer=initializer_d, num_channels=256, is_wgangp=is_wgangp) D = Conv2D(512, kernel_size=4, strides=2, padding="same", kernel_initializer=initializer_d, use_bias=False, name=base_name + "_conv4")(D) D = BatchNormalization(momentum=0.9, epsilon=1e-5, name=base_name + "_bn3")(D) D = LeakyReLU(0.2)(D) if not is_wgangp: D = Flatten()(D) D = Dense(units=128, name=base_name + "_dense1")(D) D = LeakyReLU(0.2)(D) out = Dense(units=1, activation="sigmoid", name=base_name + "_out")(D) else: #D = GlobalAveragePooling2D()(D) D = Flatten()(D) D = Dense(units=128, name=base_name + "_dense1")(D) D = LeakyReLU(0.2)(D) out = Dense(units=1, activation=None, name=base_name + "_out")(D) network = Network(in_D, out, name=base_name) return network