def test_gan_custom_layer_graph(): z_shape = (1, 8, 8) z = Input(shape=z_shape, name='z') gen_cond = Input(shape=(1, 8, 8), name='gen_cond') inputs = [z, gen_cond] gen_input = merge(inputs, mode='concat', concat_axis=1) gen_output = Convolution2D(1, 2, 2, activation='relu', name='g1', border_mode='same')(gen_input) generator = Container(inputs, gen_output) f, r = Input(z_shape, name='fake'), Input(z_shape, name='real') inputs = [f, r] dis_input = merge(inputs, mode='concat', concat_axis=0) dis_conv = Convolution2D(5, 2, 2, name='d1', activation='relu')(dis_input) dis_flatten = Flatten()(dis_conv) dis = Dense(1, activation='sigmoid')(dis_flatten) discriminator = Container(inputs, gan_outputs(dis)) gan = GAN(generator, discriminator, z_shape=z_shape, real_shape=z_shape) gan.build('adam', 'adam', gan_binary_crossentropy) fn = gan.compile_custom_layers(['g1', 'd1']) z = np.random.uniform(-1, 1, (64,) + z_shape) real = np.random.uniform(-1, 1, (64,) + z_shape) cond = np.random.uniform(-1, 1, (64,) + z_shape) print(z.shape) print(real.shape) print(cond.shape) fn({'z': z, 'gen_cond': cond, 'real': real})
def from_config(cls, config): return cls( Container.from_config(config['generator']), Container.from_config(config['discriminator']), config['z_shape'], config['real_shape'], )
def obsModule(self, kpm=True, neighEnc=True, visEnc=False, vStt=False): if (neighEnc): self.neighStateEncoder(vStt=vStt) if (visEnc): self.neighVisEncoder() if (kpm): self.kpmGate() mod_inp = [] mod_out = [] if (neighEnc): mod_inp += [self.pose, self.gaze, self.modeN] mod_out = Multiply()([ self.kpm_mod([self.pose, self.gaze]), self.nstate_mod([self.modeN]) ]) if (visEnc): mult_out = Multiply()([ self.kpm_mod([self.pose, self.gaze]), self.nvis_mod([self.visP, self.visG]) ]) if (mod_out != []): mod_out = Cat()([mod_out, mult_out]) else: mod_out = mult_out mod_inp += [self.visP, self.visG] self.obs_mod = Container(mod_inp, mod_out) else: if (neighEnc and not visEnc): self.obs_mod = self.nstate_mod elif (not neighEnc and visEnc): self.obs_mod = self.nvis_mod elif (neighEnc and visEnc): mod_out = Cat()([ self.nstate_mod([self.modeN]), self.nvis_mod([self.visP, self.visG]) ]) self.obs_mod = Container([self.modeN, self.visP, self.visG], mod_out) self.obs_mod.name = 'ObsMod'
def test_orderings(): masked_input_layer, input_layer, mask_layer = create_input_layers(6) mog = Container(inputs=[masked_input_layer, input_layer, mask_layer], outputs=mog_layer(input_layer, Activation("relu") (Add()([Dense(16)(Lambda(lambda x: x[:, :2], output_shape=(2,))(masked_input_layer)), Dense(16, use_bias=False)(mask_layer)])), 1)) inner_model = Container(inputs=[masked_input_layer, input_layer, mask_layer], outputs=mog([masked_input_layer, input_layer, mask_layer])) model = training_model(inner_model) model.compile(loss=utils.maximize_prediction, optimizer=optimizers.SGD(lr=0.1),) model.fit(np.random.normal(0, size=(1000, 6)), np.zeros(shape=(1000, 1)), batch_size=10, verbose=0, epochs=1) test_set = np.random.normal(0, size=(5, 6)) real_ld = np.log(np.sum(np.exp(-0.5 * test_set ** 2 - 0.5 * np.log(2 * np.pi)))) test_1_model = logdensity_model(inner_model) test_1_model.compile(optimizer="sgd", loss=utils.maximize_prediction) test_8_model = logdensity_model(inner_model, num_of_orderings=8) test_8_model.compile(optimizer="sgd", loss=utils.maximize_prediction) ld_1 = test_1_model.evaluate(test_set, np.zeros(shape=(5, 1)), verbose=0) ld_8 = test_8_model.evaluate(test_set, np.zeros(shape=(5, 1)), verbose=0) assert abs(-real_ld - ld_8) < abs(-real_ld - ld_1)
def build_autoencoder(config: BEGANConfig): n_filters = config.n_filters hidden_size = config.hidden_size dx = image_input = Input((config.image_height, config.image_width, 3)) dx = convolution_image_for_encoding( dx, n_filters, strides=(2, 2)) # output: (N, 32, 32, n_filters) dx = convolution_image_for_encoding( dx, n_filters * 2, strides=(2, 2)) # output: (N, 16, 16, n_filters*2) dx = convolution_image_for_encoding( dx, n_filters * 3, strides=(2, 2)) # output: (N, 8, 8, n_filters*3) dx = convolution_image_for_encoding( dx, n_filters * 4, strides=(1, 1)) # output: (N, 8, 8, n_filters*4) dx = Flatten()(dx) hidden = Dense(hidden_size, activation='linear')(dx) image_output = build_decoder_layer(config, hidden) autoencoder = Container(image_input, image_output, name="autoencoder") autoencoder_not_trainable = Container(image_input, image_output, name="autoencoder_not_trainable") autoencoder_not_trainable.trainable = False return autoencoder, autoencoder_not_trainable
def test_gan_get_config(tmpdir): z_shape = (1, 8, 8) z = Input(z_shape, name='z') g_out = Convolution2D(10, 2, 2, activation='relu', border_mode='same')(z) generator = Container(z, g_out) f, r = Input(z_shape, name='f'), Input(z_shape, name='r') dis_input = merge([f, r], mode='concat', concat_axis=1) dis_conv = Convolution2D(5, 2, 2, activation='relu')(dis_input) dis_flatten = Flatten()(dis_conv) dis = Dense(1, activation='sigmoid')(dis_flatten) discriminator = Container([f, r], gan_outputs(dis)) gan = GAN(generator, discriminator, z_shape, z_shape) weights_fname = str(tmpdir.mkdir("weights").join("{}.hdf5")) gan.save_weights(weights_fname) true_config = gan.get_config() import json with open(os.path.join(TEST_OUTPUT_DIR, "true_config.json"), 'w+') as f: json.dump(true_config, f, indent=2) gan_from_config = layer_from_config(true_config, custom_objects={ 'GAN': GAN, 'Split': Split, }) with open(os.path.join(TEST_OUTPUT_DIR, "loaded_config.json"), 'w+') as f: json.dump(gan_from_config.get_config(), f, indent=2) gan_from_config.load_weights(weights_fname)
def neighStateEncoder(self, vStt=False): self.modeN = Input(shape=(self.n_enc, self.n_states), name='mode') if (vStt): self.pose = Input(shape=(self.n_enc, 2), name='pose') self.gaze = Input(shape=(self.n_enc, 2), name='gaze') gru_out = GRU(64, name='StateEnc-gru')( Cat()([self.pose, self.gaze, self.modeN])) elu_out = Dense(64, activation='elu', name='StateEnc-elu')(gru_out) self.nstate_mod = Container([self.pose, self.gaze, self.modeN], elu_out, name='StateEnc') else: gru_out = GRU(64, name='StateEnc-gru')(self.modeN) elu_out = Dense(64, activation='elu', name='StateEnc-elu')(gru_out) self.nstate_mod = Container([self.modeN], elu_out, name='StateEnc')
def build_decoder(self): K.set_learning_phase(1) decoder_input = Input(shape=(None, NNVAL.input_dim)) state_h = Input(shape=(NNVAL.encoder_latent_dim, )) state_c = Input(shape=(NNVAL.encoder_latent_dim, )) decoder_dense_outputs = Dense(NNVAL.decoder_latent_dim, activation='sigmoid')(decoder_input) decoder_bi_lstm = LSTM(NNVAL.decoder_latent_dim, return_sequences=True, dropout=0.6, recurrent_dropout=0.6) decoder_bi_outputs = Bi(decoder_bi_lstm)(decoder_dense_outputs) decoder_lstm = LSTM(NNVAL.decoder_latent_dim, return_sequences=True, return_state=True, dropout=0.4, recurrent_dropout=0.4) encoder_states = [state_h, state_c] decoder_output, output_h, output_c = decoder_lstm( decoder_bi_outputs, initial_state=encoder_states) decoder_output = Dense(NNVAL.decoder_latent_dim, activation='tanh')(decoder_output) decoder_output = Dropout(0.2)(decoder_output) decoder_output = Dense(NNVAL.output_dim, activation='linear')(decoder_output) return Container([decoder_input, state_h, state_c], [decoder_output, output_h, output_c])
def build_decoder(config: BEGANConfig, name): """ generator and decoder( of discriminator) have same network structure, but don't share weights. This function takes different input layer, flow another network, and return different output layer. """ n_filters = config.n_filters n_layer = config.n_layer_in_conv dx = input_z = Input((64, )) dx = Dense((8*8*n_filters), activation='linear', name="%s/Dense" % name)(dx) dx = Reshape((8, 8, n_filters))(dx) # output: (N, 16, 16, n_filters) dx = convolution_image_for_decoding(dx, n_filters, upsample=True, name="%s/L1" % name, n_layer=n_layer) # output: (N, 32, 32, n_filters) dx = convolution_image_for_decoding(dx, n_filters, upsample=True, name="%s/L2" % name, n_layer=n_layer) # output: (N, 64, 64, n_filters) dx = convolution_image_for_decoding(dx, n_filters, upsample=True, name="%s/L3" % name, n_layer=n_layer) # output: (N, 64, 64, n_filters) dx = convolution_image_for_decoding(dx, n_filters, upsample=False, name="%s/L4" % name, n_layer=n_layer) # output: (N, 64, 64, 3) image_output = Convolution2D(3, (3, 3), padding="same", activation="linear", name="%s/FinalConv" % name)(dx) decoder = Container(input_z, image_output, name=name) return decoder
def from_config(cls, config, custom_objects=None): step_model = Container.from_config(config, custom_objects) config.pop('name') config.pop('layers') config.pop('input_layers') config.pop('output_layers') return cls(_step_model=step_model, **config)
def build_autoencoder(config, name="autoencoder"): encoder = build_encoder(config, name="%s/encoder" % name) decoder = build_decoder(config, name="%s/decoder" % name) autoencoder = Container(encoder.inputs, decoder(encoder.outputs), name=name) return autoencoder
def test_compare_original_nade(): """ Compare output computation with github.com/MarcCote/NADE This test use weights learned with reference implementation. Following parameters where used orderlessNADE.py --theano --form MoG --dataset simple.hdf5 --samples_name 0 --hlayers 1 --n_components 1 --epoch_size 10000 --momentum 0.0 --units 16 --training_route training --no_validation --batch_size 5 Training data consisted of 10000 samples drawn from normal(mean=0, sigma=1) The architecture here is the same as it would be created by reference implementation. """ import h5py masked_input_layer, input_layer, mask_layer = create_input_layers(2) mog = Container(inputs=[masked_input_layer, input_layer, mask_layer], outputs=mog_layer(input_layer, Activation("relu") (add([Dense(16)(Lambda(lambda x: x[:, :2])(masked_input_layer)), Dense(16, use_bias=False)(mask_layer)])), 1)) inner_model = Container(inputs=[masked_input_layer, input_layer, mask_layer], outputs=mog([masked_input_layer, input_layer, mask_layer])) model = training_model(inner_model, mask_seed=1) model.compile(loss=utils.maximize_prediction, optimizer="sgd") with h5py.File("tests/original_nade_weights.hdf5") as h: model.set_weights([ h["final_model/parameters/W1"][()].astype(np.float32), h["final_model/parameters/b1"][()].astype(np.float32), h["final_model/parameters/Wflags"][()].astype(np.float32), h["final_model/parameters/V_alpha"][()].T.reshape((16, 2)).astype(np.float32), h["final_model/parameters/b_alpha"][()].reshape(2).astype(np.float32), h["final_model/parameters/V_sigma"][()].T.reshape((16, 2)).astype(np.float32), h["final_model/parameters/b_sigma"][()].reshape(2).astype(np.float32), h["final_model/parameters/V_mu"][()].T.reshape((16, 2)).astype(np.float32), h["final_model/parameters/b_mu"][()].reshape(2).astype(np.float32) ]) np.random.seed(1) output = model.predict(np.random.normal(size=(5, 2))) # Different random generation leads to different masks if K.backend() == "tensorflow": assert np.allclose(np.array([-2.20870864, -2.12633744, -4.85813326, -3.63397837, -1.89778014]), output) elif K.backend() == "theano": assert np.allclose(np.array([-3.33089394, -2.55555928, -4.85813281, -4.85442475, -1.92244674]), output) else: raise NotImplementedError()
def get_generator(): z = Input(shape=z_shape, name='z') inputs = [z, gen_cond] gen_input = merge(inputs, mode='concat', concat_axis=1) gen_output = Convolution2D(10, 2, 2, activation='relu', border_mode='same')(gen_input) return Container(inputs, gen_output)
def neighVisEncoder(self): self.visP = Input(shape=(self.n_vis, 2), name='visP') self.visG = Input(shape=(self.n_vis, 2), name='visG') gru_out = GRU(64, name='VisEnc-gru')(Cat()([self.visP, self.visG])) elu_out = Dense(64, activation='elu', name='VisEnc-elu')(gru_out) self.nvis_mod = Container([self.visP, self.visG], elu_out, name='VisEnc')
def get_discriminator(): f, r = Input(z_shape, name='f'), Input(z_shape, name='r') inputs = [f, r] dis_input = merge(inputs, mode='concat', concat_axis=1) dis_conv = Convolution2D(5, 2, 2, activation='relu')(dis_input) dis_flatten = Flatten()(dis_conv) dis = Dense(1, activation='sigmoid')(dis_flatten) return Container(inputs, gan_outputs(dis))
def simple_gan(): z = Input(batch_shape=simple_gan_z_shape, name='z') generator = sequential([ Dense(simple_gan_nb_z, activation='relu', name='g1'), Dense(simple_gan_nb_z, activation='relu', name='g2'), Dense(simple_gan_nb_out, activation='sigmoid', name='g3'), ])(z) fake = Input(batch_shape=simple_gan_real_shape, name='fake') real = Input(batch_shape=simple_gan_real_shape, name='real') discriminator = sequential([ Dense(20, activation='relu', input_dim=2, name='d1'), Dense(1, activation='sigmoid', name='d2') ])(concat([fake, real], axis=0)) return GAN(Container(z, generator), Container([fake, real], gan_outputs(discriminator)), simple_gan_z_shape[1:], simple_gan_real_shape[1:])
def Discriminator(self, input_shape): # Assuming model is not recurrent model = Sequential() # Architecture Based on EEG Classification Model at https://arxiv.org/pdf/1703.05051.pdf model.add(layers.Conv2D(25, kernel_size=(1,11), strides=(1,1), padding='same', input_shape=input_shape, kernel_initializer='he_normal')) #kernel_regularizer=regularizers.l2(.001))) model.add(layers.Conv2D(25, kernel_size=(22,1), strides=(1,1), padding='valid', kernel_initializer='he_normal')) #kernel_regularizer=regularizers.l2(.001))) model.add(layers.LeakyReLU(alpha=0.2)) model.add(layers.AveragePooling2D(pool_size=(1,3))) model.add(layers.Dropout(rate=0.2)) model.add(layers.Conv2D(50, kernel_size=(1,11), padding='same', kernel_initializer='he_normal')) #kernel_regularizer=regularizers.l2(.001))) model.add(layers.LeakyReLU(alpha=0.2)) model.add(layers.AveragePooling2D(pool_size=(1,3))) model.add(layers.Dropout(rate=0.2)) model.add(layers.Conv2D(100, kernel_size=(1,11), strides=(1,1), padding='same', kernel_initializer='he_normal')) #kernel_regularizer=regularizers.l2(.001))) model.add(layers.LeakyReLU(alpha=0.2)) model.add(layers.AveragePooling2D(pool_size=(1,3))) model.add(layers.Dropout(rate=0.2)) model.add(layers.Conv2D(200, kernel_size=(1,11), strides=(1,1), padding='same', kernel_initializer='he_normal')) #kernel_regularizer=regularizers.l2(.001))) model.add(layers.LeakyReLU(alpha=0.2)) model.add(layers.AveragePooling2D(pool_size=(1,3))) model.add(layers.Dropout(rate=0.2)) model.add(layers.Flatten()) model.add(layers.Dense(1, activation='sigmoid')) #model.summary() eeg = layers.Input(shape=input_shape) validity = model(eeg) fixed = Container(eeg, validity) return Model(eeg, validity), fixed
def Discriminator(self, input_shape): model = Sequential() model.add(LSTM(100, input_shape=input_shape)) model.add(Dense(1, activation='sigmoid')) inp = Input(shape=input_shape) validity = model(inp) return Model(inp, validity), Container(inp, validity)
def test_train_logdensity(): masked_input_layer, input_layer, mask_layer = create_input_layers(2) mog = Container(inputs=[masked_input_layer, input_layer, mask_layer], outputs=mog_layer(input_layer, Activation("relu") (Add()([Dense(16)(Lambda(lambda x: x[:, :2], output_shape=(2,))(masked_input_layer)), Dense(16, use_bias=False)(mask_layer)])), 1)) inner_model = Container(inputs=[masked_input_layer, input_layer, mask_layer], outputs=mog([masked_input_layer, input_layer, mask_layer])) model = training_model(inner_model) model.compile(loss=utils.maximize_prediction, optimizer=optimizers.SGD(lr=0.1),) model.fit(np.random.normal(0, size=(200, 2)), np.zeros(shape=(200, 1)), batch_size=10, verbose=0, epochs=5) test_model = logdensity_model(inner_model) test_model.compile(optimizer="sgd", loss=utils.maximize_prediction) real = test_model.evaluate(np.random.normal(0, size=(50, 2)), np.zeros(shape=(50, 1)), verbose=0) wrong = test_model.evaluate(np.random.normal(1, size=(50, 2)), np.zeros(shape=(50, 1)), verbose=0) assert real < wrong
def init_models(Cnet_path=None, Dnet_path=None): assert ((Cnet_path == None) ^ (Dnet_path == None)) is False #----------------------------- completion_model ---------------------------- holed_origins_inp = Input(shape=IMG_SHAPE, name='holed_origins_inp') complnet_inp = Input(shape=IMG_SHAPE, name='complnet_inp') masks_inp = Input(shape=MASK_SHAPE, name='masks_inp') complnet_out = completion_net(VAR_IMG_SHAPE)(complnet_inp) merged_out = Add()([holed_origins_inp, Multiply()([complnet_out, masks_inp])]) compl_model = Model([holed_origins_inp, complnet_inp, masks_inp], merged_out) if Cnet_path: compl_model.load_weights(Cnet_path, by_name=True) compl_model.compile(loss='mse', optimizer=Adadelta()) #compl_model.summary() #plot_model(compl_model, to_file='C_model.png', show_shapes=True) #--------------------------- discrimination_model -------------------------- origins_inp = Input(shape=IMG_SHAPE, name='origins_inp') crop_yxhw_inp = Input(shape=(4,), dtype=np.int32, name='yxhw_inp') local_cropped = merge([origins_inp,crop_yxhw_inp], mode=cropping, output_shape=MASK_SHAPE, name='local_crop') discrim_out = discrimination_net(IMG_SHAPE, LD_CROP_SHAPE)([origins_inp, local_cropped]) discrim_model = Model([origins_inp,crop_yxhw_inp], discrim_out) if Dnet_path: discrim_model.load_weights(Dnet_path, by_name=True) discrim_model.compile(loss='binary_crossentropy', optimizer=Adadelta(lr=D_MODEL_LR)) # good? lol #optimizer=Adam(lr=0.000001)) #discrim_model.summary() #plot_model(discrim_model, to_file='D_model.png', show_shapes=True) #------------------------------- joint_model ------------------------------- d_container = Container([origins_inp,crop_yxhw_inp], discrim_out, name='D_container') d_container.trainable = False joint_model = Model([holed_origins_inp, complnet_inp, masks_inp, crop_yxhw_inp], [merged_out, d_container([merged_out,crop_yxhw_inp])]) joint_model.compile(loss=['mse', 'binary_crossentropy'], loss_weights=[1.0, ALPHA], optimizer=Adadelta()) #joint_model.summary() #plot_model(joint_model, to_file='joint_model.png', show_shapes=True) return compl_model, discrim_model, joint_model
def test_compare_original_nade_reg(): """ Same as test_compare_original_nade, but with regularization of activities in MOG layer enabled. Should not influence the output """ import h5py masked_input_layer, input_layer, mask_layer = create_input_layers(2) mog = Container(inputs=[masked_input_layer, input_layer, mask_layer], outputs=mog_layer(input_layer, Activation("relu") (add([Dense(16)(Lambda(lambda x: x[:, :2])(masked_input_layer)), Dense(16, use_bias=False)(mask_layer)])), 1, True)) inner_model = Container(inputs=[masked_input_layer, input_layer, mask_layer], outputs=mog([masked_input_layer, input_layer, mask_layer])) model = training_model(inner_model, mask_seed=1) model.compile(loss=utils.maximize_prediction, optimizer="sgd") with h5py.File("tests/original_nade_weights.hdf5") as h: model.set_weights([ h["final_model/parameters/W1"][()].astype(np.float32), h["final_model/parameters/b1"][()].astype(np.float32), h["final_model/parameters/Wflags"][()].astype(np.float32), h["final_model/parameters/V_alpha"][()].T.reshape((16, 2)).astype(np.float32), h["final_model/parameters/b_alpha"][()].reshape(2).astype(np.float32), h["final_model/parameters/V_sigma"][()].T.reshape((16, 2)).astype(np.float32), h["final_model/parameters/b_sigma"][()].reshape(2).astype(np.float32), h["final_model/parameters/V_mu"][()].T.reshape((16, 2)).astype(np.float32), h["final_model/parameters/b_mu"][()].reshape(2).astype(np.float32) ]) np.random.seed(1) output = model.predict(np.random.normal(size=(5, 2))) # Different random generation leads to different masks if K.backend() == "tensorflow": assert np.allclose(np.array([-2.20870864, -2.12633744, -4.85813326, -3.63397837, -1.89778014]), output) elif K.backend() == "theano": assert np.allclose(np.array([-3.33089394, -2.55555928, -4.85813281, -4.85442475, -1.92244674]), output) else: raise NotImplementedError()
def test_gan_graph(): z_shape = (1, 8, 8) z = Input(shape=z_shape, name='z') gen_cond = Input(shape=(1, 8, 8), name='gen_cond') inputs = [z, gen_cond] gen_input = merge(inputs, mode='concat', concat_axis=1) gen_output = Convolution2D(10, 2, 2, activation='relu', border_mode='same')(gen_input) generator = Container(inputs, gen_output) f, r = Input(z_shape, name='f'), Input(z_shape, name='r') inputs = [f, r] dis_input = merge(inputs, mode='concat', concat_axis=1) dis_conv = Convolution2D(5, 2, 2, activation='relu')(dis_input) dis_flatten = Flatten()(dis_conv) dis = Dense(1, activation='sigmoid')(dis_flatten) discriminator = Container(inputs, gan_outputs(dis)) gan = GAN(generator, discriminator, z_shape=z_shape, real_shape=z_shape) gan.build('adam', 'adam', gan_binary_crossentropy) gan.compile() gan.generate({'gen_cond': np.zeros((64,) + z_shape)}, nb_samples=64)
def build_encoder(self): K.set_learning_phase(1) encoder_input = Input(shape=(None, NNVAL.input_dim)) encoder_dense_outputs = Dense(NNVAL.encoder_latent_dim, activation='sigmoid')(encoder_input) encoder_bi_lstm = LSTM(NNVAL.encoder_latent_dim, return_sequences=True, dropout=0.4, recurrent_dropout=0.4) encoder_bi_outputs = Bi(encoder_bi_lstm)(encoder_dense_outputs) _, state_h, state_c = LSTM(NNVAL.encoder_latent_dim, return_state=True, dropout=0.2, recurrent_dropout=0.2)(encoder_bi_outputs) return Container(encoder_input, [state_h, state_c])
def kpmGate(self): if (self.n_kpm > 1): self.pose = Input(shape=(self.n_kpm, 2), name='pose') self.gaze = Input(shape=(self.n_kpm, 2), name='gaze') vis_out = GRU(64, name='KPM-gru')(Cat()([self.pose, self.gaze])) else: self.pose = Input(shape=(2, ), name='pose') self.gaze = Input(shape=(2, ), name='gaze') vis_out = Dense(64, activation='elu', name='KPM-elu')(Cat()([self.pose, self.gaze])) sig_out = Dense(1, activation='hard_sigmoid', name='KPM-sig', kernel_initializer='zero')(vis_out) rep_out = Flatten()(RepeatVector(64)(sig_out)) self.kpm_gate = Model([self.pose, self.gaze], sig_out) self.kpm_mod = Container([self.pose, self.gaze], rep_out, name='KPM-Gate')
def make_decoder(input_size, fixed=False): # Reveal network reveal_input = Input(shape=(input_size)) # Adding Gaussian noise with 0.01 standard deviation. input_with_noise = GaussianNoise(0.01, name='output_C_noise')(reveal_input) x3 = Conv2D(50, (3, 3), strides = (1, 1), padding='same', activation='relu', name='conv_rev0_3x3')(input_with_noise) x4 = Conv2D(10, (4, 4), strides = (1, 1), padding='same', activation='relu', name='conv_rev0_4x4')(input_with_noise) x5 = Conv2D(5, (5, 5), strides = (1, 1), padding='same', activation='relu', name='conv_rev0_5x5')(input_with_noise) x = concatenate([x3, x4, x5]) x3 = Conv2D(50, (3, 3), strides = (1, 1), padding='same', activation='relu', name='conv_rev1_3x3')(x) x4 = Conv2D(10, (4, 4), strides = (1, 1), padding='same', activation='relu', name='conv_rev1_4x4')(x) x5 = Conv2D(5, (5, 5), strides = (1, 1), padding='same', activation='relu', name='conv_rev1_5x5')(x) x = concatenate([x3, x4, x5]) x3 = Conv2D(50, (3, 3), strides = (1, 1), padding='same', activation='relu', name='conv_rev2_3x3')(x) x4 = Conv2D(10, (4, 4), strides = (1, 1), padding='same', activation='relu', name='conv_rev2_4x4')(x) x5 = Conv2D(5, (5, 5), strides = (1, 1), padding='same', activation='relu', name='conv_rev2_5x5')(x) x = concatenate([x3, x4, x5]) x3 = Conv2D(50, (3, 3), strides = (1, 1), padding='same', activation='relu', name='conv_rev3_3x3')(x) x4 = Conv2D(10, (4, 4), strides = (1, 1), padding='same', activation='relu', name='conv_rev3_4x4')(x) x5 = Conv2D(5, (5, 5), strides = (1, 1), padding='same', activation='relu', name='conv_rev3_5x5')(x) x = concatenate([x3, x4, x5]) x3 = Conv2D(50, (3, 3), strides = (1, 1), padding='same', activation='relu', name='conv_rev4_3x3')(x) x4 = Conv2D(10, (4, 4), strides = (1, 1), padding='same', activation='relu', name='conv_rev4_4x4')(x) x5 = Conv2D(5, (5, 5), strides = (1, 1), padding='same', activation='relu', name='conv_rev5_5x5')(x) x = concatenate([x3, x4, x5]) output_Sprime = Conv2D(3, (3, 3), strides = (1, 1), padding='same', activation='relu', name='output_S')(x) if not fixed: return Model(inputs=reveal_input, outputs=output_Sprime, name = 'Decoder') else: return Container(inputs=reveal_input, outputs=output_Sprime, name = 'DecoderFixed')
def build_encoder(config, name="encoder"): n_filters = config.n_filters hidden_size = config.hidden_size n_layer = config.n_layer_in_conv dx = image_input = Input((config.image_height, config.image_width, 3)) # output: (N, 32, 32, n_filters) dx = convolution_image_for_encoding(dx, n_filters, strides=(2, 2), name="%s/L1" % name, n_layer=n_layer) # output: (N, 16, 16, n_filters*2) dx = convolution_image_for_encoding(dx, n_filters * 2, strides=(2, 2), name="%s/L2" % name, n_layer=n_layer) # output: (N, 8, 8, n_filters*3) dx = convolution_image_for_encoding(dx, n_filters * 3, strides=(2, 2), name="%s/L3" % name, n_layer=n_layer) # output: (N, 8, 8, n_filters*4) dx = convolution_image_for_encoding(dx, n_filters * 4, strides=(1, 1), name="%s/L4" % name, n_layer=n_layer) dx = Flatten()(dx) hidden = Dense(hidden_size, activation='linear', name="%s/Dense" % name)(dx) encoder = Container(image_input, hidden, name=name) return encoder
def sequential_to_gan(generator: Sequential, discriminator: Sequential, nb_real=32, nb_fake=96): generator fake = Input(shape=discriminator.input_shape[1:], name='fake') real = Input(shape=discriminator.input_shape[1:], name='real') dis_in = merge([fake, real], concat_axis=0, mode='concat', name='concat_fake_real') dis = discriminator(dis_in) dis_outputs = gan_outputs(dis, fake_for_gen=(0, nb_fake), fake_for_dis=(nb_fake - nb_real, nb_real), real=(nb_fake, nb_fake + nb_real)) dis_container = Container([fake, real], dis_outputs) return GAN(generator, dis_container, z_shape=generator.input_shape[1:], real_shape=discriminator.input_shape[1:])
model2 = model_list[1]['autoencoder'] model3 = model_list[2]['autoencoder'] encode_layer = list() decode_layer = list() for i in range(len(model_list)): encode_layer.append(model_list[i]['autoencoder'].get_layer('encode_layer' + str(i + 1))) decode_layer.append(model_list[i]['autoencoder'].get_layer('decode_layer' + str(i + 1))) o = encode_layer[0](input_encode) o = encode_layer[1](o) o = encode_layer[2](o) encode_layers = Container(input_encode, o, name='encode_container') o = decode_layer[2](input_decode) o = decode_layer[1](o) o = decode_layer[0](o) decode_layers = Container(input_decode, o, name='decode_container') o = GaussianNoise(0.01)(input_encode) o = encode_layers(o) o = decode_layers(o) stacked_autoencoder = Model(input_encode, o) encoder_sa = Model(input_encode, encode_layers(input_encode)) decoder_sa = Model(input_decode, decode_layers(input_decode)) stacked_autoencoder.summary()
def __init__(self, lr_D=2e-4, lr_G=2e-4, image_shape=(140, 168, 144, 1), date_time_string_addition='', image_folder='MR_crop'): self.img_shape = image_shape self.channels = self.img_shape[-1] self.normalization = InstanceNormalization # Hyper parameters self.lambda_1 = 10.0 # Cyclic loss weight A_2_B self.lambda_2 = 10.0 # Cyclic loss weight B_2_A self.lambda_D = 1.0 # Weight for loss from discriminator guess on synthetic images self.learning_rate_D = lr_D self.learning_rate_G = lr_G self.generator_iterations = 2 # Number of generator training iterations in each training loop self.discriminator_iterations = 1 # Number of generator training iterations in each training loop self.beta_1 = 0.5 self.beta_2 = 0.999 self.batch_size = 1 self.epochs = 200 # choose multiples of 25 since the models are save each 25th epoch self.save_interval = 1 self.synthetic_pool_size = 50 # Linear decay of learning rate, for both discriminators and generators self.use_linear_decay = True self.decay_epoch = 75 # The epoch where the linear decay of the learning rates start # Identity loss - sometimes send images from B to G_A2B (and the opposite) to teach identity mappings self.use_identity_learning = True self.identity_mapping_modulus = 10 # Identity mapping will be done each time the iteration number is divisable with this number # PatchGAN - if false the discriminator learning rate should be decreased self.use_patchgan = True # Multi scale discriminator - if True the generator have an extra encoding/decoding step to match discriminator information access self.use_multiscale_discriminator = False # Resize convolution - instead of transpose convolution in deconvolution layers (uk) - can reduce checkerboard artifacts but the blurring might affect the cycle-consistency self.use_resize_convolution = False # Supervised learning part - for MR images - comparison self.use_supervised_learning = True self.supervised_weight = 40.0 # Fetch data during training instead of pre caching all images - might be necessary for large datasets self.use_data_generator = True # Tweaks self.REAL_LABEL = 1.0 # Use e.g. 0.9 to avoid training the discriminators to zero loss # Used as storage folder name self.date_time = time.strftime('%Y%m%d-%H%M%S', time.localtime()) + date_time_string_addition # optimizer self.opt_D = Adam(self.learning_rate_D, self.beta_1, self.beta_2) self.opt_G = Adam(self.learning_rate_G, self.beta_1, self.beta_2) # ======= Data ========== # Use 'None' to fetch all available images nr_A_train_imgs = None nr_B_train_imgs = None nr_A_test_imgs = None nr_B_test_imgs = None if self.use_data_generator: print('--- Using dataloader during training ---') #self.data_generator = load_data.load_data( # nr_of_channels=self.batch_size, generator=True, subfolder=image_folder) self.data_generator = load_crop_data.load_data(self.img_shape, nr_of_channels=self.channels, batch_size=self.batch_size, nr_A_train_imgs=nr_A_train_imgs, nr_B_train_imgs=nr_B_train_imgs, nr_A_test_imgs=nr_A_test_imgs, nr_B_test_imgs=nr_B_test_imgs, generator=True, subfolder=image_folder) # Only store test images nr_A_train_imgs = 0 nr_B_train_imgs = 0 else: print('--- Caching data ---') data = load_crop_data.load_data(img_shape=self.img_shape, nr_of_channels=self.channels, batch_size=self.batch_size, nr_A_train_imgs=nr_A_train_imgs, nr_B_train_imgs=nr_B_train_imgs, nr_A_test_imgs=nr_A_test_imgs, nr_B_test_imgs=nr_B_test_imgs, subfolder=image_folder) X = data["train_A_images"] print("A_train shape: ", X.shape) Y = data["train_B_images"] print("B_train shape: ", Y.shape) Z = data["test_A_images"] print("A_test shape: ", Z.shape) W = data["test_B_images"] print("B_train shape: ", W.shape) self.A_train = data["train_A_images"] self.B_train = data["train_B_images"] self.A_test = data["test_A_images"] self.B_test = data["test_B_images"] self.testA_image_names = data["test_A_image_names"] self.testB_image_names = data["test_B_image_names"] print('Data has been loaded') sys.stdout.flush() # Set up parallel processing strategy = tf.contrib.distribute.MirroredStrategy() with strategy.scope(): # ======= Discriminator model ========== if self.use_multiscale_discriminator: D_A = self.modelMultiScaleDiscriminator() D_B = self.modelMultiScaleDiscriminator() loss_weights_D = [0.5, 0.5] # 0.5 since we train on real and synthetic images else: D_A = self.modelDiscriminator() D_B = self.modelDiscriminator() loss_weights_D = [0.5] # 0.5 since we train on real and synthetic images D_A.summary() # Discriminator builds image_A = Input(shape=self.img_shape) image_B = Input(shape=self.img_shape) guess_A = D_A(image_A) guess_B = D_B(image_B) self.D_A = Model(inputs=image_A, outputs=guess_A, name='D_A_model') self.D_B = Model(inputs=image_B, outputs=guess_B, name='D_B_model') #self.D_A.summary() #self.D_B.summary() self.D_A.compile(optimizer=self.opt_D, loss=self.lse, loss_weights=loss_weights_D) self.D_B.compile(optimizer=self.opt_D, loss=self.lse, loss_weights=loss_weights_D) # Use containers to avoid falsy keras error about weight descripancies self.D_A_static = Container(inputs=image_A, outputs=guess_A, name='D_A_static_model') self.D_B_static = Container(inputs=image_B, outputs=guess_B, name='D_B_static_model') # ======= Generator model ========== # Do note update discriminator weights during generator training self.D_A_static.trainable = False self.D_B_static.trainable = False # Generators self.G_A2B = self.modelGenerator(name='G_A2B_model') self.G_B2A = self.modelGenerator(name='G_B2A_model') self.G_A2B.summary() self.G_B2A.summary() if self.use_identity_learning: self.G_A2B.compile(optimizer=self.opt_G, loss=self.cycle_loss) self.G_B2A.compile(optimizer=self.opt_G, loss=self.cycle_loss) # Generator builds real_A = Input(shape=self.img_shape, name='real_A') real_B = Input(shape=self.img_shape, name='real_B') synthetic_B = self.G_A2B(real_A) synthetic_A = self.G_B2A(real_B) dA_guess_synthetic = self.D_A_static(synthetic_A) dB_guess_synthetic = self.D_B_static(synthetic_B) reconstructed_A = self.G_B2A(synthetic_B) reconstructed_B = self.G_A2B(synthetic_A) model_outputs = [reconstructed_A, reconstructed_B] compile_losses = [self.cycle_loss, self.cycle_loss, self.lse, self.lse] compile_weights = [self.lambda_1, self.lambda_2, self.lambda_D, self.lambda_D] if self.use_multiscale_discriminator: for _ in range(2): compile_losses.append(self.lse) compile_weights.append(self.lambda_D) # * 1e-3) # Lower weight to regularize the model for i in range(2): model_outputs.append(dA_guess_synthetic[i]) model_outputs.append(dB_guess_synthetic[i]) else: model_outputs.append(dA_guess_synthetic) model_outputs.append(dB_guess_synthetic) if self.use_supervised_learning: model_outputs.append(synthetic_A) model_outputs.append(synthetic_B) compile_losses.append('MAE') compile_losses.append('MAE') compile_weights.append(self.supervised_weight) compile_weights.append(self.supervised_weight) self.G_model = Model(inputs=[real_A, real_B], outputs=model_outputs, name='G_model') self.G_model.compile(optimizer=self.opt_G, loss=compile_losses, loss_weights=compile_weights) self.G_A2B.summary() # ======= Create designated run folder and store meta data ========== directory = os.path.join('/outdata', 'images', self.date_time) if not os.path.exists(directory): os.makedirs(directory) self.writeMetaDataToJSON() # ======= Avoid pre-allocating GPU memory ========== # TensorFlow wizardry config = tf.ConfigProto() # Don't pre-allocate memory; allocate as-needed config.gpu_options.allow_growth = True # Create a session with the above options specified. K.tensorflow_backend.set_session(tf.Session(config=config)) # ===== Tests ====== # Simple Model # self.G_A2B = self.modelSimple('simple_T1_2_T2_model') # self.G_B2A = self.modelSimple('simple_T2_2_T1_model') # self.G_A2B.compile(optimizer=Adam(), loss='MAE') # self.G_B2A.compile(optimizer=Adam(), loss='MAE') # # self.trainSimpleModel() # self.load_model_and_generate_synthetic_images() # ======= Initialize training ========== sys.stdout.flush() #plot_model(self.G_A2B, to_file='GA2B_expanded_model_new.png', show_shapes=True) self.train(epochs=self.epochs, batch_size=self.batch_size, save_interval=self.save_interval)
def example_gan(result_dir="output", data_dir="data"): input_shape = (128, 128, 3) local_shape = (64, 64, 3) batch_size = 32 n_epoch = 10 #tc = int(n_epoch * 0.18) #td = int(n_epoch * 0.02) tc = 2 td = 2 alpha = 0.0004 train_datagen = DataGenerator(input_shape[:2], local_shape[:2]) generator = model_generator(input_shape) discriminator = model_discriminator(input_shape, local_shape) optimizer = Adadelta() # build model org_img = Input(shape=input_shape) mask = Input(shape=(input_shape[0], input_shape[1], 1)) in_img = merge([org_img, mask], mode=lambda x: x[0] * (1 - x[1]), output_shape=input_shape) imitation = generator(in_img) completion = merge([imitation, org_img, mask], mode=lambda x: x[0] * x[2] + x[1] * (1 - x[2]), output_shape=input_shape) cmp_container = Container([org_img, mask], completion, name='g_container') cmp_out = cmp_container([org_img, mask]) cmp_model = Model([org_img, mask], cmp_out) cmp_model.compile(loss='mse', optimizer=optimizer) local_img = Input(shape=local_shape) d_container = Container([org_img, local_img], discriminator([org_img, local_img]), name='d_container') d_model = Model([org_img, local_img], d_container([org_img, local_img])) d_model.compile(loss='binary_crossentropy', optimizer=optimizer) ''' ''' cmp_model.summary() d_model.summary() from keras.utils import plot_model plot_model(cmp_model, to_file='cmp_model.png', show_shapes=True) plot_model(d_model, to_file='d_model.png', show_shapes=True) def random_cropping(x, x1, y1, x2, y2): out = [] for idx in range(batch_size): out.append(x[idx, y1[idx]:y2[idx], x1[idx]:x2[idx], :]) return K.stack(out, axis=0) cropping = Lambda(random_cropping, output_shape=local_shape) for n in range(n_epoch): '''''' org_img = Input(shape=input_shape) mask = Input(shape=(input_shape[0], input_shape[1], 1)) in_img = merge([org_img, mask], mode=lambda x: x[0] * (1 - x[1]), output_shape=input_shape) imitation = generator(in_img) completion = merge([imitation, org_img, mask], mode=lambda x: x[0] * x[2] + x[1] * (1 - x[2]), output_shape=input_shape) cmp_container = Container([org_img, mask], completion, name='g_container') cmp_out = cmp_container([org_img, mask]) cmp_model = Model([org_img, mask], cmp_out) cmp_model.compile(loss='mse', optimizer=optimizer) local_img = Input(shape=local_shape) d_container = Container([org_img, local_img], discriminator([org_img, local_img]), name='d_container') d_model = Model([org_img, local_img], d_container([org_img, local_img])) d_model.compile(loss='binary_crossentropy', optimizer=optimizer) cmp_model = Model([org_img, mask], cmp_out) local_img = Input(shape=local_shape) d_container = Container([org_img, local_img], discriminator([org_img, local_img]), name='d_container') d_model = Model([org_img, local_img], d_container([org_img, local_img])) ''' for inputs, points, masks in train_datagen.flow_from_directory(data_dir, batch_size, hole_min=48, hole_max=64): cmp_image = cmp_model.predict([inputs, masks]) local = [] local_cmp = [] for i in range(batch_size): x1, y1, x2, y2 = points[i] local.append(inputs[i][y1:y2, x1:x2, :]) local_cmp.append(cmp_image[i][y1:y2, x1:x2, :]) valid = np.ones((batch_size, 1)) fake = np.zeros((batch_size, 1)) g_loss = 0.0 d_loss = 0.0 if n < tc: g_loss = cmp_model.train_on_batch([inputs, masks], inputs) print("epoch: %d < %d [D loss: %e] [G mse: %e]" % (n,tc, d_loss, g_loss)) else: #d_model.trainable = True d_loss_real = d_model.train_on_batch([inputs, np.array(local)], valid) d_loss_fake = d_model.train_on_batch([cmp_image, np.array(local_cmp)], fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) print('train D',n,(tc+td),'|',d_loss,'|',g_loss) if n >= tc + td: d_container.trainable = False cropping.arguments = {'x1': points[:, 0], 'y1': points[:, 1], 'x2': points[:, 2], 'y2': points[:, 3]} all_model = Model([org_img, mask], [cmp_out, d_container([cmp_out, cropping(cmp_out)])]) all_model.compile(loss=['mse', 'binary_crossentropy'], loss_weights=[1.0, alpha], optimizer=optimizer) g_loss = all_model.train_on_batch([inputs, masks], [inputs, valid]) #print("epoch: %d [D loss: %e] [G all: %e]" % (n, d_loss, g_loss)) print(all_model.metrics_names) print('train ALL',n,'|',d_loss,'|',g_loss) ''' if n < tc: print("epoch: %d < %d [D loss: %e] [G mse: %e]" % (n,tc, d_loss, g_loss)) else: print('train D',n,(tc+td),'|',d_loss,'|',g_loss) if n >= tc + td: print(all_model.metrics_names) print('train ALL',n,'|',d_loss,'|',g_loss) ''' num_img = min(5, batch_size) fig, axs = plt.subplots(num_img, 3) for i in range(num_img): axs[i, 0].imshow(inputs[i] * (1 - masks[i])) axs[i, 0].axis('off') axs[i, 0].set_title('Input') axs[i, 1].imshow(cmp_image[i]) axs[i, 1].axis('off') axs[i, 1].set_title('Output') axs[i, 2].imshow(inputs[i]) axs[i, 2].axis('off') axs[i, 2].set_title('Ground Truth') fig.savefig(os.path.join(result_dir, "result_%d.png" % n)) plt.close() # save model generator.save(os.path.join(result_dir, "generator_%d.h5" % n)) discriminator.save(os.path.join(result_dir, "discriminator_%d.h5" % n)) K.clear_session()