def generator(inputs): z, = inputs z_driver = Split(*z_for_driver, axis=1)(z) z_offset = Split(*z_for_offset, axis=1)(z) z_bits = Split(*z_for_bits, axis=1)(z) bits = get_bits(z_bits) driver = mask_driver(z_driver) driver_norm = NormSinCosAngle(0, name='driver_norm')(driver) mask_input = concat([bits, driver_norm], name='mask_gen_input') mask, mask_depth_map = mask_generator(mask_input) mask = name_tensor(mask, 'mask') mask_depth_map = name_tensor(mask_depth_map, 'mask_depth_map') selection = with_regularizer(Selection(threshold=-0.08, smooth_threshold=0.2, sigma=1.5, name='selection'), MinCoveredRegularizer()) mask_down = PyramidReduce()(mask) mask_selection = selection(mask) out_offset_front = offset_front([z_offset, ZeroGradient()(driver_norm)]) light_outs = list(lighting_generator( [out_offset_front, light_merge_mask16(mask_depth_map)])) mask_with_lighting = AddLighting( scale_factor=0.6, shift_factor=0.75, name='mask_with_lighting')([mask] + light_outs) out_offset_middle = offset_middle( [out_offset_front, offset_merge_mask16(mask_depth_map), offset_merge_light16(concat(light_outs))]) offset_back_feature_map, out_offset_back = offset_back( [out_offset_middle, offset_merge_mask32(mask_down)]) mask_weight64 = mask_weight_blending64(out_offset_middle) blending = PyramidBlending(offset_pyramid_layers=2, mask_pyramid_layers=2, mask_weights=['variable', 1], offset_weights=[1, 1], use_selection=[True, True], name='blending')( [out_offset_back, mask_with_lighting, mask_selection, mask_weight64 ]) mask_post = mask_postprocess( [blending, mask_selection, mask, out_offset_back, offset_back_feature_map] + light_outs) mask_post = name_tensor(mask_post, 'mask_post') mask_post_high = HighPass(4, nb_steps=4, name='mask_post_high')(mask_post) blending_post = merge([mask_post_high, blending], mode='sum', name='blending_post') return LinearInBounds(-1.2, 1.2, name='generator')(blending_post)
def get_mask_postprocess(inputs, nb_units): n = nb_units return sequential([ conv(n, 3, 3), conv(n, 3, 3), Convolution2D(1, 5, 5, border_mode='same', init='normal'), ], ns='mask_post')(concat(inputs))
def mask_blending_discriminator(x, n=32, conv_repeat=1, dense=[], out_activation='sigmoid'): def conv(n): layers = [ Convolution2D(n, 3, 3, subsample=(2, 2), border_mode='same'), BatchNormalization(axis=1), LeakyReLU(0.2), ] return layers + [[ Convolution2D(n, 3, 3, border_mode='same'), BatchNormalization(axis=1), LeakyReLU(0.2), ] for _ in range(conv_repeat-1)] def get_dense(nb): return [ Dense(nb), BatchNormalization(axis=1), LeakyReLU(0.2), ] return sequential([ Convolution2D(n, 5, 5, subsample=(2, 2), border_mode='same'), LeakyReLU(0.2), conv(2*n), conv(4*n), conv(8*n), Flatten(), [get_dense(nb) for nb in dense], Dense(1, activation=out_activation) ], ns='dis')(concat(x, axis=0, name='concat_fake_real'))
def get_mask_weight_blending(inputs, min=0, max=2): input = concat(inputs) return sequential([ Convolution2D(1, 3, 3), Flatten(), Dense(1), LinearInBounds(min, max, clip=True), ], ns='mask_weight_blending')(input)
def get_mask_postprocess(inputs, nb_units): n = nb_units return sequential([ conv(n, 3, 3), conv(n, 3, 3), Deconvolution2D(1, 5, 5, border_mode=(2, 2)), LinearInBounds(-1, 1, clip=True), ], ns='mask_post')(concat(inputs))
def test_concat(): x = Input(shape=(20, )) y = Input(shape=(20, )) model = Model([x, y], concat([x, y])) in_x = np.random.sample((32, 20)) in_y = np.random.sample((32, 20)) model.compile('adam', 'mse') assert model.predict_on_batch([in_x, in_y]).shape == (32, 40)
def test_concat(): x = Input(shape=(20,)) y = Input(shape=(20,)) model = Model([x, y], concat([x, y])) in_x = np.random.sample((32, 20)) in_y = np.random.sample((32, 20)) model.compile('adam', 'mse') assert model.predict_on_batch([in_x, in_y]).shape == (32, 40)
def get_offset_middle(inputs, nb_units): n = nb_units input = concat(inputs) return sequential([ UpSampling2D(), # 32x32 conv(2*n, 3, 3), conv(2*n, 3, 3), conv(2*n, 3, 3), ], ns='offset.middle')(input)
def get_mask_weight_blending(inputs, min=0, max=2): input = concat(inputs) return sequential([ Convolution2D(1, 3, 3), Flatten(), Dense(1), LinearInBounds(K.variable(min), K.variable(2), clip=True), ], ns='mask_weight_blending')(input)
def get_offset_middle(inputs, nb_units): n = nb_units input = concat(inputs) return sequential( [ UpSampling2D(), # 32x32 conv(2 * n, 3, 3), conv(2 * n, 3, 3), conv(2 * n, 3, 3), ], ns='offset.middle')(input)
def get_offset_back(inputs, nb_units): n = nb_units input = concat(inputs) back_feature_map = sequential([ UpSampling2D(), # 64x64 conv(n, 3, 3), conv(n, 3, 3), ], ns='offset.back')(input) return back_feature_map, sequential([ Convolution2D(1, 3, 3, border_mode='same'), LinearInBounds(-1, 1, clip=True), ], ns='offset.back_out')(back_feature_map)
def get_offset_back(inputs, nb_units): n = nb_units input = concat(inputs) back_feature_map = sequential( [ UpSampling2D(), # 64x64 conv(n, 3, 3), conv(n, 3, 3), ], ns='offset.back')(input) return back_feature_map, sequential([ Convolution2D(1, 3, 3, border_mode='same'), LinearInBounds(-1, 1, clip=True), ], ns='offset.back_out')(back_feature_map)
def get_offset_front(inputs, nb_units): n = nb_units input = concat(inputs) return sequential([ Dense(8*n*4*4), BatchNormalization(), Activation('relu'), Reshape((8*n, 4, 4)), UpSampling2D(), # 8x8 conv(4*n, 3, 3), conv(4*n, 3, 3), UpSampling2D(), # 16x16 conv(2*n, 3, 3), conv(2*n, 3, 3), ], ns='offset.front')(input)
def mask_blending_discriminator(x, n=32, out_activation='sigmoid'): def conv(n): return [ Convolution2D(n, 5, 5, subsample=(2, 2), border_mode='same'), BatchNormalization(), LeakyReLU(0.2), ] return sequential([ Convolution2D(n, 5, 5, subsample=(2, 2), border_mode='same'), LeakyReLU(0.2), conv(2*n), conv(4*n), conv(8*n), Flatten(), Dense(1, activation=out_activation) ], ns='dis')(concat(x, axis=0, name='concat_fake_real'))
def mask_blending_discriminator(x, n=32, out_activation='sigmoid'): def conv(n): return [ Convolution2D(n, 5, 5, subsample=(2, 2), border_mode='same'), BatchNormalization(), LeakyReLU(0.2), ] return sequential([ Convolution2D(n, 5, 5, subsample=(2, 2), border_mode='same'), LeakyReLU(0.2), conv(2 * n), conv(4 * n), conv(8 * n), Flatten(), Dense(1, activation=out_activation) ], ns='dis')(concat(x, axis=0, name='concat_fake_real'))
def simple_gan(): z = Input(batch_shape=simple_gan_z_shape, name='z') generator = sequential([ Dense(simple_gan_nb_z, activation='relu', name='g1'), Dense(simple_gan_nb_z, activation='relu', name='g2'), Dense(simple_gan_nb_out, activation='sigmoid', name='g3'), ])(z) fake = Input(batch_shape=simple_gan_real_shape, name='fake') real = Input(batch_shape=simple_gan_real_shape, name='real') discriminator = sequential([ Dense(20, activation='relu', input_dim=2, name='d1'), Dense(1, activation='sigmoid', name='d2') ])(concat([fake, real], axis=0)) return GAN(Container(z, generator), Container([fake, real], gan_outputs(discriminator)), simple_gan_z_shape[1:], simple_gan_real_shape[1:])
def get_offset_front(inputs, nb_units): n = nb_units input = concat(inputs) return sequential( [ Dense(8 * n * 4 * 4), BatchNormalization(), Activation('relu'), Reshape((8 * n, 4, 4)), UpSampling2D(), # 8x8 conv(4 * n, 3, 3), conv(4 * n, 3, 3), UpSampling2D(), # 16x16 conv(2 * n, 3, 3), conv(2 * n, 3, 3), ], ns='offset.front')(input)
def get_lighting_generator(inputs, nb_units): n = nb_units input = concat(inputs) light_conv = sequential([ conv(n, 5, 5), conv(n, 5, 5), conv(n, 3, 3), UpSampling2D(), # 32x32 conv(n, 5, 5), Convolution2D(2, 1, 1, border_mode='same'), UpSampling2D(), # 64x64 LinearInBounds(-1, 1, clip=True), GaussianBlur(sigma=4), ], ns='lighting')(input) shift = Split(0, 1, axis=1)(light_conv) scale = Split(1, 2, axis=1)(light_conv) return shift, scale
def get_lighting_generator(inputs, nb_units): n = nb_units input = concat(inputs) light_conv = sequential( [ conv(n, 5, 5), conv(n, 5, 5), conv(n, 3, 3), UpSampling2D(), # 32x32 conv(n, 5, 5), Convolution2D(2, 1, 1, border_mode='same'), UpSampling2D(), # 64x64 LinearInBounds(-1, 1, clip=True), GaussianBlur(sigma=4), ], ns='lighting')(input) shift = Split(0, 1, axis=1)(light_conv) scale = Split(1, 2, axis=1)(light_conv) return shift, scale
def light_generator(ins): seq = sequential([ Convolution2D(1, 3, 3, border_mode='same') ])(concat(ins)) return UpSampling2D((4, 4))(seq), UpSampling2D((4, 4))(seq), \ UpSampling2D((4, 4))(seq),
def offset_back(x): feature_map = sequential([ UpSampling2D(), ])(concat(x)) return feature_map, Convolution2D(1, 3, 3, border_mode='same')(feature_map)
def offset_middle(x): return UpSampling2D()(concat(x))
def mask_post(x): return sequential([ Convolution2D(1, 3, 3, border_mode='same') ])(concat(x))
def discriminator_fn(x): return gan_outputs(sequential([ Flatten(), Dense(1), ])(concat(x)), fake_for_gen=(0, 10), fake_for_dis=(0, 10), real=(10, 20))
def generator(inputs): z, = inputs z_driver = Split(*z_for_driver, axis=1)(z) z_offset = Split(*z_for_offset, axis=1)(z) z_bits = Split(*z_for_bits, axis=1)(z) bits = get_bits(z_bits) driver = mask_driver(z_driver) driver_norm = NormSinCosAngle(0)(driver) mask_input = concat([bits, driver_norm], name='mask_gen_in') mask = mask_generator(mask_input) if mask_generator_weights: mask_layers = collect_layers(mask_input, mask) load_weights(mask_layers, mask_generator_weights) selection = with_regularizer(Selection(threshold=-0.08, smooth_threshold=0.2, sigma=1.5, name='selection'), MinCoveredRegularizer()) mask_down = PyramidReduce()(mask) mask_selection = selection(mask) mask_selection_down = PyramidReduce(scale=4)(mask_selection) out_offset_front = offset_front([z_offset, ZeroGradient()(driver_norm)]) light_scale64, light_shift64 = \ lighting_generator([out_offset_front, light_merge_mask16(mask_selection_down)]) mask_with_lighting = AddLighting( scale_factor=0.5, shift_factor=0.75)( [mask, light_scale64, light_shift64]) out_offset_middle = offset_middle( [out_offset_front, offset_merge_mask16(mask_selection_down), offset_merge_light16(concat(light_scale64, light_shift64)) ]) offset_back_feature_map, out_offset_back = offset_back( [out_offset_middle, offset_merge_mask32(mask_down)]) mask_weight32 = mask_weight_blending32(out_offset_middle) mask_weight64 = mask_weight_blending64(out_offset_middle) blending = PyramidBlending(offset_pyramid_layers=3, mask_pyramid_layers=3, mask_weights=['variable', 'variable', 1], offset_weights=[1, 1, 1], use_selection=[True, True, True], name='blending')( [out_offset_back, mask_with_lighting, mask_selection, mask_weight32, mask_weight64]) mask_post = mask_postprocess([ blending, mask_selection, light_scale64, light_shift64, mask, out_offset_back, offset_back_feature_map ]) mask_post_high = HighFrequencies(4, nb_steps=5, name='mask_post_high')(mask_post) blending_post = merge([mask_post_high, blending], mode='sum', name='blending_post') return LinearInBounds(-1.2, 1.2)(blending_post)
def offset_front(x): return sequential([ Dense(16), Reshape((1, 4, 4)), UpSampling2D((4, 4)) ])(concat(x))
def generator(inputs): z, = inputs z_driver = Split(*z_for_driver, axis=1)(z) z_offset = Split(*z_for_offset, axis=1)(z) z_bits = Split(*z_for_bits, axis=1)(z) bits = get_bits(z_bits) driver = mask_driver(z_driver) driver_norm = NormSinCosAngle(0)(driver) mask_input = concat([bits, driver_norm], name='mask_gen_in') mask = mask_generator(mask_input) if mask_generator_weights: mask_layers = collect_layers(mask_input, mask) load_weights(mask_layers, mask_generator_weights) selection = with_regularizer( Selection(threshold=-0.08, smooth_threshold=0.2, sigma=1.5, name='selection'), MinCoveredRegularizer()) mask_down = PyramidReduce()(mask) mask_selection = selection(mask) mask_selection_down = PyramidReduce(scale=4)(mask_selection) out_offset_front = offset_front( [z_offset, ZeroGradient()(driver_norm)]) light_scale64, light_shift64 = \ lighting_generator([out_offset_front, light_merge_mask16(mask_selection_down)]) mask_with_lighting = AddLighting(scale_factor=0.5, shift_factor=0.75)( [mask, light_scale64, light_shift64]) out_offset_middle = offset_middle([ out_offset_front, offset_merge_mask16(mask_selection_down), offset_merge_light16(concat(light_scale64, light_shift64)) ]) offset_back_feature_map, out_offset_back = offset_back( [out_offset_middle, offset_merge_mask32(mask_down)]) mask_weight32 = mask_weight_blending32(out_offset_middle) mask_weight64 = mask_weight_blending64(out_offset_middle) blending = PyramidBlending(offset_pyramid_layers=3, mask_pyramid_layers=3, mask_weights=['variable', 'variable', 1], offset_weights=[1, 1, 1], use_selection=[True, True, True], name='blending')([ out_offset_back, mask_with_lighting, mask_selection, mask_weight32, mask_weight64 ]) mask_post = mask_postprocess([ blending, mask_selection, light_scale64, light_shift64, mask, out_offset_back, offset_back_feature_map ]) mask_post_high = HighFrequencies(4, nb_steps=5, name='mask_post_high')(mask_post) blending_post = merge([mask_post_high, blending], mode='sum', name='blending_post') return LinearInBounds(-1.2, 1.2)(blending_post)