def generator(inputs):
z, = inputs
z_driver = Split(*z_for_driver, axis=1)(z)
z_offset = Split(*z_for_offset, axis=1)(z)
z_bits = Split(*z_for_bits, axis=1)(z)
bits = get_bits(z_bits)
driver = mask_driver(z_driver)
driver_norm = NormSinCosAngle(0, name='driver_norm')(driver)
mask_input = concat([bits, driver_norm], name='mask_gen_input')
mask, mask_depth_map = mask_generator(mask_input)
mask = name_tensor(mask, 'mask')
mask_depth_map = name_tensor(mask_depth_map, 'mask_depth_map')
selection = with_regularizer(Selection(threshold=-0.08,
smooth_threshold=0.2,
sigma=1.5, name='selection'),
MinCoveredRegularizer())
mask_down = PyramidReduce()(mask)
mask_selection = selection(mask)
out_offset_front = offset_front([z_offset,
ZeroGradient()(driver_norm)])
light_outs = list(lighting_generator(
[out_offset_front, light_merge_mask16(mask_depth_map)]))
mask_with_lighting = AddLighting(
scale_factor=0.6, shift_factor=0.75,
name='mask_with_lighting')([mask] + light_outs)
out_offset_middle = offset_middle(
[out_offset_front, offset_merge_mask16(mask_depth_map),
offset_merge_light16(concat(light_outs))])
offset_back_feature_map, out_offset_back = offset_back(
[out_offset_middle, offset_merge_mask32(mask_down)])
mask_weight64 = mask_weight_blending64(out_offset_middle)
blending = PyramidBlending(offset_pyramid_layers=2,
mask_pyramid_layers=2,
mask_weights=['variable', 1],
offset_weights=[1, 1],
use_selection=[True, True],
name='blending')(
[out_offset_back, mask_with_lighting, mask_selection,
mask_weight64
])
mask_post = mask_postprocess(
[blending, mask_selection, mask, out_offset_back,
offset_back_feature_map] + light_outs)
mask_post = name_tensor(mask_post, 'mask_post')
mask_post_high = HighPass(4, nb_steps=4,
name='mask_post_high')(mask_post)
blending_post = merge([mask_post_high, blending], mode='sum',
name='blending_post')
return LinearInBounds(-1.2, 1.2, name='generator')(blending_post)
def get_mask_postprocess(inputs, nb_units):
n = nb_units
return sequential([
conv(n, 3, 3),
conv(n, 3, 3),
Convolution2D(1, 5, 5, border_mode='same', init='normal'),
], ns='mask_post')(concat(inputs))
def mask_blending_discriminator(x, n=32, conv_repeat=1,
dense=[],
out_activation='sigmoid'):
def conv(n):
layers = [
Convolution2D(n, 3, 3, subsample=(2, 2), border_mode='same'),
BatchNormalization(axis=1),
LeakyReLU(0.2),
]
return layers + [[
Convolution2D(n, 3, 3, border_mode='same'),
BatchNormalization(axis=1),
LeakyReLU(0.2),
] for _ in range(conv_repeat-1)]
def get_dense(nb):
return [
Dense(nb),
BatchNormalization(axis=1),
LeakyReLU(0.2),
]
return sequential([
Convolution2D(n, 5, 5, subsample=(2, 2), border_mode='same'),
LeakyReLU(0.2),
conv(2*n),
conv(4*n),
conv(8*n),
Flatten(),
[get_dense(nb) for nb in dense],
Dense(1, activation=out_activation)
], ns='dis')(concat(x, axis=0, name='concat_fake_real'))
def get_mask_weight_blending(inputs, min=0, max=2):
input = concat(inputs)
return sequential([
Convolution2D(1, 3, 3),
Flatten(),
Dense(1),
LinearInBounds(min, max, clip=True),
], ns='mask_weight_blending')(input)
def get_mask_postprocess(inputs, nb_units):
n = nb_units
return sequential([
conv(n, 3, 3),
conv(n, 3, 3),
Deconvolution2D(1, 5, 5, border_mode=(2, 2)),
LinearInBounds(-1, 1, clip=True),
], ns='mask_post')(concat(inputs))
def test_concat():
x = Input(shape=(20, ))
y = Input(shape=(20, ))
model = Model([x, y], concat([x, y]))
in_x = np.random.sample((32, 20))
in_y = np.random.sample((32, 20))
model.compile('adam', 'mse')
assert model.predict_on_batch([in_x, in_y]).shape == (32, 40)
def test_concat():
x = Input(shape=(20,))
y = Input(shape=(20,))
model = Model([x, y], concat([x, y]))
in_x = np.random.sample((32, 20))
in_y = np.random.sample((32, 20))
model.compile('adam', 'mse')
assert model.predict_on_batch([in_x, in_y]).shape == (32, 40)
def get_offset_middle(inputs, nb_units):
n = nb_units
input = concat(inputs)
return sequential([
UpSampling2D(), # 32x32
conv(2*n, 3, 3),
conv(2*n, 3, 3),
conv(2*n, 3, 3),
], ns='offset.middle')(input)
def get_mask_postprocess(inputs, nb_units):
n = nb_units
return sequential([
conv(n, 3, 3),
conv(n, 3, 3),
Deconvolution2D(1, 5, 5, border_mode=(2, 2)),
LinearInBounds(-1, 1, clip=True),
],
ns='mask_post')(concat(inputs))
def get_mask_weight_blending(inputs, min=0, max=2):
input = concat(inputs)
return sequential([
Convolution2D(1, 3, 3),
Flatten(),
Dense(1),
LinearInBounds(K.variable(min), K.variable(2), clip=True),
],
ns='mask_weight_blending')(input)
def get_offset_middle(inputs, nb_units):
n = nb_units
input = concat(inputs)
return sequential(
[
UpSampling2D(), # 32x32
conv(2 * n, 3, 3),
conv(2 * n, 3, 3),
conv(2 * n, 3, 3),
],
ns='offset.middle')(input)
def get_offset_back(inputs, nb_units):
n = nb_units
input = concat(inputs)
back_feature_map = sequential([
UpSampling2D(), # 64x64
conv(n, 3, 3),
conv(n, 3, 3),
], ns='offset.back')(input)
return back_feature_map, sequential([
Convolution2D(1, 3, 3, border_mode='same'),
LinearInBounds(-1, 1, clip=True),
], ns='offset.back_out')(back_feature_map)
def get_offset_back(inputs, nb_units):
n = nb_units
input = concat(inputs)
back_feature_map = sequential(
[
UpSampling2D(), # 64x64
conv(n, 3, 3),
conv(n, 3, 3),
],
ns='offset.back')(input)
return back_feature_map, sequential([
Convolution2D(1, 3, 3, border_mode='same'),
LinearInBounds(-1, 1, clip=True),
],
ns='offset.back_out')(back_feature_map)
def get_offset_front(inputs, nb_units):
n = nb_units
input = concat(inputs)
return sequential([
Dense(8*n*4*4),
BatchNormalization(),
Activation('relu'),
Reshape((8*n, 4, 4)),
UpSampling2D(), # 8x8
conv(4*n, 3, 3),
conv(4*n, 3, 3),
UpSampling2D(), # 16x16
conv(2*n, 3, 3),
conv(2*n, 3, 3),
], ns='offset.front')(input)
def mask_blending_discriminator(x, n=32, out_activation='sigmoid'):
def conv(n):
return [
Convolution2D(n, 5, 5, subsample=(2, 2), border_mode='same'),
BatchNormalization(),
LeakyReLU(0.2),
]
return sequential([
Convolution2D(n, 5, 5, subsample=(2, 2), border_mode='same'),
LeakyReLU(0.2),
conv(2*n),
conv(4*n),
conv(8*n),
Flatten(),
Dense(1, activation=out_activation)
], ns='dis')(concat(x, axis=0, name='concat_fake_real'))
def mask_blending_discriminator(x, n=32, out_activation='sigmoid'):
def conv(n):
return [
Convolution2D(n, 5, 5, subsample=(2, 2), border_mode='same'),
BatchNormalization(),
LeakyReLU(0.2),
]
return sequential([
Convolution2D(n, 5, 5, subsample=(2, 2), border_mode='same'),
LeakyReLU(0.2),
conv(2 * n),
conv(4 * n),
conv(8 * n),
Flatten(),
Dense(1, activation=out_activation)
],
ns='dis')(concat(x, axis=0, name='concat_fake_real'))
def simple_gan():
z = Input(batch_shape=simple_gan_z_shape, name='z')
generator = sequential([
Dense(simple_gan_nb_z, activation='relu', name='g1'),
Dense(simple_gan_nb_z, activation='relu', name='g2'),
Dense(simple_gan_nb_out, activation='sigmoid', name='g3'),
])(z)
fake = Input(batch_shape=simple_gan_real_shape, name='fake')
real = Input(batch_shape=simple_gan_real_shape, name='real')
discriminator = sequential([
Dense(20, activation='relu', input_dim=2, name='d1'),
Dense(1, activation='sigmoid', name='d2')
])(concat([fake, real], axis=0))
return GAN(Container(z, generator),
Container([fake, real], gan_outputs(discriminator)),
simple_gan_z_shape[1:], simple_gan_real_shape[1:])
def get_offset_front(inputs, nb_units):
n = nb_units
input = concat(inputs)
return sequential(
[
Dense(8 * n * 4 * 4),
BatchNormalization(),
Activation('relu'),
Reshape((8 * n, 4, 4)),
UpSampling2D(), # 8x8
conv(4 * n, 3, 3),
conv(4 * n, 3, 3),
UpSampling2D(), # 16x16
conv(2 * n, 3, 3),
conv(2 * n, 3, 3),
],
ns='offset.front')(input)
def get_lighting_generator(inputs, nb_units):
n = nb_units
input = concat(inputs)
light_conv = sequential([
conv(n, 5, 5),
conv(n, 5, 5),
conv(n, 3, 3),
UpSampling2D(), # 32x32
conv(n, 5, 5),
Convolution2D(2, 1, 1, border_mode='same'),
UpSampling2D(), # 64x64
LinearInBounds(-1, 1, clip=True),
GaussianBlur(sigma=4),
], ns='lighting')(input)
shift = Split(0, 1, axis=1)(light_conv)
scale = Split(1, 2, axis=1)(light_conv)
return shift, scale
def get_lighting_generator(inputs, nb_units):
n = nb_units
input = concat(inputs)
light_conv = sequential(
[
conv(n, 5, 5),
conv(n, 5, 5),
conv(n, 3, 3),
UpSampling2D(), # 32x32
conv(n, 5, 5),
Convolution2D(2, 1, 1, border_mode='same'),
UpSampling2D(), # 64x64
LinearInBounds(-1, 1, clip=True),
GaussianBlur(sigma=4),
],
ns='lighting')(input)
shift = Split(0, 1, axis=1)(light_conv)
scale = Split(1, 2, axis=1)(light_conv)
return shift, scale
def light_generator(ins):
seq = sequential([
Convolution2D(1, 3, 3, border_mode='same')
])(concat(ins))
return UpSampling2D((4, 4))(seq), UpSampling2D((4, 4))(seq), \
UpSampling2D((4, 4))(seq),
def offset_back(x):
feature_map = sequential([
UpSampling2D(),
])(concat(x))
return feature_map, Convolution2D(1, 3, 3,
border_mode='same')(feature_map)
def offset_middle(x):
return UpSampling2D()(concat(x))
def mask_post(x):
return sequential([
Convolution2D(1, 3, 3, border_mode='same')
])(concat(x))
def discriminator_fn(x):
return gan_outputs(sequential([
Flatten(),
Dense(1),
])(concat(x)), fake_for_gen=(0, 10), fake_for_dis=(0, 10),
real=(10, 20))
def generator(inputs):
z, = inputs
z_driver = Split(*z_for_driver, axis=1)(z)
z_offset = Split(*z_for_offset, axis=1)(z)
z_bits = Split(*z_for_bits, axis=1)(z)
bits = get_bits(z_bits)
driver = mask_driver(z_driver)
driver_norm = NormSinCosAngle(0)(driver)
mask_input = concat([bits, driver_norm], name='mask_gen_in')
mask = mask_generator(mask_input)
if mask_generator_weights:
mask_layers = collect_layers(mask_input, mask)
load_weights(mask_layers, mask_generator_weights)
selection = with_regularizer(Selection(threshold=-0.08,
smooth_threshold=0.2,
sigma=1.5, name='selection'),
MinCoveredRegularizer())
mask_down = PyramidReduce()(mask)
mask_selection = selection(mask)
mask_selection_down = PyramidReduce(scale=4)(mask_selection)
out_offset_front = offset_front([z_offset,
ZeroGradient()(driver_norm)])
light_scale64, light_shift64 = \
lighting_generator([out_offset_front,
light_merge_mask16(mask_selection_down)])
mask_with_lighting = AddLighting(
scale_factor=0.5, shift_factor=0.75)(
[mask, light_scale64, light_shift64])
out_offset_middle = offset_middle(
[out_offset_front, offset_merge_mask16(mask_selection_down),
offset_merge_light16(concat(light_scale64, light_shift64))
])
offset_back_feature_map, out_offset_back = offset_back(
[out_offset_middle, offset_merge_mask32(mask_down)])
mask_weight32 = mask_weight_blending32(out_offset_middle)
mask_weight64 = mask_weight_blending64(out_offset_middle)
blending = PyramidBlending(offset_pyramid_layers=3,
mask_pyramid_layers=3,
mask_weights=['variable', 'variable', 1],
offset_weights=[1, 1, 1],
use_selection=[True, True, True],
name='blending')(
[out_offset_back, mask_with_lighting, mask_selection,
mask_weight32, mask_weight64])
mask_post = mask_postprocess([
blending, mask_selection, light_scale64, light_shift64, mask,
out_offset_back, offset_back_feature_map
])
mask_post_high = HighFrequencies(4, nb_steps=5,
name='mask_post_high')(mask_post)
blending_post = merge([mask_post_high, blending], mode='sum',
name='blending_post')
return LinearInBounds(-1.2, 1.2)(blending_post)
def offset_front(x):
return sequential([
Dense(16),
Reshape((1, 4, 4)),
UpSampling2D((4, 4))
])(concat(x))
def generator(inputs):
z, = inputs
z_driver = Split(*z_for_driver, axis=1)(z)
z_offset = Split(*z_for_offset, axis=1)(z)
z_bits = Split(*z_for_bits, axis=1)(z)
bits = get_bits(z_bits)
driver = mask_driver(z_driver)
driver_norm = NormSinCosAngle(0)(driver)
mask_input = concat([bits, driver_norm], name='mask_gen_in')
mask = mask_generator(mask_input)
if mask_generator_weights:
mask_layers = collect_layers(mask_input, mask)
load_weights(mask_layers, mask_generator_weights)
selection = with_regularizer(
Selection(threshold=-0.08,
smooth_threshold=0.2,
sigma=1.5,
name='selection'), MinCoveredRegularizer())
mask_down = PyramidReduce()(mask)
mask_selection = selection(mask)
mask_selection_down = PyramidReduce(scale=4)(mask_selection)
out_offset_front = offset_front(
[z_offset, ZeroGradient()(driver_norm)])
light_scale64, light_shift64 = \
lighting_generator([out_offset_front,
light_merge_mask16(mask_selection_down)])
mask_with_lighting = AddLighting(scale_factor=0.5, shift_factor=0.75)(
[mask, light_scale64, light_shift64])
out_offset_middle = offset_middle([
out_offset_front,
offset_merge_mask16(mask_selection_down),
offset_merge_light16(concat(light_scale64, light_shift64))
])
offset_back_feature_map, out_offset_back = offset_back(
[out_offset_middle,
offset_merge_mask32(mask_down)])
mask_weight32 = mask_weight_blending32(out_offset_middle)
mask_weight64 = mask_weight_blending64(out_offset_middle)
blending = PyramidBlending(offset_pyramid_layers=3,
mask_pyramid_layers=3,
mask_weights=['variable', 'variable', 1],
offset_weights=[1, 1, 1],
use_selection=[True, True, True],
name='blending')([
out_offset_back, mask_with_lighting,
mask_selection, mask_weight32,
mask_weight64
])
mask_post = mask_postprocess([
blending, mask_selection, light_scale64, light_shift64, mask,
out_offset_back, offset_back_feature_map
])
mask_post_high = HighFrequencies(4, nb_steps=5,
name='mask_post_high')(mask_post)
blending_post = merge([mask_post_high, blending],
mode='sum',
name='blending_post')
return LinearInBounds(-1.2, 1.2)(blending_post)