def get_lighting_generator(inputs, nb_units):
n = nb_units
input = concat(inputs)
light_conv = sequential(
[
conv(n, 3, 3), # 16x16
MaxPooling2D(), # 8x8
conv(n, 3, 3),
conv(n, 3, 3),
UpSampling2D(), # 16x16
conv(n, 3, 3),
UpSampling2D(), # 32x32
conv(n, 3, 3),
Convolution2D(3, 1, 1, border_mode='same'),
UpSampling2D(), # 64x64
GaussianBlur(sigma=2.5),
],
ns='lighting')(input)
shift = Subtensor(0, 1, axis=1)(light_conv)
shift = InBounds(-1, 1)(shift)
in_bounds = InBounds(0, 2)
scale_black = in_bounds(Subtensor(1, 2, axis=1)(light_conv))
scale_white = in_bounds(Subtensor(2, 3, axis=1)(light_conv))
return [scale_black, scale_white, shift]
def test_in_bounds_clip():
layer = InBounds(-1, 1, clip=True)
shape = (1, 1)
layer.build(shape)
arr = np.array([[0]], dtype=np.float32)
output = layer(theano.shared(arr)).eval()
assert (output == arr).all()
arr = np.array([[0]], dtype=np.float32)
output = layer(theano.shared(arr)).eval()
assert (output == arr).all()
arr = np.array([[2]], dtype=np.float32)
output = layer(theano.shared(arr)).eval()
assert float(output) == 1.
def get_offset_back(inputs, nb_units):
n = nb_units
input = concat(inputs)
back_feature_map = sequential(
[
UpSampling2D(), # 64x64
conv(n, 3, 3),
conv(n, 3, 3),
InBounds(-1, 1),
],
ns='offset.back')(input)
return back_feature_map, sequential([
Convolution2D(1, 3, 3, border_mode='same'),
InBounds(-1, 1),
],
ns='offset.back_out')(back_feature_map)
def dcgan_generator_conv(n=32,
input_dim=50,
nb_output_channels=1,
init=normal(0.02)):
def conv(nb_filter, h, w):
model.add(Convolution2D(nb_filter, h, w, border_mode='same',
init=init))
model.add(batch_norm())
model.add(Activation('relu'))
def deconv(nb_filter, h, w):
deconv_layer = Deconvolution2D(nb_filter,
h,
w,
border_mode=(1, 1),
init=init)
model.add(deconv_layer)
w = np.random.normal(0, 0.02, deconv_layer.W_shape).astype(np.float32)
w *= np.random.uniform(0, 1, (1, w.shape[1], 1, 1))
deconv_layer.W.set_value(w)
model.add(batch_norm())
model.add(Activation('relu'))
def up():
model.add(UpSampling2D())
z = Layer(input_shape=(input_dim, ))
model = Sequential()
model.add(z)
model.add(Dense(8 * n * 4 * 4, init=init))
model.add(batch_norm())
model.add(Reshape((
8 * n,
4,
4,
)))
model.add(Activation('relu'))
up() # 8
conv(4 * n, 3, 3)
up() # 16
conv(2 * n, 3, 3)
conv(2 * n, 3, 3)
up() # 32
conv(n, 3, 3)
conv(n, 3, 3)
up() # 64
conv(n, 3, 3)
model.add(
Deconvolution2D(nb_output_channels,
3,
3,
border_mode=(1, 1),
init=init))
model.add(InBounds(-1, 1))
return model
def get_blur_factor(inputs, min=0, max=2):
input = concat(inputs)
return sequential([
Convolution2D(1, 3, 3),
Flatten(),
Dense(1),
InBounds(min, max),
],
ns='mask_weight_blending')(input)
def get_details(inputs, nb_units):
n = nb_units
return sequential([
conv(n, 3, 3),
conv(n, 3, 3),
conv(n, 3, 3),
Convolution2D(1, 3, 3, border_mode='same', init='normal'),
InBounds(-2, 2)
],
ns='details')(concat(inputs))
def _build_generator_given_z_offset_and_labels(self):
labels = Input(shape=self.labels_shape, name='input_labels')
z_offset = Input(shape=(self.z_dim_offset, ), name='input_z_offset')
outputs = OrderedDict()
labels_without_bits = Subtensor(self.nb_bits,
self.labels_shape[0],
axis=1)(labels)
# build tag3d tensors
tag3d, tag3d_depth_map = self.tag3d_network(labels)
tag3d_segmented = Segmentation(threshold=-0.08,
smooth_threshold=0.2,
sigma=1.5,
name='segmentation')(tag3d)
tag3d_segmented_blur = GaussianBlur(sigma=3.0)(tag3d_segmented)
# get generator params
blur_factor, lights, background, details = \
simple_gan_generator(self.generator_units, z_offset, labels_without_bits,
tag3d_depth_map, tag3d, depth=self.generator_depth)
tag3d_blur = BlendingBlur(sigma=1)([tag3d, blur_factor])
tag3d_lightin = AddLighting(scale_factor=0.85,
shift_factor=0.75)([tag3d_blur] + lights)
fake_without_noise = Background(name='bg')(
[background, tag3d_lightin, tag3d_segmented_blur])
details_high_pass = HighPass(4, nb_steps=4)(details)
fake = InBounds(-1.0,
1.0)(merge([details_high_pass, fake_without_noise],
mode='sum'))
outputs = [
('tag3d', tag3d),
('tag3d_blur', tag3d_blur),
('tag3d_lightin', tag3d_lightin),
('fake_without_noise', fake_without_noise),
('fake', fake),
]
outputs = OrderedDict([(name, name_tensor(x, name))
for name, x in outputs])
self.generator_given_z_and_labels = Model([z_offset, labels], [fake])
self.sample_generator_given_z_and_labels_output_names = list(
outputs.keys())
self.sample_generator_given_z_and_labels = Model([z_offset, labels],
list(
outputs.values()))
def dcgan_generator(n=32,
input_dim=50,
nb_output_channels=1,
use_dct=False,
init=normal(0.02)):
def deconv(nb_filter, h, w):
return Deconvolution2D(nb_filter,
h,
w,
subsample=(2, 2),
border_mode=(2, 2),
init=init)
model = Sequential()
model.add(Dense(8 * n * 4 * 4, input_dim=input_dim, init=init))
model.add(batch_norm())
model.add(Reshape((
8 * n,
4,
4,
)))
if use_dct:
model.add(iDCT())
model.add(Activation('relu'))
model.add(deconv(4 * n, 5, 5))
model.add(batch_norm())
model.add(Activation('relu'))
model.add(deconv(2 * n, 5, 5))
model.add(batch_norm())
model.add(Activation('relu'))
model.add(deconv(n, 5, 5))
model.add(batch_norm())
model.add(Activation('relu'))
model.add(
Deconvolution2D(nb_output_channels,
5,
5,
subsample=(2, 2),
border_mode=(2, 2),
init=init))
model.add(InBounds(-1, 1))
return model
def get_label_generator(x, nb_units, nb_output_units):
n = nb_units
driver = sequential([
Dense(n),
batch_norm(),
Dropout(0.25),
Activation('relu'),
Dense(n),
batch_norm(),
Dropout(0.25),
Activation('relu'),
Dense(nb_output_units),
batch_norm(gamma_init=constant_init(0.25)),
InBounds(-1, 1),
],
ns='driver')
return driver(x)
def simple_gan_generator(nb_units, z, labels, depth_map, tag3d, depth=2):
n = nb_units
depth_map_features = sequential([
conv2d_block(n),
conv2d_block(2 * n),
])(depth_map)
tag3d_features = sequential([
conv2d_block(n, subsample=2),
conv2d_block(2 * n, subsample=2),
])(tag3d)
x = sequential([
Dense(5 * n),
BatchNormalization(mode=2),
Activation('relu'),
Dense(5 * n),
BatchNormalization(mode=2),
Activation('relu'),
])(concat([z, labels]))
blur = InBounds(0, 1, clip=True)(Dense(1)(x))
x = sequential([
Dense(8 * 4 * 4 * n),
Activation('relu'),
BatchNormalization(mode=2),
Reshape((8 * n, 4, 4)),
])(x)
x = sequential([
conv2d_block(8 * n, filters=1, depth=1, up=True), # 4x4 -> 8x8
conv2d_block(8 * n, depth=depth, up=True), # 8x8 -> 16x16
])(x)
off_depth_map = sequential([
conv2d_block(2 * n, depth=depth),
])(concat([x, depth_map_features]))
light = sequential([
conv2d_block(2 * n, depth=depth, up=True), # 16x16 -> 32x32
conv2d_block(n, depth=depth, up=True), # 32x32 -> 64x64
])(off_depth_map)
def get_light(x):
return sequential([
conv2d_block(1, filters=1, batchnorm=False),
GaussianBlur(sigma=4),
InBounds(0, 1, clip=True),
])(x)
light_sb = get_light(light)
light_sw = get_light(light)
light_t = get_light(light)
background = sequential([
conv2d_block(2 * n, depth=depth, up=True), # 16x16 -> 32x32
conv2d_block(n, depth=depth, up=True), # 32x32 -> 64x64
conv2d_block(1, batchnorm=False),
InBounds(-1, 1, clip=True),
])(off_depth_map)
details = sequential([
conv2d_block(2 * n, depth=depth, up=True), # 16x16 -> 32x32
conv2d_block(n, depth=depth, up=True), # 32x32 -> 64x64
conv2d_block(1, depth=1, batchnorm=False),
InBounds(-1, 1, clip=True)
])(concat(tag3d_features, off_depth_map))
return blur, [light_sb, light_sw, light_t], background, details
def get_light(x):
return sequential([
conv2d_block(1, filters=1, batchnorm=False),
GaussianBlur(sigma=4),
InBounds(0, 1, clip=True),
])(x)
def _build_generator_given_z_offset_and_labels(self):
labels = Input(shape=self.labels_shape, name='input_labels')
z_offset = Input(shape=(self.z_dim_offset, ), name='input_z_offset')
outputs = OrderedDict()
labels_without_bits = Subtensor(self.nb_bits,
self.labels_shape[0],
axis=1)(labels)
raw_tag3d, tag3d_depth_map = self.tag3d_network(labels)
tag3d = ScaleUnitIntervalTo(-1, 1)(raw_tag3d)
outputs['tag3d'] = tag3d
outputs['tag3d_depth_map'] = tag3d_depth_map
segmentation = Segmentation(threshold=-0.08,
smooth_threshold=0.2,
sigma=1.5,
name='segmentation')
tag3d_downsampled = PyramidReduce()(tag3d)
tag3d_segmented = segmentation(raw_tag3d)
outputs['tag3d_segmented'] = tag3d_segmented
tag3d_segmented_blur = GaussianBlur(sigma=0.66)(tag3d_segmented)
out_offset_front = get_offset_front(
[z_offset, ZeroGradient()(labels_without_bits)],
self.generator_units)
light_depth_map = get_preprocess(tag3d_depth_map,
self.preprocess_units,
nb_conv_layers=2)
light_outs = get_lighting_generator(
[out_offset_front, light_depth_map], self.generator_units)
offset_depth_map = get_preprocess(tag3d_depth_map,
self.preprocess_units,
nb_conv_layers=2)
offset_middle_light = get_preprocess(concat(light_outs),
self.preprocess_units,
resize=['down', 'down'])
offset_middle_tag3d = get_preprocess(tag3d_downsampled,
self.preprocess_units // 2,
resize=['down', ''],
nb_conv_layers=2)
out_offset_middle = get_offset_middle([
out_offset_front, offset_depth_map, offset_middle_light,
offset_middle_tag3d
], self.generator_units)
offset_back_tag3d_downsampled = get_preprocess(tag3d_downsampled,
self.preprocess_units //
2,
nb_conv_layers=2)
offset_back_feature_map, out_offset_back = get_offset_back(
[out_offset_middle, offset_back_tag3d_downsampled],
self.generator_units)
blur_factor = get_blur_factor(out_offset_middle, min=0.25, max=1.)
outputs['blur_factor'] = blur_factor
tag3d_blur = BlendingBlur(sigma=2.0)([tag3d, blur_factor])
outputs['tag3d_blur'] = tag3d_blur
outputs['light_black'] = light_outs[0]
outputs['light_white'] = light_outs[1]
outputs['light_shift'] = light_outs[2]
tag3d_lighten = AddLighting(
scale_factor=0.90, shift_factor=0.90)([tag3d_blur] + light_outs)
tag3d_lighten = InBounds(clip=True, weight=15)(tag3d_lighten)
outputs['tag3d_lighten'] = tag3d_lighten
outputs['background_offset'] = out_offset_back
blending = Background(name='blending')(
[out_offset_back, tag3d_lighten, tag3d_segmented_blur])
outputs['fake_without_noise'] = blending
details = get_details([
blending, tag3d_segmented_blur, tag3d, out_offset_back,
offset_back_feature_map
] + light_outs, self.generator_units)
outputs['details_offset'] = details
details_high_pass = HighPass(3.5, nb_steps=3)(details)
outputs['details_high_pass'] = details_high_pass
fake = InBounds(-2.0, 2.0)(merge([details_high_pass, blending],
mode='sum'))
outputs['fake'] = fake
for name in outputs.keys():
outputs[name] = name_tensor(outputs[name], name)
self.generator_given_z_and_labels = Model([z_offset, labels], [fake])
self.sample_generator_given_z_and_labels_output_names = list(
outputs.keys())
self.sample_generator_given_z_and_labels = Model([z_offset, labels],
list(
outputs.values()))