def _build_generator_given_z(self):
z = Input(shape=(self.z_dim, ), name='z')
z_bits = Subtensor(*self.pos_z_bits, axis=1)(z)
z_labels = Subtensor(*self.pos_z_labels, axis=1)(z)
z_offset = Subtensor(*self.pos_z_offset, axis=1)(z)
bits = ThresholdBits()(z_bits)
nb_labels_without_bits = self.labels_shape[0] - self.nb_bits
generated_labels = get_label_generator(
z_labels,
self.generator_units,
nb_output_units=nb_labels_without_bits)
labels_normed = NormSinCosAngle(0)(generated_labels)
labels = concat([bits, labels_normed], name='labels')
fake = self.generator_given_z_and_labels([z_offset, labels])
self.generator_given_z = Model([z], [fake])
sample_tensors = self.sample_generator_given_z_and_labels(
[z_offset, labels])
sample_tensors = [
name_tensor(t, n) for t, n in zip(
sample_tensors,
self.sample_generator_given_z_and_labels.output_names)
]
self.sample_generator_given_z_output_names = ['labels'] + \
self.sample_generator_given_z_and_labels_output_names
self.sample_generator_given_z = Model([z], [labels] + sample_tensors)
def tag3d_network_dense(input, nb_units=64, nb_dense_units=[512, 512],
depth=2, nb_output_channels=1, trainable=True):
n = nb_units
def conv(n, repeats=None):
def normal(shape, name=None):
return keras.initializations.normal(shape, scale=0.01, name=name)
if repeats is None:
repeats = depth
return [
[
Convolution2D(n, 3, 3, border_mode='same', init='he_normal'),
Activation('relu')
] for _ in range(repeats)
]
base = sequential([
[
Dense(nb_dense, activation='relu')
for nb_dense in nb_dense_units
],
Dense(8*n*4*4),
Activation('relu'),
Reshape((8*n, 4, 4,)),
conv(8*n),
UpSampling2D(), # 8x8
conv(4*n),
UpSampling2D(), # 16x16
conv(2*n),
], ns='tag3d_gen.base', trainable=trainable)(input)
tag3d = sequential([
conv(2*n),
UpSampling2D(), # 32x32
conv(n),
UpSampling2D(), # 64x64
conv(n, 1),
Convolution2D(1, 3, 3, border_mode='same', init='he_normal'),
], ns='tag3d', trainable=trainable)(base)
depth_map = sequential([
conv(n // 2, depth - 1),
Convolution2D(1, 3, 3, border_mode='same', init='he_normal'),
], ns='depth_map', trainable=trainable)(base)
return name_tensor(tag3d, 'tag3d'), name_tensor(depth_map, 'depth_map')
def _build_generator_given_z_offset_and_labels(self):
labels = Input(shape=self.labels_shape, name='input_labels')
z_offset = Input(shape=(self.z_dim_offset, ), name='input_z_offset')
outputs = OrderedDict()
labels_without_bits = Subtensor(self.nb_bits,
self.labels_shape[0],
axis=1)(labels)
# build tag3d tensors
tag3d, tag3d_depth_map = self.tag3d_network(labels)
tag3d_segmented = Segmentation(threshold=-0.08,
smooth_threshold=0.2,
sigma=1.5,
name='segmentation')(tag3d)
tag3d_segmented_blur = GaussianBlur(sigma=3.0)(tag3d_segmented)
# get generator params
blur_factor, lights, background, details = \
simple_gan_generator(self.generator_units, z_offset, labels_without_bits,
tag3d_depth_map, tag3d, depth=self.generator_depth)
tag3d_blur = BlendingBlur(sigma=1)([tag3d, blur_factor])
tag3d_lightin = AddLighting(scale_factor=0.85,
shift_factor=0.75)([tag3d_blur] + lights)
fake_without_noise = Background(name='bg')(
[background, tag3d_lightin, tag3d_segmented_blur])
details_high_pass = HighPass(4, nb_steps=4)(details)
fake = InBounds(-1.0,
1.0)(merge([details_high_pass, fake_without_noise],
mode='sum'))
outputs = [
('tag3d', tag3d),
('tag3d_blur', tag3d_blur),
('tag3d_lightin', tag3d_lightin),
('fake_without_noise', fake_without_noise),
('fake', fake),
]
outputs = OrderedDict([(name, name_tensor(x, name))
for name, x in outputs])
self.generator_given_z_and_labels = Model([z_offset, labels], [fake])
self.sample_generator_given_z_and_labels_output_names = list(
outputs.keys())
self.sample_generator_given_z_and_labels = Model([z_offset, labels],
list(
outputs.values()))
def _build_generator_given_z(self):
z = Input(shape=(self.z_dim,), name='z')
z_bits = Subtensor(*self.pos_z_bits, axis=1)(z)
z_labels = Subtensor(*self.pos_z_labels, axis=1)(z)
z_offset = Subtensor(*self.pos_z_offset, axis=1)(z)
bits = ThresholdBits()(z_bits)
nb_labels_without_bits = self.labels_shape[0] - self.nb_bits
generated_labels = get_label_generator(
z_labels, self.generator_units, nb_output_units=nb_labels_without_bits)
labels_normed = NormSinCosAngle(0)(generated_labels)
labels = concat([bits, labels_normed], name='labels')
fake = self.generator_given_z_and_labels([z_offset, labels])
self.generator_given_z = Model([z], [fake])
sample_tensors = self.sample_generator_given_z_and_labels([z_offset, labels])
sample_tensors = [name_tensor(t, n)
for t, n in zip(sample_tensors,
self.sample_generator_given_z_and_labels.output_names)]
self.sample_generator_given_z_output_names = ['labels'] + \
self.sample_generator_given_z_and_labels_output_names
self.sample_generator_given_z = Model([z], [labels] + sample_tensors)
def _build_generator_given_z_offset_and_labels(self):
labels = Input(shape=self.labels_shape, name='input_labels')
z_offset = Input(shape=(self.z_dim_offset,), name='input_z_offset')
outputs = OrderedDict()
labels_without_bits = Subtensor(self.nb_bits, self.labels_shape[0], axis=1)(labels)
# build tag3d tensors
tag3d, tag3d_depth_map = self.tag3d_network(labels)
tag3d_segmented = Segmentation(threshold=-0.08, smooth_threshold=0.2,
sigma=1.5, name='segmentation')(tag3d)
tag3d_segmented_blur = GaussianBlur(sigma=3.0)(tag3d_segmented)
# get generator params
blur_factor, lights, background, details = \
simple_gan_generator(self.generator_units, z_offset, labels_without_bits,
tag3d_depth_map, tag3d, depth=self.generator_depth)
tag3d_blur = BlendingBlur(sigma=1)([tag3d, blur_factor])
tag3d_lightin = AddLighting(scale_factor=0.85, shift_factor=0.75)([tag3d_blur] + lights)
fake_without_noise = Background(name='bg')(
[background, tag3d_lightin, tag3d_segmented_blur])
details_high_pass = HighPass(4, nb_steps=4)(details)
fake = InBounds(-1.0, 1.0)(merge([details_high_pass, fake_without_noise], mode='sum'))
outputs = [
('tag3d', tag3d),
('tag3d_blur', tag3d_blur),
('tag3d_lightin', tag3d_lightin),
('fake_without_noise', fake_without_noise),
('fake', fake),
]
outputs = OrderedDict([(name, name_tensor(x, name)) for name, x in outputs])
self.generator_given_z_and_labels = Model([z_offset, labels], [fake])
self.sample_generator_given_z_and_labels_output_names = list(outputs.keys())
self.sample_generator_given_z_and_labels = Model([z_offset, labels],
list(outputs.values()))
def tag3d_network_dense(input,
nb_units=64,
nb_dense_units=[512, 512],
depth=2,
nb_output_channels=1,
trainable=True):
n = nb_units
def conv(n, repeats=None):
def normal(shape, name=None):
return keras.initializations.normal(shape, scale=0.01, name=name)
if repeats is None:
repeats = depth
return [[
Convolution2D(n, 3, 3, border_mode='same', init='he_normal'),
Activation('relu')
] for _ in range(repeats)]
base = sequential(
[
[
Dense(nb_dense, activation='relu')
for nb_dense in nb_dense_units
],
Dense(8 * n * 4 * 4),
Activation('relu'),
Reshape((
8 * n,
4,
4,
)),
conv(8 * n),
UpSampling2D(), # 8x8
conv(4 * n),
UpSampling2D(), # 16x16
conv(2 * n),
],
ns='tag3d_gen.base',
trainable=trainable)(input)
tag3d = sequential(
[
conv(2 * n),
UpSampling2D(), # 32x32
conv(n),
UpSampling2D(), # 64x64
conv(n, 1),
Convolution2D(1, 3, 3, border_mode='same', init='he_normal'),
],
ns='tag3d',
trainable=trainable)(base)
depth_map = sequential([
conv(n // 2, depth - 1),
Convolution2D(1, 3, 3, border_mode='same', init='he_normal'),
],
ns='depth_map',
trainable=trainable)(base)
return name_tensor(tag3d, 'tag3d'), name_tensor(depth_map, 'depth_map')
def _build_generator_given_z_offset_and_labels(self):
labels = Input(shape=self.labels_shape, name='input_labels')
z_offset = Input(shape=(self.z_dim_offset, ), name='input_z_offset')
outputs = OrderedDict()
labels_without_bits = Subtensor(self.nb_bits,
self.labels_shape[0],
axis=1)(labels)
raw_tag3d, tag3d_depth_map = self.tag3d_network(labels)
tag3d = ScaleUnitIntervalTo(-1, 1)(raw_tag3d)
outputs['tag3d'] = tag3d
outputs['tag3d_depth_map'] = tag3d_depth_map
segmentation = Segmentation(threshold=-0.08,
smooth_threshold=0.2,
sigma=1.5,
name='segmentation')
tag3d_downsampled = PyramidReduce()(tag3d)
tag3d_segmented = segmentation(raw_tag3d)
outputs['tag3d_segmented'] = tag3d_segmented
tag3d_segmented_blur = GaussianBlur(sigma=0.66)(tag3d_segmented)
out_offset_front = get_offset_front(
[z_offset, ZeroGradient()(labels_without_bits)],
self.generator_units)
light_depth_map = get_preprocess(tag3d_depth_map,
self.preprocess_units,
nb_conv_layers=2)
light_outs = get_lighting_generator(
[out_offset_front, light_depth_map], self.generator_units)
offset_depth_map = get_preprocess(tag3d_depth_map,
self.preprocess_units,
nb_conv_layers=2)
offset_middle_light = get_preprocess(concat(light_outs),
self.preprocess_units,
resize=['down', 'down'])
offset_middle_tag3d = get_preprocess(tag3d_downsampled,
self.preprocess_units // 2,
resize=['down', ''],
nb_conv_layers=2)
out_offset_middle = get_offset_middle([
out_offset_front, offset_depth_map, offset_middle_light,
offset_middle_tag3d
], self.generator_units)
offset_back_tag3d_downsampled = get_preprocess(tag3d_downsampled,
self.preprocess_units //
2,
nb_conv_layers=2)
offset_back_feature_map, out_offset_back = get_offset_back(
[out_offset_middle, offset_back_tag3d_downsampled],
self.generator_units)
blur_factor = get_blur_factor(out_offset_middle, min=0.25, max=1.)
outputs['blur_factor'] = blur_factor
tag3d_blur = BlendingBlur(sigma=2.0)([tag3d, blur_factor])
outputs['tag3d_blur'] = tag3d_blur
outputs['light_black'] = light_outs[0]
outputs['light_white'] = light_outs[1]
outputs['light_shift'] = light_outs[2]
tag3d_lighten = AddLighting(
scale_factor=0.90, shift_factor=0.90)([tag3d_blur] + light_outs)
tag3d_lighten = InBounds(clip=True, weight=15)(tag3d_lighten)
outputs['tag3d_lighten'] = tag3d_lighten
outputs['background_offset'] = out_offset_back
blending = Background(name='blending')(
[out_offset_back, tag3d_lighten, tag3d_segmented_blur])
outputs['fake_without_noise'] = blending
details = get_details([
blending, tag3d_segmented_blur, tag3d, out_offset_back,
offset_back_feature_map
] + light_outs, self.generator_units)
outputs['details_offset'] = details
details_high_pass = HighPass(3.5, nb_steps=3)(details)
outputs['details_high_pass'] = details_high_pass
fake = InBounds(-2.0, 2.0)(merge([details_high_pass, blending],
mode='sum'))
outputs['fake'] = fake
for name in outputs.keys():
outputs[name] = name_tensor(outputs[name], name)
self.generator_given_z_and_labels = Model([z_offset, labels], [fake])
self.sample_generator_given_z_and_labels_output_names = list(
outputs.keys())
self.sample_generator_given_z_and_labels = Model([z_offset, labels],
list(
outputs.values()))
def _build_generator_given_z_offset_and_labels(self):
labels = Input(shape=self.labels_shape, name='input_labels')
z_offset = Input(shape=(self.z_dim_offset,), name='input_z_offset')
outputs = OrderedDict()
labels_without_bits = Subtensor(self.nb_bits, self.labels_shape[0], axis=1)(labels)
raw_tag3d, tag3d_depth_map = self.tag3d_network(labels)
tag3d = ScaleUnitIntervalTo(-1, 1)(raw_tag3d)
outputs['tag3d'] = tag3d
outputs['tag3d_depth_map'] = tag3d_depth_map
segmentation = Segmentation(threshold=-0.08, smooth_threshold=0.2,
sigma=1.5, name='segmentation')
tag3d_downsampled = PyramidReduce()(tag3d)
tag3d_segmented = segmentation(raw_tag3d)
outputs['tag3d_segmented'] = tag3d_segmented
tag3d_segmented_blur = GaussianBlur(sigma=0.66)(tag3d_segmented)
out_offset_front = get_offset_front([z_offset, ZeroGradient()(labels_without_bits)],
self.generator_units)
light_depth_map = get_preprocess(tag3d_depth_map, self.preprocess_units,
nb_conv_layers=2)
light_outs = get_lighting_generator([out_offset_front, light_depth_map],
self.generator_units)
offset_depth_map = get_preprocess(tag3d_depth_map, self.preprocess_units,
nb_conv_layers=2)
offset_middle_light = get_preprocess(concat(light_outs), self.preprocess_units,
resize=['down', 'down'])
offset_middle_tag3d = get_preprocess(tag3d_downsampled,
self.preprocess_units // 2,
resize=['down', ''],
nb_conv_layers=2)
out_offset_middle = get_offset_middle(
[out_offset_front, offset_depth_map,
offset_middle_light, offset_middle_tag3d],
self.generator_units)
offset_back_tag3d_downsampled = get_preprocess(tag3d_downsampled,
self.preprocess_units // 2,
nb_conv_layers=2)
offset_back_feature_map, out_offset_back = get_offset_back(
[out_offset_middle, offset_back_tag3d_downsampled], self.generator_units)
blur_factor = get_blur_factor(out_offset_middle, min=0.25, max=1.)
outputs['blur_factor'] = blur_factor
tag3d_blur = BlendingBlur(sigma=2.0)([tag3d, blur_factor])
outputs['tag3d_blur'] = tag3d_blur
outputs['light_black'] = light_outs[0]
outputs['light_white'] = light_outs[1]
outputs['light_shift'] = light_outs[2]
tag3d_lighten = AddLighting(
scale_factor=0.90, shift_factor=0.90)([tag3d_blur] + light_outs)
tag3d_lighten = InBounds(clip=True, weight=15)(tag3d_lighten)
outputs['tag3d_lighten'] = tag3d_lighten
outputs['background_offset'] = out_offset_back
blending = Background(name='blending')([out_offset_back, tag3d_lighten,
tag3d_segmented_blur])
outputs['fake_without_noise'] = blending
details = get_details(
[blending, tag3d_segmented_blur, tag3d, out_offset_back,
offset_back_feature_map] + light_outs, self.generator_units)
outputs['details_offset'] = details
details_high_pass = HighPass(3.5, nb_steps=3)(details)
outputs['details_high_pass'] = details_high_pass
fake = InBounds(-2.0, 2.0)(
merge([details_high_pass, blending], mode='sum'))
outputs['fake'] = fake
for name in outputs.keys():
outputs[name] = name_tensor(outputs[name], name)
self.generator_given_z_and_labels = Model([z_offset, labels], [fake])
self.sample_generator_given_z_and_labels_output_names = list(outputs.keys())
self.sample_generator_given_z_and_labels = Model([z_offset, labels],
list(outputs.values()))
