def _build_generator_given_z(self): z = Input(shape=(self.z_dim, ), name='z') z_bits = Subtensor(*self.pos_z_bits, axis=1)(z) z_labels = Subtensor(*self.pos_z_labels, axis=1)(z) z_offset = Subtensor(*self.pos_z_offset, axis=1)(z) bits = ThresholdBits()(z_bits) nb_labels_without_bits = self.labels_shape[0] - self.nb_bits generated_labels = get_label_generator( z_labels, self.generator_units, nb_output_units=nb_labels_without_bits) labels_normed = NormSinCosAngle(0)(generated_labels) labels = concat([bits, labels_normed], name='labels') fake = self.generator_given_z_and_labels([z_offset, labels]) self.generator_given_z = Model([z], [fake]) sample_tensors = self.sample_generator_given_z_and_labels( [z_offset, labels]) sample_tensors = [ name_tensor(t, n) for t, n in zip( sample_tensors, self.sample_generator_given_z_and_labels.output_names) ] self.sample_generator_given_z_output_names = ['labels'] + \ self.sample_generator_given_z_and_labels_output_names self.sample_generator_given_z = Model([z], [labels] + sample_tensors)
def tag3d_network_dense(input, nb_units=64, nb_dense_units=[512, 512], depth=2, nb_output_channels=1, trainable=True): n = nb_units def conv(n, repeats=None): def normal(shape, name=None): return keras.initializations.normal(shape, scale=0.01, name=name) if repeats is None: repeats = depth return [ [ Convolution2D(n, 3, 3, border_mode='same', init='he_normal'), Activation('relu') ] for _ in range(repeats) ] base = sequential([ [ Dense(nb_dense, activation='relu') for nb_dense in nb_dense_units ], Dense(8*n*4*4), Activation('relu'), Reshape((8*n, 4, 4,)), conv(8*n), UpSampling2D(), # 8x8 conv(4*n), UpSampling2D(), # 16x16 conv(2*n), ], ns='tag3d_gen.base', trainable=trainable)(input) tag3d = sequential([ conv(2*n), UpSampling2D(), # 32x32 conv(n), UpSampling2D(), # 64x64 conv(n, 1), Convolution2D(1, 3, 3, border_mode='same', init='he_normal'), ], ns='tag3d', trainable=trainable)(base) depth_map = sequential([ conv(n // 2, depth - 1), Convolution2D(1, 3, 3, border_mode='same', init='he_normal'), ], ns='depth_map', trainable=trainable)(base) return name_tensor(tag3d, 'tag3d'), name_tensor(depth_map, 'depth_map')
def _build_generator_given_z_offset_and_labels(self): labels = Input(shape=self.labels_shape, name='input_labels') z_offset = Input(shape=(self.z_dim_offset, ), name='input_z_offset') outputs = OrderedDict() labels_without_bits = Subtensor(self.nb_bits, self.labels_shape[0], axis=1)(labels) # build tag3d tensors tag3d, tag3d_depth_map = self.tag3d_network(labels) tag3d_segmented = Segmentation(threshold=-0.08, smooth_threshold=0.2, sigma=1.5, name='segmentation')(tag3d) tag3d_segmented_blur = GaussianBlur(sigma=3.0)(tag3d_segmented) # get generator params blur_factor, lights, background, details = \ simple_gan_generator(self.generator_units, z_offset, labels_without_bits, tag3d_depth_map, tag3d, depth=self.generator_depth) tag3d_blur = BlendingBlur(sigma=1)([tag3d, blur_factor]) tag3d_lightin = AddLighting(scale_factor=0.85, shift_factor=0.75)([tag3d_blur] + lights) fake_without_noise = Background(name='bg')( [background, tag3d_lightin, tag3d_segmented_blur]) details_high_pass = HighPass(4, nb_steps=4)(details) fake = InBounds(-1.0, 1.0)(merge([details_high_pass, fake_without_noise], mode='sum')) outputs = [ ('tag3d', tag3d), ('tag3d_blur', tag3d_blur), ('tag3d_lightin', tag3d_lightin), ('fake_without_noise', fake_without_noise), ('fake', fake), ] outputs = OrderedDict([(name, name_tensor(x, name)) for name, x in outputs]) self.generator_given_z_and_labels = Model([z_offset, labels], [fake]) self.sample_generator_given_z_and_labels_output_names = list( outputs.keys()) self.sample_generator_given_z_and_labels = Model([z_offset, labels], list( outputs.values()))
def _build_generator_given_z(self): z = Input(shape=(self.z_dim,), name='z') z_bits = Subtensor(*self.pos_z_bits, axis=1)(z) z_labels = Subtensor(*self.pos_z_labels, axis=1)(z) z_offset = Subtensor(*self.pos_z_offset, axis=1)(z) bits = ThresholdBits()(z_bits) nb_labels_without_bits = self.labels_shape[0] - self.nb_bits generated_labels = get_label_generator( z_labels, self.generator_units, nb_output_units=nb_labels_without_bits) labels_normed = NormSinCosAngle(0)(generated_labels) labels = concat([bits, labels_normed], name='labels') fake = self.generator_given_z_and_labels([z_offset, labels]) self.generator_given_z = Model([z], [fake]) sample_tensors = self.sample_generator_given_z_and_labels([z_offset, labels]) sample_tensors = [name_tensor(t, n) for t, n in zip(sample_tensors, self.sample_generator_given_z_and_labels.output_names)] self.sample_generator_given_z_output_names = ['labels'] + \ self.sample_generator_given_z_and_labels_output_names self.sample_generator_given_z = Model([z], [labels] + sample_tensors)
def _build_generator_given_z_offset_and_labels(self): labels = Input(shape=self.labels_shape, name='input_labels') z_offset = Input(shape=(self.z_dim_offset,), name='input_z_offset') outputs = OrderedDict() labels_without_bits = Subtensor(self.nb_bits, self.labels_shape[0], axis=1)(labels) # build tag3d tensors tag3d, tag3d_depth_map = self.tag3d_network(labels) tag3d_segmented = Segmentation(threshold=-0.08, smooth_threshold=0.2, sigma=1.5, name='segmentation')(tag3d) tag3d_segmented_blur = GaussianBlur(sigma=3.0)(tag3d_segmented) # get generator params blur_factor, lights, background, details = \ simple_gan_generator(self.generator_units, z_offset, labels_without_bits, tag3d_depth_map, tag3d, depth=self.generator_depth) tag3d_blur = BlendingBlur(sigma=1)([tag3d, blur_factor]) tag3d_lightin = AddLighting(scale_factor=0.85, shift_factor=0.75)([tag3d_blur] + lights) fake_without_noise = Background(name='bg')( [background, tag3d_lightin, tag3d_segmented_blur]) details_high_pass = HighPass(4, nb_steps=4)(details) fake = InBounds(-1.0, 1.0)(merge([details_high_pass, fake_without_noise], mode='sum')) outputs = [ ('tag3d', tag3d), ('tag3d_blur', tag3d_blur), ('tag3d_lightin', tag3d_lightin), ('fake_without_noise', fake_without_noise), ('fake', fake), ] outputs = OrderedDict([(name, name_tensor(x, name)) for name, x in outputs]) self.generator_given_z_and_labels = Model([z_offset, labels], [fake]) self.sample_generator_given_z_and_labels_output_names = list(outputs.keys()) self.sample_generator_given_z_and_labels = Model([z_offset, labels], list(outputs.values()))
def tag3d_network_dense(input, nb_units=64, nb_dense_units=[512, 512], depth=2, nb_output_channels=1, trainable=True): n = nb_units def conv(n, repeats=None): def normal(shape, name=None): return keras.initializations.normal(shape, scale=0.01, name=name) if repeats is None: repeats = depth return [[ Convolution2D(n, 3, 3, border_mode='same', init='he_normal'), Activation('relu') ] for _ in range(repeats)] base = sequential( [ [ Dense(nb_dense, activation='relu') for nb_dense in nb_dense_units ], Dense(8 * n * 4 * 4), Activation('relu'), Reshape(( 8 * n, 4, 4, )), conv(8 * n), UpSampling2D(), # 8x8 conv(4 * n), UpSampling2D(), # 16x16 conv(2 * n), ], ns='tag3d_gen.base', trainable=trainable)(input) tag3d = sequential( [ conv(2 * n), UpSampling2D(), # 32x32 conv(n), UpSampling2D(), # 64x64 conv(n, 1), Convolution2D(1, 3, 3, border_mode='same', init='he_normal'), ], ns='tag3d', trainable=trainable)(base) depth_map = sequential([ conv(n // 2, depth - 1), Convolution2D(1, 3, 3, border_mode='same', init='he_normal'), ], ns='depth_map', trainable=trainable)(base) return name_tensor(tag3d, 'tag3d'), name_tensor(depth_map, 'depth_map')
def _build_generator_given_z_offset_and_labels(self): labels = Input(shape=self.labels_shape, name='input_labels') z_offset = Input(shape=(self.z_dim_offset, ), name='input_z_offset') outputs = OrderedDict() labels_without_bits = Subtensor(self.nb_bits, self.labels_shape[0], axis=1)(labels) raw_tag3d, tag3d_depth_map = self.tag3d_network(labels) tag3d = ScaleUnitIntervalTo(-1, 1)(raw_tag3d) outputs['tag3d'] = tag3d outputs['tag3d_depth_map'] = tag3d_depth_map segmentation = Segmentation(threshold=-0.08, smooth_threshold=0.2, sigma=1.5, name='segmentation') tag3d_downsampled = PyramidReduce()(tag3d) tag3d_segmented = segmentation(raw_tag3d) outputs['tag3d_segmented'] = tag3d_segmented tag3d_segmented_blur = GaussianBlur(sigma=0.66)(tag3d_segmented) out_offset_front = get_offset_front( [z_offset, ZeroGradient()(labels_without_bits)], self.generator_units) light_depth_map = get_preprocess(tag3d_depth_map, self.preprocess_units, nb_conv_layers=2) light_outs = get_lighting_generator( [out_offset_front, light_depth_map], self.generator_units) offset_depth_map = get_preprocess(tag3d_depth_map, self.preprocess_units, nb_conv_layers=2) offset_middle_light = get_preprocess(concat(light_outs), self.preprocess_units, resize=['down', 'down']) offset_middle_tag3d = get_preprocess(tag3d_downsampled, self.preprocess_units // 2, resize=['down', ''], nb_conv_layers=2) out_offset_middle = get_offset_middle([ out_offset_front, offset_depth_map, offset_middle_light, offset_middle_tag3d ], self.generator_units) offset_back_tag3d_downsampled = get_preprocess(tag3d_downsampled, self.preprocess_units // 2, nb_conv_layers=2) offset_back_feature_map, out_offset_back = get_offset_back( [out_offset_middle, offset_back_tag3d_downsampled], self.generator_units) blur_factor = get_blur_factor(out_offset_middle, min=0.25, max=1.) outputs['blur_factor'] = blur_factor tag3d_blur = BlendingBlur(sigma=2.0)([tag3d, blur_factor]) outputs['tag3d_blur'] = tag3d_blur outputs['light_black'] = light_outs[0] outputs['light_white'] = light_outs[1] outputs['light_shift'] = light_outs[2] tag3d_lighten = AddLighting( scale_factor=0.90, shift_factor=0.90)([tag3d_blur] + light_outs) tag3d_lighten = InBounds(clip=True, weight=15)(tag3d_lighten) outputs['tag3d_lighten'] = tag3d_lighten outputs['background_offset'] = out_offset_back blending = Background(name='blending')( [out_offset_back, tag3d_lighten, tag3d_segmented_blur]) outputs['fake_without_noise'] = blending details = get_details([ blending, tag3d_segmented_blur, tag3d, out_offset_back, offset_back_feature_map ] + light_outs, self.generator_units) outputs['details_offset'] = details details_high_pass = HighPass(3.5, nb_steps=3)(details) outputs['details_high_pass'] = details_high_pass fake = InBounds(-2.0, 2.0)(merge([details_high_pass, blending], mode='sum')) outputs['fake'] = fake for name in outputs.keys(): outputs[name] = name_tensor(outputs[name], name) self.generator_given_z_and_labels = Model([z_offset, labels], [fake]) self.sample_generator_given_z_and_labels_output_names = list( outputs.keys()) self.sample_generator_given_z_and_labels = Model([z_offset, labels], list( outputs.values()))
def _build_generator_given_z_offset_and_labels(self): labels = Input(shape=self.labels_shape, name='input_labels') z_offset = Input(shape=(self.z_dim_offset,), name='input_z_offset') outputs = OrderedDict() labels_without_bits = Subtensor(self.nb_bits, self.labels_shape[0], axis=1)(labels) raw_tag3d, tag3d_depth_map = self.tag3d_network(labels) tag3d = ScaleUnitIntervalTo(-1, 1)(raw_tag3d) outputs['tag3d'] = tag3d outputs['tag3d_depth_map'] = tag3d_depth_map segmentation = Segmentation(threshold=-0.08, smooth_threshold=0.2, sigma=1.5, name='segmentation') tag3d_downsampled = PyramidReduce()(tag3d) tag3d_segmented = segmentation(raw_tag3d) outputs['tag3d_segmented'] = tag3d_segmented tag3d_segmented_blur = GaussianBlur(sigma=0.66)(tag3d_segmented) out_offset_front = get_offset_front([z_offset, ZeroGradient()(labels_without_bits)], self.generator_units) light_depth_map = get_preprocess(tag3d_depth_map, self.preprocess_units, nb_conv_layers=2) light_outs = get_lighting_generator([out_offset_front, light_depth_map], self.generator_units) offset_depth_map = get_preprocess(tag3d_depth_map, self.preprocess_units, nb_conv_layers=2) offset_middle_light = get_preprocess(concat(light_outs), self.preprocess_units, resize=['down', 'down']) offset_middle_tag3d = get_preprocess(tag3d_downsampled, self.preprocess_units // 2, resize=['down', ''], nb_conv_layers=2) out_offset_middle = get_offset_middle( [out_offset_front, offset_depth_map, offset_middle_light, offset_middle_tag3d], self.generator_units) offset_back_tag3d_downsampled = get_preprocess(tag3d_downsampled, self.preprocess_units // 2, nb_conv_layers=2) offset_back_feature_map, out_offset_back = get_offset_back( [out_offset_middle, offset_back_tag3d_downsampled], self.generator_units) blur_factor = get_blur_factor(out_offset_middle, min=0.25, max=1.) outputs['blur_factor'] = blur_factor tag3d_blur = BlendingBlur(sigma=2.0)([tag3d, blur_factor]) outputs['tag3d_blur'] = tag3d_blur outputs['light_black'] = light_outs[0] outputs['light_white'] = light_outs[1] outputs['light_shift'] = light_outs[2] tag3d_lighten = AddLighting( scale_factor=0.90, shift_factor=0.90)([tag3d_blur] + light_outs) tag3d_lighten = InBounds(clip=True, weight=15)(tag3d_lighten) outputs['tag3d_lighten'] = tag3d_lighten outputs['background_offset'] = out_offset_back blending = Background(name='blending')([out_offset_back, tag3d_lighten, tag3d_segmented_blur]) outputs['fake_without_noise'] = blending details = get_details( [blending, tag3d_segmented_blur, tag3d, out_offset_back, offset_back_feature_map] + light_outs, self.generator_units) outputs['details_offset'] = details details_high_pass = HighPass(3.5, nb_steps=3)(details) outputs['details_high_pass'] = details_high_pass fake = InBounds(-2.0, 2.0)( merge([details_high_pass, blending], mode='sum')) outputs['fake'] = fake for name in outputs.keys(): outputs[name] = name_tensor(outputs[name], name) self.generator_given_z_and_labels = Model([z_offset, labels], [fake]) self.sample_generator_given_z_and_labels_output_names = list(outputs.keys()) self.sample_generator_given_z_and_labels = Model([z_offset, labels], list(outputs.values()))