Example #1
0
    def build_loss(self, a_image, ap_image, b_image):
        '''Create an expression for the loss as a function of the image inputs.'''
        print('Building loss...')
        loss = super(NNFModel, self).build_loss(a_image, ap_image, b_image)
        # Precompute static features for performance
        print('Precomputing static features...')
        all_a_features, all_ap_image_features, all_b_features = self.precompute_static_features(a_image, ap_image, b_image)
        print('Building and combining losses...')
        if self.args.analogy_weight:
            for layer_name in self.args.analogy_layers:
                a_features = all_a_features[layer_name][0]
                ap_image_features = all_ap_image_features[layer_name][0]
                b_features = all_b_features[layer_name][0]
                # current combined output
                layer_features = self.get_layer_output(layer_name)
                combination_features = layer_features[0, :, :, :]
                al = nnf_analogy_loss(
                    a_features, ap_image_features, b_features, combination_features,
                    num_steps=self.args.analogy_nnf_steps, patch_size=self.args.patch_size,
                    patch_stride=self.args.patch_stride, jump_size=1.0)
                loss += (self.args.analogy_weight / len(self.args.analogy_layers)) * al

        existing_feature_nnfs = getattr(self, 'feature_nnfs', [None] * len(self.args.mrf_layers))
        self.feature_nnfs = []
        if self.args.mrf_weight:
            for layer_name, existing_nnf in zip(self.args.mrf_layers, existing_feature_nnfs):
                ap_image_features = all_ap_image_features[layer_name][0]
                # current combined output
                layer_features = self.get_layer_output(layer_name)
                combination_features = layer_features[0, :, :, :]
                input_shape = self.get_layer_output_shape(layer_name)
                if existing_nnf:
                    matcher = existing_nnf.matcher.scale((input_shape[3], input_shape[2], input_shape[1]), ap_image_features)
                else:
                    matcher = PatchMatcher(
                        (input_shape[3], input_shape[2], input_shape[1]), ap_image_features,
                        patch_size=self.args.patch_size, jump_size=1.0, patch_stride=self.args.patch_stride)
                nnf = NNFState(matcher, self.get_f_layer(layer_name))
                self.feature_nnfs.append(nnf)
                sl = content_loss(combination_features, nnf.placeholder)
                loss += (self.args.mrf_weight / len(self.args.mrf_layers)) * sl

        if self.args.b_bp_content_weight:
            for layer_name in self.args.b_content_layers:
                b_features = K.variable(all_b_features[layer_name][0])
                # current combined output
                bp_features = self.get_layer_output(layer_name)
                cl = content_loss(bp_features, b_features)
                loss += self.args.b_bp_content_weight / len(self.args.b_content_layers) * cl

        return loss
Example #2
0
    def build_loss(self, a_sem_image, a_image, b_sem_image, b_image):
        '''Create an expression for the loss as a function of the image inputs.'''
        print('Building loss...')
        loss = super(AnalogyModel, self).build_loss(a_sem_image, a_image,
                                                    b_sem_image, b_image)
        # Precompute static features for performance
        print('Precomputing static features...')
        all_asem_features, all_a_image_features, all_bsem_features, all_b_features = self.precompute_static_features(
            a_sem_image, a_image, b_sem_image, b_image)

        print('Building DSS losses...')
        if self.args.analogy_weight:  #PatchMatch   -image-analogies, a kind of content loss (if B is not a photograph but a [0,1] mask)
            for layer_name in self.args.analogy_layers:
                asem_features = all_asem_features[layer_name][0]  #array
                a_image_features = all_a_image_features[layer_name][0]  #array
                bsem_features = all_bsem_features[layer_name][0]  #array
                # current combined output
                layer_features = self.get_layer_output(layer_name)  #variable
                combination_features = layer_features[0, :, :, :]  #variable
                al = nnf_analogy_loss(asem_features,
                                      a_image_features,
                                      bsem_features,
                                      combination_features,
                                      num_steps=self.args.analogy_nnf_steps,
                                      patch_size=self.args.patch_size,
                                      patch_stride=self.args.patch_stride,
                                      jump_size=1.0)
                loss += (self.args.analogy_weight /
                         len(self.args.analogy_layers)) * al

        print('Building Content losses...')
        if self.args.content_weight:  #content loss(now b is seg)  (default=0.0)
            for layer_name in self.args.b_content_layers:
                b_features = K.variable(all_b_features[layer_name][0])
                # current combined output
                bp_features = self.get_layer_output(layer_name)
                cl = content_loss(bp_features, b_features)
                loss += self.args.content_weight / len(
                    self.args.b_content_layers) * cl

        print('Building Style losses...')
        if self.args.neural_style_weight != 0.0:  #Gram  (default=0.0)
            for layer_name in self.args.neural_style_layers:
                a_image_features = K.variable(
                    all_a_image_features[layer_name][0])
                layer_features = self.get_layer_output(layer_name)
                layer_shape = self.get_layer_output_shape(layer_name)
                # current combined output
                combination_features = layer_features[0, :, :, :]
                nsl = neural_style_loss(a_image_features, combination_features,
                                        3, self.output_shape[-2],
                                        self.output_shape[-1])
                loss += (self.args.neural_style_weight /
                         len(self.args.neural_style_layers)) * nsl

        return loss
Example #3
0
    def build_loss(self, a_image, ap_image, b_image):
        '''Create an expression for the loss as a function of the image inputs.'''
        print('Building loss...')
        loss = super(AnalogyModel, self).build_loss(a_image, ap_image, b_image)
        # Precompute static features for performance
        print('Precomputing static features...')
        all_a_features, all_ap_image_features, all_b_features = self.precompute_static_features(a_image, ap_image, b_image)
        print('Building and combining losses...')
        if self.args.analogy_weight != 0.0:
            for layer_name in self.args.analogy_layers:
                a_features = all_a_features[layer_name][0]
                ap_image_features = all_ap_image_features[layer_name][0]
                b_features = all_b_features[layer_name][0]
                # current combined output
                layer_features = self.get_layer_output(layer_name)
                combination_features = layer_features[0, :, :, :]
                al = analogy_loss(a_features, ap_image_features,
                    b_features, combination_features,
                    use_full_analogy=self.args.use_full_analogy,
                    patch_size=self.args.patch_size,
                    patch_stride=self.args.patch_stride)
                loss += (self.args.analogy_weight / len(self.args.analogy_layers)) * al

        if self.args.mrf_weight != 0.0:
            for layer_name in self.args.mrf_layers:
                ap_image_features = K.variable(all_ap_image_features[layer_name][0])
                layer_features = self.get_layer_output(layer_name)
                # current combined output
                combination_features = layer_features[0, :, :, :]
                sl = mrf_loss(ap_image_features, combination_features,
                    patch_size=self.args.patch_size,
                    patch_stride=self.args.patch_stride)
                loss += (self.args.mrf_weight / len(self.args.mrf_layers)) * sl

        if self.args.b_bp_content_weight != 0.0:
            for layer_name in self.args.b_content_layers:
                b_features = K.variable(all_b_features[layer_name][0])
                # current combined output
                bp_features = self.get_layer_output(layer_name)
                cl = content_loss(bp_features, b_features)
                loss += self.args.b_bp_content_weight / len(self.args.b_content_layers) * cl

        if self.args.neural_style_weight != 0.0:
            for layer_name in self.args.neural_style_layers:
                ap_image_features = K.variable(all_ap_image_features[layer_name][0])
                layer_features = self.get_layer_output(layer_name)
                layer_shape = self.get_layer_output_shape(layer_name)
                # current combined output
                combination_features = layer_features[0, :, :, :]
                nsl = neural_style_loss(ap_image_features, combination_features, 3, self.output_shape[-2], self.output_shape[-1])
                loss += (self.args.neural_style_weight / len(self.args.neural_style_layers)) * nsl
        return loss
Example #4
0
    def build_loss(self, a_sem_image, a_image, b_sem_image,b_image):
        '''Create an expression for the loss as a function of the image inputs.'''
        print('Building loss...')
        loss = super(NNFModel, self).build_loss(a_sem_image, a_image, b_sem_image,b_image)#viriable
        # Precompute static features for performance
        print('Precomputing static features...')
        all_asem_features, all_a_image_features, all_bsem_features,all_b_features = self.precompute_static_features(a_sem_image, a_image, b_sem_image,b_image)
        print('Building and combining losses...')
        
        print('Building DSS losses...')
        if self.args.analogy_weight:#Deep Semantic Space-guided Loss
            for layer_name in self.args.analogy_layers:
                asem_features = all_asem_features[layer_name][0]#array
                a_image_features = all_a_image_features[layer_name][0]#array
                bsem_features = all_bsem_features[layer_name][0]#array
                # current combined output
                layer_features = self.get_layer_output(layer_name)#variable
                combination_features = layer_features[0, :, :, :]#variable
                al = nnf_analogy_loss(
                    asem_features, a_image_features, bsem_features, combination_features,
                    num_steps=self.args.analogy_nnf_steps, patch_size=self.args.patch_size,
                    patch_stride=self.args.patch_stride, jump_size=1.0)
                loss += (self.args.analogy_weight / len(self.args.analogy_layers)) * al

        print('Building Content losses...')
        if self.args.content_weight:#content loss
            for layer_name in self.args.b_content_layers:
                b_features = K.variable(all_b_features[layer_name][0])
                # current combined output
                bp_features = self.get_layer_output(layer_name)
                cl = content_loss(bp_features, b_features)
                loss += self.args.content_weight / len(self.args.b_content_layers) * cl

        print('Building Style losses...')
        if self.args.neural_style_weight != 0.0:#Gram-based Style loss
            for layer_name in self.args.neural_style_layers:
                a_image_features = K.variable(all_a_image_features[layer_name][0])
                layer_features = self.get_layer_output(layer_name)
                layer_shape = self.get_layer_output_shape(layer_name)
                # current combined output
                combination_features = layer_features[0, :, :, :]
                nsl = neural_style_loss(a_image_features, combination_features, 3, self.output_shape[-2], self.output_shape[-1])
                loss += (self.args.neural_style_weight / len(self.args.neural_style_layers)) * nsl

        return loss
Example #5
0
    def build_loss(self, a_image, ap_image, b_image):
        '''Create an expression for the loss as a function of the image inputs.'''
        print('Building loss...')
        loss = super(AnalogyModel, self).build_loss(a_image, ap_image, b_image)
        # Precompute static features for performance
        print('Precomputing static features...')
        all_a_features, all_ap_image_features, all_b_features = self.precompute_static_features(
            a_image, ap_image, b_image)
        print('Building and combining losses...')
        if self.args.analogy_weight != 0.0:
            for layer_name in self.args.analogy_layers:
                a_features = all_a_features[layer_name][0]
                ap_image_features = all_ap_image_features[layer_name][0]
                b_features = all_b_features[layer_name][0]
                # current combined output
                layer_features = self.get_layer_output(layer_name)
                combination_features = layer_features[0, :, :, :]
                al = analogy_loss(a_features,
                                  ap_image_features,
                                  b_features,
                                  combination_features,
                                  use_full_analogy=self.args.use_full_analogy,
                                  patch_size=self.args.patch_size,
                                  patch_stride=self.args.patch_stride)
                loss += (self.args.analogy_weight /
                         len(self.args.analogy_layers)) * al

        if self.args.mrf_weight != 0.0:
            for layer_name in self.args.mrf_layers:
                ap_image_features = K.variable(
                    all_ap_image_features[layer_name][0])
                layer_features = self.get_layer_output(layer_name)
                # current combined output
                combination_features = layer_features[0, :, :, :]
                sl = mrf_loss(ap_image_features,
                              combination_features,
                              patch_size=self.args.patch_size,
                              patch_stride=self.args.patch_stride)
                loss += (self.args.mrf_weight / len(self.args.mrf_layers)) * sl

        if self.args.b_bp_content_weight != 0.0:
            for layer_name in self.args.b_content_layers:
                b_features = K.variable(all_b_features[layer_name][0])
                # current combined output
                bp_features = self.get_layer_output(layer_name)
                cl = content_loss(bp_features, b_features)
                loss += self.args.b_bp_content_weight / len(
                    self.args.b_content_layers) * cl

        if self.args.neural_style_weight != 0.0:
            for layer_name in self.args.neural_style_layers:
                ap_image_features = K.variable(
                    all_ap_image_features[layer_name][0])
                layer_features = self.get_layer_output(layer_name)
                layer_shape = self.get_layer_output_shape(layer_name)
                # current combined output
                combination_features = layer_features[0, :, :, :]
                nsl = neural_style_loss(ap_image_features,
                                        combination_features, 3,
                                        self.output_shape[-2],
                                        self.output_shape[-1])
                loss += (self.args.neural_style_weight /
                         len(self.args.neural_style_layers)) * nsl
        return loss
Example #6
0
    def build_loss(self, a_image, ap_image, b_image):
        '''Create an expression for the loss as a function of the image inputs.'''
        print('Building loss...')
        loss = super(NNFModel, self).build_loss(a_image, ap_image, b_image)
        # Precompute static features for performance
        print('Precomputing static features...')
        all_a_features, all_ap_image_features, all_b_features = self.precompute_static_features(
            a_image, ap_image, b_image)
        print('Building and combining losses...')
        if self.args.analogy_weight:
            for layer_name in self.args.analogy_layers:
                a_features = all_a_features[layer_name][0]
                ap_image_features = all_ap_image_features[layer_name][0]
                b_features = all_b_features[layer_name][0]
                # current combined output
                layer_features = self.get_layer_output(layer_name)
                combination_features = layer_features[0, :, :, :]
                al = nnf_analogy_loss(a_features,
                                      ap_image_features,
                                      b_features,
                                      combination_features,
                                      num_steps=self.args.analogy_nnf_steps,
                                      patch_size=self.args.patch_size,
                                      patch_stride=self.args.patch_stride,
                                      jump_size=1.0)
                loss += (self.args.analogy_weight /
                         len(self.args.analogy_layers)) * al

        existing_feature_nnfs = getattr(self, 'feature_nnfs',
                                        [None] * len(self.args.mrf_layers))
        self.feature_nnfs = []
        if self.args.mrf_weight:
            for layer_name, existing_nnf in zip(self.args.mrf_layers,
                                                existing_feature_nnfs):
                ap_image_features = all_ap_image_features[layer_name][0]
                # current combined output
                layer_features = self.get_layer_output(layer_name)
                combination_features = layer_features[0, :, :, :]
                input_shape = self.get_layer_output_shape(layer_name)
                if existing_nnf and not self.args.randomize_mnf_nnf:
                    matcher = existing_nnf.matcher.scale(
                        (input_shape[3], input_shape[2], input_shape[1]),
                        ap_image_features)
                else:
                    matcher = PatchMatcher(
                        (input_shape[3], input_shape[2], input_shape[1]),
                        ap_image_features,
                        patch_size=self.args.patch_size,
                        jump_size=1.0,
                        patch_stride=self.args.patch_stride)
                nnf = NNFState(matcher, self.get_f_layer(layer_name))
                self.feature_nnfs.append(nnf)
                sl = content_loss(combination_features, nnf.placeholder)
                loss += (self.args.mrf_weight / len(self.args.mrf_layers)) * sl

        if self.args.b_bp_content_weight:
            for layer_name in self.args.b_content_layers:
                b_features = K.variable(all_b_features[layer_name][0])
                # current combined output
                bp_features = self.get_layer_output(layer_name)
                cl = content_loss(bp_features, b_features)
                loss += self.args.b_bp_content_weight / len(
                    self.args.b_content_layers) * cl

        if self.args.neural_style_weight != 0.0:
            for layer_name in self.args.neural_style_layers:
                ap_image_features = K.variable(
                    all_ap_image_features[layer_name][0])
                layer_features = self.get_layer_output(layer_name)
                layer_shape = self.get_layer_output_shape(layer_name)
                # current combined output
                combination_features = layer_features[0, :, :, :]
                nsl = neural_style_loss(ap_image_features,
                                        combination_features, 3,
                                        self.output_shape[-2],
                                        self.output_shape[-1])
                loss += (self.args.neural_style_weight /
                         len(self.args.neural_style_layers)) * nsl

        return loss