def make_perceptual_loss( args: argparse.Namespace, mle: enc.MultiLayerEncoder, ) -> loss.PerceptualLoss: content_loss = make_loss(args.content_loss, args.content_layers, args.content_weight, mle) style_loss = make_loss(args.style_loss, args.style_layers, args.style_weight, mle) return loss.PerceptualLoss(content_loss, style_loss)
def test_PerceptualLoss(self): op = TotalVariationOperator() required_components = {"content_loss", "style_loss"} all_components = {*required_components, "regularization"} for components in powerset(all_components): if not set(components).intersection(required_components): with self.assertRaises(RuntimeError): loss.PerceptualLoss() continue perceptual_loss = loss.PerceptualLoss( **{component: op for component in components}) for component in components: self.assertTrue(getattr(perceptual_loss, f"has_{component}")) self.assertIs(getattr(perceptual_loss, component), op) for component in all_components - set(components): self.assertFalse(getattr(perceptual_loss, f"has_{component}"))
def test_PerceptualLoss_set_style_image(self): torch.manual_seed(0) image = torch.rand(1, 1, 100, 100) content_loss = FeatureReconstructionOperator( SequentialEncoder((nn.Conv2d(1, 1, 1), ))) style_loss = FeatureReconstructionOperator( SequentialEncoder((nn.Conv2d(1, 1, 1), ))) perceptual_loss = loss.PerceptualLoss(content_loss=content_loss) with self.assertRaises(RuntimeError): perceptual_loss.set_style_image(image) perceptual_loss = loss.PerceptualLoss(content_loss=content_loss, style_loss=style_loss) perceptual_loss.set_style_image(image) self.assertTrue(style_loss.has_target_image) actual = style_loss.target_image desired = image self.assertTensorAlmostEqual(actual, desired)
def test_PerceptualLoss_set_content_image(): torch.manual_seed(0) image = torch.rand(1, 1, 100, 100) content_loss = ops.FeatureReconstructionOperator( enc.SequentialEncoder((nn.Conv2d(1, 1, 1), ))) style_loss = ops.FeatureReconstructionOperator( enc.SequentialEncoder((nn.Conv2d(1, 1, 1), ))) perceptual_loss = loss.PerceptualLoss(content_loss, style_loss) perceptual_loss.set_content_image(image) actual = content_loss.target_image desired = image ptu.assert_allclose(actual, desired)
def test_model_default_optimization_criterion_update_fn( transformer, test_image, ): image_loader = data.DataLoader(Dataset(test_image)) content_loss = MSEOperator() style_loss = MSEOperator() criterion = loss.PerceptualLoss(content_loss, style_loss) content_loss.set_target_image(torch.rand_like(test_image)) style_loss.set_target_image(torch.rand_like(test_image)) optim.model_optimization(image_loader, transformer, criterion) ptu.assert_allclose(content_loss.target_image, test_image)
def perceptual_loss( impl_params: bool = True, instance_norm: bool = True, multi_layer_encoder: Optional[enc.MultiLayerEncoder] = None, hyper_parameters: Optional[HyperParameters] = None, ) -> loss.PerceptualLoss: r"""Perceptual loss from :cite:`ULVL2016,UVL2017`. Args: impl_params: Switch the behavior and hyper-parameters between the reference implementation of the original authors and what is described in the paper. For details see :ref:`here <li_wand_2016-impl_params>`. instance_norm: Switch the behavior and hyper-parameters between both publications of the original authors. For details see :ref:`here <ulyanov_et_al_2016-instance_norm>`. multi_layer_encoder: Pretrained :class:`~pystiche.enc.MultiLayerEncoder`. If omitted, :func:`~pystiche_papers.ulyanov_et_al_2016.multi_layer_encoder` is used. hyper_parameters: Hyper parameters. If omitted, :func:`~pystiche_papers.ulyanov_et_al_2016.hyper_parameters` is used. .. seealso:: - :func:`pystiche_papers.ulyanov_et_al_2016.content_loss` - :func:`pystiche_papers.ulyanov_et_al_2016.style_loss` """ if multi_layer_encoder is None: multi_layer_encoder = _multi_layer_encoder() if hyper_parameters is None: hyper_parameters = _hyper_parameters(impl_params=impl_params, instance_norm=instance_norm) return loss.PerceptualLoss( content_loss( impl_params=impl_params, instance_norm=instance_norm, multi_layer_encoder=multi_layer_encoder, hyper_parameters=hyper_parameters, ), style_loss( impl_params=impl_params, instance_norm=instance_norm, multi_layer_encoder=multi_layer_encoder, hyper_parameters=hyper_parameters, ), )
def _get_perceptual_loss( self, *, backbone: str, content_layer: str, content_weight: float, style_layers: Sequence[str], style_weight: float, ) -> loss.PerceptualLoss: mle, _ = cast(enc.MultiLayerEncoder, self.backbones.get(backbone)()) content_loss = loss.FeatureReconstructionLoss(mle.extract_encoder(content_layer), score_weight=content_weight) style_loss = loss.MultiLayerEncodingLoss( mle, style_layers, lambda encoder, layer_weight: self._modified_gram_loss(encoder, score_weight=layer_weight), layer_weights="sum", score_weight=style_weight, ) return loss.PerceptualLoss(content_loss, style_loss)
def perceptual_loss( impl_params: bool = True, multi_layer_encoder: Optional[enc.MultiLayerEncoder] = None, hyper_parameters: Optional[HyperParameters] = None, ) -> loss.PerceptualLoss: r"""Perceptual loss from :cite:`LW2016`. Args: impl_params: Switch the behavior and hyper-parameters between the reference implementation of the original authors and what is described in the paper. For details see :ref:`here <li_wand_2016-impl_params>`. multi_layer_encoder: Pretrained multi-layer encoder. If omitted, :func:`~pystiche_papers.li_wand_2016.multi_layer_encoder` is used. hyper_parameters: Hyper parameters. If omitted, :func:`~pystiche_papers.li_wand_2016.hyper_parameters` is used. .. seealso:: - :func:`pystiche_papers.li_wand_2016.content_loss` - :func:`pystiche_papers.li_wand_2016.style_loss` - :func:`pystiche_papers.li_wand_2016.regularization` """ if multi_layer_encoder is None: multi_layer_encoder = _multi_layer_encoder() if hyper_parameters is None: hyper_parameters = _hyper_parameters() return loss.PerceptualLoss( content_loss( impl_params=impl_params, multi_layer_encoder=multi_layer_encoder, hyper_parameters=hyper_parameters, ), style_loss( impl_params=impl_params, multi_layer_encoder=multi_layer_encoder, hyper_parameters=hyper_parameters, ), regularization(impl_params=impl_params, hyper_parameters=hyper_parameters), )
def perceptual_loss( impl_params: bool = True, multi_layer_encoder: Optional[enc.MultiLayerEncoder] = None, hyper_parameters: Optional[HyperParameters] = None, ) -> loss.PerceptualLoss: r"""Perceptual loss comprising content and style loss as well as a regularization. Args: impl_params: Switch the behavior and hyper-parameters between the reference implementation of the original authors and what is described in the paper. For details see :ref:`here <johnson_alahi_li_2016-impl_params>`. multi_layer_encoder: Pretrained :class:`~pystiche.enc.MultiLayerEncoder`. If omitted, the default :func:`~pystiche_papers.johnson_alahi_li_2016.multi_layer_encoder` is used. hyper_parameters: If omitted, :func:`~pystiche_papers.johnson_alahi_li_2016.hyper_parameters` is used. """ if multi_layer_encoder is None: multi_layer_encoder = _multi_layer_encoder(impl_params=impl_params) if hyper_parameters is None: hyper_parameters = _hyper_parameters() return loss.PerceptualLoss( content_loss( impl_params=impl_params, multi_layer_encoder=multi_layer_encoder, hyper_parameters=hyper_parameters, ), style_loss( impl_params=impl_params, multi_layer_encoder=multi_layer_encoder, hyper_parameters=hyper_parameters, ), regularization(hyper_parameters=hyper_parameters, ), )
def get_style_op(encoder, layer_weight): return loss.GramLoss(encoder, score_weight=layer_weight) style_loss = loss.MultiLayerEncodingLoss( multi_layer_encoder, style_layers, get_style_op, score_weight=style_weight, ) print(style_loss) ######################################################################################## # We combine the ``content_loss`` and ``style_loss`` into a joined # :class:`~pystiche.loss.PerceptualLoss`, which will serve as optimization criterion. perceptual_loss = loss.PerceptualLoss(content_loss, style_loss).to(device) print(perceptual_loss) ######################################################################################## # Images # ------ # # We now load and show the images that will be used in the NST. The images will be # resized to ``size=500`` pixels. images = demo.images() images.download() size = 500
style_layers = ("relu1_1", "relu2_1", "relu3_1", "relu4_1", "relu5_1") style_weight = 1e4 def get_style_op(encoder, layer_weight): return ops.GramOperator(encoder, score_weight=layer_weight) style_loss = ops.MultiLayerEncodingOperator( multi_layer_encoder, style_layers, get_style_op, score_weight=style_weight, ) criterion = loss.PerceptualLoss(content_loss, style_loss).to(device) print(criterion) ######################################################################################## # We set the target images for the optimization ``criterion``. criterion.set_content_image(content_image) criterion.set_style_image(style_image) ######################################################################################## # We perform the unguided NST and show the result. starting_point = "content" input_image = get_input_image(starting_point, content_image=content_image) output_image = optim.image_optimization(input_image, criterion, num_steps=500)
repr = super().enc_to_repr(enc) num_channels = repr.size()[1] return repr / num_channels style_layers = ("relu1_2", "relu2_2", "relu3_3", "relu4_3") style_weight = 1e10 style_loss = loss.MultiLayerEncodingLoss( multi_layer_encoder, style_layers, lambda encoder, layer_weight: GramOperator(encoder, score_weight=layer_weight), layer_weights="sum", score_weight=style_weight, ) perceptual_loss = loss.PerceptualLoss(content_loss, style_loss) perceptual_loss = perceptual_loss.to(device) print(perceptual_loss) ######################################################################################## # Training # -------- # # In a first step we load the style image that will be used to train the # ``transformer``. images = demo.images() size = 500 style_image = images["paint"].read(size=size, device=device)