def content_loss( impl_params: bool = True, multi_layer_encoder: Optional[enc.MultiLayerEncoder] = None, hyper_parameters: Optional[HyperParameters] = None, ) -> loss.FeatureReconstructionLoss: r"""Content loss from :cite:`JAL2016`. Args: impl_params: Switch the behavior and hyper-parameters between the reference implementation of the original authors and what is described in the paper. For details see :ref:`here <johnson_alahi_li_2016-impl_params>`. multi_layer_encoder: Pretrained :class:`~pystiche.enc.MultiLayerEncoder`. If omitted, the default :func:`~pystiche_papers.johnson_alahi_li_2016.multi_layer_encoder` is used. hyper_parameters: If omitted, :func:`~pystiche_papers.johnson_alahi_li_2016.hyper_parameters` is used. """ if multi_layer_encoder is None: multi_layer_encoder = _multi_layer_encoder(impl_params=impl_params) if hyper_parameters is None: hyper_parameters = _hyper_parameters() return loss.FeatureReconstructionLoss( multi_layer_encoder.extract_encoder( hyper_parameters.content_loss.layer), score_weight=hyper_parameters.content_loss.score_weight, )
def test_set_style_image(self): torch.manual_seed(0) image = torch.rand(1, 1, 100, 100) content_loss = loss.FeatureReconstructionLoss( enc.SequentialEncoder((nn.Conv2d(1, 1, 1),)) ) style_loss = loss.FeatureReconstructionLoss( enc.SequentialEncoder((nn.Conv2d(1, 1, 1),)) ) perceptual_loss = loss.PerceptualLoss(content_loss, style_loss) perceptual_loss.set_style_image(image) actual = style_loss.target_image desired = image ptu.assert_allclose(actual, desired)
def test_call(self, encoder): torch.manual_seed(0) target_image = torch.rand(1, 3, 128, 128) input_image = torch.rand(1, 3, 128, 128) loss = loss_.FeatureReconstructionLoss(encoder) loss.set_target_image(target_image) actual = loss(input_image) desired = F.mse_loss(encoder(input_image), encoder(target_image)) ptu.assert_allclose(actual, desired)
def _get_perceptual_loss( self, *, backbone: str, content_layer: str, content_weight: float, style_layers: Sequence[str], style_weight: float, ) -> loss.PerceptualLoss: mle, _ = cast(enc.MultiLayerEncoder, self.backbones.get(backbone)()) content_loss = loss.FeatureReconstructionLoss(mle.extract_encoder(content_layer), score_weight=content_weight) style_loss = loss.MultiLayerEncodingLoss( mle, style_layers, lambda encoder, layer_weight: self._modified_gram_loss(encoder, score_weight=layer_weight), layer_weights="sum", score_weight=style_weight, ) return loss.PerceptualLoss(content_loss, style_loss)
def content_loss( impl_params: bool = True, instance_norm: bool = True, multi_layer_encoder: Optional[enc.MultiLayerEncoder] = None, hyper_parameters: Optional[HyperParameters] = None, ) -> loss.FeatureReconstructionLoss: r"""Content loss from :cite:`ULVL2016,UVL2017`. Args: impl_params: Switch the behavior and hyper-parameters between the reference implementation of the original authors and what is described in the paper. For details see :ref:`here <li_wand_2016-impl_params>`. instance_norm: Switch the behavior and hyper-parameters between both publications of the original authors. For details see :ref:`here <ulyanov_et_al_2016-instance_norm>`. multi_layer_encoder: Pretrained :class:`~pystiche.enc.MultiLayerEncoder`. If omitted, :func:`~pystiche_papers.ulyanov_et_al_2016.multi_layer_encoder` is used. hyper_parameters: Hyper parameters. If omitted, :func:`~pystiche_papers.ulyanov_et_al_2016.hyper_parameters` is used. .. seealso:: - :class:`pystiche.loss.FeatureReconstructionLoss` """ if multi_layer_encoder is None: multi_layer_encoder = _multi_layer_encoder() if hyper_parameters is None: hyper_parameters = _hyper_parameters(impl_params=impl_params, instance_norm=instance_norm) return loss.FeatureReconstructionLoss( multi_layer_encoder.extract_encoder( hyper_parameters.content_loss.layer), score_weight=hyper_parameters.content_loss.score_weight, )
# Perceptual Loss # --------------- # # The core components of every NST are the ``content_loss`` and the ``style_loss``. # Combined they make up the perceptual loss, i.e. the optimization criterion. # # In this example we use the :class:`~pystiche.loss.FeatureReconstructionLoss` # introduced by Mahendran and Vedaldi :cite:`MV2015` as ``content_loss``. We first # extract the ``content_encoder`` that generates encodings from the ``content_layer``. # Together with the ``content_weight`` we can construct the ``content_loss``. content_layer = "relu4_2" content_encoder = multi_layer_encoder.extract_encoder(content_layer) content_weight = 1e0 content_loss = loss.FeatureReconstructionLoss( content_encoder, score_weight=content_weight ) print(content_loss) ######################################################################################## # We use the :class:`~pystiche.loss.GramLoss` introduced by Gatys, Ecker, and Bethge # :cite:`GEB2016` as ``style_loss``. Unlike before, we use multiple ``style_layers``. # The individual losses can be conveniently bundled in a # :class:`~pystiche.loss.MultiLayerEncodingLoss`. style_layers = ("relu1_1", "relu2_1", "relu3_1", "relu4_1", "relu5_1") style_weight = 1e3 def get_style_op(encoder, layer_weight):
def get_guided_perceptual_loss(): content_loss = loss.FeatureReconstructionLoss( enc.SequentialEncoder((nn.Conv2d(1, 1, 1),)) ) style_loss = ops.MultiRegionOperator(regions, get_op) return loss.GuidedPerceptualLoss(content_loss, style_loss)
raise ValueError( add_suggestion( f"Unknown multi-layer encoder '{mle_str}'.", word=mle_str, possibilities=MLES.keys(), )) from error return mle_fn() # This needs to be filled manually, since some losses such as MRF need more parameters # than just encoder and score_weight LOSSES = { "featurereconstruction": ( "FeatureReconstruction", lambda encoder, score_weight: loss.FeatureReconstructionLoss( encoder, score_weight=score_weight), ), "gram": ( "Gram", lambda encoder, score_weight: loss.GramLoss(encoder, score_weight=score_weight), ), "mrf": ( "MRF", lambda encoder, score_weight: loss.MRFLoss( encoder, patch_size=3, stride=2, score_weight=score_weight), ), } def make_loss(
def test_repr_smoke(self, encoder): assert isinstance(repr(loss_.FeatureReconstructionLoss(encoder)), str)