def make_perceptual_loss(
args: argparse.Namespace,
mle: enc.MultiLayerEncoder,
) -> loss.PerceptualLoss:
content_loss = make_loss(args.content_loss, args.content_layers,
args.content_weight, mle)
style_loss = make_loss(args.style_loss, args.style_layers,
args.style_weight, mle)
return loss.PerceptualLoss(content_loss, style_loss)
def test_PerceptualLoss(self):
op = TotalVariationOperator()
required_components = {"content_loss", "style_loss"}
all_components = {*required_components, "regularization"}
for components in powerset(all_components):
if not set(components).intersection(required_components):
with self.assertRaises(RuntimeError):
loss.PerceptualLoss()
continue
perceptual_loss = loss.PerceptualLoss(
**{component: op
for component in components})
for component in components:
self.assertTrue(getattr(perceptual_loss, f"has_{component}"))
self.assertIs(getattr(perceptual_loss, component), op)
for component in all_components - set(components):
self.assertFalse(getattr(perceptual_loss, f"has_{component}"))
def test_PerceptualLoss_set_style_image(self):
torch.manual_seed(0)
image = torch.rand(1, 1, 100, 100)
content_loss = FeatureReconstructionOperator(
SequentialEncoder((nn.Conv2d(1, 1, 1), )))
style_loss = FeatureReconstructionOperator(
SequentialEncoder((nn.Conv2d(1, 1, 1), )))
perceptual_loss = loss.PerceptualLoss(content_loss=content_loss)
with self.assertRaises(RuntimeError):
perceptual_loss.set_style_image(image)
perceptual_loss = loss.PerceptualLoss(content_loss=content_loss,
style_loss=style_loss)
perceptual_loss.set_style_image(image)
self.assertTrue(style_loss.has_target_image)
actual = style_loss.target_image
desired = image
self.assertTensorAlmostEqual(actual, desired)
def test_PerceptualLoss_set_content_image():
torch.manual_seed(0)
image = torch.rand(1, 1, 100, 100)
content_loss = ops.FeatureReconstructionOperator(
enc.SequentialEncoder((nn.Conv2d(1, 1, 1), )))
style_loss = ops.FeatureReconstructionOperator(
enc.SequentialEncoder((nn.Conv2d(1, 1, 1), )))
perceptual_loss = loss.PerceptualLoss(content_loss, style_loss)
perceptual_loss.set_content_image(image)
actual = content_loss.target_image
desired = image
ptu.assert_allclose(actual, desired)
def test_model_default_optimization_criterion_update_fn(
transformer,
test_image,
):
image_loader = data.DataLoader(Dataset(test_image))
content_loss = MSEOperator()
style_loss = MSEOperator()
criterion = loss.PerceptualLoss(content_loss, style_loss)
content_loss.set_target_image(torch.rand_like(test_image))
style_loss.set_target_image(torch.rand_like(test_image))
optim.model_optimization(image_loader, transformer, criterion)
ptu.assert_allclose(content_loss.target_image, test_image)
def perceptual_loss(
impl_params: bool = True,
instance_norm: bool = True,
multi_layer_encoder: Optional[enc.MultiLayerEncoder] = None,
hyper_parameters: Optional[HyperParameters] = None,
) -> loss.PerceptualLoss:
r"""Perceptual loss from :cite:`ULVL2016,UVL2017`.
Args:
impl_params: Switch the behavior and hyper-parameters between the reference
implementation of the original authors and what is described in the paper.
For details see :ref:`here <li_wand_2016-impl_params>`.
instance_norm: Switch the behavior and hyper-parameters between both
publications of the original authors. For details see
:ref:`here <ulyanov_et_al_2016-instance_norm>`.
multi_layer_encoder: Pretrained :class:`~pystiche.enc.MultiLayerEncoder`. If
omitted, :func:`~pystiche_papers.ulyanov_et_al_2016.multi_layer_encoder`
is used.
hyper_parameters: Hyper parameters. If omitted,
:func:`~pystiche_papers.ulyanov_et_al_2016.hyper_parameters` is used.
.. seealso::
- :func:`pystiche_papers.ulyanov_et_al_2016.content_loss`
- :func:`pystiche_papers.ulyanov_et_al_2016.style_loss`
"""
if multi_layer_encoder is None:
multi_layer_encoder = _multi_layer_encoder()
if hyper_parameters is None:
hyper_parameters = _hyper_parameters(impl_params=impl_params,
instance_norm=instance_norm)
return loss.PerceptualLoss(
content_loss(
impl_params=impl_params,
instance_norm=instance_norm,
multi_layer_encoder=multi_layer_encoder,
hyper_parameters=hyper_parameters,
),
style_loss(
impl_params=impl_params,
instance_norm=instance_norm,
multi_layer_encoder=multi_layer_encoder,
hyper_parameters=hyper_parameters,
),
)
def _get_perceptual_loss(
self,
*,
backbone: str,
content_layer: str,
content_weight: float,
style_layers: Sequence[str],
style_weight: float,
) -> loss.PerceptualLoss:
mle, _ = cast(enc.MultiLayerEncoder, self.backbones.get(backbone)())
content_loss = loss.FeatureReconstructionLoss(mle.extract_encoder(content_layer), score_weight=content_weight)
style_loss = loss.MultiLayerEncodingLoss(
mle,
style_layers,
lambda encoder, layer_weight: self._modified_gram_loss(encoder, score_weight=layer_weight),
layer_weights="sum",
score_weight=style_weight,
)
return loss.PerceptualLoss(content_loss, style_loss)
def perceptual_loss(
impl_params: bool = True,
multi_layer_encoder: Optional[enc.MultiLayerEncoder] = None,
hyper_parameters: Optional[HyperParameters] = None,
) -> loss.PerceptualLoss:
r"""Perceptual loss from :cite:`LW2016`.
Args:
impl_params: Switch the behavior and hyper-parameters between the reference
implementation of the original authors and what is described in the paper.
For details see :ref:`here <li_wand_2016-impl_params>`.
multi_layer_encoder: Pretrained multi-layer encoder. If
omitted, :func:`~pystiche_papers.li_wand_2016.multi_layer_encoder` is used.
hyper_parameters: Hyper parameters. If omitted,
:func:`~pystiche_papers.li_wand_2016.hyper_parameters` is used.
.. seealso::
- :func:`pystiche_papers.li_wand_2016.content_loss`
- :func:`pystiche_papers.li_wand_2016.style_loss`
- :func:`pystiche_papers.li_wand_2016.regularization`
"""
if multi_layer_encoder is None:
multi_layer_encoder = _multi_layer_encoder()
if hyper_parameters is None:
hyper_parameters = _hyper_parameters()
return loss.PerceptualLoss(
content_loss(
impl_params=impl_params,
multi_layer_encoder=multi_layer_encoder,
hyper_parameters=hyper_parameters,
),
style_loss(
impl_params=impl_params,
multi_layer_encoder=multi_layer_encoder,
hyper_parameters=hyper_parameters,
),
regularization(impl_params=impl_params,
hyper_parameters=hyper_parameters),
)
def perceptual_loss(
impl_params: bool = True,
multi_layer_encoder: Optional[enc.MultiLayerEncoder] = None,
hyper_parameters: Optional[HyperParameters] = None,
) -> loss.PerceptualLoss:
r"""Perceptual loss comprising content and style loss as well as a regularization.
Args:
impl_params: Switch the behavior and hyper-parameters between the reference
implementation of the original authors and what is described in the paper.
For details see :ref:`here <johnson_alahi_li_2016-impl_params>`.
multi_layer_encoder: Pretrained :class:`~pystiche.enc.MultiLayerEncoder`. If
omitted, the default
:func:`~pystiche_papers.johnson_alahi_li_2016.multi_layer_encoder`
is used.
hyper_parameters: If omitted,
:func:`~pystiche_papers.johnson_alahi_li_2016.hyper_parameters` is used.
"""
if multi_layer_encoder is None:
multi_layer_encoder = _multi_layer_encoder(impl_params=impl_params)
if hyper_parameters is None:
hyper_parameters = _hyper_parameters()
return loss.PerceptualLoss(
content_loss(
impl_params=impl_params,
multi_layer_encoder=multi_layer_encoder,
hyper_parameters=hyper_parameters,
),
style_loss(
impl_params=impl_params,
multi_layer_encoder=multi_layer_encoder,
hyper_parameters=hyper_parameters,
),
regularization(hyper_parameters=hyper_parameters, ),
)
def get_style_op(encoder, layer_weight):
return loss.GramLoss(encoder, score_weight=layer_weight)
style_loss = loss.MultiLayerEncodingLoss(
multi_layer_encoder, style_layers, get_style_op, score_weight=style_weight,
)
print(style_loss)
########################################################################################
# We combine the ``content_loss`` and ``style_loss`` into a joined
# :class:`~pystiche.loss.PerceptualLoss`, which will serve as optimization criterion.
perceptual_loss = loss.PerceptualLoss(content_loss, style_loss).to(device)
print(perceptual_loss)
########################################################################################
# Images
# ------
#
# We now load and show the images that will be used in the NST. The images will be
# resized to ``size=500`` pixels.
images = demo.images()
images.download()
size = 500
style_layers = ("relu1_1", "relu2_1", "relu3_1", "relu4_1", "relu5_1")
style_weight = 1e4
def get_style_op(encoder, layer_weight):
return ops.GramOperator(encoder, score_weight=layer_weight)
style_loss = ops.MultiLayerEncodingOperator(
multi_layer_encoder,
style_layers,
get_style_op,
score_weight=style_weight,
)
criterion = loss.PerceptualLoss(content_loss, style_loss).to(device)
print(criterion)
########################################################################################
# We set the target images for the optimization ``criterion``.
criterion.set_content_image(content_image)
criterion.set_style_image(style_image)
########################################################################################
# We perform the unguided NST and show the result.
starting_point = "content"
input_image = get_input_image(starting_point, content_image=content_image)
output_image = optim.image_optimization(input_image, criterion, num_steps=500)
repr = super().enc_to_repr(enc)
num_channels = repr.size()[1]
return repr / num_channels
style_layers = ("relu1_2", "relu2_2", "relu3_3", "relu4_3")
style_weight = 1e10
style_loss = loss.MultiLayerEncodingLoss(
multi_layer_encoder,
style_layers,
lambda encoder, layer_weight: GramOperator(encoder, score_weight=layer_weight),
layer_weights="sum",
score_weight=style_weight,
)
perceptual_loss = loss.PerceptualLoss(content_loss, style_loss)
perceptual_loss = perceptual_loss.to(device)
print(perceptual_loss)
########################################################################################
# Training
# --------
#
# In a first step we load the style image that will be used to train the
# ``transformer``.
images = demo.images()
size = 500
style_image = images["paint"].read(size=size, device=device)
