def gatys_ecker_bethge_2015_multi_layer_encoder(
impl_params: bool = True,
) -> MultiLayerEncoder:
multi_layer_encoder = vgg19_multi_layer_encoder(
weights="caffe", preprocessing=False, allow_inplace=True
)
if impl_params:
return multi_layer_encoder
for name, module in multi_layer_encoder.named_children():
if isinstance(module, nn.MaxPool2d):
multi_layer_encoder._modules[name] = nn.AvgPool2d(
**pystiche.pool_module_meta(module)
)
return multi_layer_encoder
def multi_layer_encoder(impl_params: bool = True, ) -> enc.MultiLayerEncoder:
r"""Multi-layer encoder from :cite:`GEB2016`.
Args:
impl_params: If ``True``, the :class:`~torch.nn.MaxPool2d` in
the ``multi_layer_encoder`` are exchanged for :class:`~torch.nn.AvgPool2d`.
"""
# TODO: check if the encoder used inplace ops
multi_layer_encoder_ = enc.vgg19_multi_layer_encoder(
framework="caffe", internal_preprocessing=False, allow_inplace=True)
if impl_params:
return multi_layer_encoder_
for name, module in multi_layer_encoder_.named_children():
if isinstance(module, nn.MaxPool2d):
multi_layer_encoder_._modules[name] = nn.AvgPool2d(
**meta.pool_module_meta(module))
return multi_layer_encoder_
from pystiche.optim import default_image_pyramid_optim_loop
from pystiche.pyramid import ImagePyramid
print(f"I'm working with pystiche=={pystiche.__version__}")
device = get_device()
print(f"I'm working with {device}")
images = demo_images()
images.download()
########################################################################################
# At first we define a :class:`~pystiche.loss.perceptual.PerceptualLoss` that is used
# as optimization ``criterion``.
multi_layer_encoder = vgg19_multi_layer_encoder()
content_layer = "relu4_2"
content_encoder = multi_layer_encoder.extract_single_layer_encoder(
content_layer)
content_weight = 1e0
content_loss = FeatureReconstructionOperator(content_encoder,
score_weight=content_weight)
style_layers = ("relu3_1", "relu4_1")
style_weight = 2e0
def get_style_op(encoder, layer_weight):
patch_size = 3
return MRFOperator(encoder,
#
# By default, :func:`~pystiche.enc.vgg19_multi_layer_encoder` adds an
# ``internal_preprocessing`` so that the user can simply pass the image as is,
# without worrying about it. We disable this here to enable a comparison.
#
# .. note::
#
# By default, :func:`~pystiche.enc.vgg19_multi_layer_encoder` disallows in-place
# operations since after they are carried out, the previous encoding is no longer
# accessible. In order to enable a fair performance comparison, we allow them here,
# since they are also used in :func:`~torchvision.models.vgg19`.
#
# .. note::
#
# The fully connected stage of the original VGG19 architecture requires the input to
# be exactly 224 pixels wide and high :cite:`SZ2014`. Since this requirement can
# usually not be met in an NST, the builtin multi-layer encoder only comprises the
# size invariant convolutional stage. Thus, we only use ``vgg19().features`` to
# enable a comparison.
seq = models.vgg19()
mle = enc.vgg19_multi_layer_encoder(
pretrained=False, internal_preprocessing=False, allow_inplace=True
)
mle.load_state_dict(seq.state_dict())
input = torch.rand((4, 3, 256, 256), device=device)
assert torch.allclose(mle(input), seq.features(input))
print(fdifftimeit(lambda: seq.features(input), lambda: mle(input)))
def li_wand_2016_multi_layer_encoder() -> MultiLayerEncoder:
return vgg19_multi_layer_encoder(weights="caffe",
preprocessing=False,
allow_inplace=True)
def ulyanov_et_al_2016_multi_layer_encoder() -> MultiLayerEncoder:
return vgg19_multi_layer_encoder(weights="caffe", allow_inplace=True)
def multi_layer_encoder() -> enc.MultiLayerEncoder:
r"""Multi-layer encoder from :cite:`LW2016`."""
return enc.vgg19_multi_layer_encoder(framework="caffe",
internal_preprocessing=False,
allow_inplace=True)
device = get_device()
print(f"I'm working with {device}")
########################################################################################
# Multi-layer Encoder
# -------------------
# The ``content_loss`` and the ``style_loss`` operate on the encodings of an image
# rather than on the image itself. These encodings are generated by a pretrained
# encoder. Since we will be using encodings from multiple layers we load a
# multi-layer encoder. In this example we use the
# :func:`~pystiche.enc.vgg19_multi_layer_encoder` that is based on the VGG19
# architecture introduced by Simonyan and Zisserman :cite:`SZ2014` .
multi_layer_encoder = enc.vgg19_multi_layer_encoder()
print(multi_layer_encoder)
########################################################################################
# Perceptual Loss
# ---------------
#
# The core components of every NST are the ``content_loss`` and the ``style_loss``.
# Combined they make up the perceptual loss, i.e. the optimization criterion.
#
# In this example we use the :class:`~pystiche.loss.FeatureReconstructionLoss`
# introduced by Mahendran and Vedaldi :cite:`MV2015` as ``content_loss``. We first
# extract the ``content_encoder`` that generates encodings from the ``content_layer``.
# Together with the ``content_weight`` we can construct the ``content_loss``.
def gatys_et_al_2017_multi_layer_encoder() -> MultiLayerEncoder:
return vgg19_multi_layer_encoder(weights="caffe",
preprocessing=False,
allow_inplace=True)
