def project_on_generator(
G: Generator,
pix2pix: networks.UnetGenerator,
target_image: np.ndarray,
E: Encoder,
dcgan_img_size: int = 64,
pix2pix_img_size: int = 128) -> Tuple[np.ndarray, torch.Tensor]:
"""Projects the input image onto the manifold span by the GAN. It operates as follows:
1. reshape and normalize the image
2. run the encoder to obtain a latent vector
3. run the DCGAN generator to obtain a low resolution image
4. run the Pix2Pix model to obtain a high resulution image
Arguments:
G {Generator} -- DCGAN generator
pix2pix {networks.UnetGenerator} -- Low resolution to high resolution Pix2Pix model
target_image {np.ndarray} -- The image to project
E {Encoder} -- The DCGAN encoder
Keyword Arguments:
dcgan_img_size {int} -- Low resolution image size (default: {64})
pix2pix_img_size {int} -- High resolution image size (default: {128})
Returns:
Tuple[np.ndarray, torch.Tensor] -- The projected high resolution image and the latent vector that was used to generate it.
"""
# reshape and normalize image
target_image = torch.Tensor(target_image).cuda().reshape(
1, 3, pix2pix_img_size, pix2pix_img_size)
target_image = F.interpolate(target_image,
scale_factor=dcgan_img_size /
pix2pix_img_size,
mode='bilinear')
target_image = target_image.clamp(min=0)
target_image = target_image / target_image.max()
target_image = (target_image - 0.5) / 0.5
# Run dcgan
z = E(target_image)
dcgan_image = G(z)
# run pix2pix
pix_input = F.interpolate(dcgan_image,
scale_factor=pix2pix_img_size / dcgan_img_size,
mode='bilinear')
pix_outputs = pix2pix(pix_input)
out_image = utils.denorm(pix_outputs.detach()).clamp(
0, 1).cpu().numpy().reshape(3, -1, 1)
return out_image, z
def __call__(
self,
kspace: np.ndarray,
mask: np.ndarray,
target: np.ndarray,
attrs: Dict,
fname: str,
slice_num: int,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, str,
int, float]:
"""
Args:
kspace: Input k-space of shape (num_coils, rows, cols) for
multi-coil data or (rows, cols) for single coil data.
mask: Mask from the test dataset.
target: Target image.
attrs: Acquisition related information stored in the HDF5 object.
fname: File name.
slice_num: Serial number of the slice.
Returns:
tuple containing:
image: Zero-filled input image.
target: Target image converted to a torch.Tensor.
mean: Mean value used for normalization.
std: Standard deviation value used for normalization.
fname: File name.
slice_num: Serial number of the slice.
"""
kspace = to_tensor(kspace)
# check for max value
max_value = attrs["max"] if "max" in attrs.keys() else 0.0
# apply mask
if self.mask_func:
seed = None if not self.use_seed else tuple(map(ord, fname))
masked_kspace, mask = apply_mask(kspace, self.mask_func, seed)
else:
masked_kspace = kspace
# inverse Fourier transform to get zero filled solution
image = fastmri.ifft2c(masked_kspace)
# crop input to correct size
if target is not None:
crop_size = (target.shape[-2], target.shape[-1])
else:
crop_size = (attrs["recon_size"][0], attrs["recon_size"][1])
# check for FLAIR 203
if image.shape[-2] < crop_size[1]:
crop_size = (image.shape[-2], image.shape[-2])
image = complex_center_crop(image, crop_size)
# absolute value
image = fastmri.complex_abs(image)
# apply Root-Sum-of-Squares if multicoil data
if self.which_challenge == "multicoil":
image = fastmri.rss(image)
# normalize input
image, mean, std = normalize_instance(image, eps=1e-11)
image = image.clamp(-6, 6)
# normalize target
if target is not None:
target = to_tensor(target)
target = center_crop(target, crop_size)
target = normalize(target, mean, std, eps=1e-11)
target = target.clamp(-6, 6)
else:
target = torch.Tensor([0])
return image, target, mean, std, fname, slice_num, max_value