def project_on_generator( G: Generator, pix2pix: networks.UnetGenerator, target_image: np.ndarray, E: Encoder, dcgan_img_size: int = 64, pix2pix_img_size: int = 128) -> Tuple[np.ndarray, torch.Tensor]: """Projects the input image onto the manifold span by the GAN. It operates as follows: 1. reshape and normalize the image 2. run the encoder to obtain a latent vector 3. run the DCGAN generator to obtain a low resolution image 4. run the Pix2Pix model to obtain a high resulution image Arguments: G {Generator} -- DCGAN generator pix2pix {networks.UnetGenerator} -- Low resolution to high resolution Pix2Pix model target_image {np.ndarray} -- The image to project E {Encoder} -- The DCGAN encoder Keyword Arguments: dcgan_img_size {int} -- Low resolution image size (default: {64}) pix2pix_img_size {int} -- High resolution image size (default: {128}) Returns: Tuple[np.ndarray, torch.Tensor] -- The projected high resolution image and the latent vector that was used to generate it. """ # reshape and normalize image target_image = torch.Tensor(target_image).cuda().reshape( 1, 3, pix2pix_img_size, pix2pix_img_size) target_image = F.interpolate(target_image, scale_factor=dcgan_img_size / pix2pix_img_size, mode='bilinear') target_image = target_image.clamp(min=0) target_image = target_image / target_image.max() target_image = (target_image - 0.5) / 0.5 # Run dcgan z = E(target_image) dcgan_image = G(z) # run pix2pix pix_input = F.interpolate(dcgan_image, scale_factor=pix2pix_img_size / dcgan_img_size, mode='bilinear') pix_outputs = pix2pix(pix_input) out_image = utils.denorm(pix_outputs.detach()).clamp( 0, 1).cpu().numpy().reshape(3, -1, 1) return out_image, z
def __call__( self, kspace: np.ndarray, mask: np.ndarray, target: np.ndarray, attrs: Dict, fname: str, slice_num: int, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, str, int, float]: """ Args: kspace: Input k-space of shape (num_coils, rows, cols) for multi-coil data or (rows, cols) for single coil data. mask: Mask from the test dataset. target: Target image. attrs: Acquisition related information stored in the HDF5 object. fname: File name. slice_num: Serial number of the slice. Returns: tuple containing: image: Zero-filled input image. target: Target image converted to a torch.Tensor. mean: Mean value used for normalization. std: Standard deviation value used for normalization. fname: File name. slice_num: Serial number of the slice. """ kspace = to_tensor(kspace) # check for max value max_value = attrs["max"] if "max" in attrs.keys() else 0.0 # apply mask if self.mask_func: seed = None if not self.use_seed else tuple(map(ord, fname)) masked_kspace, mask = apply_mask(kspace, self.mask_func, seed) else: masked_kspace = kspace # inverse Fourier transform to get zero filled solution image = fastmri.ifft2c(masked_kspace) # crop input to correct size if target is not None: crop_size = (target.shape[-2], target.shape[-1]) else: crop_size = (attrs["recon_size"][0], attrs["recon_size"][1]) # check for FLAIR 203 if image.shape[-2] < crop_size[1]: crop_size = (image.shape[-2], image.shape[-2]) image = complex_center_crop(image, crop_size) # absolute value image = fastmri.complex_abs(image) # apply Root-Sum-of-Squares if multicoil data if self.which_challenge == "multicoil": image = fastmri.rss(image) # normalize input image, mean, std = normalize_instance(image, eps=1e-11) image = image.clamp(-6, 6) # normalize target if target is not None: target = to_tensor(target) target = center_crop(target, crop_size) target = normalize(target, mean, std, eps=1e-11) target = target.clamp(-6, 6) else: target = torch.Tensor([0]) return image, target, mean, std, fname, slice_num, max_value