def test_get_predictions(self): inception_utils.create_inception_graph('./metrics/inception_model') images = np.ones((4, 32, 32, 3)) preds = inception_score_utils.get_predictions(images) assert preds.shape == (4, 1008)
def test_get_activations(self): inception_path = './metrics/inception_model' inception_utils.create_inception_graph(inception_path) images = np.ones((4, 32, 32, 3)) with tf.compat.v1.Session() as sess: feat = inception_utils.get_activations(images=images, sess=sess) assert feat.shape == (4, 2048)
def test_calculate_activation_statistics(self): inception_path = './metrics/inception_model' inception_utils.create_inception_graph(inception_path) mu, sigma = fid_utils.calculate_activation_statistics( images=self.images, sess=self.sess) assert mu.shape == (2048, ) assert sigma.shape == (2048, 2048)
def setup(self): self.netG = ExampleGen() self.num_samples = 50 self.device = torch.device("cpu") # Create inception graph once. self.inception_path = './metrics/inception_model' if not os.path.exists(self.inception_path): os.makedirs(self.inception_path) inception_utils.create_inception_graph(self.inception_path) # Directory self.log_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test_log") if not os.path.exists(self.log_dir): os.makedirs(self.log_dir)
def fid_score(num_real_samples, num_fake_samples, netG, dataset, seed=0, device=None, batch_size=50, verbose=True, stats_file=None, log_dir='./log'): """ Computes FID stats using functions that store images in memory for speed and fidelity. Fidelity since by storing images in memory, we don't subject the scores to different read/write implementations of imaging libraries. Args: num_real_samples (int): The number of real images to use for FID. num_fake_samples (int): The number of fake images to use for FID. netG (Module): Torch Module object representing the generator model. device (str/torch.device): Device identifier to use for computation. seed (int): The random seed to use. dataset (str/Dataset): The name of the dataset to load if known, or a custom Dataset object batch_size (int): The batch size to feedforward for inference. verbose (bool): If True, prints progress. stats_file (str): The statistics file to load from if there is already one. log_dir (str): Directory where feature statistics can be stored. Returns: float: Scalar FID score. """ start_time = time.time() # Check inputs if device is None: device = torch.device('cuda:0' if torch.cuda.is_available() else "cpu") if isinstance(dataset, str): default_datasets = { 'cifar10', 'cifar100', 'stl10_48', 'imagenet_32', 'imagenet_128', 'celeba_64', 'celeba_128', 'lsun_bedroom', 'fake_data', } if dataset not in default_datasets: raise ValueError('For default datasets, must be one of {}'.format( default_datasets)) elif issubclass(type(dataset), torch.utils.data.Dataset): if stats_file is None: raise ValueError( "stats_file to save/load from cannot be empty if using a custom dataset." ) if not stats_file.endswith('.npz'): stats_file = stats_file + '.npz' else: raise ValueError( 'dataset must be either a Dataset object or a string.') # Make sure the random seeds are fixed torch.manual_seed(seed) random.seed(seed) np.random.seed(seed) # Setup directories inception_path = os.path.join(log_dir, 'metrics', 'inception_model') # Setup the inception graph inception_utils.create_inception_graph(inception_path) # Start producing statistics for real and fake images # if device and device.index is not None: # # Avoid unbounded memory usage # gpu_options = tf.compat.v1.GPUOptions(allow_growth=True, # per_process_gpu_memory_fraction=0.15, # visible_device_list=str(device.index)) # config = tf.compat.v1.ConfigProto(gpu_options=gpu_options) # else: # config = tf.compat.v1.ConfigProto(device_count={'GPU': 0}) config = tf.compat.v1.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.2 config.gpu_options.allow_growth = True with tf.compat.v1.Session(config=config) as sess: sess.run(tf.compat.v1.global_variables_initializer()) m_real, s_real = compute_real_dist_stats(num_samples=num_real_samples, sess=sess, dataset=dataset, batch_size=batch_size, verbose=verbose, stats_file=stats_file, log_dir=log_dir, seed=seed) m_fake, s_fake = compute_gen_dist_stats(netG=netG, num_samples=num_fake_samples, sess=sess, device=device, seed=seed, batch_size=batch_size, verbose=verbose) FID_score = fid_utils.calculate_frechet_distance(mu1=m_real, sigma1=s_real, mu2=m_fake, sigma2=s_fake) print("INFO: FID: {} [Time Taken: {:.4f} secs]".format( FID_score, time.time() - start_time)) return float(FID_score)
def kid_score(num_samples, netG, dataset, seed=0, device=None, num_subsets=10, batch_size=50, verbose=True, feat_file=None, log_dir='./log', **kwargs): """ Computes KID score. Args: num_samples (int): The number of real and fake images to use for KID. num_subsets (int): Number of subsets to compute average MMD. netG (Module): Torch Module object representing the generator model. device (str): Device identifier to use for computation. seed (int): The random seed to use. dataset (str/Dataset): The name of the dataset to load if known, or a custom Dataset object batch_size (int): The batch size to feedforward for inference. feat_file (str): The path to specific inception features for real images. log_dir (str): Directory where features can be stored. verbose (bool): If True, prints progress. Returns: tuple: Scalar mean and std of KID scores computed. """ start_time = time.time() # Check inputs if device is None: device = torch.device('cuda:0' if torch.cuda.is_available() else "cpu") if isinstance(dataset, str): default_datasets = { 'cifar10', 'cifar100', 'stl10_48', 'imagenet_32', 'imagenet_128', 'celeba_64', 'celeba_128', 'lsun_bedroom', 'fake_data', } if dataset not in default_datasets: raise ValueError('For default datasets, must be one of {}'.format( default_datasets)) elif issubclass(type(dataset), torch.utils.data.Dataset): if feat_file is None: raise ValueError( "feat_file to save/load from cannot be empty if using a custom dataset." ) if not feat_file.endswith('.npz'): feat_file = feat_file + '.npz' else: raise ValueError( 'dataset must be either a Dataset object or a string.') # Make sure the random seeds are fixed torch.manual_seed(seed) random.seed(seed) np.random.seed(seed) # Directories inception_path = os.path.join('./inception_model') # Setup the inception graph inception_utils.create_inception_graph(inception_path) # Start producing features for real and fake images # if device.index is not None: # # Avoid unbounded memory usage # gpu_options = tf.compat.v1.GPUOptions(allow_growth=True, # per_process_gpu_memory_fraction=0.15, # visible_device_list=str(device.index)) # config = tf.compat.v1.ConfigProto(gpu_options=gpu_options) # else: # config = tf.compat.v1.ConfigProto(device_count={'GPU': 0}) config = tf.compat.v1.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.2 config.gpu_options.allow_growth = True with tf.compat.v1.Session(config=config) as sess: sess.run(tf.compat.v1.global_variables_initializer()) real_feat = compute_real_dist_feat(num_samples=num_samples, sess=sess, dataset=dataset, batch_size=batch_size, verbose=verbose, feat_file=feat_file, log_dir=log_dir, seed=seed, **kwargs) fake_feat = compute_gen_dist_feat(netG=netG, num_samples=num_samples, sess=sess, device=device, seed=seed, batch_size=batch_size, verbose=verbose) # Compute the KID score subset_size = num_samples // num_subsets scores = kid_utils.polynomial_mmd_averages(real_feat, fake_feat, n_subsets=num_subsets, subset_size=subset_size) mmd_score, mmd_std = float(np.mean(scores)), float(np.std(scores)) print("INFO: KID: {:.4f} ± {:.4f} [Time Taken: {:.4f} secs]".format( mmd_score, mmd_std, time.time() - start_time)) return mmd_score, mmd_std
def kid_score(num_subsets, subset_size, netG, device, seed, dataset_name, batch_size=50, verbose=True, feat_file=None, log_dir='./log'): """ Computes KID score. Args: num_subsets (int): Number of subsets to compute average MMD. subset_size (int): Size of subset for computing MMD. netG (Module): Torch Module object representing the generator model. device (str): Device identifier to use for computation. seed (int): The random seed to use. dataset_name (str): The name of the dataset to load. batch_size (int): The batch size to feedforward for inference. feat_file (str): The path to specific inception features for real images. log_dir (str): Directory where features can be stored. verbose (bool): If True, prints progress. Returns: tuple: Scalar mean and std of KID scores computed. """ start_time = time.time() # Make sure the random seeds are fixed torch.manual_seed(seed) random.seed(seed) np.random.seed(seed) # Directories inception_path = os.path.join(log_dir, 'metrics', 'inception_model') # Setup the inception graph inception_utils.create_inception_graph(inception_path) # Decide sample size num_samples = int(num_subsets * subset_size) # Start producing features for real and fake images if device.index is not None: # Avoid unbounded memory usage gpu_options = tf.GPUOptions(allow_growth=True, per_process_gpu_memory_fraction=0.15, visible_device_list=str(device.index)) config = tf.compat.v1.ConfigProto(gpu_options=gpu_options) else: config = tf.compat.v1.ConfigProto(device_count={'GPU': 0}) with tf.compat.v1.Session(config=config) as sess: sess.run(tf.compat.v1.global_variables_initializer()) real_feat = compute_real_dist_feat(num_samples=num_samples, sess=sess, dataset_name=dataset_name, batch_size=batch_size, verbose=verbose, feat_file=feat_file, log_dir=log_dir, seed=seed) fake_feat = compute_gen_dist_feat(netG=netG, num_samples=num_samples, sess=sess, device=device, seed=seed, batch_size=batch_size, verbose=verbose) # Compute the KID score scores = kid_utils.polynomial_mmd_averages(real_feat, fake_feat, n_subsets=num_subsets, subset_size=subset_size) mmd_score, mmd_std = float(np.mean(scores)), float(np.std(scores)) print("INFO: KID: {:.4f} ± {:.4f} [Time Taken: {:.4f} secs]".format( mmd_score, mmd_std, time.time() - start_time)) return mmd_score, mmd_std
def pr_score(num_real_samples, num_fake_samples, netG, dataset, nearest_k=3, seed=0, device=None, batch_size=50, verbose=True, feat_file=None, log_dir='./log'): """ Computes precision and recall score. Args: num_real_samples (int): The number of real images to use for PR. num_fake_samples (int): The number of fake images to use for PR. netG (Module): Torch Module object representing the generator model. device (str/torch.device): Device identifier to use for computation. seed (int): The random seed to use. dataset (str/Dataset): The name of the dataset to load if known, or a custom Dataset object batch_size (int): The batch size to feedforward for inference. nearest_k (int): The number of nearest neighborhood to use for PR. verbose (bool): If True, prints progress. feat_file (str): The features file to load from if there is already one. log_dir (str): Directory where feature statistics can be stored. Returns: dictionary: precision, recall score dictionary. """ start_time = time.time() # Check inputs if device is None: device = torch.device('cuda:0' if torch.cuda.is_available() else "cpu") if isinstance(dataset, str): default_datasets = { 'cifar10', 'cifar100', 'stl10_48', 'imagenet_32', 'imagenet_128', 'celeba_64', 'celeba_128', 'lsun_bedroom', 'fake_data', } if dataset not in default_datasets: raise ValueError('For default datasets, must be one of {}'.format( default_datasets)) elif issubclass(type(dataset), torch.utils.data.Dataset): if feat_file is None: raise ValueError( "feat_file cannot be empty if using a custom dataset.") if not feat_file.endswith('.npy'): feat_file = feat_file + '.npy' else: raise ValueError( 'dataset must be either a Dataset object or a string.') # Make sure the random seeds are fixed torch.manual_seed(seed) random.seed(seed) np.random.seed(seed) # Setup directories inception_path = os.path.join(log_dir, 'metrics', 'inception_model') # Setup the inception graph inception_utils.create_inception_graph(inception_path) # Start producing statistics for real and fake images # if device and device.index is not None: # # Avoid unbounded memory usage # gpu_options = tf.compat.v1.GPUOptions(allow_growth=True, # per_process_gpu_memory_fraction=0.15, # visible_device_list=str(device.index)) # config = tf.compat.v1.ConfigProto(gpu_options=gpu_options) # else: # config = tf.compat.v1.ConfigProto(device_count={'GPU': 0}) config = tf.compat.v1.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.2 config.gpu_options.allow_growth = True with tf.compat.v1.Session(config=config) as sess: sess.run(tf.compat.v1.global_variables_initializer()) real_features = compute_real_features(num_samples=num_real_samples, sess=sess, dataset=dataset, batch_size=batch_size, verbose=verbose, feat_file=feat_file, log_dir=log_dir, seed=seed) fake_features = compute_fake_features(netG=netG, num_samples=num_fake_samples, sess=sess, device=device, seed=seed, batch_size=batch_size, verbose=verbose) metrics = compute_pr(real_features=real_features, fake_features=fake_features, nearest_k=nearest_k, device=device) for key in metrics: print("INFO: {}: {} [Time Taken: {:.4f} secs]".format( key, metrics[key], time.time() - start_time)) return metrics
def fid_score(num_real_samples, num_fake_samples, netG, device, seed, dataset_name, batch_size=50, verbose=True, stats_file=None, log_dir='./log'): """ Computes FID stats using functions that store images in memory for speed and fidelity. Fidelity since by storing images in memory, we don't subject the scores to different read/write implementations of imaging libraries. Args: num_real_samples (int): The number of real images to use for FID. num_fake_samples (int): The number of fake images to use for FID. netG (Module): Torch Module object representing the generator model. device (str): Device identifier to use for computation. seed (int): The random seed to use. dataset_name (str): The name of the dataset to load. batch_size (int): The batch size to feedforward for inference. verbose (bool): If True, prints progress. stats_file (str): The statistics file to load from if there is already one. log_dir (str): Directory where feature statistics can be stored. Returns: float: Scalar FID score. """ start_time = time.time() # Make sure the random seeds are fixed torch.manual_seed(seed) random.seed(seed) np.random.seed(seed) # Setup directories inception_path = os.path.join(log_dir, 'metrics', 'inception_model') # Setup the inception graph inception_utils.create_inception_graph(inception_path) # Start producing statistics for real and fake images if device and device.index is not None: # Avoid unbounded memory usage gpu_options = tf.GPUOptions(allow_growth=True, per_process_gpu_memory_fraction=0.15, visible_device_list=str(device.index)) config = tf.ConfigProto(gpu_options=gpu_options) else: config = tf.ConfigProto(device_count={'GPU': 0}) with tf.Session(config=config) as sess: sess.run(tf.global_variables_initializer()) m_real, s_real = compute_real_dist_stats(num_samples=num_real_samples, sess=sess, dataset_name=dataset_name, batch_size=batch_size, verbose=verbose, stats_file=stats_file, log_dir=log_dir, seed=seed) m_fake, s_fake = compute_gen_dist_stats(netG=netG, num_samples=num_fake_samples, sess=sess, device=device, seed=seed, batch_size=batch_size, verbose=verbose) FID_score = fid_utils.calculate_frechet_distance(mu1=m_real, sigma1=s_real, mu2=m_fake, sigma2=s_fake) print("INFO: FID Score: {} [Time Taken: {:.4f} secs]".format( FID_score, time.time() - start_time)) return float(FID_score)
def inception_score(num_samples, netG, device=None, batch_size=50, splits=10, log_dir='./log', seed=0, print_every=20): """ Computes the inception score of generated images. Args: netG (Module): The generator model to use for generating images. device (str/torch.device): Device identifier to use for computation. num_samples (int): The number of samples to generate. batch_size (int): Batch size per feedforward step for inception model. splits (int): The number of splits to use for computing IS. log_dir (str): Path to store metric computation objects. seed (int): Random seed for generation. Returns: Mean and standard deviation of the inception score computed from using num_samples generated images. """ start_time = time.time() if device is None: device = torch.device('cuda:0' if torch.cuda.is_available() else "cpu") # Make sure the random seeds are fixed torch.manual_seed(seed) random.seed(seed) np.random.seed(seed) # Build inception inception_path = os.path.join(log_dir, 'metrics/inception_model') inception_utils.create_inception_graph(inception_path) # Inference variables batch_size = min(batch_size, num_samples) num_batches = num_samples // batch_size # Get images images = [] with torch.no_grad(): start_time = time.time() for idx in range(num_batches): # noise = torch.randn((batch_size, netG.nz), device=device) # fake_images = netG(noise) fake_images = netG.generate_images(num_images=batch_size, device=device).detach().cpu() fake_images = _normalize_images(fake_images) images.append(fake_images) if (idx + 1) % min(print_every, num_batches) == 0: end_time = time.time() print( "INFO: Generated image {}/{} [Random Seed {}] ({:.4f} sec/idx)" .format( (idx + 1) * batch_size, num_samples, seed, (end_time - start_time) / (print_every * batch_size))) start_time = end_time images = np.concatenate(images, axis=0) is_mean, is_std = tf_inception_score.get_inception_score(images, splits=splits, device=device) print("INFO: Inception Score: {:.4f} ± {:.4f} [Time Taken: {:.4f} secs]". format(is_mean, is_std, time.time() - start_time)) return is_mean, is_std