def test_get_predictions(self):
inception_utils.create_inception_graph('./metrics/inception_model')
images = np.ones((4, 32, 32, 3))
preds = inception_score_utils.get_predictions(images)
assert preds.shape == (4, 1008)
def test_get_activations(self):
inception_path = './metrics/inception_model'
inception_utils.create_inception_graph(inception_path)
images = np.ones((4, 32, 32, 3))
with tf.compat.v1.Session() as sess:
feat = inception_utils.get_activations(images=images, sess=sess)
assert feat.shape == (4, 2048)
def test_calculate_activation_statistics(self):
inception_path = './metrics/inception_model'
inception_utils.create_inception_graph(inception_path)
mu, sigma = fid_utils.calculate_activation_statistics(
images=self.images, sess=self.sess)
assert mu.shape == (2048, )
assert sigma.shape == (2048, 2048)
def setup(self):
self.netG = ExampleGen()
self.num_samples = 50
self.device = torch.device("cpu")
# Create inception graph once.
self.inception_path = './metrics/inception_model'
if not os.path.exists(self.inception_path):
os.makedirs(self.inception_path)
inception_utils.create_inception_graph(self.inception_path)
# Directory
self.log_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"test_log")
if not os.path.exists(self.log_dir):
os.makedirs(self.log_dir)
def fid_score(num_real_samples,
num_fake_samples,
netG,
dataset,
seed=0,
device=None,
batch_size=50,
verbose=True,
stats_file=None,
log_dir='./log'):
"""
Computes FID stats using functions that store images in memory for speed and fidelity.
Fidelity since by storing images in memory, we don't subject the scores to different read/write
implementations of imaging libraries.
Args:
num_real_samples (int): The number of real images to use for FID.
num_fake_samples (int): The number of fake images to use for FID.
netG (Module): Torch Module object representing the generator model.
device (str/torch.device): Device identifier to use for computation.
seed (int): The random seed to use.
dataset (str/Dataset): The name of the dataset to load if known, or a custom Dataset object
batch_size (int): The batch size to feedforward for inference.
verbose (bool): If True, prints progress.
stats_file (str): The statistics file to load from if there is already one.
log_dir (str): Directory where feature statistics can be stored.
Returns:
float: Scalar FID score.
"""
start_time = time.time()
# Check inputs
if device is None:
device = torch.device('cuda:0' if torch.cuda.is_available() else "cpu")
if isinstance(dataset, str):
default_datasets = {
'cifar10',
'cifar100',
'stl10_48',
'imagenet_32',
'imagenet_128',
'celeba_64',
'celeba_128',
'lsun_bedroom',
'fake_data',
}
if dataset not in default_datasets:
raise ValueError('For default datasets, must be one of {}'.format(
default_datasets))
elif issubclass(type(dataset), torch.utils.data.Dataset):
if stats_file is None:
raise ValueError(
"stats_file to save/load from cannot be empty if using a custom dataset."
)
if not stats_file.endswith('.npz'):
stats_file = stats_file + '.npz'
else:
raise ValueError(
'dataset must be either a Dataset object or a string.')
# Make sure the random seeds are fixed
torch.manual_seed(seed)
random.seed(seed)
np.random.seed(seed)
# Setup directories
inception_path = os.path.join(log_dir, 'metrics', 'inception_model')
# Setup the inception graph
inception_utils.create_inception_graph(inception_path)
# Start producing statistics for real and fake images
# if device and device.index is not None:
# # Avoid unbounded memory usage
# gpu_options = tf.compat.v1.GPUOptions(allow_growth=True,
# per_process_gpu_memory_fraction=0.15,
# visible_device_list=str(device.index))
# config = tf.compat.v1.ConfigProto(gpu_options=gpu_options)
# else:
# config = tf.compat.v1.ConfigProto(device_count={'GPU': 0})
config = tf.compat.v1.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.2
config.gpu_options.allow_growth = True
with tf.compat.v1.Session(config=config) as sess:
sess.run(tf.compat.v1.global_variables_initializer())
m_real, s_real = compute_real_dist_stats(num_samples=num_real_samples,
sess=sess,
dataset=dataset,
batch_size=batch_size,
verbose=verbose,
stats_file=stats_file,
log_dir=log_dir,
seed=seed)
m_fake, s_fake = compute_gen_dist_stats(netG=netG,
num_samples=num_fake_samples,
sess=sess,
device=device,
seed=seed,
batch_size=batch_size,
verbose=verbose)
FID_score = fid_utils.calculate_frechet_distance(mu1=m_real,
sigma1=s_real,
mu2=m_fake,
sigma2=s_fake)
print("INFO: FID: {} [Time Taken: {:.4f} secs]".format(
FID_score,
time.time() - start_time))
return float(FID_score)
def kid_score(num_samples,
netG,
dataset,
seed=0,
device=None,
num_subsets=10,
batch_size=50,
verbose=True,
feat_file=None,
log_dir='./log',
**kwargs):
"""
Computes KID score.
Args:
num_samples (int): The number of real and fake images to use for KID.
num_subsets (int): Number of subsets to compute average MMD.
netG (Module): Torch Module object representing the generator model.
device (str): Device identifier to use for computation.
seed (int): The random seed to use.
dataset (str/Dataset): The name of the dataset to load if known, or a custom Dataset object
batch_size (int): The batch size to feedforward for inference.
feat_file (str): The path to specific inception features for real images.
log_dir (str): Directory where features can be stored.
verbose (bool): If True, prints progress.
Returns:
tuple: Scalar mean and std of KID scores computed.
"""
start_time = time.time()
# Check inputs
if device is None:
device = torch.device('cuda:0' if torch.cuda.is_available() else "cpu")
if isinstance(dataset, str):
default_datasets = {
'cifar10',
'cifar100',
'stl10_48',
'imagenet_32',
'imagenet_128',
'celeba_64',
'celeba_128',
'lsun_bedroom',
'fake_data',
}
if dataset not in default_datasets:
raise ValueError('For default datasets, must be one of {}'.format(
default_datasets))
elif issubclass(type(dataset), torch.utils.data.Dataset):
if feat_file is None:
raise ValueError(
"feat_file to save/load from cannot be empty if using a custom dataset."
)
if not feat_file.endswith('.npz'):
feat_file = feat_file + '.npz'
else:
raise ValueError(
'dataset must be either a Dataset object or a string.')
# Make sure the random seeds are fixed
torch.manual_seed(seed)
random.seed(seed)
np.random.seed(seed)
# Directories
inception_path = os.path.join('./inception_model')
# Setup the inception graph
inception_utils.create_inception_graph(inception_path)
# Start producing features for real and fake images
# if device.index is not None:
# # Avoid unbounded memory usage
# gpu_options = tf.compat.v1.GPUOptions(allow_growth=True,
# per_process_gpu_memory_fraction=0.15,
# visible_device_list=str(device.index))
# config = tf.compat.v1.ConfigProto(gpu_options=gpu_options)
# else:
# config = tf.compat.v1.ConfigProto(device_count={'GPU': 0})
config = tf.compat.v1.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.2
config.gpu_options.allow_growth = True
with tf.compat.v1.Session(config=config) as sess:
sess.run(tf.compat.v1.global_variables_initializer())
real_feat = compute_real_dist_feat(num_samples=num_samples,
sess=sess,
dataset=dataset,
batch_size=batch_size,
verbose=verbose,
feat_file=feat_file,
log_dir=log_dir,
seed=seed,
**kwargs)
fake_feat = compute_gen_dist_feat(netG=netG,
num_samples=num_samples,
sess=sess,
device=device,
seed=seed,
batch_size=batch_size,
verbose=verbose)
# Compute the KID score
subset_size = num_samples // num_subsets
scores = kid_utils.polynomial_mmd_averages(real_feat,
fake_feat,
n_subsets=num_subsets,
subset_size=subset_size)
mmd_score, mmd_std = float(np.mean(scores)), float(np.std(scores))
print("INFO: KID: {:.4f} ± {:.4f} [Time Taken: {:.4f} secs]".format(
mmd_score, mmd_std,
time.time() - start_time))
return mmd_score, mmd_std
def kid_score(num_subsets,
subset_size,
netG,
device,
seed,
dataset_name,
batch_size=50,
verbose=True,
feat_file=None,
log_dir='./log'):
"""
Computes KID score.
Args:
num_subsets (int): Number of subsets to compute average MMD.
subset_size (int): Size of subset for computing MMD.
netG (Module): Torch Module object representing the generator model.
device (str): Device identifier to use for computation.
seed (int): The random seed to use.
dataset_name (str): The name of the dataset to load.
batch_size (int): The batch size to feedforward for inference.
feat_file (str): The path to specific inception features for real images.
log_dir (str): Directory where features can be stored.
verbose (bool): If True, prints progress.
Returns:
tuple: Scalar mean and std of KID scores computed.
"""
start_time = time.time()
# Make sure the random seeds are fixed
torch.manual_seed(seed)
random.seed(seed)
np.random.seed(seed)
# Directories
inception_path = os.path.join(log_dir, 'metrics', 'inception_model')
# Setup the inception graph
inception_utils.create_inception_graph(inception_path)
# Decide sample size
num_samples = int(num_subsets * subset_size)
# Start producing features for real and fake images
if device.index is not None:
# Avoid unbounded memory usage
gpu_options = tf.GPUOptions(allow_growth=True,
per_process_gpu_memory_fraction=0.15,
visible_device_list=str(device.index))
config = tf.compat.v1.ConfigProto(gpu_options=gpu_options)
else:
config = tf.compat.v1.ConfigProto(device_count={'GPU': 0})
with tf.compat.v1.Session(config=config) as sess:
sess.run(tf.compat.v1.global_variables_initializer())
real_feat = compute_real_dist_feat(num_samples=num_samples,
sess=sess,
dataset_name=dataset_name,
batch_size=batch_size,
verbose=verbose,
feat_file=feat_file,
log_dir=log_dir,
seed=seed)
fake_feat = compute_gen_dist_feat(netG=netG,
num_samples=num_samples,
sess=sess,
device=device,
seed=seed,
batch_size=batch_size,
verbose=verbose)
# Compute the KID score
scores = kid_utils.polynomial_mmd_averages(real_feat,
fake_feat,
n_subsets=num_subsets,
subset_size=subset_size)
mmd_score, mmd_std = float(np.mean(scores)), float(np.std(scores))
print("INFO: KID: {:.4f} ± {:.4f} [Time Taken: {:.4f} secs]".format(
mmd_score, mmd_std,
time.time() - start_time))
return mmd_score, mmd_std
def pr_score(num_real_samples,
num_fake_samples,
netG,
dataset,
nearest_k=3,
seed=0,
device=None,
batch_size=50,
verbose=True,
feat_file=None,
log_dir='./log'):
"""
Computes precision and recall score.
Args:
num_real_samples (int): The number of real images to use for PR.
num_fake_samples (int): The number of fake images to use for PR.
netG (Module): Torch Module object representing the generator model.
device (str/torch.device): Device identifier to use for computation.
seed (int): The random seed to use.
dataset (str/Dataset): The name of the dataset to load if known, or a custom Dataset object
batch_size (int): The batch size to feedforward for inference.
nearest_k (int): The number of nearest neighborhood to use for PR.
verbose (bool): If True, prints progress.
feat_file (str): The features file to load from if there is already one.
log_dir (str): Directory where feature statistics can be stored.
Returns:
dictionary: precision, recall score dictionary.
"""
start_time = time.time()
# Check inputs
if device is None:
device = torch.device('cuda:0' if torch.cuda.is_available() else "cpu")
if isinstance(dataset, str):
default_datasets = {
'cifar10',
'cifar100',
'stl10_48',
'imagenet_32',
'imagenet_128',
'celeba_64',
'celeba_128',
'lsun_bedroom',
'fake_data',
}
if dataset not in default_datasets:
raise ValueError('For default datasets, must be one of {}'.format(
default_datasets))
elif issubclass(type(dataset), torch.utils.data.Dataset):
if feat_file is None:
raise ValueError(
"feat_file cannot be empty if using a custom dataset.")
if not feat_file.endswith('.npy'):
feat_file = feat_file + '.npy'
else:
raise ValueError(
'dataset must be either a Dataset object or a string.')
# Make sure the random seeds are fixed
torch.manual_seed(seed)
random.seed(seed)
np.random.seed(seed)
# Setup directories
inception_path = os.path.join(log_dir, 'metrics', 'inception_model')
# Setup the inception graph
inception_utils.create_inception_graph(inception_path)
# Start producing statistics for real and fake images
# if device and device.index is not None:
# # Avoid unbounded memory usage
# gpu_options = tf.compat.v1.GPUOptions(allow_growth=True,
# per_process_gpu_memory_fraction=0.15,
# visible_device_list=str(device.index))
# config = tf.compat.v1.ConfigProto(gpu_options=gpu_options)
# else:
# config = tf.compat.v1.ConfigProto(device_count={'GPU': 0})
config = tf.compat.v1.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.2
config.gpu_options.allow_growth = True
with tf.compat.v1.Session(config=config) as sess:
sess.run(tf.compat.v1.global_variables_initializer())
real_features = compute_real_features(num_samples=num_real_samples,
sess=sess,
dataset=dataset,
batch_size=batch_size,
verbose=verbose,
feat_file=feat_file,
log_dir=log_dir,
seed=seed)
fake_features = compute_fake_features(netG=netG,
num_samples=num_fake_samples,
sess=sess,
device=device,
seed=seed,
batch_size=batch_size,
verbose=verbose)
metrics = compute_pr(real_features=real_features,
fake_features=fake_features,
nearest_k=nearest_k,
device=device)
for key in metrics:
print("INFO: {}: {} [Time Taken: {:.4f} secs]".format(
key, metrics[key],
time.time() - start_time))
return metrics
def fid_score(num_real_samples,
num_fake_samples,
netG,
device,
seed,
dataset_name,
batch_size=50,
verbose=True,
stats_file=None,
log_dir='./log'):
"""
Computes FID stats using functions that store images in memory for speed and fidelity.
Fidelity since by storing images in memory, we don't subject the scores to different read/write
implementations of imaging libraries.
Args:
num_real_samples (int): The number of real images to use for FID.
num_fake_samples (int): The number of fake images to use for FID.
netG (Module): Torch Module object representing the generator model.
device (str): Device identifier to use for computation.
seed (int): The random seed to use.
dataset_name (str): The name of the dataset to load.
batch_size (int): The batch size to feedforward for inference.
verbose (bool): If True, prints progress.
stats_file (str): The statistics file to load from if there is already one.
log_dir (str): Directory where feature statistics can be stored.
Returns:
float: Scalar FID score.
"""
start_time = time.time()
# Make sure the random seeds are fixed
torch.manual_seed(seed)
random.seed(seed)
np.random.seed(seed)
# Setup directories
inception_path = os.path.join(log_dir, 'metrics', 'inception_model')
# Setup the inception graph
inception_utils.create_inception_graph(inception_path)
# Start producing statistics for real and fake images
if device and device.index is not None:
# Avoid unbounded memory usage
gpu_options = tf.GPUOptions(allow_growth=True,
per_process_gpu_memory_fraction=0.15,
visible_device_list=str(device.index))
config = tf.ConfigProto(gpu_options=gpu_options)
else:
config = tf.ConfigProto(device_count={'GPU': 0})
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
m_real, s_real = compute_real_dist_stats(num_samples=num_real_samples,
sess=sess,
dataset_name=dataset_name,
batch_size=batch_size,
verbose=verbose,
stats_file=stats_file,
log_dir=log_dir,
seed=seed)
m_fake, s_fake = compute_gen_dist_stats(netG=netG,
num_samples=num_fake_samples,
sess=sess,
device=device,
seed=seed,
batch_size=batch_size,
verbose=verbose)
FID_score = fid_utils.calculate_frechet_distance(mu1=m_real,
sigma1=s_real,
mu2=m_fake,
sigma2=s_fake)
print("INFO: FID Score: {} [Time Taken: {:.4f} secs]".format(
FID_score,
time.time() - start_time))
return float(FID_score)
def inception_score(num_samples,
netG,
device=None,
batch_size=50,
splits=10,
log_dir='./log',
seed=0,
print_every=20):
"""
Computes the inception score of generated images.
Args:
netG (Module): The generator model to use for generating images.
device (str/torch.device): Device identifier to use for computation.
num_samples (int): The number of samples to generate.
batch_size (int): Batch size per feedforward step for inception model.
splits (int): The number of splits to use for computing IS.
log_dir (str): Path to store metric computation objects.
seed (int): Random seed for generation.
Returns:
Mean and standard deviation of the inception score computed from using
num_samples generated images.
"""
start_time = time.time()
if device is None:
device = torch.device('cuda:0' if torch.cuda.is_available() else "cpu")
# Make sure the random seeds are fixed
torch.manual_seed(seed)
random.seed(seed)
np.random.seed(seed)
# Build inception
inception_path = os.path.join(log_dir, 'metrics/inception_model')
inception_utils.create_inception_graph(inception_path)
# Inference variables
batch_size = min(batch_size, num_samples)
num_batches = num_samples // batch_size
# Get images
images = []
with torch.no_grad():
start_time = time.time()
for idx in range(num_batches):
# noise = torch.randn((batch_size, netG.nz), device=device)
# fake_images = netG(noise)
fake_images = netG.generate_images(num_images=batch_size,
device=device).detach().cpu()
fake_images = _normalize_images(fake_images)
images.append(fake_images)
if (idx + 1) % min(print_every, num_batches) == 0:
end_time = time.time()
print(
"INFO: Generated image {}/{} [Random Seed {}] ({:.4f} sec/idx)"
.format(
(idx + 1) * batch_size, num_samples, seed,
(end_time - start_time) / (print_every * batch_size)))
start_time = end_time
images = np.concatenate(images, axis=0)
is_mean, is_std = tf_inception_score.get_inception_score(images,
splits=splits,
device=device)
print("INFO: Inception Score: {:.4f} ± {:.4f} [Time Taken: {:.4f} secs]".
format(is_mean, is_std,
time.time() - start_time))
return is_mean, is_std
