def test_isc_pt_tf_fidelity(self):
cuda = os.environ.get('CUDA_VISIBLE_DEVICES', '') != ''
limit = 5000
cifar10_root = os.path.join(tempfile.gettempdir(), f'cifar10-train-img-{limit}')
res = subprocess.run(
('python3', 'utils/util_dump_dataset_as_images.py', 'cifar10-train', cifar10_root,
'-l', str(limit)),
)
self.assertEqual(res.returncode, 0, msg=res)
print(f'Running reference ISC...', file=sys.stderr)
res_ref = self.call_ref_isc(cifar10_root, cuda, determinism=True)
self.assertEqual(res_ref.returncode, 0, msg=res_ref)
res_ref = json_decode_string(res_ref.stdout.decode())
print('Reference ISC result:', res_ref, file=sys.stderr)
print(f'Running fidelity ISC...', file=sys.stderr)
res_fidelity = self.call_fidelity_isc(cifar10_root)
self.assertEqual(res_fidelity.returncode, 0, msg=res_fidelity)
res_fidelity = json_decode_string(res_fidelity.stdout.decode())
print('Fidelity ISC result:', res_fidelity, file=sys.stderr)
err_abs_mean = abs(res_ref[KEY_METRIC_ISC_MEAN] - res_fidelity[KEY_METRIC_ISC_MEAN])
err_abs_std = abs(res_ref[KEY_METRIC_ISC_STD] - res_fidelity[KEY_METRIC_ISC_STD])
print(f'Error absolute mean={err_abs_mean} std={err_abs_std}')
err_rel_mean = err_abs_mean / res_ref[KEY_METRIC_ISC_MEAN]
err_rel_std = err_abs_std / res_ref[KEY_METRIC_ISC_STD]
print(f'Error relative mean={err_rel_mean} std={err_rel_std}')
self.assertLess(err_rel_mean, 1e-3)
self.assertLess(err_rel_std, 5e-1)
self.assertAlmostEqual(res_fidelity[KEY_METRIC_ISC_MEAN], 10.7504627, delta=1e-5)
def sample_runs(isc_mean, isc_std, prefix, determinism):
for i in range(num_sample_runs):
print(f'{prefix} run {i+1} of {num_sample_runs}...', file=sys.stderr)
res = self.call_ref_isc(cifar10_root, cuda, determinism)
self.assertEqual(res.returncode, 0, msg=res)
out = json_decode_string(res.stdout.decode())
isc_mean.append(out[KEY_METRIC_ISC_MEAN])
isc_std.append(out[KEY_METRIC_ISC_STD])
def sample_runs(fid, prefix, determinism):
for i in range(num_sample_runs):
print(f'{prefix} run {i+1} of {num_sample_runs}...',
file=sys.stderr)
res = self.call_ref_fid(cifar10train_root, cifar10valid_root,
cuda, determinism)
self.assertEqual(res.returncode, 0, msg=res)
out = json_decode_string(res.stdout.decode())
fid.append(out[KEY_METRIC_FID])
def test_fid_pt_tf_fidelity(self):
cuda = os.environ.get('CUDA_VISIBLE_DEVICES', '') != ''
limit = 5000
cifar10train_root = os.path.join(tempfile.gettempdir(),
f'cifar10-train-img-{limit}')
cifar10valid_root = os.path.join(tempfile.gettempdir(),
f'cifar10-valid-img-{limit}')
res = subprocess.run(
('python3', 'utils/util_dump_dataset_as_images.py',
'cifar10-train', cifar10train_root, '-l', str(limit)), )
self.assertEqual(res.returncode, 0, msg=res)
res = subprocess.run(
('python3', 'utils/util_dump_dataset_as_images.py',
'cifar10-valid', cifar10valid_root, '-l', str(limit)), )
self.assertEqual(res.returncode, 0, msg=res)
print(f'Running reference FID...', file=sys.stderr)
res_ref = self.call_ref_fid(cifar10train_root,
cifar10valid_root,
cuda,
determinism=True)
self.assertEqual(res_ref.returncode, 0, msg=res_ref)
res_ref = json_decode_string(res_ref.stdout.decode())
print('Reference FID result:', res_ref, file=sys.stderr)
print(f'Running fidelity FID...', file=sys.stderr)
res_fidelity = self.call_fidelity_fid(cifar10train_root,
cifar10valid_root)
self.assertEqual(res_fidelity.returncode, 0, msg=res_fidelity)
res_fidelity = json_decode_string(res_fidelity.stdout.decode())
print('Fidelity FID result:', res_fidelity, file=sys.stderr)
err_abs = abs(res_ref[KEY_METRIC_FID] - res_fidelity[KEY_METRIC_FID])
print(f'Error absolute={err_abs}')
err_rel = err_abs / res_ref[KEY_METRIC_FID]
print(f'Error relative={err_rel}')
self.assertLess(err_rel, 1e-6)
self.assertAlmostEqual(res_fidelity[KEY_METRIC_FID],
10.3233274,
delta=1e-6)
def sample_runs(kid_mean, kid_std, prefix, determinism):
for i in range(num_sample_runs):
print(f'{prefix} run {i+1} of {num_sample_runs}...',
file=sys.stderr)
res = self.call_ref_kid(cifar10validnoise_root,
cifar10trainorig_codes, cuda,
determinism)
self.assertEqual(res.returncode, 0, msg=res)
res = json_decode_string(res.stdout.decode())
kid_mean.append(res[KEY_METRIC_KID_MEAN])
kid_std.append(res[KEY_METRIC_KID_STD])
def test_ppl_gen_model(self):
model = os.path.basename(URL_SNGAN_MODEL)
model = os.path.realpath(os.path.join(tempfile.gettempdir(), model))
download_url_to_file(URL_SNGAN_MODEL, model, progress=True)
self.assertTrue(os.path.isfile(model))
print(f'Running fidelity PPL...', file=sys.stderr)
res_fidelity = self.call_fidelity_ppl(model, 100)
self.assertEqual(res_fidelity.returncode, 0, msg=res_fidelity)
res_fidelity = json_decode_string(res_fidelity.stdout.decode())
print('Fidelity PPL result:', res_fidelity, file=sys.stderr)
self.assertAlmostEqual(res_fidelity[KEY_METRIC_PPL_MEAN],
2560.187255859375,
delta=1e-5)
def test_kid_pt_tf_fidelity(self):
cuda = os.environ.get('CUDA_VISIBLE_DEVICES', '') != ''
limit = 5000
cifar10trainorig_root = os.path.join(tempfile.gettempdir(),
f'cifar10-train-img-{limit}')
cifar10validnoise_root = os.path.join(
tempfile.gettempdir(), f'cifar10-valid-img-noise-{limit}')
cifar10trainorig_codes = os.path.join(
tempfile.gettempdir(), f'cifar10-train-codes-{limit}.pth')
res = subprocess.run(
('python3', 'utils/util_dump_dataset_as_images.py',
'cifar10-train', cifar10trainorig_root, '-l', str(limit)), )
self.assertEqual(res.returncode, 0, msg=res)
res = subprocess.run(
('python3', 'utils/util_dump_dataset_as_images.py',
'cifar10-valid', cifar10validnoise_root, '-l', str(limit),
'-n'), )
self.assertEqual(res.returncode, 0, msg=res)
# this partucular reference app needs features pre-extracted
if not os.path.exists(cifar10trainorig_codes):
args = [
'python3', 'tests/reference/reference_metric_kid_mmdgan.py',
cifar10trainorig_root, '--save-codes', cifar10trainorig_codes,
'--determinism'
]
if cuda:
args.append('--gpu')
args.append(os.environ['CUDA_VISIBLE_DEVICES'])
res = subprocess.run(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
self.assertEqual(res.returncode, 0, msg=res)
print(f'Running reference KID...', file=sys.stderr)
res_ref = self.call_ref_kid(cifar10validnoise_root,
cifar10trainorig_codes,
cuda,
determinism=True)
self.assertEqual(res_ref.returncode, 0, msg=res_ref)
res_ref = json_decode_string(res_ref.stdout.decode())
print('Reference KID result:', res_ref, file=sys.stderr)
print(f'Running fidelity KID...', file=sys.stderr)
res_fidelity = self.call_fidelity_kid(cifar10validnoise_root,
cifar10trainorig_root)
self.assertEqual(res_fidelity.returncode, 0, msg=res_fidelity)
res_fidelity = json_decode_string(res_fidelity.stdout.decode())
print('Fidelity KID result:', res_fidelity, file=sys.stderr)
err_abs_mean = abs(res_ref[KEY_METRIC_KID_MEAN] -
res_fidelity[KEY_METRIC_KID_MEAN])
err_abs_std = abs(res_ref[KEY_METRIC_KID_STD] -
res_fidelity[KEY_METRIC_KID_STD])
print(f'Error absolute mean={err_abs_mean} std={err_abs_std}')
err_rel_mean = err_abs_mean / max(1e-6,
abs(res_ref[KEY_METRIC_KID_MEAN]))
err_rel_std = err_abs_std / res_ref[KEY_METRIC_KID_STD]
print(f'Error relative mean={err_rel_mean} std={err_rel_std}')
self.assertLess(err_rel_mean, 1e-6)
self.assertLess(err_rel_std, 1e-4)
self.assertAlmostEqual(res_fidelity[KEY_METRIC_KID_MEAN],
0.4718520,
delta=1e-6)