def test_isc_pt_tf_fidelity(self): cuda = os.environ.get('CUDA_VISIBLE_DEVICES', '') != '' limit = 5000 cifar10_root = os.path.join(tempfile.gettempdir(), f'cifar10-train-img-{limit}') res = subprocess.run( ('python3', 'utils/util_dump_dataset_as_images.py', 'cifar10-train', cifar10_root, '-l', str(limit)), ) self.assertEqual(res.returncode, 0, msg=res) print(f'Running reference ISC...', file=sys.stderr) res_ref = self.call_ref_isc(cifar10_root, cuda, determinism=True) self.assertEqual(res_ref.returncode, 0, msg=res_ref) res_ref = json_decode_string(res_ref.stdout.decode()) print('Reference ISC result:', res_ref, file=sys.stderr) print(f'Running fidelity ISC...', file=sys.stderr) res_fidelity = self.call_fidelity_isc(cifar10_root) self.assertEqual(res_fidelity.returncode, 0, msg=res_fidelity) res_fidelity = json_decode_string(res_fidelity.stdout.decode()) print('Fidelity ISC result:', res_fidelity, file=sys.stderr) err_abs_mean = abs(res_ref[KEY_METRIC_ISC_MEAN] - res_fidelity[KEY_METRIC_ISC_MEAN]) err_abs_std = abs(res_ref[KEY_METRIC_ISC_STD] - res_fidelity[KEY_METRIC_ISC_STD]) print(f'Error absolute mean={err_abs_mean} std={err_abs_std}') err_rel_mean = err_abs_mean / res_ref[KEY_METRIC_ISC_MEAN] err_rel_std = err_abs_std / res_ref[KEY_METRIC_ISC_STD] print(f'Error relative mean={err_rel_mean} std={err_rel_std}') self.assertLess(err_rel_mean, 1e-3) self.assertLess(err_rel_std, 5e-1) self.assertAlmostEqual(res_fidelity[KEY_METRIC_ISC_MEAN], 10.7504627, delta=1e-5)
def sample_runs(isc_mean, isc_std, prefix, determinism): for i in range(num_sample_runs): print(f'{prefix} run {i+1} of {num_sample_runs}...', file=sys.stderr) res = self.call_ref_isc(cifar10_root, cuda, determinism) self.assertEqual(res.returncode, 0, msg=res) out = json_decode_string(res.stdout.decode()) isc_mean.append(out[KEY_METRIC_ISC_MEAN]) isc_std.append(out[KEY_METRIC_ISC_STD])
def sample_runs(fid, prefix, determinism): for i in range(num_sample_runs): print(f'{prefix} run {i+1} of {num_sample_runs}...', file=sys.stderr) res = self.call_ref_fid(cifar10train_root, cifar10valid_root, cuda, determinism) self.assertEqual(res.returncode, 0, msg=res) out = json_decode_string(res.stdout.decode()) fid.append(out[KEY_METRIC_FID])
def test_fid_pt_tf_fidelity(self): cuda = os.environ.get('CUDA_VISIBLE_DEVICES', '') != '' limit = 5000 cifar10train_root = os.path.join(tempfile.gettempdir(), f'cifar10-train-img-{limit}') cifar10valid_root = os.path.join(tempfile.gettempdir(), f'cifar10-valid-img-{limit}') res = subprocess.run( ('python3', 'utils/util_dump_dataset_as_images.py', 'cifar10-train', cifar10train_root, '-l', str(limit)), ) self.assertEqual(res.returncode, 0, msg=res) res = subprocess.run( ('python3', 'utils/util_dump_dataset_as_images.py', 'cifar10-valid', cifar10valid_root, '-l', str(limit)), ) self.assertEqual(res.returncode, 0, msg=res) print(f'Running reference FID...', file=sys.stderr) res_ref = self.call_ref_fid(cifar10train_root, cifar10valid_root, cuda, determinism=True) self.assertEqual(res_ref.returncode, 0, msg=res_ref) res_ref = json_decode_string(res_ref.stdout.decode()) print('Reference FID result:', res_ref, file=sys.stderr) print(f'Running fidelity FID...', file=sys.stderr) res_fidelity = self.call_fidelity_fid(cifar10train_root, cifar10valid_root) self.assertEqual(res_fidelity.returncode, 0, msg=res_fidelity) res_fidelity = json_decode_string(res_fidelity.stdout.decode()) print('Fidelity FID result:', res_fidelity, file=sys.stderr) err_abs = abs(res_ref[KEY_METRIC_FID] - res_fidelity[KEY_METRIC_FID]) print(f'Error absolute={err_abs}') err_rel = err_abs / res_ref[KEY_METRIC_FID] print(f'Error relative={err_rel}') self.assertLess(err_rel, 1e-6) self.assertAlmostEqual(res_fidelity[KEY_METRIC_FID], 10.3233274, delta=1e-6)
def sample_runs(kid_mean, kid_std, prefix, determinism): for i in range(num_sample_runs): print(f'{prefix} run {i+1} of {num_sample_runs}...', file=sys.stderr) res = self.call_ref_kid(cifar10validnoise_root, cifar10trainorig_codes, cuda, determinism) self.assertEqual(res.returncode, 0, msg=res) res = json_decode_string(res.stdout.decode()) kid_mean.append(res[KEY_METRIC_KID_MEAN]) kid_std.append(res[KEY_METRIC_KID_STD])
def test_ppl_gen_model(self): model = os.path.basename(URL_SNGAN_MODEL) model = os.path.realpath(os.path.join(tempfile.gettempdir(), model)) download_url_to_file(URL_SNGAN_MODEL, model, progress=True) self.assertTrue(os.path.isfile(model)) print(f'Running fidelity PPL...', file=sys.stderr) res_fidelity = self.call_fidelity_ppl(model, 100) self.assertEqual(res_fidelity.returncode, 0, msg=res_fidelity) res_fidelity = json_decode_string(res_fidelity.stdout.decode()) print('Fidelity PPL result:', res_fidelity, file=sys.stderr) self.assertAlmostEqual(res_fidelity[KEY_METRIC_PPL_MEAN], 2560.187255859375, delta=1e-5)
def test_kid_pt_tf_fidelity(self): cuda = os.environ.get('CUDA_VISIBLE_DEVICES', '') != '' limit = 5000 cifar10trainorig_root = os.path.join(tempfile.gettempdir(), f'cifar10-train-img-{limit}') cifar10validnoise_root = os.path.join( tempfile.gettempdir(), f'cifar10-valid-img-noise-{limit}') cifar10trainorig_codes = os.path.join( tempfile.gettempdir(), f'cifar10-train-codes-{limit}.pth') res = subprocess.run( ('python3', 'utils/util_dump_dataset_as_images.py', 'cifar10-train', cifar10trainorig_root, '-l', str(limit)), ) self.assertEqual(res.returncode, 0, msg=res) res = subprocess.run( ('python3', 'utils/util_dump_dataset_as_images.py', 'cifar10-valid', cifar10validnoise_root, '-l', str(limit), '-n'), ) self.assertEqual(res.returncode, 0, msg=res) # this partucular reference app needs features pre-extracted if not os.path.exists(cifar10trainorig_codes): args = [ 'python3', 'tests/reference/reference_metric_kid_mmdgan.py', cifar10trainorig_root, '--save-codes', cifar10trainorig_codes, '--determinism' ] if cuda: args.append('--gpu') args.append(os.environ['CUDA_VISIBLE_DEVICES']) res = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) self.assertEqual(res.returncode, 0, msg=res) print(f'Running reference KID...', file=sys.stderr) res_ref = self.call_ref_kid(cifar10validnoise_root, cifar10trainorig_codes, cuda, determinism=True) self.assertEqual(res_ref.returncode, 0, msg=res_ref) res_ref = json_decode_string(res_ref.stdout.decode()) print('Reference KID result:', res_ref, file=sys.stderr) print(f'Running fidelity KID...', file=sys.stderr) res_fidelity = self.call_fidelity_kid(cifar10validnoise_root, cifar10trainorig_root) self.assertEqual(res_fidelity.returncode, 0, msg=res_fidelity) res_fidelity = json_decode_string(res_fidelity.stdout.decode()) print('Fidelity KID result:', res_fidelity, file=sys.stderr) err_abs_mean = abs(res_ref[KEY_METRIC_KID_MEAN] - res_fidelity[KEY_METRIC_KID_MEAN]) err_abs_std = abs(res_ref[KEY_METRIC_KID_STD] - res_fidelity[KEY_METRIC_KID_STD]) print(f'Error absolute mean={err_abs_mean} std={err_abs_std}') err_rel_mean = err_abs_mean / max(1e-6, abs(res_ref[KEY_METRIC_KID_MEAN])) err_rel_std = err_abs_std / res_ref[KEY_METRIC_KID_STD] print(f'Error relative mean={err_rel_mean} std={err_rel_std}') self.assertLess(err_rel_mean, 1e-6) self.assertLess(err_rel_std, 1e-4) self.assertAlmostEqual(res_fidelity[KEY_METRIC_KID_MEAN], 0.4718520, delta=1e-6)