def evaluate(val_loaders, fake_loader, net, criterion, args, log,
num_mc_samples, num_mc_samples2):
freeze(net)
if args.gpu == 0:
print("-----------------deterministic-----------------")
deter_rets = ens_validate(val_loaders, net, criterion, args, log, 1)
unfreeze(net)
if args.gpu == 0:
print("-----------------ensemble {} times-----------------".format(
num_mc_samples2))
rets = ens_validate(val_loaders, net, criterion, args, log,
num_mc_samples2)
ens_attack(val_loaders, net, criterion, args, log, num_mc_samples,
min(num_mc_samples, 8))
if args.gpu == 0:
for k in val_loaders:
print_log(
'{} vs. adversarial: AP {}'.format(
k[0], plot_mi(args.save_path, 'adv_' + k[0], k[0])), log)
ens_validate(fake_loader,
net,
criterion,
args,
log,
num_mc_samples,
suffix='fake')
if args.gpu == 0:
for k in val_loaders:
print_log(
'{} vs. DeepFake: AP {}'.format(
k[0], plot_mi(args.save_path, 'fake', k[0])), log)
def evaluate(test_loader, adv_loader, fake_loader, fake_loader2, net,
criterion, args, log, num_mc_samples, num_mc_samples2):
net.apply(freeze)
deter_rets = ens_validate(test_loader, net, criterion, args, log, 1)
net.apply(unfreeze)
rets = ens_validate(test_loader, net, criterion, args, log, num_mc_samples2)
print_log('TOP1 average: {:.4f}, ensemble: {:.4f}, deter: {:.4f}'.format(
rets[:,2].mean(), rets[-1][-3], deter_rets[0][2]), log)
print_log('TOP5 average: {:.4f}, ensemble: {:.4f}, deter: {:.4f}'.format(
rets[:,3].mean(), rets[-1][-2], deter_rets[0][3]), log)
print_log('LOS average: {:.4f}, ensemble: {:.4f}, deter: {:.4f}'.format(
rets[:,1].mean(), rets[-1][-4], deter_rets[0][1]), log)
print_log('ECE ensemble: {:.4f}, deter: {:.4f}'.format(
rets[-1][-1], deter_rets[-1][-1]), log)
if args.gpu == 0: plot_ens(args.save_path, rets, deter_rets[0][2])
ens_attack(adv_loader, net, criterion, args, log, num_mc_samples)
if args.gpu == 0:
print_log('NAT vs. ADV: AP {}'.format(plot_mi(args.save_path, 'advg')), log)
ens_validate(fake_loader, net, criterion, args, log, num_mc_samples, suffix='_fake')
if args.gpu == 0:
print_log('NAT vs. Fake (SNGAN): AP {}'.format(plot_mi(args.save_path, 'fake')), log)
ens_validate(fake_loader2, net, criterion, args, log, num_mc_samples, suffix='_fake2')
if args.gpu == 0:
print_log('NAT vs. Fake (PGGAN): AP {}'.format(plot_mi(args.save_path, 'fake2')), log)
return rets[-1][-3], rets[-1][-4]
def evaluate(test_loader, test_loader1, fake_loader, fake_loader2, net,
criterion, inv_factors, args, log):
rets = ens_validate(test_loader, net, criterion, inv_factors, args, log,
100)
print_log(
'TOP1 average: {:.4f}, ensemble: {:.4f}'.format(
rets[:, 2].mean(), rets[-1][-3]), log)
print_log(
'TOP5 average: {:.4f}, ensemble: {:.4f}'.format(
rets[:, 3].mean(), rets[-1][-2]), log)
print_log(
'LOS average: {:.4f}, ensemble: {:.4f}'.format(
rets[:, 1].mean(), rets[-1][-4]), log)
print_log('ECE ensemble: {:.4f}'.format(rets[-1][-1]), log)
ens_validate(fake_loader,
net,
criterion,
inv_factors,
args,
log,
20,
suffix='_fake')
if args.gpu == 0:
print_log(
'NAT vs. Fake (SNGAN): AP {}'.format(
plot_mi(args.save_path, 'fake')), log)
ens_validate(fake_loader2,
net,
criterion,
inv_factors,
args,
log,
20,
suffix='_fake2')
if args.gpu == 0:
print_log(
'NAT vs. Fake (PGGAN): AP {}'.format(
plot_mi(args.save_path, 'fake2')), log)
ens_attack(test_loader1, net, criterion, inv_factors, args, log, 20)
if args.gpu == 0:
print_log('NAT vs. ADV: AP {}'.format(plot_mi(args.save_path, 'advg')),
log)
def evaluate(test_loader, test_loader1, fake_loader, fake_loader2, net, criterion, args, log, nums=100, nums2=None):
if args.bayes: net.apply(freeze)
deter_rets = ens_validate(test_loader, net, criterion, args, log, False, 1)
if args.bayes: net.apply(unfreeze)
if not args.bayes and args.dropout_rate == 0: nums = 1; nums2=1
if not nums2: nums2 = nums
rets = ens_validate(test_loader, net, criterion, args, log, True, nums2)
print_log('TOP1 average: {:.4f}, ensemble: {:.4f}, deter: {:.4f}'.format(rets[:,2].mean(), rets[-1][-3], deter_rets[0][2]), log)
print_log('TOP5 average: {:.4f}, ensemble: {:.4f}, deter: {:.4f}'.format(rets[:,3].mean(), rets[-1][-2], deter_rets[0][3]), log)
print_log('LOS average: {:.4f}, ensemble: {:.4f}, deter: {:.4f}'.format(rets[:,1].mean(), rets[-1][-4], deter_rets[0][1]), log)
print_log('ECE ensemble: {:.4f}, deter: {:.4f}'.format(rets[-1][-1], deter_rets[-1][-1]), log)
if args.gpu == 0: plot_ens(args.save_path, rets, deter_rets[0][2])
ens_attack(test_loader1, net, criterion, args, log, nums)
if args.gpu == 0: print_log('NAT vs. ADV: AP {}'.format(plot_mi(args.save_path, 'advg')), log)
ens_validate(fake_loader, net, criterion, args, log, True, nums, suffix='_fake')
if args.gpu == 0: print_log('NAT vs. Fake (SNGAN): AP {}'.format(plot_mi(args.save_path, 'fake')), log)
ens_validate(fake_loader2, net, criterion, args, log, True, nums, suffix='_fake2')
if args.gpu == 0: print_log('NAT vs. Fake (PGGAN): AP {}'.format(plot_mi(args.save_path, 'fake2')), log)
return rets[-1][-3], rets[-1][-4]