def evaluate(description):
load_cfg_fom_args(description)
assert cfg.CORRUPTION.DATASET == 'cifar10'
base_model = load_model(cfg.MODEL.ARCH, cfg.CKPT_DIR,
'cifar10', ThreatModel.Linf).cuda()
if cfg.MODEL.ADAPTATION == "dent":
assert cfg.MODEL.EPISODIC
dent_model = Dent(base_model, cfg.OPTIM)
logger.info(dent_model.model)
x_test, y_test = load_cifar10(cfg.CORRUPTION.NUM_EX, cfg.DATA_DIR)
x_test, y_test = x_test.cuda(), y_test.cuda()
adversary = AutoAttack(
dent_model, norm='Linf', eps=8./255., version='standard',
log_path=osp.join(cfg.SAVE_DIR, cfg.LOG_DEST))
adversary.run_standard_evaluation(
x_test, y_test, bs=cfg.TEST.BATCH_SIZE)
def eval(model, args, dataset, logger=None):
def out(s):
if logger is not None:
logger.info(s)
else:
print(s)
model.eval()
args.attack_iters = 10
attack = PGD(args, model=model)
out("Test pgd10")
pgd10_acc, nat_acc = pgd(model,
dataset.test_loader,
attack,
need_nature_acc=True)
out(f"Nature: {nat_acc}")
out(f"PGD-10: {pgd10_acc}")
args.attack_iters = 20
out("Test pgd20")
attack = PGD(args, model=model)
pgd20_acc, _ = pgd(model, dataset.test_loader, attack)
out(f"PGD-20: {pgd20_acc}")
args.attack_iters = 1
args.pgd_alpha = 10
out("Test fgsm")
attack = PGD(args, model=model)
fgsm_acc, _ = pgd(model, dataset.test_loader, attack)
out("=" * 70)
out(f"Nature: {nat_acc}")
out(f"FGSM: {fgsm_acc}")
out(f"PGD-10: {pgd10_acc}")
out(f"PGD-20: {pgd20_acc}")
### Evaluate AutoAttack ###
l = [x for (x, y) in dataset.test_loader]
x_test = torch.cat(l, 0)
l = [y for (x, y) in dataset.test_loader]
y_test = torch.cat(l, 0)
epsilon = 8 / 255.0
adversary = AutoAttack(model, norm="Linf", eps=epsilon, version="standard")
adversary.run_standard_evaluation(x_test, y_test, bs=256)
def eval_batch(model, x_test, y_test, eps):
n_correct = {}
num_test = x_test.shape[0]
eps = eps/255
adversary = AutoAttack(model, norm='dftinf', eps=eps, version='custom',
attacks_to_run=['apgd-ce', 'apgd-dlr'])
adversary.apgd.n_restarts = 1
# adversary.apgd.n_iter = 1
x_adv = adversary.run_standard_evaluation(x_test, y_test)
n_correct['dftinf'] = adversary.clean_accuracy(x_adv, y_test)*num_test
n_correct['clean'] = adversary.clean_accuracy(x_test, y_test)*num_test
return n_correct
def evaluate_aa(args, model):
test_transform_nonorm = transforms.Compose([transforms.ToTensor()])
num_workers = 2
test_dataset_nonorm = datasets.CIFAR10(args.data_dir,
train=False,
transform=test_transform_nonorm,
download=True)
test_loader_nonorm = torch.utils.data.DataLoader(
dataset=test_dataset_nonorm,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2,
)
model.eval()
l = [x for (x, y) in test_loader_nonorm]
x_test = torch.cat(l, 0)
l = [y for (x, y) in test_loader_nonorm]
y_test = torch.cat(l, 0)
class normalize_model():
def __init__(self, model):
self.model_test = model
def __call__(self, x):
x_norm = normalize(x)
return self.model_test(x_norm)
new_model = normalize_model(model)
epsilon = 8 / 255.
adversary = AutoAttack(new_model,
norm='Linf',
eps=epsilon,
version='standard')
X_adv = adversary.run_standard_evaluation(x_test, y_test, bs=128)
def auto(model, device, val_loader, criterion, args, epoch=0):
"""
Evaluate on atuo-attack adversarial validation set inputs.
"""
batch_time = AverageMeter("Time", ":6.3f")
losses = AverageMeter("Loss", ":.4f")
adv_losses = AverageMeter("Adv_Loss", ":.4f")
top1 = AverageMeter("Acc_1", ":6.2f")
top5 = AverageMeter("Acc_5", ":6.2f")
top1_adv = AverageMeter("Adv-Acc_1", ":6.2f")
top5_adv = AverageMeter("Adv-Acc_5", ":6.2f")
progress = ProgressMeter(
len(val_loader),
[batch_time, losses, adv_losses, top1, top5, top1_adv, top5_adv],
prefix="Test: ",
)
# switch to evaluation mode
model.eval()
assert args.distance in ["linf", "l2"]
print(
"USING ONLY APGD_CE & APGD-T attacks. Rest of them don't change robust accuracy much but take super long to finish."
)
adversary = AutoAttack(model,
norm="Linf" if args.distance == "linf" else "L2",
eps=args.epsilon)
adversary.attacks_to_run = ['apgd-ce']
with torch.no_grad():
end = time.time()
for i, data in enumerate(val_loader):
images, target = data[0].to(device), data[1].to(device)
# clean images
output = model(images)
loss = criterion(output, target)
acc1, acc5 = accuracy(output, target, topk=(1, 2))
losses.update(loss.item(), images.size(0))
top1.update(acc1[0], images.size(0))
top5.update(acc5[0], images.size(0))
images = adversary.run_standard_evaluation(images,
target,
bs=len(images))
# compute output
output = model(images)
loss = criterion(output, target)
# measure accuracy and record loss
acc1, acc5 = accuracy(output, target, topk=(1, 2))
adv_losses.update(loss.item(), images.size(0))
top1_adv.update(acc1[0], images.size(0))
top5_adv.update(acc5[0], images.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if (i + 1) % args.print_freq == 0 and args.local_rank == 0:
progress.display(i)
if args.local_rank == 0:
progress.display(i) # print final results
result = {
"top1": top1.avg,
"top5": top5.avg,
"loss": losses.avg,
"top1_adv": top1_adv.avg,
"top5_adv": top5_adv.avg,
"adv_loss": adv_losses.avg
}
return result
clean_y_test,
batch_size=batch_size,
device=device)
print(f'Clean accuracy: {accuracy:.2%}')
if threat_model_ in {ThreatModel.Linf, ThreatModel.L2}:
if eps is None:
raise ValueError(
"If the threat model is L2 or Linf, `eps` must be specified.")
adversary = AutoAttack(model,
norm=threat_model_.value,
eps=eps,
version='standard',
device=device)
x_adv = adversary.run_standard_evaluation(clean_x_test, clean_y_test)
adv_accuracy = clean_accuracy(model,
x_adv,
clean_y_test,
batch_size=batch_size,
device=device)
elif threat_model_ == ThreatModel.corruptions:
print(f"Evaluating over {len(CORRUPTIONS)} corruptions")
# Save into a dict to make a Pandas DF with nested index
adv_accuracy = corruptions_evaluation(batch_size, data_dir, dataset_,
device, model, n_examples,
to_disk, model_name)
else:
raise NotImplementedError
print(f'Adversarial accuracy: {adv_accuracy:.2%}')
x_test = torch.cat(l, 0)
l = [y for (x, y) in test_loader]
y_test = torch.cat(l, 0)
# cheap version
if args.cheap:
adversary.cheap()
# plus version
if args.plus:
adversary.plus = True
# run attack and save images
with torch.no_grad():
if not args.individual:
adv_complete = adversary.run_standard_evaluation(
x_test[:args.n_ex], y_test[:args.n_ex], bs=args.batch_size)
torch.save({'adv_complete': adv_complete},
'{}/{}_1_{}_eps_{:.5f}_plus_{}_cheap_{}.pth'.format(
args.save_dir, 'aa', adv_complete.shape[0],
args.epsilon, args.plus, args.cheap))
else:
# individual version, each attack is run on all test points
# specify attacks to run with
# adversary.attacks_to_run = ['apgd-ce']
adv_complete = adversary.run_standard_evaluation_individual(
x_test[:args.n_ex], y_test[:args.n_ex], bs=args.batch_size)
torch.save(
adv_complete,
testY = np.argmax(testY, axis=1)
torch_testX = torch.from_numpy(np.transpose(testX,
(0, 3, 1, 2))).float().cuda()
torch_testY = torch.from_numpy(testY).float()
# load model from saved weights
print('[INFO] MODEL_PATH: {:s}'.format(MODEL_PATH))
tf_model = mnist_model()
tf_model.load_weights(MODEL_PATH)
# remove 'softmax layer' and put it into adapter
atk_model = tf.keras.models.Model(
inputs=tf_model.input, outputs=tf_model.get_layer(index=-2).output)
atk_model.summary()
y_input = tf.placeholder(tf.int64, shape=[None])
x_input = atk_model.input
logits = atk_model.output
model_adapted = utils_tf.ModelAdapter(logits, x_input, y_input, sess)
# run attack
adversary = AutoAttack(model_adapted,
norm='Linf',
eps=epsilon,
plus=False,
is_tf_model=True)
x_adv = adversary.run_standard_evaluation(torch_testX,
torch_testY,
bs=batch_size)
np_x_adv = np.moveaxis(x_adv.cpu().numpy(), 1, 3)
np.save("./output/mnist_adv.npy", np_x_adv)
test_loader,
0,
adversary=adversary_t,
logger=None,
ret='adv')
elif P.mode == 'test_auto_attack':
from autoattack import AutoAttack
auto_adversary = AutoAttack(model,
norm=P.distance,
eps=P.epsilon,
version='standard')
x_test, y_test = extract_dataset(test_loader)
x_adv = auto_adversary.run_standard_evaluation(x_test, y_test)
elif P.mode == 'test_mce':
from evals import test_classifier
mean_corruption_error = 0.
for name in corruption_list:
error = test_classifier(P,
model,
corruption_loader[name],
0,
logger=None)
mean_corruption_error += error
print(f'Error of {name}: {error}%\n')
print(f'MCE: {mean_corruption_error/len(corruption_list)} %')
def robust_test(model,
loss_fn,
loader,
args,
att_dir,
epoch=0,
training_output_dir_name=None,
figure_dir_name=None,
n_batches=0,
train_data=False,
training_time=False):
"""
n_batches (int): Number of batches for evaluation.
"""
model.eval()
num_correct, num_correct_adv, num_samples = 0, 0, 0
steps = 1
losses_adv = []
losses_ben = []
prob_dict = {}
if args.track_hard:
loss_dict = collections.OrderedDict()
pred_dict = {}
if training_time and args.track_hard:
f_name = training_output_dir_name + 'losses.json'
f = open(f_name, 'a+')
loss_dict['epoch'] = epoch
if 'PGD' in att_dir['attack']:
for t, (x, y, idx, ez, m) in enumerate(loader):
x = x.cuda()
y = y.cuda()
x_var = Variable(x, requires_grad=True)
y_var = Variable(y, requires_grad=False)
if att_dir['targeted']:
y_target = generate_target_label_tensor(y_var.cpu(),
args).cuda()
else:
y_target = y_var
if 'PGD_linf' in att_dir['attack']:
adv_x = pgd_attack(model, x, x_var, y_target,
att_dir['attack_iter'], att_dir['epsilon'],
att_dir['eps_step'], att_dir['clip_min'],
att_dir['clip_max'], att_dir['targeted'],
att_dir['rand_init'])
elif 'PGD_l2' in att_dir['attack']:
adv_x = pgd_l2_attack(model, x, x_var, y_target,
att_dir['attack_iter'],
att_dir['epsilon'], att_dir['eps_step'],
att_dir['clip_min'], att_dir['clip_max'],
att_dir['targeted'],
att_dir['rand_init'],
att_dir['num_restarts'])
# Predictions
scores = model(x.cuda())
_, preds = scores.data.max(1)
scores_adv = model(adv_x)
_, preds_adv = scores_adv.data.max(1)
# Losses
batch_loss_adv = loss_fn(scores_adv, y)
loss_adv = torch.mean(batch_loss_adv)
losses_adv.append(loss_adv.data.cpu().numpy())
batch_loss_ben = loss_fn(scores, y)
loss_ben = torch.mean(batch_loss_ben)
losses_ben.append(loss_ben.data.cpu().numpy())
# Correct count
num_correct += (preds == y).sum()
num_correct_adv += (preds_adv == y).sum()
num_samples += len(preds)
# Adding probs to dict
count = 0
for i in idx.numpy():
score_curr = scores_adv[count].cpu().detach().numpy()
prob_dict[str(i)] = softmax(score_curr)
# print(count)
count += 1
if args.track_hard:
idx_matched = idx[~ez].numpy()
m_matched = m[~ez].numpy()
preds_adv_matched = preds_adv[~ez].cpu().numpy()
loss_adv_matched = batch_loss_adv[~ez].cpu().detach().numpy()
y_np_matched = y[~ez].cpu().numpy()
ez_np = ez.data.cpu().numpy()
for i in range(len(y_np_matched)):
pred_dict[str(idx_matched[i])] = [
m_matched[i], y_np_matched[i], preds_adv_matched[i]
]
loss_dict = track_hard_losses(ez_np, batch_loss_adv,
batch_loss_ben, loss_dict, t)
if not training_time:
if args.viz and steps == 1:
if not os.path.exists(figure_dir_name):
os.makedirs(figure_dir_name)
custom_save_image(adv_x, preds_adv, y, args,
figure_dir_name, train_data)
if n_batches > 0 and steps == n_batches:
break
steps += 1
elif 'AA' in att_dir['attack']:
x_test = []
y_test = []
idx_all = []
for t, (x, y, idx, ez, m) in enumerate(loader):
x_test.append(x)
y_test.append(y)
idx_all.extend(idx.numpy())
x_test = torch.cat(x_test, 0)
y_test = torch.cat(y_test, 0)
print(x_test.size())
print(y_test.size())
adversary = AutoAttack(model,
norm='L2',
eps=att_dir['epsilon'],
version='standard')
adversary.attacks_to_run = ['apgd-ce', 'fab']
adv_x = adversary.run_standard_evaluation(x_test, y_test)
loss_adv, loss_ben, num_correct, num_correct_adv, num_samples, scores_adv = compute_scores_and_losses(
model, loss_fn, x_test, y_test, adv_x)
losses_adv.append(loss_adv.data.cpu().numpy())
losses_ben.append(loss_ben.data.cpu().numpy())
# Adding probs to dict
count = 0
for i in idx_all:
score_curr = scores_adv[count].cpu().detach().numpy()
prob_dict[str(i)] = softmax(score_curr)
# print(count)
count += 1
acc = float(num_correct) / num_samples
acc_adv = float(num_correct_adv) / num_samples
print('Clean accuracy: {:.2f}% ({}/{})'.format(
100. * acc,
num_correct,
num_samples,
))
print('Adversarial accuracy: {:.2f}% ({}/{})'.format(
100. * acc_adv,
num_correct_adv,
num_samples,
))
if args.track_hard:
if not training_time:
print('Counting hard points')
hard_point_class_count(pred_dict)
if training_time:
json.dump(loss_dict, f)
f.write('\n')
if len(loss_dict['batch_losses_hard']) > 0:
print('Adv loss easy: %.8f' %
np.mean(loss_dict['batch_losses_easy']))
print('Adv loss hard: %.8f' %
np.mean(loss_dict['batch_losses_hard']))
print('Ben loss easy: %.8f' %
np.mean(loss_dict['batch_losses_ben_easy']))
print('Ben loss hard: %.8f' %
np.mean(loss_dict['batch_losses_ben_hard']))
return 100. * acc, 100. * acc_adv, np.mean(losses_ben), np.mean(
losses_adv), prob_dict
def main(config):
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = config['meta']['gpu_id']
model_name = config['meta']['model_name']
# Set all random seeds
seed = config['meta']['seed']
np.random.seed(seed)
torch.manual_seed(seed)
device = 'cuda' if torch.cuda.is_available() else 'cpu'
if device == 'cuda':
cudnn.benchmark = True
# Set up model directory
save_dir = os.path.join(config['meta']['save_path'], './')
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
model_path = os.path.join(save_dir, model_name)
timenow = datetime.datetime.now().strftime("%m-%d-%Y-%H:%M:%S")
logfile = 'test_' + model_name + '_' + timenow
log = get_logger(logfile, '')
log.info('\n%s', yaml.dump(config))
log.info('Preparing data...')
if config['test']['dataset'] == 'cifar10':
_, _, (x_test,
y_test) = load_cifar10_all(data_dir=config['meta']['data_path'],
val_size=0.1,
shuffle=False,
seed=seed)
num_classes = 10
elif config['test']['dataset'] == 'cifar100':
_, _, (x_test, y_test) = load_cifar100_all(
data_dir=config['meta']['data_path'],
val_size=0.1,
shuffle=False,
seed=seed)
num_classes = 100
elif config['test']['dataset'] == 'mnist':
_, _, (x_test,
y_test) = load_mnist_all(data_dir=config['meta']['data_path'],
val_size=0.1,
shuffle=True,
seed=seed)
num_classes = 10
else:
raise NotImplementedError('invalid dataset.')
log.info('Building model...')
if config['test']['network'] == 'resnet':
# use ResNetV2-20
net = PreActResNet(PreActBlock, [2, 2, 2, 2], num_classes=num_classes)
elif config['test']['network'] == 'wideresnet':
# use WideResNet-34-10
net = WideResNet(num_classes=num_classes)
elif config['test']['network'] == 'basic':
net = BasicModel(num_classes=num_classes)
else:
raise NotImplementedError('Specified network not implemented.')
net.load_state_dict(torch.load(model_path + '.pt'))
net = net.eval().to(device)
num_test_samples = config['test']['num_test_samples']
log.info('Starting attack...')
adversary = AutoAttack(net,
norm='Linf',
eps=config['test']['epsilon'],
version='standard',
log_path=logfile + '.log')
x_adv = adversary.run_standard_evaluation(x_test[:num_test_samples],
y_test[:num_test_samples],
bs=config['test']['batch_size'])
y_pred = classify(net, x_adv)
adv_acc = get_acc(y_pred, y_test[:num_test_samples])
log.info('AA acc: %.4f.', adv_acc)
def main():
args = get_args()
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
logger = logging.getLogger(__name__)
logging.basicConfig(
format='[%(asctime)s] - %(message)s',
datefmt='%Y/%m/%d %H:%M:%S',
level=logging.DEBUG,
handlers=[
logging.StreamHandler()
])
logger.info(args)
_, test_loader = get_loaders(args.data_dir, args.batch_size)
best_state_dict = torch.load(os.path.join(args.out_dir, 'model_best.pth'))
# Evaluation
model_test = PreActResNet18().cuda()
# model_test = WideResNet(34, 10, widen_factor=10, dropRate=0.0)
model_test = nn.DataParallel(model_test).cuda()
if 'state_dict' in best_state_dict.keys():
model_test.load_state_dict(best_state_dict['state_dict'])
else:
model_test.load_state_dict(best_state_dict)
model_test.float()
model_test.eval()
### Evaluate clean acc ###
_, test_acc = evaluate_standard(test_loader, model_test)
print('Clean acc: ', test_acc)
### Evaluate PGD (CE loss) acc ###
_, pgd_acc_CE = evaluate_pgd(test_loader, model_test, attack_iters=10, restarts=1, eps=8, step=2, use_CWloss=False)
print('PGD-10 (10 restarts, step 2, CE loss) acc: ', pgd_acc_CE)
### Evaluate PGD (CW loss) acc ###
_, pgd_acc_CW = evaluate_pgd(test_loader, model_test, attack_iters=10, restarts=1, eps=8, step=2, use_CWloss=True)
print('PGD-10 (10 restarts, step 2, CW loss) acc: ', pgd_acc_CW)
### Evaluate AutoAttack ###
l = [x for (x, y) in test_loader]
x_test = torch.cat(l, 0)
l = [y for (x, y) in test_loader]
y_test = torch.cat(l, 0)
class normalize_model():
def __init__(self, model):
self.model_test = model
def __call__(self, x):
return self.model_test(normalize(x))
new_model = normalize_model(model_test)
epsilon = 8 / 255.
adversary = AutoAttack(new_model, norm='Linf', eps=epsilon, version='standard')
X_adv = adversary.run_standard_evaluation(x_test, y_test, bs=128)
shuffle=False,
num_workers=0)
from autoattack import AutoAttack
log = 'store/' + log_folder + '/AA_eval-new.txt'
model = model.cuda()
adversary = AutoAttack(model,
norm='Linf',
eps=8 / 255,
log_path=log,
version='standard')
adversary.attacks_to_run = ['apgd-t']
l = [x for (x, y) in test_loader]
x_test = torch.cat(l, 0).cuda()
l = [y for (x, y) in test_loader]
y_test = torch.cat(l, 0).cuda()
adv_complete = adversary.run_standard_evaluation(x_test, y_test, bs=128)
save_dir = 'store/' + log_folder
print(adv_complete)
torch.save({'adv_complete': adv_complete},
'{}/{}_{}_1_{}_eps_{:.5f}.pth'.format(save_dir, 'aa', 'standard',
adv_complete.shape[0],
8 / 255))
## PGD20 EVAL ##
out_store = cox.store.Store('store', exp_id=log_folder)
eval_pgd20_args = Parameters({
'constraint': 'inf',
'eps': 8 / 255,
'attack_lr': 2 / 255,
'epochs': 10,
dataloader = dl.get_CIFAR100(False,
batch_size=ROBUSTNESS_DATAPOINTS,
augm_type='none')
else:
raise NotImplementedError()
data_iterator = iter(dataloader)
ref_data, target = next(data_iterator)
if L2:
print('Eps: 0.5')
attack = AutoAttack(model,
device=device,
norm='L2',
eps=0.5,
verbose=True)
attack.run_standard_evaluation(ref_data, target, bs=bs)
# print('Eps: 1.0')
# attack = AutoAttack(model, device=device, norm='L2', eps=1.0, attacks_to_run=attacks_to_run,verbose=True)
# attack.run_standard_evaluation(ref_data, target, bs=bs)
if LINF:
print('Eps: 8/255')
attack = AutoAttack(model,
device=device,
norm='Linf',
eps=8. / 255.,
verbose=True)
attack.run_standard_evaluation(ref_data, target, bs=bs)
def get_all_cifar10_data(constants, is_test=True):
ds = constants.get_dataset()
if is_test:
_, data_loader = ds.make_loaders(batch_size=16,
only_val=True,
workers=8,
data_aug=False)
else:
data_loader, _ = ds.make_loaders(batch_size=16,
workers=8,
data_aug=False)
X, Y = [], []
for (x, y) in data_loader:
X.append(x)
Y.append(y)
X, Y = ch.cat(X), ch.cat(Y)
return (X, Y)
if __name__ == "__main__":
epsilon = 8 / 255
constants = utils.CIFAR10()
ds = constants.get_dataset()
model = constants.get_model(
"/p/adversarialml/as9rw/models_cifar10_vgg19/custom_adv_train_try_10.000000_100.000000_16_0.010000_3_fast_1/checkpoint.pt.best",
"vgg19")
wrapped_model = ModelWrapper(model)
images, labels = get_all_cifar10_data(constants)
adversary = AutoAttack(wrapped_model, norm='Linf', eps=epsilon, plus=False)
x_adv = adversary.run_standard_evaluation(images, labels, bs=128)
model_adapted = utils_tf2.ModelAdapter(model_ensemble)
adversary = AutoAttack(model_adapted,
norm='Linf',
eps=eps,
version='standard',
is_tf_model=True,
log_path=logfile)
# Convert to required format: NHWC --> NCHW
_x = torch.from_numpy(np.moveaxis(x_test, 3, 1))
_y = torch.from_numpy(np.argmax(y_test, axis=-1))
_x.float().to("cuda")
_y.to("cuda")
x_adv = adversary.run_standard_evaluation(_x, _y, bs=bs)
x_adv = x_adv.cpu().numpy()
# Convert back to NHWC
x_adv = np.moveaxis(x_adv, 1, 3)
pred_adv = model_ensemble.predict(x=x_adv)
pred_nat = model_ensemble.predict(x=x_test)
acc_nat_test = np.mean(
np.argmax(pred_nat, axis=-1) == np.argmax(y_test, axis=-1))
acc_adv_test = np.mean(
np.argmax(pred_adv, axis=-1) == np.argmax(y_test, axis=-1))
h_nat, m_nat, s_nat = get_statistic(pred_nat)
h_adv, m_adv, s_adv = get_statistic(pred_adv)
