def cal(refn, out_fn, base_folder='data/round2models', example_folder_name='example_data'):
"""
:param refn:
:param out_fn:
:param base_folder:
:return:
"""
from sklearn.isotonic import IsotonicRegression
from sklearn.metrics import log_loss, roc_auc_score
import os
calpath = 'calibration/data/' + out_fn + '_caldata.p'
if os.path.exists(calpath):
try:
with open(calpath, 'rb')as f:
ldirs, pcal, mags = pickle.load(f)
return ldirs, pcal, mags
except:
with open(calpath, 'rb')as f:
ldirs, pcal = pickle.load(f)
return ldirs, pcal
mags = []
y = []
dirs = os.listdir(path=base_folder)
for dir in dirs:
adv_path = os.path.join(base_folder, dir, example_folder_name, refn)
if os.path.exists(adv_path):
mag = get_blur_mag(adv_path, sigma=2.0)
truth_fn = os.path.join(base_folder, dir, 'config.json')
cls = utils.get_class(truth_fn, classtype='binary', file=True)
mags.append(mag)
y.append(cls)
ir_model = IsotonicRegression(out_of_bounds='clip')
pcal = ir_model.fit_transform(mags, y)
kld = log_loss(y, pcal)
# print(kld)
roc1 = roc_auc_score(y, np.array(pcal))
print(out_fn, 'AUC:', roc1, 'KLD:', kld)
# dump(ir_model, 'data/classifiers/blur' + '_ir.joblib')
dump(ir_model, 'calibration/fitted/' + out_fn)
pcal = pcal[np.argsort(dirs)]
dirs.sort()
with open(calpath,'wb') as f:
pickle.dump([dirs, pcal, mags], f)
return dirs, pcal, mags
config.batch_size,
config.num_steps,
shuffle=False):
data_feed = {
inputs: batch_X,
targets: batch_Y,
learning_rate: 1.0,
keep_prob: 1.0
}
predict = sess.run([prediction], data_feed)
predict = predict[0]
final_predict = np.append(final_predict, predict)
final_predict = np.array(final_predict).reshape(
len(final_predict), 1)
print("0.5 threshold")
classified = [utils.get_class(x, 0.5) for x in final_predict]
f, _ = utils.get_score_and_confusion_matrix(labels, classified)
print("Best threshold")
best_threshold = utils.find_best_threshold(
labels, final_predict)
best_classified = [
utils.get_class(x, best_threshold) for x in final_predict
]
best_f, _ = utils.get_score_and_confusion_matrix(
labels, best_classified)
if which_data == "training":
train_score.append(round(sp.average(best_f) * 100, 2))
else:
validation_score.append(round(sp.average(best_f) * 100, 2))
def cal(out_fn,
base_folder='data/round2models',
example_folder_name='example_data',
pert_scale=10.,
diff_fn=None,
filt_fn=None,
use_confl=False,
training=False):
"""
Calibrates a uap detector, saves it, and returns the calibrated probabilities
:param out_fn: filename of output for calibration model
:param base_folder: directory containing all the models (id-XXXXXXXX folders)
:param pert_scale: (float) desired scaling factor of the filters/perturbation
:param diff_fn: (string) filename of the numpy files containing the original and perturbed images
:param filt_fn: (string) filename of the numpy files containing the adversarial filters
:param use_confl: (bool) use the slightly better "confluence" score provided by get_maxtarget
:return:
"""
assert (diff_fn
is not None) != (filt_fn
is not None), "set either diff_fn or filt_fn"
from sklearn.isotonic import IsotonicRegression
from sklearn.metrics import log_loss, roc_auc_score
import os
from utils import utils
calpath = 'calibration/data/' + out_fn + '_caldata.p'
if os.path.exists(calpath):
try:
with open(calpath, 'rb') as f:
ldirs, pcal, mags = pickle.load(f)
return ldirs, pcal, mags
except:
with open(calpath, 'rb') as f:
ldirs, pcal = pickle.load(f)
return ldirs, pcal
mags = []
y = []
dirs = os.listdir(path=base_folder)
# dirs=dirs[:100] # debugging
for dir in dirs:
example_path = os.path.join(base_folder, dir, example_folder_name)
model_path = os.path.join(base_folder, dir, 'model.pt')
if diff_fn is not None:
adv_path = os.path.join(base_folder, dir, example_folder_name,
diff_fn)
mag = get_foolrate_diff(model_path,
example_path,
adv_path,
pert_scale=pert_scale,
use_confl=use_confl,
training=training)
else:
filt_path = os.path.join(base_folder, dir, example_folder_name,
filt_fn)
mag = get_foolrate_filt(model_path,
example_path,
filt_path,
pert_scale=pert_scale,
use_confl=use_confl,
training=training)
truth_fn = os.path.join(base_folder, dir, 'config.json')
cls = utils.get_class(truth_fn, classtype='binary', file=True)
mags.append(mag)
y.append(cls)
mags = np.clip(mags, np.percentile(mags, 5), np.percentile(mags, 95))
ir_model = IsotonicRegression(out_of_bounds='clip')
pcal = ir_model.fit_transform(mags, y)
kld = log_loss(y, pcal)
roc1 = roc_auc_score(y, pcal)
print(out_fn, 'AUC:', roc1, 'KLD:', kld)
dump(ir_model, 'calibration/fitted/' + out_fn)
pcal = pcal[np.argsort(dirs)]
mags = np.array(mags)
mags = mags[np.argsort(dirs)]
dirs.sort()
with open(calpath, 'wb') as f:
pickle.dump([dirs, pcal, mags], f)
return dirs, pcal, mags