def pair_plot_regions(lgkm,
save=False,
show=True,
inc_x=True,
outdir=default_outdir,
palette='rainbow_r'):
logger.info('plotting regions...')
regions = lgkm.get_regions()
n_cats = len(lgkm.get_categories())
X = ([x for r in regions
for x in r.X] if inc_x else []) + [r.centroid for r in regions]
y = ([y for r in regions for y in r.y]
if inc_x else []) + [n_cats + r.category for r in regions]
df = pd.DataFrame(X, columns=TOTUtils.get_feature_names())
df['y'] = y
colors = [
tohex(r, g, b)
for r, g, b in sns.color_palette('rainbow_r', n_cats)
]
palette = {
i: colors[i if i < n_cats else i - n_cats]
for i in range(n_cats * (2 if inc_x else 1))
}
markers = [
'o' if i < n_cats else 'D'
for i in range(n_cats * (2 if inc_x else 1))
]
g = sns.pairplot(df,
hue='y',
corner=True,
palette=palette,
markers=markers,
plot_kws=dict(alpha=0.5, s=10))
g = g.add_legend(
{i: l
for i, l in enumerate(TOTUtils.get_category_names())})
if save:
savepath = os.path.join(outdir, 'lgkm.png')
g.savefig(savepath, dpi=300)
logger.info(f'regions plot saved to {savepath}')
if show:
plt.show()
def test_local_robustness(nnet_path,
samples,
e_min=0.00001,
e_max=100,
e_prec=None,
asym=False,
coarse_pass=True,
save_results=False,
save_samples=False,
outdir=default_outdir,
timeout=default_timeout,
verbose=0):
if not e_prec:
dp_prec = count_decimal_places(e_min) + 1
e_prec = round(1 / (10**dp_prec), dp_prec)
assert (e_prec < e_min)
net = TOTNet(nnet_path)
start = ms_since_1970()
epsilons = []
for s, sample in enumerate(samples):
sid = f's{s}'
if asym:
l_ce_lb, l_ce_ub = (e_min, e_max), (e_min, e_max)
if coarse_pass:
# find coarse bounds for lower epsilon
step_start = ms_since_1970()
l_ce_lb, l_ce_ub = find_epsilon_bounds(net,
sample,
e_min,
e_max,
e_prec,
asym_side='l',
timeout=timeout,
verbose=verbose)
if verbose > 1:
logger.info(
f'{sid} lower epsilon coarse bounds: {l_ce_lb, l_ce_ub} ({ms_since_1970() - step_start}ms)'
)
# find coarse bounds for upper epsilon
step_start = ms_since_1970()
u_ce_lb, u_ce_ub = find_epsilon_bounds(net,
sample,
e_min,
e_max,
e_prec,
asym_side='u',
timeout=timeout,
verbose=verbose)
if verbose > 1:
logger.info(
f'{sid} upper epsilon coarse bounds: {u_ce_lb, u_ce_ub} ({ms_since_1970() - step_start}ms)'
)
# find lower epsilon within coarse bounds
step_start = ms_since_1970()
le, counterexample = find_epsilon(net,
sample,
l_ce_lb,
l_ce_ub,
e_prec,
asym_side='l',
timeout=timeout,
verbose=verbose)
if verbose > 0:
logger.info(
f'{sid} lower epsilon: {le} ({ms_since_1970() - step_start}ms)'
)
# find upper epsilon within coarse bounds
step_start = ms_since_1970()
ue, counterexample = find_epsilon(net,
sample,
u_ce_lb,
u_ce_ub,
e_prec,
asym_side='u',
timeout=timeout,
verbose=verbose)
if verbose > 0:
logger.info(
f'{sid} upper epsilon: {ue} ({ms_since_1970() - step_start}ms)'
)
epsilons.append((le, ue, counterexample))
else:
ce_lb, ce_ub = e_min, e_max
if coarse_pass:
# find coarse bounds for epsilon
step_start = ms_since_1970()
ce_lb, ce_ub = find_epsilon_bounds(net,
sample,
e_min,
e_max,
e_prec,
timeout=timeout,
verbose=verbose)
if verbose > 1:
logger.info(
f'{sid} coarse epsilon bounds: {ce_lb, ce_ub} ({ms_since_1970() - step_start}ms)'
)
# find epsilon within coarse bounds
step_start = ms_since_1970()
epsilon, counterexample = find_epsilon(net,
sample,
ce_lb,
ce_ub,
e_prec,
timeout=timeout,
verbose=verbose)
if verbose > 0:
logger.info(
f'{sid} epsilon: {epsilon} ({ms_since_1970() - step_start}ms)'
)
# update running min epislon
epsilons.append((epsilon, epsilon, counterexample))
# save and return test results
leps = [le for le, _, _ in epsilons if le != 0]
ueps = [ue for _, ue, _ in epsilons if ue != 0]
summary = (-min(leps if leps else [0]), min(ueps if ueps else [0]))
results = (summary, epsilons)
logger.info(('asymm ' if asym else '') +
f'local robustness: {summary} ({ms_since_1970() - start}ms)')
if save_results:
save_local_robustness_results_to_csv(results, samples, outdir)
if save_samples: TOTUtils.save_samples_to_csv(samples, outdir)
return results
parser.add_argument('-ck', '--checkresults', action='store_true')
parser.add_argument('-sr', '--saveresults', action='store_true')
parser.add_argument('-ss', '--savesamples', action='store_true')
parser.add_argument('-sl', '--savelogs', action='store_true')
parser.add_argument('-o', '--outdir', default=default_outdir)
parser.add_argument('-v', '--verbose', type=int, default=0)
args = parser.parse_args()
# configure logger
for handler in logger.handlers[:]:
logger.removeHandler(handler)
logger = create_logger('robustness',
to_file=args.savelogs,
logdir=args.outdir)
# load % of samples, and filter out incorrect predictions
samples = TOTUtils.filter_samples(
TOTUtils.load_samples(args.datapath, frac=args.datafrac),
args.nnetpath)
logger.info(f'starting local robustness test on {len(samples)} samples')
results = test_local_robustness(args.nnetpath,
samples,
e_min=args.emin,
e_max=args.emax,
e_prec=args.eprec,
asym=args.asym,
coarse_pass=not args.nocoarse,
timeout=args.timeout,
save_results=args.saveresults,
save_samples=args.savesamples,
outdir=args.outdir,
verbose=args.verbose)
if args.checkresults:
def test_sensitivity(nnet_path,
samples,
x_indexes=[],
e_min=default_emin,
e_max=default_emax,
e_prec=None,
asym=False,
coarse_pass=True,
save_results=False,
save_samples=False,
outdir=default_outdir,
timeout=default_timeout,
verbose=0):
if not e_prec:
dp_prec = count_decimal_places(e_min) + 1
e_prec = round(1 / (10**dp_prec), dp_prec)
assert (e_prec <= e_min)
net = TOTNet(nnet_path)
start = ms_since_1970()
x_indexes = x_indexes if x_indexes else [
x for x in range(net.get_num_inputs())
]
results = {}
for x in x_indexes:
xid, x_start = f'x{x}', ms_since_1970()
epsilons = []
for s, sample in enumerate(samples):
sid = f's{s}'
if asym:
le_bounds, ue_bounds = (e_min, e_max), (e_min, e_max)
if coarse_pass:
# find coarse bounds for lower and upper epsilon
step_start = ms_since_1970()
le_bounds = find_epsilon_bounds(net,
sample,
x,
e_min,
e_max,
e_prec,
asym_side='l',
timeout=timeout,
verbose=verbose)
if verbose > 1:
logger.info(
f'{xid}_{sid} lower epsilon coarse bounds: {le_bounds} ({ms_since_1970() - step_start}ms)'
)
step_start = ms_since_1970()
ue_bounds = find_epsilon_bounds(net,
sample,
x,
e_min,
e_max,
e_prec,
asym_side='u',
timeout=timeout,
verbose=verbose)
if verbose > 1:
logger.info(
f'{xid}_{sid} upper epsilon coarse bounds: {ue_bounds} ({ms_since_1970() - step_start}ms)'
)
# find lower and upper epsilon within the coarse bounds
step_start = ms_since_1970()
le = find_epsilon(net,
sample,
x,
le_bounds[0],
le_bounds[1],
e_prec,
asym_side='l',
timeout=timeout,
verbose=verbose)
if verbose > 0:
logger.info(
f'{xid}_{sid} lower epsilon: {le} ({ms_since_1970() - step_start}ms)'
)
step_start = ms_since_1970()
ue = find_epsilon(net,
sample,
x,
ue_bounds[0],
ue_bounds[1],
e_prec,
asym_side='u',
timeout=timeout,
verbose=verbose)
if verbose > 0:
logger.info(
f'{xid}_{sid} upper epsilon: {ue} ({ms_since_1970() - step_start}ms)'
)
epsilons.append((le, ue))
else:
e_bounds = (e_min, e_max)
if coarse_pass:
# find coarse bounds for epsilon
step_start = ms_since_1970()
e_bounds = find_epsilon_bounds(net,
sample,
x,
e_min,
e_max,
e_prec,
timeout=timeout,
verbose=verbose)
if verbose > 1:
logger.info(
f'{xid}_{sid} coarse epsilon bounds: {e_bounds} ({ms_since_1970() - step_start}ms)'
)
# find epsilon within ebounds
step_start = ms_since_1970()
e = find_epsilon(net,
sample,
x,
e_bounds[0],
e_bounds[1],
e_prec,
timeout=timeout,
verbose=verbose)
if verbose > 0:
logger.info(
f'{xid}_{sid} epsilon: {e} ({ms_since_1970() - step_start}ms)'
)
epsilons.append((e, e))
leps = [le for le, _ in epsilons if le != 0]
ueps = [ue for _, ue in epsilons if ue != 0]
x_summary = (-min(leps if leps else [0]), min(ueps if ueps else [0]))
if verbose > 0:
logger.info(
f'{xid} epsilon: {x_summary} ({ms_since_1970() - x_start}ms)')
results[f'x{x}'] = (x_summary, epsilons)
summary = {x: r[0] for x, r in results.items()}
logger.info(('asymm ' if asym else '') +
f'sensitivity: {summary} ({ms_since_1970() - start}ms)')
if save_results: save_sensitivity_results_to_csv(results, samples, outdir)
if save_samples: TOTUtils.save_samples_to_csv(samples, outdir)
return results