def test_local_robustness(nnet_path,
samples,
e_min=0.00001,
e_max=100,
e_prec=None,
asym=False,
coarse_pass=True,
save_results=False,
save_samples=False,
outdir=default_outdir,
timeout=default_timeout,
verbose=0):
if not e_prec:
dp_prec = count_decimal_places(e_min) + 1
e_prec = round(1 / (10**dp_prec), dp_prec)
assert (e_prec < e_min)
net = TOTNet(nnet_path)
start = ms_since_1970()
epsilons = []
for s, sample in enumerate(samples):
sid = f's{s}'
if asym:
l_ce_lb, l_ce_ub = (e_min, e_max), (e_min, e_max)
if coarse_pass:
# find coarse bounds for lower epsilon
step_start = ms_since_1970()
l_ce_lb, l_ce_ub = find_epsilon_bounds(net,
sample,
e_min,
e_max,
e_prec,
asym_side='l',
timeout=timeout,
verbose=verbose)
if verbose > 1:
logger.info(
f'{sid} lower epsilon coarse bounds: {l_ce_lb, l_ce_ub} ({ms_since_1970() - step_start}ms)'
)
# find coarse bounds for upper epsilon
step_start = ms_since_1970()
u_ce_lb, u_ce_ub = find_epsilon_bounds(net,
sample,
e_min,
e_max,
e_prec,
asym_side='u',
timeout=timeout,
verbose=verbose)
if verbose > 1:
logger.info(
f'{sid} upper epsilon coarse bounds: {u_ce_lb, u_ce_ub} ({ms_since_1970() - step_start}ms)'
)
# find lower epsilon within coarse bounds
step_start = ms_since_1970()
le, counterexample = find_epsilon(net,
sample,
l_ce_lb,
l_ce_ub,
e_prec,
asym_side='l',
timeout=timeout,
verbose=verbose)
if verbose > 0:
logger.info(
f'{sid} lower epsilon: {le} ({ms_since_1970() - step_start}ms)'
)
# find upper epsilon within coarse bounds
step_start = ms_since_1970()
ue, counterexample = find_epsilon(net,
sample,
u_ce_lb,
u_ce_ub,
e_prec,
asym_side='u',
timeout=timeout,
verbose=verbose)
if verbose > 0:
logger.info(
f'{sid} upper epsilon: {ue} ({ms_since_1970() - step_start}ms)'
)
epsilons.append((le, ue, counterexample))
else:
ce_lb, ce_ub = e_min, e_max
if coarse_pass:
# find coarse bounds for epsilon
step_start = ms_since_1970()
ce_lb, ce_ub = find_epsilon_bounds(net,
sample,
e_min,
e_max,
e_prec,
timeout=timeout,
verbose=verbose)
if verbose > 1:
logger.info(
f'{sid} coarse epsilon bounds: {ce_lb, ce_ub} ({ms_since_1970() - step_start}ms)'
)
# find epsilon within coarse bounds
step_start = ms_since_1970()
epsilon, counterexample = find_epsilon(net,
sample,
ce_lb,
ce_ub,
e_prec,
timeout=timeout,
verbose=verbose)
if verbose > 0:
logger.info(
f'{sid} epsilon: {epsilon} ({ms_since_1970() - step_start}ms)'
)
# update running min epislon
epsilons.append((epsilon, epsilon, counterexample))
# save and return test results
leps = [le for le, _, _ in epsilons if le != 0]
ueps = [ue for _, ue, _ in epsilons if ue != 0]
summary = (-min(leps if leps else [0]), min(ueps if ueps else [0]))
results = (summary, epsilons)
logger.info(('asymm ' if asym else '') +
f'local robustness: {summary} ({ms_since_1970() - start}ms)')
if save_results:
save_local_robustness_results_to_csv(results, samples, outdir)
if save_samples: TOTUtils.save_samples_to_csv(samples, outdir)
return results
def test_sensitivity(nnet_path,
samples,
x_indexes=[],
e_min=default_emin,
e_max=default_emax,
e_prec=None,
asym=False,
coarse_pass=True,
save_results=False,
save_samples=False,
outdir=default_outdir,
timeout=default_timeout,
verbose=0):
if not e_prec:
dp_prec = count_decimal_places(e_min) + 1
e_prec = round(1 / (10**dp_prec), dp_prec)
assert (e_prec <= e_min)
net = TOTNet(nnet_path)
start = ms_since_1970()
x_indexes = x_indexes if x_indexes else [
x for x in range(net.get_num_inputs())
]
results = {}
for x in x_indexes:
xid, x_start = f'x{x}', ms_since_1970()
epsilons = []
for s, sample in enumerate(samples):
sid = f's{s}'
if asym:
le_bounds, ue_bounds = (e_min, e_max), (e_min, e_max)
if coarse_pass:
# find coarse bounds for lower and upper epsilon
step_start = ms_since_1970()
le_bounds = find_epsilon_bounds(net,
sample,
x,
e_min,
e_max,
e_prec,
asym_side='l',
timeout=timeout,
verbose=verbose)
if verbose > 1:
logger.info(
f'{xid}_{sid} lower epsilon coarse bounds: {le_bounds} ({ms_since_1970() - step_start}ms)'
)
step_start = ms_since_1970()
ue_bounds = find_epsilon_bounds(net,
sample,
x,
e_min,
e_max,
e_prec,
asym_side='u',
timeout=timeout,
verbose=verbose)
if verbose > 1:
logger.info(
f'{xid}_{sid} upper epsilon coarse bounds: {ue_bounds} ({ms_since_1970() - step_start}ms)'
)
# find lower and upper epsilon within the coarse bounds
step_start = ms_since_1970()
le = find_epsilon(net,
sample,
x,
le_bounds[0],
le_bounds[1],
e_prec,
asym_side='l',
timeout=timeout,
verbose=verbose)
if verbose > 0:
logger.info(
f'{xid}_{sid} lower epsilon: {le} ({ms_since_1970() - step_start}ms)'
)
step_start = ms_since_1970()
ue = find_epsilon(net,
sample,
x,
ue_bounds[0],
ue_bounds[1],
e_prec,
asym_side='u',
timeout=timeout,
verbose=verbose)
if verbose > 0:
logger.info(
f'{xid}_{sid} upper epsilon: {ue} ({ms_since_1970() - step_start}ms)'
)
epsilons.append((le, ue))
else:
e_bounds = (e_min, e_max)
if coarse_pass:
# find coarse bounds for epsilon
step_start = ms_since_1970()
e_bounds = find_epsilon_bounds(net,
sample,
x,
e_min,
e_max,
e_prec,
timeout=timeout,
verbose=verbose)
if verbose > 1:
logger.info(
f'{xid}_{sid} coarse epsilon bounds: {e_bounds} ({ms_since_1970() - step_start}ms)'
)
# find epsilon within ebounds
step_start = ms_since_1970()
e = find_epsilon(net,
sample,
x,
e_bounds[0],
e_bounds[1],
e_prec,
timeout=timeout,
verbose=verbose)
if verbose > 0:
logger.info(
f'{xid}_{sid} epsilon: {e} ({ms_since_1970() - step_start}ms)'
)
epsilons.append((e, e))
leps = [le for le, _ in epsilons if le != 0]
ueps = [ue for _, ue in epsilons if ue != 0]
x_summary = (-min(leps if leps else [0]), min(ueps if ueps else [0]))
if verbose > 0:
logger.info(
f'{xid} epsilon: {x_summary} ({ms_since_1970() - x_start}ms)')
results[f'x{x}'] = (x_summary, epsilons)
summary = {x: r[0] for x, r in results.items()}
logger.info(('asymm ' if asym else '') +
f'sensitivity: {summary} ({ms_since_1970() - start}ms)')
if save_results: save_sensitivity_results_to_csv(results, samples, outdir)
if save_samples: TOTUtils.save_samples_to_csv(samples, outdir)
return results