def verify_tasks(parser: VerifyArgParser):
""" Verify a range of tasks in the VNN-COMP2020 table. """
args = parser.parse_args()
logging.info(fmt_args(args))
assert args.task_i is not None and args.task_j is not None
info = VNN20Info()
d = info.results_all if args.series == 'all' else info.results_hard
low, high = args.task_i, min(len(d.props), args.task_j)
logging.info(f'Enumerating verification task [{low}, {high}).')
for i in range(low, high):
# parse network id
nid_nums = [int(v) for v in d.nets[i].strip().split('-')]
nid = acas.AcasNetID(*nid_nums)
# parse prop id
prop = int(d.props[i])
if prop == 6:
# 6a and 6b
all_props = AndProp([acas.AcasProp.property6a(dom), acas.AcasProp.property6b(dom)])
else:
prop_method = f'property{prop}'
prop = getattr(acas.AcasProp, prop_method)(dom)
all_props = AndProp([prop])
logging.info(f'===== Processing {nid}, verifying one property {all_props.name} =====')
t0 = timer()
res = _verify(nid, all_props, args)
logging.info(f'After {pp_time(timer() - t0)}, verify result -- CEX: {res}\n\n')
return
def test_goal_accuracy(parser: AcasArgParser):
""" Q2: Show that the safe-by-construction overhead on accuracy is mild. """
defaults = {
# 'start_abs_cnt': 5000,
'batch_size': 100, # to make it faster
'min_epochs': 25,
'max_epochs': 35
}
parser.set_defaults(**defaults)
args = parser.parse_args()
logging.info(utils.fmt_args(args))
nids = acas.AcasNetID.goal_accuracy_ids(args.dom)
_run(nids, args)
return
def test_goal_safety(parser: AcasArgParser):
""" Q1: Show that we can train previously unsafe networks to safe. """
defaults = {
# 'start_abs_cnt': 5000,
'batch_size': 100, # to make it faster
'min_epochs': 25,
'max_epochs': 35
}
parser.set_defaults(**defaults)
args = parser.parse_args()
logging.info(utils.fmt_args(args))
nids = acas.AcasNetID.goal_safety_ids(args.dom)
_run(nids, args)
return
def verify_net(parser: VerifyArgParser):
""" Verify all properties a network should hold at the same time. """
args = parser.parse_args()
logging.info(fmt_args(args))
# parse network id
nums = [int(v) for v in args.net.strip().split('-')]
nid = acas.AcasNetID(*nums)
# should hold for all props
all_props = AndProp(nid.applicable_props(dom))
logging.info(f'===== Processing {nid}, verifying all its props {all_props.name} =====')
t0 = timer()
res = _verify(nid, all_props, args)
logging.info(f'After {pp_time(timer() - t0)}, verify result -- CEX: {res}\n\n')
return res
def test_all(parser: CollisionArgParser):
""" Q: Show that we can train all previously safe/unsafe networks to safe, and evaluate on the given dataset. """
defaults = {
'min_epochs': 20,
}
parser.set_defaults(**defaults)
args = parser.parse_args()
logging.info(utils.fmt_args(args))
# since all original network params are the same, we don't need to load many times
net = args.net_fn().to(device)
all_props = [
c.CollisionProp.load(fpath, args.dom)
for fpath in c.COLLISION_DIR.glob('*.rlv')
]
outs = train_collision(net, all_props, args)
logging.info(
f'Final Summary -- Avg <epochs, train_time, certified, accuracy>: {outs}'
)
return
def verify_net_prop(parser: VerifyArgParser):
""" Verify a specific network w.r.t. a single property. """
args = parser.parse_args()
logging.info(fmt_args(args))
# parse network id
nums = [int(v) for v in args.net.strip().split('-')]
nid = acas.AcasNetID(*nums)
# parse prop id
if args.prop == 6:
# 6a and 6b
all_props = AndProp([acas.AcasProp.property6a(dom), acas.AcasProp.property6b(dom)])
else:
prop_method = f'property{args.prop}'
prop = getattr(acas.AcasProp, prop_method)(dom)
all_props = AndProp([prop])
logging.info(f'===== Processing {nid}, verifying one property {all_props.name} =====')
res = _verify(nid, all_props, args)
t0 = timer()
logging.info(f'After {pp_time(timer() - t0)}, verify result -- CEX: {res}\n\n')
return res