def prime_equillibrium(vocab, signature, aux=set()):
"""
Generates the formula:
x = x' & y = y' & n* = n*' & ...
for all the symbols in the vocabulary
@param vocab symbols to process
@param signature a (sorted) signature
@param aux symbols to skip
"""
res = []
vocab_prime = vocab.prime()
for v, vprime in vocab_prime.locals:
# HACK: Exclude non-deterministic variables
if v.literal.startswith('nondet') or v in aux: continue
if tuple(signature.sorts[v][0].from_) == ("V", "V"):
f = FolFormula(eqrel, [FolFormula(v), FolFormula(vprime)])
elif tuple(signature.sorts[v][0].from_) == ("V", ):
f = FolFormula(eqpred, [FolFormula(v), FolFormula(vprime)])
else:
f = FolFormula(FolWithEquality.eq,
[FolFormula(v), FolFormula(vprime)])
res.append(f)
return FolFormula.conjunction(res), vocab_prime
def generate_unary_pred_properties(type_decls, preds, consts):
ts = type_decls.sorts
for p in preds:
for (from_, to) in ts.ary(p, 1):
if to in ['', 'bool']:
arg_sort = from_[0]
for c in consts:
if type_decls.sorts.returns(c, arg_sort):
yield FolFormula(p, [FolFormula(c)])
def generate_named_properties(name_property_list, type_decls):
named_axioms = []
for name, property_formula in name_property_list:
p = Identifier(name, 'predicate')
type_decls.sorts[p] = [((), '')]
axiom = FolFormula(FolFormula.IFF, [FolFormula(p), property_formula])
named_axioms += [(p, axiom)]
return named_axioms
def _quantify_forall(self, t):
vars0 = [
self.Var(Identifier(t.var_name(i), 'variable'), None)
for i in reversed(xrange(t.num_vars()))
]
body = self.with_some_vars(vars0)(t.body())
phi = body
for v in vars0:
phi = FolFormula(FolFormula.FORALL,
[FolFormula(v.identifier), phi])
return phi
def generate_binary_pred_properties(type_decls, preds, consts):
ts = type_decls.sorts
for p in preds:
if p == 'F0': continue # hack
for (from_, to) in ts.ary(p, 2):
if to in ['', 'bool']:
arg_sort0, arg_sort1 = from_
for c0 in consts:
for c1 in consts:
if type_decls.sorts.returns(c0, arg_sort0) and \
type_decls.sorts.returns(c1, arg_sort1):
yield FolFormula(p,
[FolFormula(c0),
FolFormula(c1)])
def transition_system(syn, loop, vocab, signature):
fin, vocab_prime = prime_equillibrium(vocab, signature)
L = syn.expansion.parser
rho = FolScheme(L * 'wp_ea([?loopBody],[?trans])')(loop.body, fin)
rho0 = FolFormula.conjunction(syn.first_pass([rho]))
# set back in time; x->x0, x'->x
rho0 = vocab.to_past(rho0)
rho0 = vocab_prime.to_past(rho0)
return rho0
def generate_p_properties(syn, consts, vocab):
"""
Extra properties: {n(u,v) | u,v program variables + null}
"""
ptot_template = FolFormula.conjunction(
map(FolScheme, syn.expansion("ptot_([?u],[?v])")))
extras = [('p%s%s' % (u, v), ptot_template(u, v)) for u in consts
for v in consts]
return AbstractionProperties.from_named_formulas(extras, vocab)
def _id(self, symbol):
if not isinstance(symbol, Identifier):
raise FolSemanticError(symbol)
try:
return self.ids[symbol], True
except KeyError:
i = "%s%d" % ("".join(
c for c in str(symbol) if c in string.letters + "_")
or "i", len(self.ids))
self.ids[symbol] = i = FolFormula.Identifier(i, symbol.kind)
return i, True
def display_invariant(inv, abs_props):
z3g = Z3Gate()
fol_inv = z3g.formula_back(inv)
bnd = [t.subtrees[0] for t in fol_inv.nodes if t.root.kind == 'quantifier']
subst = {
v: FolFormula(Identifier("x%d" % i, 'variable'))
for i, v in enumerate(bnd)
}
subst.update(abs_props.defs)
fol_inv = FolSubstitution(subst)(fol_inv)
print " - " * 20
print fol_inv
print " - " * 20
def __call__(self, t):
if is_var(t):
return FolFormula(self._get_free_var(t))
elif is_true(t) or is_false(t):
return FolFormula(is_true(t))
elif is_const(t):
kind = 'predicate' if is_bool(t) else 'function'
return FolFormula(Identifier(unicode(t), kind))
elif is_quantifier(t) and t.is_forall():
return self._quantify_forall(t)
elif is_app(t):
r = t.decl()
for k, v in self.o.map_ops.iteritems():
if is_app_of(t, k):
r = v
break
else:
r = Identifier(unicode(r), '?')
if callable(r):
return r([self(x) for x in t.children()])
else:
return FolFormula(r, [self(x) for x in t.children()])
else:
raise NotImplementedError
def quantifier_semantics(cls, formula, domain_consts):
"""
Given a list of domain constants, unwraps the quantifier to a long
formula or term.
E.g. Я[+]x (f(x)) ---> f(1)+f(2)+f(3)+f(4)
"""
if formula.root == cls.Signature.REDUCE_GLOB:
if len(formula.subtrees) != 3:
raise ValueError, "%r requires exactly 3 arguments, in %r" % (formula.root, formula)
op, v, expr = formula.subtrees
if op.subtrees != [] or op.root.kind != 'function':
raise TypeError, "expected a binary function symbol instead of %r, in %r" % (op, formula)
qf = reduce(lambda x,y: FolFormula(op.root, [x,y]),
(TreeSubstitution({v: i})(expr) for i in domain_consts))
return qf
def redefine(self, synopsis):
"""
Defines macros 'cond', 'loopBody', 'epilogue', and 'bad' in the environment.
"""
stmts = lambda x: FolFormula.join(self.SEMI, x, self.SKIP)
delta = synopsis.expansion.delta
delta.transformers += [
DeltaReduction.Transformer(delta, name, params=[], body=expr)
for name, expr in [("cond", self.cond), (
"loopBody", self.body), ("epilogue", stmts(self.epilogue))]
]
if not any(
getattr(t, 'head', None) == "bad" for t in delta.transformers):
bad = synopsis.expansion.parser * self.BAD
delta.transformers += [
DeltaReduction.Transformer(delta, "bad", params=[], body=bad)
]
def load_data():
import argparse
a = argparse.ArgumentParser()
a.add_argument('filename', type=str, nargs='?', default=argparse.SUPPRESS)
a.add_argument('-u',
action='store_true',
help="universal invariant inference")
a.add_argument('-a',
action='store_true',
help="(experimental) use alpha-from-below")
a.add_argument('-p',
action='store_true',
help="(experimental) use partial models, not sound")
a.add_argument('-e',
action='store_true',
help="(experimental) stuff, broken")
a.add_argument('--gen-enums',
action='store_true',
help="generalize enum values")
a.add_argument('--no-preds-dll',
action='store_true',
help="do not generate abstraction predicates for dll")
a.add_argument('--no-preds-sorted',
action='store_true',
help="do not generate 'sorted' abstraction predicates")
a.add_argument('--domain', type=str, default=None)
a.add_argument('--disable-opt',
dest="o",
action='store_false',
default=True,
help="(development) disable IC3 optimizations")
a.add_argument(
'--out-inv',
action='store_true',
help=
"print the discovered invariant as a first-order formula (requires unicode support)"
)
a.add_argument('--latex',
action='store_true',
help="typeset discovered invariant using LaTeX")
a.add_argument('-v', '--verbose', action='store_true')
args = a.parse_args()
if not hasattr(args, 'filename'):
import os.path
from filesystem.paths import find_closest
here = os.path.dirname(__file__)
args.filename = os.path.join(find_closest('benchmarks', start_at=here),
'sll-last.imp')
program = open(args.filename).read().decode('utf-8')
print "* BENCHMARK"
# print os.path.basename(args.filename),
print args.filename
# Initiate proof synopsis and load required modules
syn = ProofSynopsis()
# annot = list(ProofSynopsis.get_annotations(program))
annot = list(syn.get_annotations(program))
# Get more flags from @flags annotation in input file
# (note: there is currently no way to override @flags)
for (key, val) in annot:
if key == "flags":
args = a.parse_args(a.convert_arg_line_to_args(val), args)
ms = module_system_default()
uses = [
t.strip() for (key, val) in annot for t in val.split() if key == "uses"
]
if uses == []: uses = ['dtca_ea'] # backward compat
for t in ['base', 'dtca']:
if t not in uses: uses.append(t)
syn.libs += [
open(fn).read().decode('utf-8') for module_name in uses
for fn in ms.find_module_source(module_name)
]
syn.libs += [EXTRA_PROP_MACROS]
# Construct vocabulary
t = syn.type_declarations
vocab = TwoVocabulary()
vocab.type_declarations = t
axioms = list(vocab << syn.first_pass(program))
print "* PREDICATES"
print vocab.preds
print "* CONSTANTS"
print vocab.consts
# Extract loop from program (use cond := ... and loopBody := ... definitions as fallback)
loop = MainLoop.locate_loop(syn.first_pass)
if loop.prologue:
raise NotImplementedError("loop prologue is currently not supported")
loop.redefine(syn)
trans = transition_system(syn, loop, vocab, t)
cond = loop.cond
cond0 = vocab.to_past(cond)
##############################################################
use_extra_properties = not args.u
if use_extra_properties:
extra = generate_n_properties(syn, vocab.consts, vocab)
extra.props += list(
generate_unary_pred_properties(t, vocab.preds, vocab.consts))
extra.props += list(
generate_binary_pred_properties(t, vocab.preds, vocab.consts))
extra.axioms += axioms
# Stability (absence of dangling pointers)
if FolSorts.FunctionType.parse(u'V→') in t.sorts.ary('alloc', 1):
extra += generate_stability_properties(syn, vocab)
# Properties for order and sorting
if not args.no_preds_sorted:
if FolSorts.FunctionType.parse(u'V×V→') in t.sorts.ary('R', 2):
if args.domain is None or "S" in args.domain:
extra += generate_order_and_sorting_properties(syn, vocab)
# Properties for reversal and doubly-linked lists
if not args.no_preds_dll:
if FolSorts.FunctionType.parse(u'V×V→') in t.sorts.ary('p*', 2):
if args.domain is None or "R" in args.domain:
extra += generate_rev_properties(
syn,
vocab,
binary_too=(args.domain is None
or "-" not in args.domain))
else:
extra = AbstractionProperties()
extra.axioms += axioms
# --- Now send everything to PDR + Z3
z3g = Z3Gate()
z3g.z3_decls = decls = z3g.define_symbols(t)
z3g.expansion = syn.first_pass
preds = [decls[x] for x in vocab.preds]
extra_props = [z3g.formula(FolFormula.promote(p)) for p in extra.props]
extra_axioms = [fol_formula_to_z3(phi, decls) for phi in extra.axioms]
extra_axioms0 = [fol_formula_to_z3(phi, decls) for phi in extra.axioms0]
init = z3g.formula("init")
rho = And(z3g.formula(cond0), z3g.formula(trans),
*(extra_axioms + extra_axioms0))
bad = z3g.formula("bad")
background = And(generate_gamma(syn, vocab.preds_flat, decls),
*extra_axioms)
globals = [decls[x] for x in vocab.globals
if t.sorts.ary(x, 0)] # @ReservedAssignment
locals = [(decls[x0], decls[x])
for x0, x in vocab.locals] # @ReservedAssignment
if args.u:
print "*** Using universal invariant inference"
from mini_pdr import PDR
if args.p:
print "*** Use partial models (experimental)"
if args.gen_enums:
print "*** Generalize enum values"
args.o = False # universals currently not supported in opt version
if args.o:
from mini_pdr_opt import PDR # @Reimport
elif args.a:
print "*** Using alpha from below"
from mini_pdr_opt import PDR # @Reimport
else:
from mini_pdr import PDR # @Reimport
print "* GLOBALS"
print globals
print "* LOCALS"
print locals
"""
if args.u:
pdr = PDR(init, rho, bad, background, globals, locals, preds,
universal=args.u, partial=args.p, gen_enums=args.gen_enums,
experiment=args.e)
else:
pdr = PDR(init, rho, bad, background, globals, locals, [n])
"""
return init, rho, bad, background, globals, locals, [n], args.n
def formula(self, fol_formula):
if isinstance(fol_formula, (str, unicode)):
fol_formula = FolFormula.conjunction(self.expansion([fol_formula]))
f_aux = AuxTransformers.renumber_bound_vars(fol_formula)
return fol_formula_to_z3(f_aux, self.z3_decls)
def create_f_with_implies(b,h):
f = FolFormula( FolFormula.IMPLIES )
f.subtrees = []
f.subtrees.append( b )
f.subtrees.append( h )
return f
def conjunct_formulas( fs ):
return FolFormula.conjunction( fs )
def join(self, connective=FolFormula.AND, default_value=None):
return FolFormula.join(connective, self, default_value)
def __call__(self, program):
wps = WeakestPreSynthesis()
wps.configure(detptr_libraries())
wps.syn.libs += [self.VC] #TODO - change
formulas , decls = wps_formula_get(wps,program)
I_args = inv_args_get(wps, "I")
formula = FolFormula.conjunction(formulas)
prefix = wp_forall_prefix_get(formula)| set([Identifier('null', 'const')])
bounded_prefix = AuxTransformers.get_all_bound_vars(formula)
prefix-=bounded_prefix
pprint( formulas )
print prefix
z3_formulas_with_blasting = []
from shape2horn import hoist_forall
from shape2horn import blast_horn
from shape2horn import hoist_horn
formulas_with_implies = []
ackm_bools = {}
ackm_bools_to_vars = {}
#sanity = FormulaSanity()
watch = Stopwatch()
with watch:
for f in formulas:
if f.root == 'true':
continue
head = f.subtrees[1]
body = f.subtrees[0]
head_ls = FolFormula.split(head) #splits conjunctions in head
for i in head_ls:
#creates new formulas of form: body -> i
f_with_implies = create_f_with_implies(body,i)
formulas_with_implies.append( f_with_implies )
for s in formulas_with_implies:
print '-------- Input Clause ----------------'
print s
s_with_gamma = gamma_add( s )
#renumber_bound_vars() renames bounded vars to unique numbers
s_with_gamma = AuxTransformers.renumber_bound_vars( s_with_gamma )
print s_with_gamma
z3_f = transform_formula_to_z3(prefix, s_with_gamma, decls)
for f in hoist_horn(z3_f[0]):
z3_f_hoisted = hoist_forall(f)
print '-------- Hoisted Formula ------------------'
print z3_f_hoisted
blasted_f = blast_horn(z3_f_hoisted)
print '-------- Blasted Formula ---------------------'
#print blasted_f
if 0:
blasted_f_with_replaced_n = replace_nstar_with_bools( blasted_f, \
decls, ackm_bools, ackm_bools_to_vars )
f_with_ackm_reduction = ackerman_reduction_get( \
blasted_f_with_replaced_n , ackm_bools_to_vars )
#print ackm_bools_to_vars
z3_formulas_with_blasting.append( blasted_f )
#benchmark_print_to_file(h, z3_formulas_with_blasting)
formulas_solver ( self, z3_formulas_with_blasting, decls["I"], I_args )
print 'Total sovling time: ',watch.total_time,'sec'
def generate_gamma(syn, preds, decls):
dtca = Identifier('dtca', '?')
starred = [z for z in preds if '*' in z.literal]
gamma = FolFormula.conjunction(
[FolFormula(dtca, [FolFormula(z)]) for z in starred])
return generate_z3_condition(gamma, syn, decls)
def conjunct_formulas( fs ):
return FolFormula.conjunction( fs )
def singl(self, t): return FolFormula(Identifier((t.subtrees[0] if t.subtrees else t).root.token.value, '?')) def qua(self, t): return FolFormula(self.PRIM[t.subtrees[0].root.token.value], [self.singl(t.subtrees[1]), t.subtrees[3]])
def generate_z3_condition(folformula, syn, decls):
c = list(syn.first_pass([folformula]))
f = FolFormula.conjunction(c)
f_aux = AuxTransformers.renumber_bound_vars(f)
f_z3 = fol_formula_to_z3(f_aux, decls)
return hoist_forall(f_z3)
def cal(self, t): return FolFormula(self.singl(t).root, self.split(t.subtrees[2], ('term*', 'term+', ',')))
def bin(self, t): return FolFormula(self.PRIM[t.subtrees[1].root.token.value], [t.subtrees[0], t.subtrees[2]])
def _to_formula(self, tree):
_ = self._spread_quantifiers
return _(
FolFormula(tree.root,
[self._to_formula(s) for s in tree.subtrees]))
vocab,
binary_too=(args.domain is None
or "-" not in args.domain))
else:
extra = AbstractionProperties()
extra.axioms += axioms
# --- Now send everything to PDR + Z3
z3g = Z3Gate()
z3g.z3_decls = decls = z3g.define_symbols(t)
z3g.expansion = syn.first_pass
preds = [decls[x] for x in vocab.preds]
extra_props = [z3g.formula(FolFormula.promote(p)) for p in extra.props]
extra_axioms = [fol_formula_to_z3(phi, decls) for phi in extra.axioms]
extra_axioms0 = [fol_formula_to_z3(phi, decls) for phi in extra.axioms0]
init = z3g.formula("init")
rho = And(z3g.formula(cond0), z3g.formula(trans),
*(extra_axioms + extra_axioms0))
bad = z3g.formula("bad")
background = And(generate_gamma(syn, vocab.preds_flat, decls),
*extra_axioms)
globals = [decls[x] for x in vocab.globals
if t.sorts.ary(x, 0)] # @ReservedAssignment
locals = [(decls[x0], decls[x])
for x0, x in vocab.locals] # @ReservedAssignment
def __init__(self, context, inner):
self.context = context
self.inner = inner
if isinstance(inner, self.__class__):
self.__class__ = inner.__class__
else:
class NestedException(self.__class__, inner.__class__):
__str__ = __repr__ = self.__class__.__str__
self.__class__ = NestedException
def __str__(self):
return "In: %r\n%r" % (self.context, self.inner)
# Snippet
if __name__ == '__main__':
from logic.fol import Identifier
i = Identifier('i', 'function')
sigma = FolSignature([(i, 1)], [])
m = FolStructure([1, 2, 3], {i: lambda x: x % 3 + 1})
p = FolDomainProjection(sigma, lambda t: t * 2)
n = p(m)
print n.domain
print n.interpretation['i']
print[n.interpretation['i'](x) for x in n.domain]
print FolResample().structure(n, sigma)
print m.evaluate(FolFormula(i, [FolFormula(2)]))
#print unicode(FOL_GRAMMAR)
for rule in FOL_GRAMMAR.rules:
if rule.head == 'fml': print rule
#raise SystemExit
class TX(ParserXforms):
PRIM = {'forall': FolFormula.FORALL, '|': FolFormula.OR, '&': FolFormula.AND, '~': FolFormula.NOT, '->': FolFormula.IMPLIES, '<->': FolFormula.IFF}
for v in PRIM.values(): PRIM[v.literal] = v
def singl(self, t): return FolFormula(Identifier((t.subtrees[0] if t.subtrees else t).root.token.value, '?'))
def qua(self, t): return FolFormula(self.PRIM[t.subtrees[0].root.token.value], [self.singl(t.subtrees[1]), t.subtrees[3]])
def cal(self, t): return FolFormula(self.singl(t).root, self.split(t.subtrees[2], ('term*', 'term+', ',')))
def bin(self, t): return FolFormula(self.PRIM[t.subtrees[1].root.token.value], [t.subtrees[0], t.subtrees[2]])
def un(self, t): return FolFormula(self.PRIM[t.subtrees[0].root.token.value], [t.subtrees[1]])
def sam(self, t): return t.subtrees[0]
def sam1(self, t): return t.subtrees[1]
lG = set((Tagged(tok).with_(annotation='sam'), tok) for tok in tokens)
ast = Earley(FOL_GRAMMAR.rules, lG, tokens, lexer=lambda x: x)
PrintTrees(ast)
assert len(ast) == 1
ast = FolFormula.reconstruct(ast[0])
print ast
phi = TreeTransform([TX()], dir=TreeTransform.BOTTOM_UP)(ast)
print phi
class Z3FormulaToFolFormula(object):
def __init__(self):
self.map_ops = {
Z3_OP_AND: FolFormula.conjunction,
Z3_OP_OR: FolFormula.disjunction,
Z3_OP_NOT: FolFormula.NOT,
Z3_OP_IMPLIES: FolFormula.IMPLIES,
Z3_OP_IFF: FolFormula.IFF,
Z3_OP_EQ: FolWithEquality.eq,
Z3_OP_DISTINCT: FolWithEquality.neq,
Z3_OP_ADD: FolIntegerArithmetic.add,
Z3_OP_MUL: FolIntegerArithmetic.mul,
Z3_OP_LE: FolIntegerArithmetic.le,
Z3_OP_GE: FolIntegerArithmetic.ge,
Z3_OP_SELECT: self._array_select
}
class ConversionProcess(InnerClasses.Owned):
Var = namedtuple("Var", 'identifier sort')
def __init__(self, o):
super(Z3FormulaToFolFormula.ConversionProcess, self).__init__(o)
self.vars = {}
def with_some_vars(self, vars0):
""" vars0 is a list of Var instances """
import copy
c = copy.copy(self)
nz = len(vars0)
c.vars = {k + nz: v for k, v in c.vars.iteritems()} #c.vars.copy()
for i, v in enumerate(vars0):
c.vars[i] = v
return c
def __call__(self, t):
if is_var(t):
return FolFormula(self._get_free_var(t))
elif is_true(t) or is_false(t):
return FolFormula(is_true(t))
elif is_const(t):
kind = 'predicate' if is_bool(t) else 'function'
return FolFormula(Identifier(unicode(t), kind))
elif is_quantifier(t) and t.is_forall():
return self._quantify_forall(t)
elif is_app(t):
r = t.decl()
for k, v in self.o.map_ops.iteritems():
if is_app_of(t, k):
r = v
break
else:
r = Identifier(unicode(r), '?')
if callable(r):
return r([self(x) for x in t.children()])
else:
return FolFormula(r, [self(x) for x in t.children()])
else:
raise NotImplementedError
def _get_free_var(self, var):
i = get_var_index(var)
if i in self.vars:
return self.vars[i].identifier
else:
self.vars[i] = v = self.Var(Identifier("$%d" % i, 'variable'),
var.sort())
return v.identifier
def _quantify_forall(self, t):
vars0 = [
self.Var(Identifier(t.var_name(i), 'variable'), None)
for i in reversed(xrange(t.num_vars()))
]
body = self.with_some_vars(vars0)(t.body())
phi = body
for v in vars0:
phi = FolFormula(FolFormula.FORALL,
[FolFormula(v.identifier), phi])
return phi
def __call__(self, t):
phi = self.scheme(t)
if not phi.placeholders:
return phi.formula
else:
return phi
def scheme(self, t):
cp = self.ConversionProcess(self)
phi = cp(t)
return FolScheme(phi, [v.identifier for v in cp.vars.itervalues()])
def _array_select(self, (arr, subscript)):
return FolFormula(arr.root, arr.subtrees + [subscript])
class Signature:
a = FolFormula.Identifier(u'α', 'predicate')
v = FolFormula.Identifier('v', 'variable')
u = FolFormula.Identifier('u', 'variable')
#for foo in adt_brute_force():
# print foo
pr = cProfile.Profile()
pr.enable()
for formula in adt_brute_force():
total_counter+=1
first_st[INV] = formula
s_loop = chk_inv_on_general_stmt( first_st )
if s_loop:
last_st[INV] = formula
e_loop = chk_inv_on_general_stmt( last_st )
if not e_loop:
holds_counter += 1
print 'Does inv hold?: ', FolFormula.reconstruct(formula), str(e_loop), total_counter, holds_counter
if total_counter > 50000:
break
pr.disable()
#f = open('x.prof', 'a')
sortby = 'time'
pstats.Stats(pr ).strip_dirs().sort_stats(sortby).print_stats()
#f.close()
#ps.print_results()
#w = WhileFrontend.WhileASTDeserialize()
#print w(unicode(astf))
#print astf.root
#print astf.subtrees
def un(self, t): return FolFormula(self.PRIM[t.subtrees[0].root.token.value], [t.subtrees[1]]) def sam(self, t): return t.subtrees[0]