def main(argv):
args = parseArgs(argv[1:])
# get the names of inductive assumps first
iass = set()
cf = open(args.cert_mis_file)
for line in cf:
r = re.match("\s*\(\s*assert\s+(pre![^\)|^\s]+)", line)
if r is not None: iass.add(r.group(1))
# now the invariants
ctx = z3.Context()
fmla = z3.parse_smt2_file(args.ass_inv_file, ctx=ctx)
lemmas = []
assert z3.is_and(fmla), \
"invariant file should be a set of assertions"
for l in fmla.children():
assert z3u.isZ3Implies(l), \
"assertions in the invariant file should be implications"
name = str(l.arg(0).decl())
assert name.startswith("pre!"), \
"implicants in the invariant should start with pre!"
if name in iass: lemmas.append(l.arg(1))
# dump the collected lemmas
out = sys.stdout if args.out == '-' else open(args.out, 'w')
z3u.to_smtlib(lemmas, out)
def main (argv):
args = parseArgs (argv[1:])
# get the names of inductive assumps first
iass = set()
cf = open(args.cert_mis_file)
for line in cf:
r = re.match("\s*\(\s*assert\s+(pre![^\)|^\s]+)", line)
if r is not None: iass.add(r.group(1))
# now the invariants
ctx = z3.Context()
fmla = z3.parse_smt2_file(args.ass_inv_file, ctx=ctx)
lemmas = []
assert z3.is_and(fmla), \
"invariant file should be a set of assertions"
for l in fmla.children():
assert z3u.isZ3Implies(l), \
"assertions in the invariant file should be implications"
name = str(l.arg(0).decl())
assert name.startswith("pre!"), \
"implicants in the invariant should start with pre!"
if name in iass: lemmas.append(l.arg(1))
# dump the collected lemmas
out = sys.stdout if args.out == '-' else open (args.out, 'w')
z3u.to_smtlib(lemmas, out)
def main(argv):
print 'In main'
args = parseArgs(argv[1:])
ctx = z3.Context()
fmla = z3.parse_smt2_file(args.file, ctx=ctx)
msg = 'Solving'
if not args.no_blast:
msg = msg + ' (blasted)'
blasted = bitblast(fmla)
else:
blasted = fmla
msg = msg + " ..."
if args.o <> None:
with open(args.o, 'w') as f:
f.write(blasted.sexpr())
print msg
solver = z3.Solver(ctx=ctx)
solver.add(blasted)
res = solver.check()
print res
return 0
def parse_with_z3(file):
t = z3.With(z3.Tactic("horn-simplify"), "xform.inline_eager", False)
assertions = z3.parse_smt2_file(file)
g = z3.Goal()
g.add(assertions)
r = t(g)
s = z3.Solver()
s.add(r[0])
print(s.sexpr())
def check(values = []):
createSMTInput(values)
f = z3.parse_smt2_file("./output.smt2")
s = z3.Solver()
s.add(f)
sat = str(s.check())
if sat == "sat":
return { "sat": "sat", "model": s.model() }
else:
return { "sat": "unsat", "model": "" }
def count(smt_file):
formula = z3.parse_smt2_file(smt_file)
tactic_simp = z3.With(z3.Tactic('simplify'), 'elim_and', True)
tactic_total = z3.Then(tactic_simp, z3.Tactic('elim-term-ite'))
tactic_total = z3.Then(tactic_total, z3.Tactic('tseitin-cnf'))
goals = tactic_total(formula)
if len(goals) == 1:
goal = goals[0]
return len(goal)
else:
return -1
def get_features(file_path, feature_setting, logic="", track=""):
if file_path not in cache:
cache[file_path] = {}
if logic not in cache[file_path]:
cache[file_path][logic] = {}
if track not in cache[file_path][logic]:
cache[file_path][logic][track] = {}
if feature_setting == "bow":
features = get_syntactic_count_features(file_path)
elif feature_setting == "probes":
g = z3.Goal()
g.add(z3.parse_smt2_file(file_path))
features = [z3.Probe(x)(g) for x in PROBES]
else:
g = z3.Goal()
g.add(z3.parse_smt2_file(file_path))
features = get_syntactic_count_features(file_path) + [
z3.Probe(x)(g) for x in PROBES
]
cache[file_path][logic][track] = features
return features
def from_smt2_file(smt_file: PathLike) -> 'CNFFormula':
""""
Load a CNFFormula from an SMT2 file. It uses Z3 to parse
the SMT2 and to apply the different tactics generating the
goals in CNF.
:param smt_file: SMT2 file
:return: a CNF Formula instance
"""
p = Path(smt_file)
if not p.exists():
raise FileNotFoundError('SMT2 file not found: {}'.format(p))
x = z3.parse_smt2_file(p.as_posix())
return CNFFormula.from_z3(x)
def encodeAndSolve(self):
"""Generate horn formulas and solve"""
self.setSolver()
hornFormulas = self.args.file if self.args.smt2 else self.mk_horn()
cex = None
if not hornFormulas:
self.log.error('Problem generating Horn formulae')
return
with utils.stats.timer('Parse'):
self.log.info('Successful Horn Generation ... ' +
str(hornFormulas))
q = self.fp.parse_file(hornFormulas)
preds = utils.fp_get_preds(
self.fp) # get the predicates before z3 starts playing with them
if self.args.invs:
lemmas = z3.parse_smt2_file(args.invs, sorts={}, decls={}, ctx=ctx)
if z3.is_and(lemmas):
lemmas = lemmas.children()
for l in lemmas:
if self.args.verbose: print l
fp_add_cover(self.fp, l.arg(0), l.arg(1))
contract_file, emf_file = None, None
with utils.stats.timer('Query'):
res = self.fp.query(q[0])
if res == z3.sat:
utils.stat('Result', 'CEX')
cex = self.mk_cex(preds)
elif res == z3.unsat:
utils.stat('Result', 'SAFE')
if self.args.ri: self.get_raw_invs(preds)
if self.args.cg:
contract_file, emf_file = self.mk_contract(preds)
# try:
# except Exception as e:
# print e
# self.log.warning('Failed to generate CoCoSpec')
if not self.args.save:
self.log.debug("Cleaning up temp files ...")
try:
os.remove(self.smt2_file)
os.remove(self.trace_file)
except:
self.log.info('No Cleaning of temp files ...')
if self.args.xml:
utils.stats.xml_print(self.args.node, cex, contract_file, emf_file)
else:
utils.stats.brunch_print()
def parse_smt2_file(smt2_filename):
"""Reads a smt2 file and returns the first formula.
Args:
smt2_filename: smt2 file that was generated from Sketch.
Raises:
An assertion if the original smt2 file didn't contain any assert
statements.
"""
# parse_smt2_file returns a vector of ASTs, and each element corresponds to
# one assert statement in the original file. The smt2 file generated by
# sketch only has one assertions, simply take the first.
formulas = z3.parse_smt2_file(smt2_filename)
assert len(formulas) == 1, (smt2_filename,
'contains 0 or more than 1 asserts.')
return formulas[0]
def __init__(self, filename: str):
self.filename: str = filename
self.assertions: z3.z3.AstVector = z3.parse_smt2_file(filename)
self.alphabet: Collection[str] = set()
self.variables: Collection[str] = set()
self.constraints: REConstraints = dict()
self.equations: Collection[StringEquation] = []
# Gather alphabet
for ref in self.assertions:
self._gather_symbols(ref)
self.alphabet_str: str = "".join(self.alphabet) + NONSPEC_SYMBOL
# Fresh variables
self.next_variable_id = 0
def calculate(input_smt):
formula = z3.parse_smt2_file(input_smt)
tactic_simp = z3.With(z3.Tactic('simplify'), 'elim_and', True)
tactic_total = z3.Then(tactic_simp, z3.Tactic('elim-term-ite'))
tactic_total = z3.Then(tactic_total, z3.Tactic('tseitin-cnf'))
goals = tactic_total.apply(formula)
if len(goals) == 1:
goal = goals[0]
# the goal is the list of constraints, and conjunction of which is equivalent to the original problem
manager = compute(goal)
# compute sparseness
num_visible = manager.get_visible_size()
num_var = manager.get_var_size()
num_total = manager.get_total_size()
num_min = manager.get_maximum_clause_size()
sparse = (num_visible - num_min) / (num_total - num_min)
factor_min = math.ceil((1 + math.sqrt(1 + 8 * num_visible)) / 2)
factor_max = 2 * num_total
factor = (num_var - factor_min) / (factor_max - factor_min)
return sparse, factor
else:
return '*', '*'
def load(self, filename):
self.filename = filename
print("im about to parse %s" % filename)
fs = z3.parse_smt2_file(filename)
for f in fs:
# add predicates
predicate = f.body().arg(1)
if predicate.decl().name() == 'false':
print("is query")
print(predicate.sexpr())
vars, body = stripQuantifierBlock(f)
query = z3.Exists(list(reversed(vars)), f.body().arg(0))
print("reconstructed query:", query)
self.queries.append(query)
else:
print("f: ", f)
self.predicates[predicate.decl().name()] = predicate.decl()
# add variables and sorts
self.all_var_sort[predicate.decl().name()] = self.get_var_sort(
predicate)
# add rule
self.rules.append(f)
print("=======DONE LOADING =======")
self.dump()
filename = None
for arg in sys.argv[1:]:
if not arg.startswith('-'):
filename = arg
if filename == None:
print('Error: no filename given')
exit()
outfile = filename + '.qdimacs'
for arg in sys.argv[1:]:
if arg.startswith('-o='):
outfile = arg[3:]
log('Setting output file: ' + outfile)
log('Parsing SMTLIB')
expr = parse_smt2_file(filename)
new_expr, boundvariables, quantifiers = separateQuantifiersFromConstraints(
expr)
quantifiers, constraint = reencode_quantifiers(new_expr, boundvariables,
quantifiers)
if Write2QBF:
constraint = Not(constraint)
for q in quantifiers:
assert (q[0] == 'a' or q[0] == 'e')
if q[0] == 'a':
q[0] = 'e'
elif q[0] == 'e':
q[0] = 'a'
import z3
import functools
#mc_file = open('mc.z3', 'r')
#joinlines = lambda f: functools.reduce(lambda l1,l2: l1 + l2, f)
#js = joinlines(mc_file.readlines())
#print(js)
parsed = z3.parse_smt2_file('test.z3')
def mk_symb_cex(self, smtlib_file):
vmap = self.mk_var_map_from_smtlib(smtlib_file)
imap = self.mk_init_var_map(vmap)
# mlog.debug('imap: {}'.format(imap))
self.symb_cex = z3.parse_smt2_file(str(smtlib_file))
self.imap = imap
def get_queries(filename):
queries = z3.parse_smt2_file(filename)
return queries
if len(sys.argv) == 3:
values = [int(sys.argv[1]), int(sys.argv[2])]
print("Input Values\n---------------------")
# + ", ".join(map(lambda x: str(x), values)))
print("total_cpus: " + str(values[0]))
print("mem_size: " + str(values[1]))
# print(check()["sat"])
sat = check()["sat"]
if sat == "unsat":
print emoji.emojize('sat: :x:', use_aliases=True)
print("Is unsat without initializing any variable")
exit(1)
solver = check(values)
if solver["sat"] == "sat":
print emoji.emojize('sat: :white_check_mark:', use_aliases=True)
else:
while solver["sat"] == "unsat" and len(values) > 0:
values.pop()
solver = check(values)
print emoji.emojize('sat: :x:', use_aliases=True)
print "\n" + emoji.emojize(':rotating_light: Suggestion :rotating_light:', use_aliases=True)+"\n---------------------"
for i in range(len(solver["model"])-2, 0, -1):
print solver["model"][i], ": ", solver["model"][solver["model"][i]]
# main()
f = z3.parse_smt2_file("./ouput2.smt2")
s = z3.Solver()
s.add(f)
print s.check()
def parse_with_z3(file, out_dir, check_only):
lst = file.split('/')
tmp = lst.pop()
lst = tmp.split('.')
base_name = lst[0]
assert len(lst) > 0
if len(lst) > 1:
for stuff in lst[1:-1]:
base_name = base_name + '.' + stuff
t = z3.With(
z3.Tactic("horn-simplify"),
"xform.inline_eager",
False,
"xform.inline_linear",
False,
"xform.slice",
False,
"xform.coi",
False,
"xform.compress_unbound",
False,
)
assertions = z3.parse_smt2_file(file)
goals = z3.Goal()
goals.add(assertions)
if check_only:
check_chcs(goals)
print "success"
else:
simplified = t(goals)
clauses = []
queries = []
for clause, is_query in [
fix_clause(clause) for clause in simplified[0]
]:
if is_query:
queries.append(clause)
else:
clauses.append(clause)
for cnt, query in enumerate(queries):
these_clauses = []
for clause in clauses:
these_clauses.append(clause)
these_clauses.append(query)
goals = z3.Solver()
goals.add(these_clauses)
if out_dir is None:
print('(set-logic HORN)')
print goals.sexpr()
print '(check-sat)'
print '(exit)'
else:
out_file = "{}/{}_{:0>3}.smt2".format(out_dir, base_name, cnt)
print 'Writing to {}'.format(out_file)
out_file = open(out_file, mode='w')
out_file.write('(set-logic HORN)\n\n')
out_file.write(goals.sexpr())
out_file.write('\n\n(check-sat)\n')
out_file.write('(exit)\n')
try:
check_chcs(these_clauses)
except Exception, blah:
raise Exception(
"Result of formatting is ill-formed:\n{}".format(blah))
def __call__(self, project, test, dump, fronend_source):
logger.info('inferring specification for test \'{}\''.format(test))
environment = dict(os.environ)
if self.config['klee_max_forks'] is not None:
environment['ANGELIX_KLEE_MAX_FORKS'] = str(
self.config['klee_max_forks'])
if self.config['klee_max_depth'] is not None:
environment['ANGELIX_KLEE_MAX_DEPTH'] = str(
self.config['klee_max_depth'])
if self.config['klee_search'] is not None:
environment['ANGELIX_KLEE_SEARCH'] = self.config['klee_search']
if self.config['klee_timeout'] is not None:
environment['ANGELIX_KLEE_MAX_TIME'] = str(
self.config['klee_timeout'])
if self.config['klee_solver_timeout'] is not None:
environment['ANGELIX_KLEE_MAX_SOLVER_TIME'] = str(
self.config['klee_solver_timeout'])
if self.config['klee_debug']:
environment['ANGELIX_KLEE_DEBUG'] = 'YES'
if self.config['klee_ignore_errors']:
environment['KLEE_DISABLE_MEMORY_ERROR'] = 'YES'
if self.config['use_semfix_syn']:
environment['ANGELIX_USE_SEMFIX_SYN'] = 'YES'
environment['ANGELIX_KLEE_WORKDIR'] = project.dir
test_dir = self.get_test_dir(test)
shutil.rmtree(test_dir, ignore_errors='true')
klee_dir = join(test_dir, 'klee')
os.makedirs(klee_dir)
self.run_test(project, test, klee=True, env=environment)
# loading dump
# name -> value list
oracle = dict()
vars = os.listdir(dump)
for var in vars:
instances = os.listdir(join(dump, var))
for i in range(0, len(instances)):
if str(i) not in instances:
logger.error('corrupted dump for test \'{}\''.format(test))
raise InferenceError()
oracle[var] = []
for i in range(0, len(instances)):
file = join(dump, var, str(i))
with open(file) as f:
content = f.read()
oracle[var].append(content)
# solving path constraints
angelic_paths = []
solver = Solver()
smt_glob = join(project.dir, 'klee-out-0', '*.smt2')
smt_files = glob(smt_glob)
for smt in smt_files:
logger.info('solving path {}'.format(relpath(smt)))
try:
path = z3.parse_smt2_file(smt)
except:
logger.warning('failed to parse {}'.format(smt))
continue
variables = [
str(var) for var in get_vars(path)
if str(var).startswith('int!') or str(var).startswith('bool!')
or str(var).startswith('char!')
or str(var).startswith('reachable!')
]
outputs, choices, constants, reachable, original_available = parse_variables(
variables)
# name -> value list (parsed)
oracle_constraints = dict()
def str_to_int(s):
return int(s)
def str_to_bool(s):
if s == 'false':
return False
if s == 'true':
return True
raise InferenceError()
def str_to_char(s):
if len(s) != 1:
raise InferenceError()
return s[0]
dump_parser_by_type = dict()
dump_parser_by_type['int'] = str_to_int
dump_parser_by_type['bool'] = str_to_bool
dump_parser_by_type['char'] = str_to_char
def bool_to_bv32(b):
if b:
return BitVecVal(1, 32)
else:
return BitVecVal(0, 32)
def int_to_bv32(i):
return BitVecVal(i, 32)
to_bv32_converter_by_type = dict()
to_bv32_converter_by_type['bool'] = bool_to_bv32
to_bv32_converter_by_type['int'] = int_to_bv32
def bv32_to_bool(bv):
return bv.as_long() != 0
def bv32_to_int(bv):
l = bv.as_long()
if l >> 31 == 1: # negative
l -= 4294967296
return l
from_bv32_converter_by_type = dict()
from_bv32_converter_by_type['bool'] = bv32_to_bool
from_bv32_converter_by_type['int'] = bv32_to_int
matching_path = True
for expected_variable, expected_values in oracle.items():
if expected_variable == 'reachable':
expected_reachable = set(expected_values)
if not (expected_reachable == reachable):
logger.info(
'labels \'{}\' executed while {} required'.format(
list(reachable), list(expected_reachable)))
matching_path = False
break
continue
if expected_variable not in outputs.keys():
outputs[expected_variable] = (
None, 0) # unconstraint does not mean wrong
required_executions = len(expected_values)
actual_executions = outputs[expected_variable][1]
if required_executions != actual_executions:
logger.info(
'value \'{}\' executed {} times while {} required'.
format(expected_variable, actual_executions,
required_executions))
matching_path = False
break
oracle_constraints[expected_variable] = []
for i in range(0, required_executions):
type = outputs[expected_variable][0]
try:
value = dump_parser_by_type[type](expected_values[i])
except:
logger.error(
'variable \'{}\' has incompatible type {}'.format(
expected_variable, type))
raise InferenceError()
oracle_constraints[expected_variable].append(value)
if not matching_path:
continue
solver.reset()
solver.add(path)
def array_to_bv32(array):
return Concat(Select(array, BitVecVal(3, 32)),
Select(array, BitVecVal(2, 32)),
Select(array, BitVecVal(1, 32)),
Select(array, BitVecVal(0, 32)))
def angelic_selector(expr, instance):
s = 'angelic!{}!{}!{}!{}!{}'.format(expr[0], expr[1], expr[2],
expr[3], instance)
return BitVec(s, 32)
def original_selector(expr, instance):
s = 'original!{}!{}!{}!{}!{}'.format(expr[0], expr[1], expr[2],
expr[3], instance)
return BitVec(s, 32)
def env_selector(expr, instance, name):
s = 'env!{}!{}!{}!{}!{}!{}'.format(name, expr[0], expr[1],
expr[2], expr[3], instance)
return BitVec(s, 32)
for name, values in oracle_constraints.items():
type, _ = outputs[name]
for i, value in enumerate(values):
array = self.output_variable(type, name, i)
bv_value = to_bv32_converter_by_type[type](value)
solver.add(bv_value == array_to_bv32(array))
for (expr, item) in choices.items():
type, instances, env = item
for instance in range(0, instances):
selector = angelic_selector(expr, instance)
array = self.angelic_variable(type, expr, instance)
solver.add(selector == array_to_bv32(array))
selector = original_selector(expr, instance)
array = self.original_variable(type, expr, instance)
solver.add(selector == array_to_bv32(array))
for name in env:
selector = env_selector(expr, instance, name)
env_type = 'int' #FIXME
array = self.env_variable(env_type, expr, instance,
name)
solver.add(selector == array_to_bv32(array))
result = solver.check()
if result != z3.sat:
logger.info('UNSAT')
continue
model = solver.model()
# store smt2 files
shutil.copy(smt, klee_dir)
# generate IO file
self.generate_IO_file(test, choices, oracle_constraints, outputs)
# expr -> (angelic * original * env) list
angelic_path = dict()
for (expr, item) in choices.items():
angelic_path[expr] = []
type, instances, env = item
for instance in range(0, instances):
bv_angelic = model[angelic_selector(expr, instance)]
angelic = from_bv32_converter_by_type[type](bv_angelic)
bv_original = model[original_selector(expr, instance)]
original = from_bv32_converter_by_type[type](bv_original)
if original_available:
logger.info(
'expression {}[{}]: angelic = {}, original = {}'.
format(expr, instance, angelic, original))
else:
logger.info('expression {}[{}]: angelic = {}'.format(
expr, instance, angelic))
env_values = dict()
for name in env:
bv_env = model[env_selector(expr, instance, name)]
value = from_bv32_converter_by_type['int'](bv_env)
env_values[name] = value
if original_available:
angelic_path[expr].append(
(angelic, original, env_values))
else:
angelic_path[expr].append((angelic, None, env_values))
# TODO: add constants to angelic path
angelic_paths.append(angelic_path)
# update IO files
for smt in glob(join(klee_dir, '*.smt2')):
with open(smt) as f_smt:
for line in f_smt.readlines():
if re.search("declare-fun [a-z]+!output!", line):
output_var = line.split(' ')[1]
output_var_type = output_var.split('!')[0]
for io_file in glob(join(test_dir, '*.IO')):
if not output_var in open(io_file).read():
try:
with open(io_file, "a") as f_io:
f_io.write("\n")
f_io.write("@output\n")
f_io.write(
'name {}\n'.format(output_var))
f_io.write('type {}\n'.format(
output_var_type))
except:
raise Exception(
"Error when updating io_file %s!\n" %
io_file)
finally:
f_io.close()
if self.config['max_angelic_paths'] is not None and \
len(angelic_paths) > self.config['max_angelic_paths']:
angelic_paths = self._reduce_angelic_forest(angelic_paths)
else:
logger.info('found {} angelic paths for test \'{}\''.format(
len(angelic_paths), test))
return angelic_paths
def parse_with_z3(file, out_dir, check_only, split_queries, simplify, skip_err,
datalog):
if check_only:
assertions = z3.parse_smt2_file(file)
check.check_chcs(assertions.children())
print("success")
return
lst = file.split('/')
tmp = lst.pop()
lst = tmp.split('.')
base_name = lst[0]
assert len(lst) > 0
if len(lst) > 1:
for stuff in lst[1:-1]:
base_name = base_name + '.' + stuff
# Check if the file is actually in datalog.
engine = z3.Fixedpoint()
engine.set("xform.inline_eager", simplify, "xform.inline_linear", simplify,
"xform.slice", simplify, "xform.coi", simplify,
"xform.compress_unbound", simplify, "xform.subsumption_checker",
simplify, "xform.tail_simplifier_pve", simplify)
try:
queries = engine.parse_file(file)
except z3.Z3Exception as e:
raise Exc('Parse error on file {}'.format(file))
assertions = engine.get_assertions()
for rule in engine.get_rules():
assertions.push(rule)
for query in queries:
assertions.push(z3.Implies(query, False))
# engine.get_assertions()
goals = z3.Goal()
goals.add(assertions)
#non_lin = z3.Probe('arith-max-deg')
#if non_lin(goals) > 1:
# raise Skip(
# 'found non-linear expressions'
# )
# if simplify:
tactic = z3.Tactic("horn-simplify")
simplified = tactic(goals, "xform.inline_eager", simplify,
"xform.inline_linear", simplify, "xform.slice",
simplify, "xform.coi", simplify,
"xform.compress_unbound", simplify,
"xform.subsumption_checker", simplify,
"xform.tail_simplifier_pve", simplify)
# else:
# simplified = [goals]
clauses = []
queries = []
if len(simplified) == 0:
#raise Skip(
# 'empty benchmark (possibly because of pre-processing)'
#)
print('')
pred_decls = set()
for index, clause in enumerate(simplified[0]):
try:
clause, is_query = fix.fix_clause(clause, pred_decls)
if is_query:
queries.append(clause)
else:
clauses.append(clause)
except Exc as e:
raise Exc('While fixing clause {}:\n{}'.format(index, e.value))
if len(queries) < 1:
# raise Skip('no query clause (possibly because of pre-processing)')
print('')
separated_clauses = []
if split_queries:
for cnt, query in enumerate(queries):
these_clauses = []
for clause in clauses:
these_clauses.append(clause)
these_clauses.append(query)
separated_clauses.append(these_clauses)
else:
for query in queries:
clauses.append(query)
separated_clauses.append(clauses)
cnt = 0
for clauses in separated_clauses:
if out_dir is not None:
out_file = "{}/{}_{:0>3}.smt2".format(out_dir, base_name, cnt)
cnt += 1
print('Writing to {}'.format(out_file))
writer = open(out_file, mode='w')
else:
writer = sys.stdout
if split_queries:
try:
check.check_chcs(clauses)
except Exc as e:
exc = Exc('Result of formatting is ill-formed:\n{}'.format(
e.value))
if skip_err:
eprint('Error on file {}'.format(file))
eprint(exc.value)
continue
else:
raise exc
if datalog:
write_clauses_datalog(pred_decls, clauses, writer)
else:
write_clauses_smt2(pred_decls, clauses, writer)
# args.smt2_file.close()
# with open('build/foo.c', 'w') as f:
# f.write(foo_dot_c)
# subprocess.call(['make','compile'])
# t_parse_and_compile=time.time()
# if args.genOnly: sys.exit(0)
with open ("XSAT_IN.txt") as f:
try:
expr_z3=z3.simplify(z3.parse_smt2_file(f.read().rstrip()))
except z3.Z3Exception:
sys.stderr.write("[Xsat] The Z3 fornt-end fails when verifying the model.\n")
with open ("build/foo.symbolTable","rb") as f:
symbolTable=pickle.load(f)
if len(symbolTable)==0:
print "sat"
sys.exit(0)
if not args.multi:
#round1
t_round1_start=time.time()
(X_star,R_star)=mcmc_round1(args)
t_round1_end=time.time()
assert len(sys.argv) >= 3, 'Not enough arguments provided'
def get_rlimit():
tmp = z3.Solver()
stats = tmp.statistics()
for i in range(len(stats)):
if stats[i][0] == 'rlimit count':
return stats[i][1]
return 0
# print 'Runner...'
# print sys.argv[1]
if sys.argv[1] == 'default':
solver = z3.Solver()
else:
solver = from_string(sys.argv[1]).tactic.solver()
smt_file = sys.argv[2]
f = z3.parse_smt2_file(smt_file)
solver.add(f)
rlimit_before = get_rlimit()
res = solver.check()
rlimit_after = get_rlimit()
print('{} {}'.format(res, rlimit_after - rlimit_before))
#!/usr/bin/env python3
import z3
import sys
ctx = z3.Context()
s = z3.Solver(ctx=ctx)
queries = z3.parse_smt2_file(sys.argv[1])
k = int(sys.argv[2])
outfile = sys.argv[3]
query = queries[k]
with open(outfile, "w") as fout:
fout.write("(assert\n" + query.sexpr() + "\n)\n")
def __call__(self, project, test, dump, validation_project):
logger.info('inferring specification for test \'{}\''.format(test))
environment = dict(os.environ)
if self.config['klee_max_forks'] is not None:
environment['ANGELIX_KLEE_MAX_FORKS'] = str(self.config['klee_max_forks'])
if self.config['klee_max_depth'] is not None:
environment['ANGELIX_KLEE_MAX_DEPTH'] = str(self.config['klee_max_depth'])
if self.config['klee_search'] is not None:
environment['ANGELIX_KLEE_SEARCH'] = self.config['klee_search']
if self.config['klee_timeout'] is not None:
environment['ANGELIX_KLEE_MAX_TIME'] = str(self.config['klee_timeout'])
if self.config['klee_solver_timeout'] is not None:
environment['ANGELIX_KLEE_MAX_SOLVER_TIME'] = str(self.config['klee_solver_timeout'])
if self.config['klee_debug']:
environment['ANGELIX_KLEE_DEBUG'] = 'YES'
if self.config['klee_ignore_errors']:
environment['KLEE_DISABLE_MEMORY_ERROR'] = 'YES'
if self.config['use_semfix_syn']:
environment['ANGELIX_USE_SEMFIX_SYN'] = 'YES'
environment['ANGELIX_KLEE_WORKDIR'] = project.dir
klee_start_time = time.time()
self.run_test(project, test, klee=True, env=environment)
klee_end_time = time.time()
klee_elapsed = klee_end_time - klee_start_time
statistics.data['time']['klee'] += klee_elapsed
statistics.save()
logger.info('sleeping for 1 second...')
time.sleep(1)
smt_glob = join(project.dir, 'klee-out-0', '*.smt2')
smt_files = glob(smt_glob)
err_glob = join(project.dir, 'klee-out-0', '*.err')
err_files = glob(err_glob)
err_list = []
for err in err_files:
err_list.append(os.path.basename(err).split('.')[0])
non_error_smt_files = []
for smt in smt_files:
smt_id = os.path.basename(smt).split('.')[0]
if not smt_id in err_list:
non_error_smt_files.append(smt)
if not self.config['ignore_infer_errors']:
smt_files = non_error_smt_files
if len(smt_files) == 0 and len(err_list) == 0:
logger.warning('No paths explored')
raise NoSmtError()
if len(smt_files) == 0:
logger.warning('No non-error paths explored')
raise NoSmtError()
# loading dump
# name -> value list
oracle = dict()
vars = os.listdir(dump)
for var in vars:
instances = os.listdir(join(dump, var))
for i in range(0, len(instances)):
if str(i) not in instances:
logger.error('corrupted dump for test \'{}\''.format(test))
raise InferenceError()
oracle[var] = []
for i in range(0, len(instances)):
file = join(dump, var, str(i))
with open(file) as f:
content = f.read()
oracle[var].append(content)
# solving path constraints
inference_start_time = time.time()
angelic_paths = []
z3.set_param("timeout", self.config['path_solving_timeout'])
solver = Solver()
for smt in smt_files:
logger.info('solving path {}'.format(relpath(smt)))
try:
path = z3.parse_smt2_file(smt)
except:
logger.warning('failed to parse {}'.format(smt))
continue
variables = [str(var) for var in get_vars(path)
if str(var).startswith('int!')
or str(var).startswith('long!')
or str(var).startswith('bool!')
or str(var).startswith('char!')
or str(var).startswith('reachable!')]
try:
outputs, choices, constants, reachable, original_available = parse_variables(variables)
except:
continue
# name -> value list (parsed)
oracle_constraints = dict()
def str_to_int(s):
return int(s)
def str_to_long(s):
return int(s)
def str_to_bool(s):
if s == 'false':
return False
if s == 'true':
return True
raise InferenceError()
def str_to_char(s):
if len(s) != 1:
raise InferenceError()
return s[0]
dump_parser_by_type = dict()
dump_parser_by_type['int'] = str_to_int
dump_parser_by_type['long'] = str_to_long
dump_parser_by_type['bool'] = str_to_bool
dump_parser_by_type['char'] = str_to_char
def bool_to_bv(b):
if b:
return BitVecVal(1, 32)
else:
return BitVecVal(0, 32)
def int_to_bv(i):
return BitVecVal(i, 32)
def long_to_bv(i):
return BitVecVal(i, 64)
def char_to_bv(c):
return BitVecVal(ord(c), 32)
to_bv_converter_by_type = dict()
to_bv_converter_by_type['bool'] = bool_to_bv
to_bv_converter_by_type['int'] = int_to_bv
to_bv_converter_by_type['long'] = long_to_bv
to_bv_converter_by_type['char'] = char_to_bv
def bv_to_bool(bv):
return bv.as_long() != 0
def bv_to_int(bv):
l = bv.as_long()
if l >> 31 == 1: # negative
l -= pow(2, 32)
return l
def bv_to_long(bv):
l = bv.as_long()
if l >> 63 == 1: # negative
l -= pow(2, 64)
return l
def bv_to_char(bv):
l = bv.as_long()
return chr(l)
from_bv_converter_by_type = dict()
from_bv_converter_by_type['bool'] = bv_to_bool
from_bv_converter_by_type['int'] = bv_to_int
from_bv_converter_by_type['long'] = bv_to_long
from_bv_converter_by_type['char'] = bv_to_char
matching_path = True
for expected_variable, expected_values in oracle.items():
if expected_variable == 'reachable':
expected_reachable = set(expected_values)
if not (expected_reachable == reachable):
logger.info('labels \'{}\' executed while {} required'.format(
list(reachable),
list(expected_reachable)))
matching_path = False
break
continue
if expected_variable not in outputs.keys():
outputs[expected_variable] = (None, 0) # unconstraint does not mean wrong
required_executions = len(expected_values)
actual_executions = outputs[expected_variable][1]
if required_executions != actual_executions:
logger.info('value \'{}\' executed {} times while {} required'.format(
expected_variable,
actual_executions,
required_executions))
matching_path = False
break
oracle_constraints[expected_variable] = []
for i in range(0, required_executions):
type = outputs[expected_variable][0]
try:
value = dump_parser_by_type[type](expected_values[i])
except:
logger.error('variable \'{}\' has incompatible type {}'.format(expected_variable,
type))
raise InferenceError()
oracle_constraints[expected_variable].append(value)
if not matching_path:
continue
solver.reset()
solver.add(path)
def array_to_bv32(array):
return Concat(Select(array, BitVecVal(3, 32)),
Select(array, BitVecVal(2, 32)),
Select(array, BitVecVal(1, 32)),
Select(array, BitVecVal(0, 32)))
def array_to_bv64(array):
return Concat(Select(array, BitVecVal(7, 32)),
Select(array, BitVecVal(6, 32)),
Select(array, BitVecVal(5, 32)),
Select(array, BitVecVal(4, 32)),
Select(array, BitVecVal(3, 32)),
Select(array, BitVecVal(2, 32)),
Select(array, BitVecVal(1, 32)),
Select(array, BitVecVal(0, 32)))
def angelic_variable(type, expr, instance):
pattern = '{}!choice!{}!{}!{}!{}!{}!angelic'
s = pattern.format(type, expr[0], expr[1], expr[2], expr[3], instance)
return Array(s, BitVecSort(32), BitVecSort(8))
def original_variable(type, expr, instance):
pattern = '{}!choice!{}!{}!{}!{}!{}!original'
s = pattern.format(type, expr[0], expr[1], expr[2], expr[3], instance)
return Array(s, BitVecSort(32), BitVecSort(8))
def env_variable(expr, instance, name):
pattern = 'int!choice!{}!{}!{}!{}!{}!env!{}'
s = pattern.format(expr[0], expr[1], expr[2], expr[3], instance, name)
return Array(s, BitVecSort(32), BitVecSort(8))
def output_variable(type, name, instance):
s = '{}!output!{}!{}'.format(type, name, instance)
if type == 'long':
return Array(s, BitVecSort(32), BitVecSort(8))
else:
return Array(s, BitVecSort(32), BitVecSort(8))
def angelic_selector(expr, instance):
s = 'angelic!{}!{}!{}!{}!{}'.format(expr[0], expr[1], expr[2], expr[3], instance)
return BitVec(s, 32)
def original_selector(expr, instance):
s = 'original!{}!{}!{}!{}!{}'.format(expr[0], expr[1], expr[2], expr[3], instance)
return BitVec(s, 32)
def env_selector(expr, instance, name):
s = 'env!{}!{}!{}!{}!{}!{}'.format(name, expr[0], expr[1], expr[2], expr[3], instance)
return BitVec(s, 32)
for name, values in oracle_constraints.items():
type, _ = outputs[name]
for i, value in enumerate(values):
array = output_variable(type, name, i)
bv_value = to_bv_converter_by_type[type](value)
if type == 'long':
solver.add(bv_value == array_to_bv64(array))
else:
solver.add(bv_value == array_to_bv32(array))
for (expr, item) in choices.items():
type, instances, env = item
for instance in range(0, instances):
selector = angelic_selector(expr, instance)
array = angelic_variable(type, expr, instance)
solver.add(selector == array_to_bv32(array))
selector = original_selector(expr, instance)
array = original_variable(type, expr, instance)
solver.add(selector == array_to_bv32(array))
for name in env:
selector = env_selector(expr, instance, name)
array = env_variable(expr, instance, name)
solver.add(selector == array_to_bv32(array))
result = solver.check()
if result != z3.sat:
logger.info('UNSAT') # TODO: can be timeout
continue
model = solver.model()
# expr -> (angelic * original * env) list
angelic_path = dict()
if os.path.exists(self.load[test]):
shutil.rmtree(self.load[test])
os.mkdir(self.load[test])
for (expr, item) in choices.items():
angelic_path[expr] = []
type, instances, env = item
expr_str = '{}-{}-{}-{}'.format(expr[0], expr[1], expr[2], expr[3])
expression_dir = join(self.load[test], expr_str)
if not os.path.exists(expression_dir):
os.mkdir(expression_dir)
for instance in range(0, instances):
bv_angelic = model[angelic_selector(expr, instance)]
angelic = from_bv_converter_by_type[type](bv_angelic)
bv_original = model[original_selector(expr, instance)]
original = from_bv_converter_by_type[type](bv_original)
if original_available:
logger.info('expression {}[{}]: angelic = {}, original = {}'.format(expr,
instance,
angelic,
original))
else:
logger.info('expression {}[{}]: angelic = {}'.format(expr,
instance,
angelic))
env_values = dict()
for name in env:
bv_env = model[env_selector(expr, instance, name)]
value = from_bv_converter_by_type['int'](bv_env)
env_values[name] = value
if original_available:
angelic_path[expr].append((angelic, original, env_values))
else:
angelic_path[expr].append((angelic, None, env_values))
# Dump angelic path to dump folder
instance_file = join(expression_dir, str(instance))
with open(instance_file, 'w') as file:
if isinstance(angelic, bool):
if angelic:
file.write('1')
else:
file.write('0')
else:
file.write(str(angelic))
# Run Tester to validate the dumped values
validated = self.run_test(validation_project, test, load=self.load[test])
if validated:
angelic_paths.append(angelic_path)
else:
logger.info('spurious angelic path')
if self.config['synthesis_bool_only']:
angelic_paths = self._boolean_angelic_forest(angelic_paths)
if self.config['max_angelic_paths'] is not None and \
len(angelic_paths) > self.config['max_angelic_paths']:
angelic_paths = self._reduce_angelic_forest(angelic_paths)
else:
logger.info('found {} angelic paths for test \'{}\''.format(len(angelic_paths), test))
inference_end_time = time.time()
inference_elapsed = inference_end_time - inference_start_time
statistics.data['time']['inference'] += inference_elapsed
iter_stat = dict()
iter_stat['time'] = dict()
iter_stat['time']['klee'] = klee_elapsed
iter_stat['time']['inference'] = inference_elapsed
iter_stat['paths'] = dict()
iter_stat['paths']['explored'] = len(smt_files)
iter_stat['paths']['angelic'] = len(angelic_paths)
statistics.data['iterations']['klee'].append(iter_stat)
statistics.save()
return angelic_paths
import sys
import z3
def usage():
print("%s <query.smt2> <id1> [<id2> ... <idn>]" % sys.argv[0])
exit(1)
if len(sys.argv) < 3:
usage()
query_f = sys.argv[1]
ids = list(map(int, sys.argv[2:]))
query = z3.parse_smt2_file(query_f)
bvs = [z3.BitVec("k!%d" % i, 8) for i in ids]
bv = bvs[0]
for i in range(1, len(bvs)):
bv = z3.Concat(bvs[i], bv)
s = z3.Solver()
s.add(query)
# import IPython; IPython.embed()
j = 1
while s.check().r == 1:
m = s.model()
v = m.eval(bv, model_completion=True)
def find_addresses(filename):
fin = open(filename, "r")
addresses = []
for line in fin:
if line.startswith("; ADDRESS "):
address = int(line.strip().split(" ")[2], 16)
addresses.append(address)
fin.close()
return addresses
smt_file = sys.argv[1]
directory = sys.argv[2]
queries = z3.parse_smt2_file(smt_file)
addresses = find_addresses(smt_file)
has_address = len(addresses) > 0
if not has_address:
print("WARNING: no addresses")
else:
assert len(addresses) == len(queries)
if not os.path.exists(directory):
os.mkdir(directory)
i = 0
for query in queries:
# query = deduplicate_query(query)
filename = "query_%03d_@_0x%x.smt2" % (i,
def __call__(self, project, test, dump):
logger.info('inferring specification for test \'{}\''.format(test))
environment = dict(os.environ)
if self.config['klee_max_forks'] is not None:
environment['ANGELIX_KLEE_MAX_FORKS'] = str(self.config['klee_max_forks'])
if self.config['klee_max_depth'] is not None:
environment['ANGELIX_KLEE_MAX_DEPTH'] = str(self.config['klee_max_depth'])
if self.config['klee_search'] is not None:
environment['ANGELIX_KLEE_SEARCH'] = self.config['klee_search']
if self.config['klee_timeout'] is not None:
environment['ANGELIX_KLEE_MAX_TIME'] = str(self.config['klee_timeout'])
if self.config['klee_solver_timeout'] is not None:
environment['ANGELIX_KLEE_MAX_SOLVER_TIME'] = str(self.config['klee_solver_timeout'])
if self.config['klee_debug']:
environment['ANGELIX_KLEE_DEBUG'] = 'YES'
if self.config['klee_ignore_errors']:
environment['KLEE_DISABLE_MEMORY_ERROR'] = 'YES'
if self.config['use_semfix_syn']:
environment['ANGELIX_USE_SEMFIX_SYN'] = 'YES'
environment['ANGELIX_KLEE_WORKDIR'] = project.dir
test_dir = self.get_test_dir(test)
shutil.rmtree(test_dir, ignore_errors='true')
klee_dir = join(test_dir, 'klee')
os.makedirs(klee_dir)
self.run_test(project, test, klee=True, env=environment)
# loading dump
# name -> value list
oracle = dict()
vars = os.listdir(dump)
for var in vars:
instances = os.listdir(join(dump, var))
for i in range(0, len(instances)):
if str(i) not in instances:
logger.error('corrupted dump for test \'{}\''.format(test))
raise InferenceError()
oracle[var] = []
for i in range(0, len(instances)):
file = join(dump, var, str(i))
with open(file) as f:
content = f.read()
oracle[var].append(content)
# solving path constraints
angelic_paths = []
solver = Solver()
smt_glob = join(project.dir, 'klee-out-0', '*.smt2')
smt_files = glob(smt_glob)
for smt in smt_files:
logger.info('solving path {}'.format(relpath(smt)))
try:
path = z3.parse_smt2_file(smt)
except:
logger.warning('failed to parse {}'.format(smt))
continue
variables = [str(var) for var in get_vars(path)
if str(var).startswith('int!')
or str(var).startswith('bool!')
or str(var).startswith('char!')
or str(var).startswith('reachable!')]
outputs, choices, constants, reachable, original_available = parse_variables(variables)
# name -> value list (parsed)
oracle_constraints = dict()
def str_to_int(s):
return int(s)
def str_to_bool(s):
if s == 'false':
return False
if s == 'true':
return True
raise InferenceError()
def str_to_char(s):
if len(s) != 1:
raise InferenceError()
return s[0]
dump_parser_by_type = dict()
dump_parser_by_type['int'] = str_to_int
dump_parser_by_type['bool'] = str_to_bool
dump_parser_by_type['char'] = str_to_char
def bool_to_bv32(b):
if b:
return BitVecVal(1, 32)
else:
return BitVecVal(0, 32)
def int_to_bv32(i):
return BitVecVal(i, 32)
to_bv32_converter_by_type = dict()
to_bv32_converter_by_type['bool'] = bool_to_bv32
to_bv32_converter_by_type['int'] = int_to_bv32
def bv32_to_bool(bv):
return bv.as_long() != 0
def bv32_to_int(bv):
l = bv.as_long()
if l >> 31 == 1: # negative
l -= 4294967296
return l
from_bv32_converter_by_type = dict()
from_bv32_converter_by_type['bool'] = bv32_to_bool
from_bv32_converter_by_type['int'] = bv32_to_int
matching_path = True
for expected_variable, expected_values in oracle.items():
if expected_variable == 'reachable':
expected_reachable = set(expected_values)
if not (expected_reachable == reachable):
logger.info('labels \'{}\' executed while {} required'.format(
list(reachable),
list(expected_reachable)))
matching_path = False
break
continue
if expected_variable not in outputs.keys():
outputs[expected_variable] = (None, 0) # unconstraint does not mean wrong
required_executions = len(expected_values)
actual_executions = outputs[expected_variable][1]
if required_executions != actual_executions:
logger.info('value \'{}\' executed {} times while {} required'.format(
expected_variable,
actual_executions,
required_executions))
matching_path = False
break
oracle_constraints[expected_variable] = []
for i in range(0, required_executions):
type = outputs[expected_variable][0]
try:
value = dump_parser_by_type[type](expected_values[i])
except:
logger.error('variable \'{}\' has incompatible type {}'.format(expected_variable,
type))
raise InferenceError()
oracle_constraints[expected_variable].append(value)
if not matching_path:
continue
solver.reset()
solver.add(path)
def array_to_bv32(array):
return Concat(Select(array, BitVecVal(3, 32)),
Select(array, BitVecVal(2, 32)),
Select(array, BitVecVal(1, 32)),
Select(array, BitVecVal(0, 32)))
def angelic_selector(expr, instance):
s = 'angelic!{}!{}!{}!{}!{}'.format(expr[0], expr[1], expr[2], expr[3], instance)
return BitVec(s, 32)
def original_selector(expr, instance):
s = 'original!{}!{}!{}!{}!{}'.format(expr[0], expr[1], expr[2], expr[3], instance)
return BitVec(s, 32)
def env_selector(expr, instance, name):
s = 'env!{}!{}!{}!{}!{}!{}'.format(name, expr[0], expr[1], expr[2], expr[3], instance)
return BitVec(s, 32)
for name, values in oracle_constraints.items():
type, _ = outputs[name]
for i, value in enumerate(values):
array = self.output_variable(type, name, i)
bv_value = to_bv32_converter_by_type[type](value)
solver.add(bv_value == array_to_bv32(array))
for (expr, item) in choices.items():
type, instances, env = item
for instance in range(0, instances):
selector = angelic_selector(expr, instance)
array = self.angelic_variable(type, expr, instance)
solver.add(selector == array_to_bv32(array))
selector = original_selector(expr, instance)
array = self.original_variable(type, expr, instance)
solver.add(selector == array_to_bv32(array))
for name in env:
selector = env_selector(expr, instance, name)
env_type = 'int' #FIXME
array = self.env_variable(env_type, expr, instance, name)
solver.add(selector == array_to_bv32(array))
result = solver.check()
if result != z3.sat:
logger.info('UNSAT')
continue
model = solver.model()
# store smt2 files
shutil.copy(smt, klee_dir)
# generate IO file
self.generate_IO_file(test, choices, oracle_constraints, outputs)
# expr -> (angelic * original * env) list
angelic_path = dict()
for (expr, item) in choices.items():
angelic_path[expr] = []
type, instances, env = item
for instance in range(0, instances):
bv_angelic = model[angelic_selector(expr, instance)]
angelic = from_bv32_converter_by_type[type](bv_angelic)
bv_original = model[original_selector(expr, instance)]
original = from_bv32_converter_by_type[type](bv_original)
if original_available:
logger.info('expression {}[{}]: angelic = {}, original = {}'.format(expr,
instance,
angelic,
original))
else:
logger.info('expression {}[{}]: angelic = {}'.format(expr,
instance,
angelic))
env_values = dict()
for name in env:
bv_env = model[env_selector(expr, instance, name)]
value = from_bv32_converter_by_type['int'](bv_env)
env_values[name] = value
if original_available:
angelic_path[expr].append((angelic, original, env_values))
else:
angelic_path[expr].append((angelic, None, env_values))
# TODO: add constants to angelic path
angelic_paths.append(angelic_path)
# update IO files
for smt in glob(join(klee_dir, '*.smt2')):
with open(smt) as f_smt:
for line in f_smt.readlines():
if re.search("declare-fun [a-z]+!output!", line):
output_var = line.split(' ')[1]
output_var_type = output_var.split('!')[0]
for io_file in glob(join(test_dir, '*.IO')):
if not output_var in open(io_file).read():
with open(io_file, "a") as f_io:
f_io.write("\n")
f_io.write("@output\n")
f_io.write('name {}\n'.format(output_var))
f_io.write('type {}\n'.format(output_var_type))
if self.config['max_angelic_paths'] is not None and \
len(angelic_paths) > self.config['max_angelic_paths']:
angelic_paths = self._reduce_angelic_forest(angelic_paths)
else:
logger.info('found {} angelic paths for test \'{}\''.format(len(angelic_paths), test))
return angelic_paths
def solve(): f = z3.parse_smt2_file(OUTPUT_PATH) s = z3.Solver() s.add(f) return (s, str(s.check()))
#!/usr/bin/python
import sys
import z3
if len(sys.argv) < 2:
print("Usage: {} <smt2 file>".format(sys.argv[0]))
quit()
s = z3.Solver()
f = z3.parse_smt2_file(sys.argv[1])
print(f)
def __call__(self, project, test, dump, validation_project):
logger.info('inferring specification for test \'{}\''.format(test))
environment = dict(os.environ)
if self.config['klee_max_forks'] is not None:
environment['ANGELIX_KLEE_MAX_FORKS'] = str(
self.config['klee_max_forks'])
if self.config['klee_max_depth'] is not None:
environment['ANGELIX_KLEE_MAX_DEPTH'] = str(
self.config['klee_max_depth'])
if self.config['klee_search'] is not None:
environment['ANGELIX_KLEE_SEARCH'] = self.config['klee_search']
if self.config['klee_timeout'] is not None:
environment['ANGELIX_KLEE_MAX_TIME'] = str(
self.config['klee_timeout'])
if self.config['klee_solver_timeout'] is not None:
environment['ANGELIX_KLEE_MAX_SOLVER_TIME'] = str(
self.config['klee_solver_timeout'])
if self.config['klee_debug']:
environment['ANGELIX_KLEE_DEBUG'] = 'YES'
if self.config['klee_ignore_errors']:
environment['KLEE_DISABLE_MEMORY_ERROR'] = 'YES'
if self.config['use_semfix_syn']:
environment['ANGELIX_USE_SEMFIX_SYN'] = 'YES'
environment['ANGELIX_KLEE_WORKDIR'] = project.dir
klee_start_time = time.time()
self.run_test(project, test, klee=True, env=environment)
klee_end_time = time.time()
klee_elapsed = klee_end_time - klee_start_time
statistics.data['time']['klee'] += klee_elapsed
statistics.save()
logger.info('sleeping for 1 second...')
time.sleep(1)
smt_glob = join(project.dir, 'klee-out-0', '*.smt2')
smt_files = glob(smt_glob)
err_glob = join(project.dir, 'klee-out-0', '*.err')
err_files = glob(err_glob)
err_list = []
for err in err_files:
err_list.append(os.path.basename(err).split('.')[0])
non_error_smt_files = []
for smt in smt_files:
smt_id = os.path.basename(smt).split('.')[0]
if not smt_id in err_list:
non_error_smt_files.append(smt)
if not self.config['ignore_infer_errors']:
smt_files = non_error_smt_files
if len(smt_files) == 0 and len(err_list) == 0:
logger.warning('No paths explored')
raise NoSmtError()
if len(smt_files) == 0:
logger.warning('No non-error paths explored')
raise NoSmtError()
# loading dump
# name -> value list
oracle = dict()
vars = os.listdir(dump)
for var in vars:
instances = os.listdir(join(dump, var))
for i in range(0, len(instances)):
if str(i) not in instances:
logger.error('corrupted dump for test \'{}\''.format(test))
raise InferenceError()
oracle[var] = []
for i in range(0, len(instances)):
file = join(dump, var, str(i))
with open(file) as f:
content = f.read()
oracle[var].append(content)
# solving path constraints
inference_start_time = time.time()
angelic_paths = []
z3.set_param("timeout", self.config['path_solving_timeout'])
solver = Solver()
for smt in smt_files:
logger.info('solving path {}'.format(relpath(smt)))
try:
path = z3.parse_smt2_file(smt)
except:
logger.warning('failed to parse {}'.format(smt))
continue
variables = [
str(var) for var in get_vars(path)
if str(var).startswith('int!') or str(var).startswith('long!')
or str(var).startswith('bool!') or str(var).startswith('char!')
or str(var).startswith('reachable!')
]
try:
outputs, choices, constants, reachable, original_available = parse_variables(
variables)
except:
continue
# name -> value list (parsed)
oracle_constraints = dict()
def str_to_int(s):
return int(s)
def str_to_long(s):
return int(s)
def str_to_bool(s):
if s == 'false':
return False
if s == 'true':
return True
raise InferenceError()
def str_to_char(s):
if len(s) != 1:
raise InferenceError()
return s[0]
dump_parser_by_type = dict()
dump_parser_by_type['int'] = str_to_int
dump_parser_by_type['long'] = str_to_long
dump_parser_by_type['bool'] = str_to_bool
dump_parser_by_type['char'] = str_to_char
def bool_to_bv(b):
if b:
return BitVecVal(1, 32)
else:
return BitVecVal(0, 32)
def int_to_bv(i):
return BitVecVal(i, 32)
def long_to_bv(i):
return BitVecVal(i, 64)
def char_to_bv(c):
return BitVecVal(ord(c), 32)
to_bv_converter_by_type = dict()
to_bv_converter_by_type['bool'] = bool_to_bv
to_bv_converter_by_type['int'] = int_to_bv
to_bv_converter_by_type['long'] = long_to_bv
to_bv_converter_by_type['char'] = char_to_bv
def bv_to_bool(bv):
return bv.as_long() != 0
def bv_to_int(bv):
l = bv.as_long()
if l >> 31 == 1: # negative
l -= pow(2, 32)
return l
def bv_to_long(bv):
l = bv.as_long()
if l >> 63 == 1: # negative
l -= pow(2, 64)
return l
def bv_to_char(bv):
l = bv.as_long()
return chr(l)
from_bv_converter_by_type = dict()
from_bv_converter_by_type['bool'] = bv_to_bool
from_bv_converter_by_type['int'] = bv_to_int
from_bv_converter_by_type['long'] = bv_to_long
from_bv_converter_by_type['char'] = bv_to_char
matching_path = True
for expected_variable, expected_values in oracle.items():
if expected_variable == 'reachable':
expected_reachable = set(expected_values)
if not (expected_reachable == reachable):
logger.info(
'labels \'{}\' executed while {} required'.format(
list(reachable), list(expected_reachable)))
matching_path = False
break
continue
if expected_variable not in outputs.keys():
outputs[expected_variable] = (
None, 0) # unconstraint does not mean wrong
required_executions = len(expected_values)
actual_executions = outputs[expected_variable][1]
if required_executions != actual_executions:
logger.info(
'value \'{}\' executed {} times while {} required'.
format(expected_variable, actual_executions,
required_executions))
matching_path = False
break
oracle_constraints[expected_variable] = []
for i in range(0, required_executions):
type = outputs[expected_variable][0]
try:
value = dump_parser_by_type[type](expected_values[i])
except:
logger.error(
'variable \'{}\' has incompatible type {}'.format(
expected_variable, type))
raise InferenceError()
oracle_constraints[expected_variable].append(value)
if not matching_path:
continue
solver.reset()
solver.add(path)
def array_to_bv32(array):
return Concat(Select(array, BitVecVal(3, 32)),
Select(array, BitVecVal(2, 32)),
Select(array, BitVecVal(1, 32)),
Select(array, BitVecVal(0, 32)))
def array_to_bv64(array):
return Concat(Select(array, BitVecVal(7, 32)),
Select(array, BitVecVal(6, 32)),
Select(array, BitVecVal(5, 32)),
Select(array, BitVecVal(4, 32)),
Select(array, BitVecVal(3, 32)),
Select(array, BitVecVal(2, 32)),
Select(array, BitVecVal(1, 32)),
Select(array, BitVecVal(0, 32)))
def angelic_variable(type, expr, instance):
pattern = '{}!choice!{}!{}!{}!{}!{}!angelic'
s = pattern.format(type, expr[0], expr[1], expr[2], expr[3],
instance)
return Array(s, BitVecSort(32), BitVecSort(8))
def original_variable(type, expr, instance):
pattern = '{}!choice!{}!{}!{}!{}!{}!original'
s = pattern.format(type, expr[0], expr[1], expr[2], expr[3],
instance)
return Array(s, BitVecSort(32), BitVecSort(8))
def env_variable(expr, instance, name):
pattern = 'int!choice!{}!{}!{}!{}!{}!env!{}'
s = pattern.format(expr[0], expr[1], expr[2], expr[3],
instance, name)
return Array(s, BitVecSort(32), BitVecSort(8))
def output_variable(type, name, instance):
s = '{}!output!{}!{}'.format(type, name, instance)
if type == 'long':
return Array(s, BitVecSort(32), BitVecSort(8))
else:
return Array(s, BitVecSort(32), BitVecSort(8))
def angelic_selector(expr, instance):
s = 'angelic!{}!{}!{}!{}!{}'.format(expr[0], expr[1], expr[2],
expr[3], instance)
return BitVec(s, 32)
def original_selector(expr, instance):
s = 'original!{}!{}!{}!{}!{}'.format(expr[0], expr[1], expr[2],
expr[3], instance)
return BitVec(s, 32)
def env_selector(expr, instance, name):
s = 'env!{}!{}!{}!{}!{}!{}'.format(name, expr[0], expr[1],
expr[2], expr[3], instance)
return BitVec(s, 32)
for name, values in oracle_constraints.items():
type, _ = outputs[name]
for i, value in enumerate(values):
array = output_variable(type, name, i)
bv_value = to_bv_converter_by_type[type](value)
if type == 'long':
solver.add(bv_value == array_to_bv64(array))
else:
solver.add(bv_value == array_to_bv32(array))
for (expr, item) in choices.items():
type, instances, env = item
for instance in range(0, instances):
selector = angelic_selector(expr, instance)
array = angelic_variable(type, expr, instance)
solver.add(selector == array_to_bv32(array))
selector = original_selector(expr, instance)
array = original_variable(type, expr, instance)
solver.add(selector == array_to_bv32(array))
for name in env:
selector = env_selector(expr, instance, name)
array = env_variable(expr, instance, name)
solver.add(selector == array_to_bv32(array))
result = solver.check()
if result != z3.sat:
logger.info('UNSAT') # TODO: can be timeout
continue
model = solver.model()
# expr -> (angelic * original * env) list
angelic_path = dict()
if os.path.exists(self.load[test]):
shutil.rmtree(self.load[test])
os.mkdir(self.load[test])
for (expr, item) in choices.items():
angelic_path[expr] = []
type, instances, env = item
expr_str = '{}-{}-{}-{}'.format(expr[0], expr[1], expr[2],
expr[3])
expression_dir = join(self.load[test], expr_str)
if not os.path.exists(expression_dir):
os.mkdir(expression_dir)
for instance in range(0, instances):
bv_angelic = model[angelic_selector(expr, instance)]
angelic = from_bv_converter_by_type[type](bv_angelic)
bv_original = model[original_selector(expr, instance)]
original = from_bv_converter_by_type[type](bv_original)
if original_available:
logger.info(
'expression {}[{}]: angelic = {}, original = {}'.
format(expr, instance, angelic, original))
else:
logger.info('expression {}[{}]: angelic = {}'.format(
expr, instance, angelic))
env_values = dict()
for name in env:
bv_env = model[env_selector(expr, instance, name)]
value = from_bv_converter_by_type['int'](bv_env)
env_values[name] = value
if original_available:
angelic_path[expr].append(
(angelic, original, env_values))
else:
angelic_path[expr].append((angelic, None, env_values))
# Dump angelic path to dump folder
instance_file = join(expression_dir, str(instance))
with open(instance_file, 'w') as file:
if isinstance(angelic, bool):
if angelic:
file.write('1')
else:
file.write('0')
else:
file.write(str(angelic))
# Run Tester to validate the dumped values
validated = self.run_test(validation_project,
test,
load=self.load[test])
if validated:
angelic_paths.append(angelic_path)
else:
logger.info('spurious angelic path')
if self.config['synthesis_bool_only']:
angelic_paths = self._boolean_angelic_forest(angelic_paths)
if self.config['max_angelic_paths'] is not None and \
len(angelic_paths) > self.config['max_angelic_paths']:
angelic_paths = self._reduce_angelic_forest(angelic_paths)
else:
logger.info('found {} angelic paths for test \'{}\''.format(
len(angelic_paths), test))
inference_end_time = time.time()
inference_elapsed = inference_end_time - inference_start_time
statistics.data['time']['inference'] += inference_elapsed
iter_stat = dict()
iter_stat['time'] = dict()
iter_stat['time']['klee'] = klee_elapsed
iter_stat['time']['inference'] = inference_elapsed
iter_stat['paths'] = dict()
iter_stat['paths']['explored'] = len(smt_files)
iter_stat['paths']['angelic'] = len(angelic_paths)
statistics.data['iterations']['klee'].append(iter_stat)
statistics.save()
return angelic_paths
return set(inputs)
if len(sys.argv) < 3:
print("USAGE: %s directory smt_file" % sys.argv[0])
exit(1)
directory = sys.argv[1]
smt_file = sys.argv[2]
if not os.path.exists(directory):
print("ERROR: %s does not exist" % directory)
exit(1)
inputs = set()
res = ""
for filename in os.listdir(directory):
filename = directory + "/" + filename
if filename.endswith(".smt2"):
print(filename)
queries = z3.parse_smt2_file(filename)
for query in queries:
res += "(assert \n" + query.sexpr() + "\n)\n"
inputs |= find_inputs(query)
fout = open(smt_file, "w")
for inp in inputs:
fout.write("(declare-const %s (_ BitVec 8))\n" % inp)
fout.write(res)
fout.close()
def main():
parser = argparse.ArgumentParser(description="Solve given formula.")
parser.add_argument("formula", metavar="formula_file", type=str,
help="path to .smt2 formula file")
parser.add_argument("-r", "--reduction", default="0", type=str,
help="determine the reduction type (0/1/s)")
args = parser.parse_args()
# Determine the type of reduction
if args.reduction == "1":
reduction_type = ReductionType.ONE_EXTENSION
elif args.reduction == "s":
reduction_type = ReductionType.SIGN_EXTENSION
else:
reduction_type = ReductionType.ZERO_EXTENSION
# File with .smt2 formula
formula_file = args.formula
# Parse SMT2 formula to Z3 format
formula = z3.parse_smt2_file(formula_file)
# Parallel run of original and approximated formula
with multiprocessing.Manager() as manager:
result_queue = multiprocessing.Queue()
# ORIGINAL FORMULA: Create process
p0 = multiprocessing.Process(target=solve_without_approx,
args=(formula, result_queue))
# APPROXIMATED FORMULA - Over-approximation: Create process
p1 = multiprocessing.Process(target=solve_with_approx,
args=(formula,
reduction_type,
Quantification.UNIVERSAL,
1,
Polarity.POSITIVE,
result_queue))
# APPROXIMATED FORMULA - Under-approximation: Create process
p2 = multiprocessing.Process(target=solve_with_approx,
args=(formula,
reduction_type,
Quantification.EXISTENTIAL,
1,
Polarity.POSITIVE,
result_queue))
# Start all
p0.start()
p1.start()
p2.start()
# Get result
try:
# Wait at most 60 seconds for a return
result = result_queue.get(timeout=300)
except multiprocessing.queues.Empty:
# If queue is empty, set result to undef
result = z3.CheckSatResult(z3.Z3_L_UNDEF)
# Terminate all
p0.terminate()
p1.terminate()
p2.terminate()
print(result)
return result
