def test_parse_len_old(self):
sample = """
In:
state foo : Bag<Int>
query fooLen()
sum [1 | _ <- foo]
"""
parse_spec(sample)
def test_parse_len(self):
sample = """
In:
state foo : Bag<Int>
query fooLen()
len foo
"""
parse_spec(sample)
def test_guard_mechanism(self):
sample = """
Test:
state foo : Bag<Int>
op add_2x_unique(i : Int)
if not (i + i) in foo {
foo.add(i + i);
}
"""
parse_spec(sample)
def test_parse_method_call_with_expr(self):
sample = """
Test:
state foo : Bag<Int>
op add2x(i : Int)
foo.add(i + i);
op add3x(i : Int)
foo.add(i + i + i);
op addIncremented(i : Int)
foo.add(i + 1);
op addNegative(i : Int)
foo.add(0 - i);
"""
parse_spec(sample)
def test_bag_elimination(self):
spec = """
MyDataStructure:
state elements : Bag<Int>
query containsZero()
exists [x | x <- elements, x == 0]
op addElement(x : Int)
elements.add(x);
"""
spec = parse_spec(spec)
errs = typecheck(spec)
assert not errs, str(errs)
spec = desugar(spec)
impl = construct_initial_implementation(spec)
impl = improve_implementation(impl, timeout=datetime.timedelta(seconds=30))
print(pprint(impl.code))
assert len(impl.concretization_functions) == 1
(v, e), = list(impl.concretization_functions.items())
print("{} = {}".format(v, pprint(e)))
assert e.type == BOOL
def test_bag_elimination(self):
spec = """
MyDataStructure:
state elements : Bag<Int>
query containsZero()
exists [x | x <- elements, x == 0]
op addElement(x : Int)
elements.add(x);
"""
spec = parse_spec(spec)
errs = typecheck(spec)
assert not errs, str(errs)
spec = desugar(spec)
impl = construct_initial_implementation(spec)
impl = improve_implementation(impl,
timeout=datetime.timedelta(seconds=30))
print(pprint(impl.code))
assert len(impl.concretization_functions) == 1
(v, e), = list(impl.concretization_functions.items())
print("{} = {}".format(v, pprint(e)))
assert e.type == BOOL
def f(self):
with open(os.path.join("examples", filename + ".ds"), "r") as f:
ast = parse_spec(f.read())
assert isinstance(ast, Spec)
errs = typecheck(ast)
for e in errs:
print(e)
assert not errs
def get_invariant_preservation_errs(spec: str):
spec = parse_spec(spec)
errs = typecheck(spec)
assert not errs, str(errs)
spec = desugar(spec)
return check_ops_preserve_invariants(spec)
assert errs
assert "modX" in errs[0]
assert "modY" in errs[1]
def get_invariant_preservation_errs(spec: str):
spec = parse_spec(spec)
errs = list(typecheck(spec))
assert not errs, str(errs)
spec = desugar(spec)
errs.extend(check_calls_wf(spec))
errs.extend(check_ops_preserve_invariants(spec))
if errs:
print("{} errors:".format(len(errs)))
for e in errs:
print(" - {}".format(e))
return errs
def get_invariant_preservation_errs(spec : str):
spec = parse_spec(spec)
errs = list(typecheck(spec))
assert not errs, str(errs)
spec = desugar(spec)
errs.extend(check_calls_wf(spec))
errs.extend(check_ops_preserve_invariants(spec))
if errs:
print("{} errors:".format(len(errs)))
for e in errs:
print(" - {}".format(e))
return errs
def test_dangling_else(self):
sample = """Test:
state f : Bag<Int>
op foo(i : Int)
if not (i in foo) {
if not ((i + i) in foo) {
foo.add(i + i + i);
} else {
foo.add(i + i + i + i);
}
}
"""
# Verify that `else` code pairs with inner `if`.
ast = parse_spec(sample)
foo = ast.methods[0]
assert isinstance(foo.body, syntax.SIf)
assert isinstance(foo.body.then_branch.else_branch, syntax.SCall)
assert isinstance(foo.body.else_branch, syntax.SNoOp)
def test_pickling(self):
i = parse_spec("""
Foo:
type ETRUE = Native "int"
extern newX(x : Int) : ETRUE = "..."
extern readX(x : ETRUE) : Int = "..."
state xs : Set<ETRUE>
state intsA : Set<Int>
state intsB : Set<Int>
invariant intsA == [readX(x) - 1 | x <- xs];
invariant intsB == [readX(x) + 1 | x <- xs];
query getA()
intsA
query getB()
intsB
""")
errs = typecheck(i)
assert not errs, errs
i = desugar(i)
i1 = construct_initial_implementation(i)
print(pprint(i1.code))
i2 = pickle.loads(pickle.dumps(i1))
assert i1.code == i2.code
def run():
"""Entry point for Cozy executable.
This procedure reads sys.argv and executes the requested tasks.
"""
parser = argparse.ArgumentParser(description='Data structure synthesizer.')
parser.add_argument("-S",
"--save",
metavar="FILE",
type=str,
default=None,
help="Save synthesis output")
parser.add_argument("-R",
"--resume",
action="store_true",
help="Resume from saved synthesis output")
parser.add_argument(
"-t",
"--timeout",
metavar="N",
type=float,
default=60,
help="Per-query synthesis timeout (in seconds); default=60")
parser.add_argument(
"-s",
"--simple",
action="store_true",
help=
"Do not synthesize improved solution; use the most trivial implementation of the spec"
)
parser.add_argument("-p",
"--port",
metavar="P",
type=int,
default=None,
help="Port to run progress-showing HTTP server")
java_opts = parser.add_argument_group("Java codegen")
java_opts.add_argument(
"--java",
metavar="FILE.java",
default=None,
help="Output file for java classes, use '-' for stdout")
java_opts.add_argument(
"--unboxed",
action="store_true",
help=
"Use unboxed primitives. NOTE: synthesized data structures may require GNU Trove (http://trove.starlight-systems.com/)"
)
cxx_opts = parser.add_argument_group("C++ codegen")
cxx_opts.add_argument(
"--c++",
metavar="FILE.h",
default=None,
help="Output file for C++ (header-only class), use '-' for stdout")
cxx_opts.add_argument(
"--use-qhash",
action="store_true",
help=
"QHash---the Qt implementation of hash maps---often outperforms the default C++ map implementations"
)
internal_opts = parser.add_argument_group("Internal parameters")
opts.setup(internal_opts)
parser.add_argument("file",
nargs="?",
default=None,
help="Input file (omit to use stdin)")
args = parser.parse_args()
opts.read(args)
if args.resume:
if args.file is None:
ast = pickle.load(sys.stdin.buffer)
else:
with open(args.file, "rb") as f:
ast = pickle.load(f)
print("Loaded implementation from {}".format(
"stdin" if args.file is None else "file {}".format(args.file)))
else:
input_text = sys.stdin.read(
) if args.file is None else common.read_file(args.file)
ast = parse.parse_spec(input_text)
errors = typecheck.typecheck(ast)
if errors:
for e in errors:
print("Error: {}".format(e))
sys.exit(1)
ast = desugar.desugar(ast)
ast = invariant_preservation.add_implicit_handle_assumptions(ast)
print("Checking assumptions...")
errors = (invariant_preservation.check_ops_preserve_invariants(ast) +
invariant_preservation.check_the_wf(ast))
if errors:
for e in errors:
print("Error: {}".format(e))
sys.exit(1)
print("Done!")
ast = synthesis.construct_initial_implementation(ast)
start = datetime.datetime.now()
if not args.simple:
callback = None
server = None
if checkpoint_prefix.value:
def callback(res):
impl, ast, state_map = res
assert isinstance(impl, synthesis.Implementation)
now = datetime.datetime.now()
elapsed = now - start
fname = "{}{:010d}.synthesized".format(
checkpoint_prefix.value, int(elapsed.total_seconds()))
with open(fname, "wb") as f:
pickle.dump(impl, f)
print("Saved checkpoint {}".format(fname))
if args.port:
from cozy import progress_server
state = ["Initializing..."]
orig_callback = callback
def callback(res):
if orig_callback is not None:
orig_callback(res)
impl, ast, state_map = res
s = "<!DOCTYPE html>\n"
s += "<html>"
s += "<head><style>"
s += ".kw { color: #909; font-weight: bold; }"
s += ".builtin { color: #009; font-weight: bold; }"
s += ".comment { color: #999; }"
s += "</style></head>"
s += "<body><pre>"
for v, e in state_map.items():
s += "{} : {} = {}\n".format(
v, syntax_tools.pprint(e.type, format="html"),
syntax_tools.pprint(e, format="html"))
s += "\n"
s += syntax_tools.pprint(ast, format="html")
s += "</pre></body></html>"
state[0] = s
server = progress_server.ProgressServer(port=args.port,
callback=lambda: state[0])
server.start_async()
# Do full synthesis
ast = synthesis.improve_implementation(
ast,
timeout=datetime.timedelta(seconds=args.timeout),
progress_callback=callback)
if server is not None:
server.join()
print("Generating IR...")
code = ast.code
print("Loading concretization functions...")
state_map = ast.concretization_functions
print()
for v, e in state_map.items():
print("{} : {} = {}".format(v, syntax_tools.pprint(e.type),
syntax_tools.pprint(e)))
print()
print(syntax_tools.pprint(code))
if args.save:
with open(args.save, "wb") as f:
pickle.dump(ast, f)
print("Saved implementation to file {}".format(args.save))
impl = code
share_info = defaultdict(list)
if do_cse.value:
impl = syntax_tools.inline_calls(impl)
impl = syntax_tools.eliminate_common_subexpressions(impl)
try:
java = args.java
if java is not None:
with common.open_maybe_stdout(java) as out:
codegen.JavaPrinter(out=out).render_complete(
spec=impl,
state_map=state_map,
share_info=share_info,
abstract_state=ast.spec.statevars)
cxx = getattr(args, "c++")
if cxx is not None:
with common.open_maybe_stdout(cxx) as out:
codegen.CxxPrinter(out=out, use_qhash=args.use_qhash).visit(
impl,
state_map,
share_info,
abstract_state=ast.spec.statevars)
except:
print("Code generation failed!")
if save_failed_codegen_inputs.value:
with open(save_failed_codegen_inputs.value, "w") as f:
f.write("impl = {}\n".format(repr(impl)))
f.write("state_map = {}\n".format(repr(state_map)))
f.write("share_info = {}\n".format(repr(share_info)))
print("Implementation was dumped to {}".format(
save_failed_codegen_inputs.value))
raise
def run():
"""Entry point for Cozy executable.
This procedure reads sys.argv and executes the requested tasks.
"""
parser = argparse.ArgumentParser(description='Data structure synthesizer.')
parser.add_argument("-S", "--save", metavar="FILE", type=str, default=None, help="Save synthesis output")
parser.add_argument("-R", "--resume", action="store_true", help="Resume from saved synthesis output")
parser.add_argument("-t", "--timeout", metavar="N", type=float, default=60,
help="Global synthesis timeout (in seconds); default=60. " +
"Smaller timeout speeds up the synthesis process, but may also result in less efficient code")
parser.add_argument("-s", "--simple", action="store_true", help="Do not synthesize improved solution; use the most trivial implementation of the spec")
parser.add_argument("-p", "--port", metavar="P", type=int, default=None, help="Port to run progress-showing HTTP server")
java_opts = parser.add_argument_group("Java codegen")
java_opts.add_argument("--java", metavar="FILE.java", default=None, help="Output file for java classes, use '-' for stdout")
java_opts.add_argument("--unboxed", action="store_true", help="Use unboxed primitives. NOTE: synthesized data structures may require GNU Trove (http://trove.starlight-systems.com/)")
cxx_opts = parser.add_argument_group("C++ codegen")
cxx_opts.add_argument("--c++", metavar="FILE.h", default=None, help="Output file for C++ (header-only class), use '-' for stdout")
cxx_opts.add_argument("--use-qhash", action="store_true", help="QHash---the Qt implementation of hash maps---often outperforms the default C++ map implementations")
internal_opts = parser.add_argument_group("Internal parameters")
opts.setup(internal_opts)
parser.add_argument("file", nargs="?", default=None, help="Input file (omit to use stdin)")
args = parser.parse_args()
opts.read(args)
improve_count = Value('i', 0)
if args.resume:
with common.open_maybe_stdin(args.file or "-", mode="rb") as f:
ast = pickle.load(f)
print("Loaded implementation from {}".format("stdin" if args.file is None else "file {}".format(args.file)))
else:
with common.open_maybe_stdin(args.file or "-") as f:
input_text = f.read()
ast = parse.parse_spec(input_text)
# Collection of errors in user-provided specification
errors = typecheck.typecheck(ast)
if errors:
for e in errors:
print("Error: {}".format(e))
sys.exit(1)
ast = desugar.desugar(ast)
ast = invariant_preservation.add_implicit_handle_assumptions(ast)
print("Checking call legality...")
call_errors = invariant_preservation.check_calls_wf(ast)
ast = syntax_tools.inline_calls(ast)
print("Checking invariant preservation...")
errors = (
invariant_preservation.check_ops_preserve_invariants(ast) +
invariant_preservation.check_the_wf(ast) +
invariant_preservation.check_minmax_wf(ast) +
call_errors)
if errors:
for e in errors:
print("Error: {}".format(e))
sys.exit(1)
print("Done!")
ast = synthesis.construct_initial_implementation(ast)
start = datetime.datetime.now()
if not args.simple:
callback = None
server = None
if checkpoint_prefix.value:
def callback(impl):
assert isinstance(impl, synthesis.Implementation)
now = datetime.datetime.now()
elapsed = now - start
fname = "{}{:010d}.synthesized".format(checkpoint_prefix.value, int(elapsed.total_seconds()))
with open(fname, "wb") as f:
pickle.dump(impl, f)
print("Saved checkpoint {}".format(fname))
if args.port:
from cozy import progress_server
state = ["Initializing..."]
orig_callback = callback
def callback(impl):
if orig_callback is not None:
orig_callback(impl)
ast = impl.code
state_map = impl.concretization_functions
s = "<!DOCTYPE html>\n"
s += "<html>"
s += "<head><style>"
s += ".kw { color: #909; font-weight: bold; }"
s += ".builtin { color: #009; font-weight: bold; }"
s += ".comment { color: #999; }"
s += "</style></head>"
s += "<body><pre>"
for v, e in state_map.items():
s += "{} : {} = {}\n".format(v, syntax_tools.pprint(e.type, format="html"), syntax_tools.pprint(e, format="html"))
s += "\n"
s += syntax_tools.pprint(ast, format="html")
s += "</pre></body></html>"
state[0] = s
server = progress_server.ProgressServer(port=args.port, callback=lambda: state[0])
server.start_async()
# Do full synthesis
ast = synthesis.improve_implementation(
ast,
timeout = datetime.timedelta(seconds=args.timeout),
progress_callback = callback,
improve_count=improve_count)
if server is not None:
server.join()
if args.save:
with open(args.save, "wb") as f:
pickle.dump(ast, f)
print("Saved implementation to file {}".format(args.save))
print("Generating IR...")
code = ast.code
print("Inlining calls...")
code = syntax_tools.inline_calls(code)
print("Generating code for extension types...")
code, state_map = rewriting.rewrite_extensions(code, ast.concretization_functions)
if do_cse.value:
print("Eliminating common subexpressions...")
code = syntax_tools.cse_replace_spec(code)
print("Concretization functions:")
print()
for v, e in state_map.items():
print("{} : {} = {}".format(v, syntax_tools.pprint(e.type), syntax_tools.pprint(e)))
print()
print(syntax_tools.pprint(code))
impl = code
share_info = defaultdict(list)
try:
java = args.java
if java is not None:
with common.open_maybe_stdout(java) as out:
codegen.JavaPrinter(out=out, boxed=(not args.unboxed)).visit(impl, state_map, share_info, abstract_state=ast.spec.statevars)
cxx = getattr(args, "c++")
if cxx is not None:
with common.open_maybe_stdout(cxx) as out:
codegen.CxxPrinter(out=out, use_qhash=args.use_qhash).visit(impl, state_map, share_info, abstract_state=ast.spec.statevars)
except:
print("Code generation failed!")
if save_failed_codegen_inputs.value:
with open(save_failed_codegen_inputs.value, "w") as f:
f.write("impl = {}\n".format(repr(impl)))
f.write("state_map = {}\n".format(repr(state_map)))
f.write("share_info = {}\n".format(repr(share_info)))
print("Implementation was dumped to {}".format(save_failed_codegen_inputs.value))
raise
print("Number of improvements done: {}".format(improve_count.value))
def run():
"""Entry point for Cozy executable.
This procedure reads sys.argv and executes the requested tasks.
"""
parser = argparse.ArgumentParser(description='Data structure synthesizer.')
parser.add_argument("-S",
"--save",
metavar="FILE",
type=str,
default=None,
help="Save synthesis output")
parser.add_argument("-R",
"--resume",
action="store_true",
help="Resume from saved synthesis output")
parser.add_argument(
"-t",
"--timeout",
metavar="N",
type=float,
default=60,
help="Global synthesis timeout (in seconds); default=60. " +
"Smaller timeout speeds up the synthesis process, but may also result in less efficient code"
)
parser.add_argument(
"-s",
"--simple",
action="store_true",
help=
"Do not synthesize improved solution; use the most trivial implementation of the spec"
)
parser.add_argument("-p",
"--port",
metavar="P",
type=int,
default=None,
help="Port to run progress-showing HTTP server")
java_opts = parser.add_argument_group("Java codegen")
java_opts.add_argument(
"--java",
metavar="FILE.java",
default=None,
help="Output file for java classes, use '-' for stdout")
java_opts.add_argument(
"--unboxed",
action="store_true",
help=
"Use unboxed primitives. NOTE: synthesized data structures may require GNU Trove (http://trove.starlight-systems.com/)"
)
cxx_opts = parser.add_argument_group("C++ codegen")
cxx_opts.add_argument(
"--c++",
metavar="FILE.h",
default=None,
help="Output file for C++ (header-only class), use '-' for stdout")
cxx_opts.add_argument(
"--use-qhash",
action="store_true",
help=
"QHash---the Qt implementation of hash maps---often outperforms the default C++ map implementations"
)
ruby_opts = parser.add_argument_group("Ruby codegen")
ruby_opts.add_argument("--ruby",
metavar="FILE.rb",
default=None,
help="Output file for Ruby, use '-' for stdout")
internal_opts = parser.add_argument_group("Internal parameters")
opts.setup(internal_opts)
parser.add_argument("file",
nargs="?",
default=None,
help="Input file (omit to use stdin)")
args = parser.parse_args()
opts.read(args)
# Install a handler for SIGINT, the signal that is delivered when you
# Ctrl+C a process. This allows Cozy to exit cleanly when it is
# interrupted. If you need to stop Cozy forcibly, use SIGTERM or SIGKILL.
jobs.install_graceful_sigint_handler()
improve_count = jobs.multiprocessing_context.Value('i', 0)
if args.resume:
with common.open_maybe_stdin(args.file or "-", mode="rb") as f:
ast = pickle.load(f)
print("Loaded implementation from {}".format(
"stdin" if args.file is None else "file {}".format(args.file)))
else:
with common.open_maybe_stdin(args.file or "-") as f:
input_text = f.read()
ast = parse.parse_spec(input_text)
# Collection of errors in user-provided specification
errors = typecheck.typecheck(ast)
if errors:
for e in errors:
print("Error: {}".format(e))
sys.exit(1)
ast = desugar.desugar(ast)
ast = invariant_preservation.add_implicit_handle_assumptions(ast)
print("Checking call legality...")
call_errors = invariant_preservation.check_calls_wf(ast)
ast = syntax_tools.inline_calls(ast)
print("Checking invariant preservation...")
errors = (invariant_preservation.check_ops_preserve_invariants(ast) +
invariant_preservation.check_the_wf(ast) +
invariant_preservation.check_minmax_wf(ast) + call_errors)
if errors:
for e in errors:
print("Error: {}".format(e))
sys.exit(1)
print("Done!")
ast = synthesis.construct_initial_implementation(ast)
start = datetime.datetime.now()
if args.simple:
if args.save:
with open(args.save, "wb") as f:
pickle.dump(ast, f)
print("Saved implementation to file {}".format(args.save))
else:
callback = None
server = None
if checkpoint_prefix.value:
def callback(impl):
assert isinstance(impl, synthesis.Implementation)
now = datetime.datetime.now()
elapsed = now - start
fname = "{}{:010d}.synthesized".format(
checkpoint_prefix.value, int(elapsed.total_seconds()))
with open(fname, "wb") as f:
pickle.dump(impl, f)
print("Saved checkpoint {}".format(fname))
if args.port:
from cozy import progress_server
state = ["Initializing..."]
orig_callback = callback
def callback(impl):
if orig_callback is not None:
orig_callback(impl)
ast = impl.code
state_map = impl.concretization_functions
s = "<!DOCTYPE html>\n"
s += "<html>"
s += "<head><style>"
s += ".kw { color: #909; font-weight: bold; }"
s += ".builtin { color: #009; font-weight: bold; }"
s += ".comment { color: #999; }"
s += "</style></head>"
s += "<body><pre>"
for v, e in state_map.items():
s += "{} : {} = {}\n".format(
v, syntax_tools.pprint(e.type, format="html"),
syntax_tools.pprint(e, format="html"))
s += "\n"
s += syntax_tools.pprint(ast, format="html")
s += "</pre></body></html>"
state[0] = s
server = progress_server.ProgressServer(port=args.port,
callback=lambda: state[0])
server.start_async()
# Do full synthesis
ast = synthesis.improve_implementation(
ast,
timeout=datetime.timedelta(seconds=args.timeout),
progress_callback=callback,
improve_count=improve_count,
dump_synthesized_in_file=args.save)
if server is not None:
server.join()
print("Generating IR...")
code = ast.code
print("Inlining calls...")
code = syntax_tools.inline_calls(code)
print("Generating code for extension types...")
code, state_map = rewriting.rewrite_extensions(
code, ast.concretization_functions)
if do_cse.value:
print("Eliminating common subexpressions...")
code = syntax_tools.cse_replace_spec(code)
print("Concretization functions:")
print()
for v, e in state_map.items():
print("{} : {} = {}".format(v, syntax_tools.pprint(e.type),
syntax_tools.pprint(e)))
print()
print(syntax_tools.pprint(code))
impl = code
share_info = defaultdict(list)
try:
java = args.java
if java is not None:
with common.open_maybe_stdout(java) as out:
codegen.JavaPrinter(out=out, boxed=(not args.unboxed)).visit(
impl,
state_map,
share_info,
abstract_state=ast.spec.statevars)
cxx = getattr(args, "c++")
if cxx is not None:
with common.open_maybe_stdout(cxx) as out:
codegen.CxxPrinter(out=out, use_qhash=args.use_qhash).visit(
impl,
state_map,
share_info,
abstract_state=ast.spec.statevars)
ruby = args.ruby
if ruby is not None:
with common.open_maybe_stdout(ruby) as out:
codegen.RubyPrinter(out=out).visit(
impl,
state_map,
share_info,
abstract_state=ast.spec.statevars)
except:
print("Code generation failed!")
if save_failed_codegen_inputs.value:
with open(save_failed_codegen_inputs.value, "w") as f:
f.write("impl = {}\n".format(repr(impl)))
f.write("state_map = {}\n".format(repr(state_map)))
f.write("share_info = {}\n".format(repr(share_info)))
print("Implementation was dumped to {}".format(
save_failed_codegen_inputs.value))
raise
print("Number of improvements done: {}".format(improve_count.value))