def compile_smt(model_filename, hypers_filename, data_filename, train_batch,
out_dir):
(parsed_model, data, hypers,
out_name) = u.read_inputs(model_filename, hypers_filename, data_filename,
train_batch)
print("Unrolling execution model.")
parsed_model = u.replace_hypers(parsed_model, hypers)
unrolled_parsed_model = unroller.unroll_and_flatten(parsed_model,
do_checks=False,
print_info=False)
input_dependents = tptv1.get_input_dependent_vars(unrolled_parsed_model)
idx = 1
constraints = []
z3compilers = []
for i in data['instances']:
print("Generating SMT constraint for I/O example %i." % idx)
z3compiler = ToZ3ConstraintsVisitor(tag="__ex%i" % idx,
variables_to_tag=input_dependents)
constraints.extend(z3compiler.visit(unrolled_parsed_model))
for var_name, vals in i.iteritems():
if vals is None:
pass
elif isinstance(vals, list):
for i, val in enumerate(vals):
if val is not None:
var_name_item = "%s_%s" % (var_name, i)
constraints.append(
z3compiler.get_expr(var_name_item) == IntVal(val))
else:
constraints.append(
z3compiler.get_expr(var_name) == IntVal(vals))
z3compilers.append(z3compiler)
idx = idx + 1
# Unify things:
z3compiler = z3compilers[0]
for i in xrange(1, len(z3compilers)):
z3compilerP = z3compilers[i]
for param in z3compiler.get_params():
constraints.append(
z3compiler.get_expr(param) == z3compilerP.get_expr(param))
out_file_name = os.path.join(out_dir, out_name + ".smt2")
print "Writing SMTLIB2 benchmark info to '%s'." % out_file_name
if not os.path.isdir(out_dir):
os.makedirs(out_dir)
solver = Solver()
idx = 0
for c in constraints:
# Debugging helper if things unexpectedly end up UNSAT:
# solver.assert_and_track(c, "c%i" % idx)
solver.add(c)
idx = idx + 1
with open(out_file_name, 'w') as f:
f.write(solver.to_smt2())
f.write("(get-model)")
def compile_smt(model_filename, hypers_filename, data_filename,
train_batch, out_dir):
(parsed_model, data, hypers, out_name) = u.read_inputs(model_filename,
hypers_filename,
data_filename,
train_batch)
print ("Unrolling execution model.")
parsed_model = u.replace_hypers(parsed_model, hypers)
unrolled_parsed_model = unroller.unroll_and_flatten(parsed_model,
do_checks=False,
print_info=False)
input_dependents = tptv1.get_input_dependent_vars(unrolled_parsed_model)
idx = 1
constraints = []
z3compilers = []
for i in data['instances']:
print ("Generating SMT constraint for I/O example %i." % idx)
z3compiler = ToZ3ConstraintsVisitor(tag="__ex%i" % idx, variables_to_tag=input_dependents)
constraints.extend(z3compiler.visit(unrolled_parsed_model))
for var_name, vals in i.iteritems():
if isinstance(vals, list):
for i, val in enumerate(vals):
var_name_item = "%s_%s" % (var_name, i)
constraints.append(
z3compiler.get_expr(var_name_item) == IntVal(val))
else:
constraints.append(
z3compiler.get_expr(var_name) == IntVal(vals))
z3compilers.append(z3compiler)
idx = idx + 1
# Unify things:
z3compiler = z3compilers[0]
for i in xrange(1, len(z3compilers)):
z3compilerP = z3compilers[i]
for param in z3compiler.get_params():
constraints.append(
z3compiler.get_expr(param) == z3compilerP.get_expr(param))
out_file_name = os.path.join(out_dir, out_name + ".smt2")
print "Writing SMTLIB2 benchmark info to '%s'." % out_file_name
solver = Solver()
idx = 0
for c in constraints:
# Debugging helper if things unexpectedly end up UNSAT:
# solver.assert_and_track(c, "c%i" % idx)
solver.add(c)
idx = idx + 1
with open(out_file_name, 'w') as f:
f.write(solver.to_smt2())
f.write("(get-model)")
def compile_to_tensorflow(self,
source_filename,
hypers_filename,
out_directory,
VERBOSE=False):
source, hypers_list, filename = self.load_source_and_hypers(
source_filename, hypers_filename)
result_filename = hypers_filename.split("/")[-1].replace(".json", "")
module_nodes = {}
step = 0
for hypers_name, hypers in hypers_list.iteritems():
module_node = ast.parse(source)
module_node = u.replace_hypers(module_node, hypers)
transforms = self.get_transforms()
for transform in transforms:
module_node = transform(module_node)
if VERBOSE:
print 80 * "*"
print unparse(module_node)
intermediate_result_filename = \
os.path.join(out_directory, "%s_compiled_step%s.py" %
(filename, step))
with open(intermediate_result_filename, 'w') as f:
f.write(unparse(module_node))
step += 1
module_node = self.class_formatting_xform(module_node, hypers_name)
module_nodes[hypers_name] = module_node
module_node = self.merge_hypers(module_nodes)
module_node = self.add_get_hypers_method(module_node, hypers_list)
if VERBOSE:
print 80 * "*"
print unparse(u.get_class_node(module_node))
filename = source_filename.split("/")[-1].replace(".py", "")
if not os.path.isdir(out_directory):
os.makedirs(out_directory)
result_filename = os.path.join(out_directory,
"%s_compiled.py" % result_filename)
print "Outputting to %s" % result_filename
with open(result_filename, 'w') as f:
f.write(unparse(module_node))
return module_node
def run(source_filename, hypers_filename=None):
source = open(source_filename, 'r').read()
if hypers_filename is None:
hypers_list = {"default": ""}
else:
with open(hypers_filename, 'r') as f:
hypers_list = json.load(f)
for hypers_name, hypers in hypers_list.iteritems():
module = ast.parse(source)
if hypers != "":
module = u.replace_hypers(module, hypers)
unrolled_node = unroller.unroll_and_flatten(module, do_checks=True)
print(astunparse.unparse(unrolled_node))
# Only do the first one...
break
return None
def run(source_filename, hypers_filename=None):
source = open(source_filename, 'r').read()
if hypers_filename is None:
hypers_list = {"default": ""}
else:
with open(hypers_filename, 'r') as f:
hypers_list = json.load(f)
for hypers_name, hypers in hypers_list.iteritems():
module = ast.parse(source)
if hypers != "":
module = u.replace_hypers(module, hypers)
unrolled_node = unroller.unroll_and_flatten(module, do_checks=True)
print(astunparse.unparse(unrolled_node))
# Only do the first one...
break
return None
def translate_to_tptv1(parsed_model, data_batch, hypers):
parsed_model = u.replace_hypers(parsed_model, hypers)
input_dependents = get_input_dependent_vars(parsed_model)
idx_var_name = "input_idx"
idx_var = ast.Name(idx_var_name, ast.Load())
input_number = len(data_batch['instances'])
range_expr = ast.Num(input_number)
input_vars = set()
output_vars = set()
var_decls = []
input_stmts = []
general_stmts = []
output_stmts = []
for stmt in parsed_model.body:
if isinstance(stmt, ast.Assign) and is_input_declaration(stmt.value):
input_vars.add(get_var_name(stmt.targets[0]))
var_decls.append(stmt)
elif isinstance(stmt, ast.Assign) and is_output_declaration(stmt.value):
output_vars.add(get_var_name(stmt.targets[0]))
var_decls.append(stmt)
elif isinstance(stmt, ast.Assign) and is_var_declaration(stmt.value):
var_decls.append(stmt)
elif ast_uses_varset(stmt, input_dependents):
input_stmts.append(add_input_indices(stmt, input_dependents, idx_var))
elif ast_uses_varset(stmt, output_vars):
output_stmts.append(stmt)
else:
general_stmts.append(stmt)
input_init = []
output_observation = []
for input_idx, instance in enumerate(data_batch['instances']):
for var_name, val in instance.iteritems():
if var_name in input_vars:
input_init.extend(generate_io_stmt(input_idx, var_name, val, "set_to_constant"))
elif var_name in output_vars:
output_observation.extend(generate_io_stmt(input_idx, var_name, val, "observe_value"))
extended_var_decls = []
for var_decl in var_decls:
# If input-dependent, extend dimension by one
if get_var_name(var_decl.targets[0]) in input_dependents:
new_decl = copy.deepcopy(var_decl)
if isinstance(new_decl.value, ast.Subscript):
new_decl.value = extend_subscript_for_input(new_decl.value,
ast.Num(input_number))
else:
new_decl.value = ast.Subscript(new_decl.value,
ast.Index(ast.Num(input_number)),
ast.Load())
extended_var_decls.append(new_decl)
else:
extended_var_decls.append(var_decl)
input_loop = ast.For(ast.Name(idx_var_name, ast.Store()),
ast.Call(ast.Name("range", ast.Load()),
[range_expr],
[], None, None),
input_stmts,
[])
parsed_model.body = general_stmts + extended_var_decls + input_init + [input_loop] + output_stmts + output_observation
ast.fix_missing_locations(parsed_model)
return parsed_model
def instantiate_model(model_filename, hypers, param_values):
class Transformer(ast.NodeTransformer):
def __init__(self):
self.__declared = set()
self.__parameters = set()
self.__runtime_arg = "tptRuntime_%s" % str(uuid.uuid4()).replace('-', '')[0:8]
def visit_Assign(self, node):
assigned_var = node.targets[0].id
if u.is_param_definition(node):
self.__parameters.add(assigned_var)
node.value = ast.Num(n=param_values[assigned_var])
elif u.is_var_definition(node):
self.__declared.add(assigned_var)
node.value.args.append(ast.Str(assigned_var))
elif u.is_input_definition(node):
self.__declared.add(assigned_var)
# Add name to call so that the runtime can choose value
runtime = ast.Name(self.__runtime_arg, ast.Load())
attr = ast.Attribute(runtime, "get_input", ast.Load())
call = ast.Call(attr,
[ast.Str(assigned_var)] + node.value.args,
[],
None,
None)
node.value = call
elif u.is_output_definition(node):
self.__declared.add(assigned_var)
# Add name to call so that the runtime can check value
runtime = ast.Name(self.__runtime_arg, ast.Load())
attr = ast.Attribute(runtime, "get_output", ast.Load())
call = ast.Call(attr,
[ast.Str(assigned_var)] + node.value.args,
[],
None,
None)
node.value = call
return node
def visit_FunctionDef(self, node):
# Don't recurse into FunctionDef. Also, filter @Inline
if len(node.decorator_list) == 1 and \
isinstance(node.decorator_list[0].func, ast.Name) and \
node.decorator_list[0].func.id == "Inline":
return []
node.decorator_list = []
return node
def visit_Call(self, node):
# Do not rewrite lhs of a set_to to .get(), i.e., don't recurse:
if u.is_set_to_call(node):
node.args = [self.visit(arg) for arg in node.args]
else:
self.generic_visit(node)
return node
def visit_Name(self, node):
if node.id in self.__parameters:
return ast.Num(n=param_values[node.id])
if node.id in self.__declared:
# Rewrite "foo" to "foo.get()"
attr = ast.Attribute(node, "get", ast.Load())
call = ast.Call(attr, [], [], None, None)
return call
else:
return node
def visit_ImportFrom(self, node):
if node.module is "dummy":
return []
return node
def visit_Module(self, node):
self.generic_visit(node)
var_alias = ast.alias("Var", None)
runtime_alias = ast.alias("Runtime", None)
importStmt = ast.ImportFrom("terpret_run_runtime",
[var_alias, runtime_alias],
0)
fun_name = "__generated_%s" % str(uuid.uuid4()).replace('-', '')
arguments = ast.arguments([ast.Name(self.__runtime_arg, ast.Param())],
None,
None,
[])
fun_def = ast.FunctionDef(name=fun_name,
args=arguments,
body=[importStmt] + node.body,
decorator_list=[])
return (fun_name, ast.Module([fun_def]))
# Load model, turn into something we can execute:
model = open(model_filename, 'r').read()
model_ast = ast.parse(model)
model_ast = u.replace_hypers(model_ast, hypers)
unrolled_model = unroll.unroll_and_flatten(model_ast,
do_checks=False,
print_info=False)
(fun_name, runnable_model) = Transformer().visit(unrolled_model)
ast.fix_missing_locations(runnable_model)
return (fun_name, runnable_model)
def instantiate_model(model_filename, hypers, param_values):
class Transformer(ast.NodeTransformer):
def __init__(self):
self.__declared = set()
self.__parameters = set()
self.__runtime_arg = "tptRuntime_%s" % str(uuid.uuid4()).replace(
'-', '')[0:8]
def visit_Assign(self, node):
assigned_var = node.targets[0].id
if u.is_param_definition(node):
self.__parameters.add(assigned_var)
node.value = ast.Num(n=param_values[assigned_var])
elif u.is_var_definition(node):
self.__declared.add(assigned_var)
node.value.args.append(ast.Str(assigned_var))
elif u.is_input_definition(node):
self.__declared.add(assigned_var)
# Add name to call so that the runtime can choose value
runtime = ast.Name(self.__runtime_arg, ast.Load())
attr = ast.Attribute(runtime, "get_input", ast.Load())
call = ast.Call(attr,
[ast.Str(assigned_var)] + node.value.args, [],
None, None)
node.value = call
elif u.is_output_definition(node):
self.__declared.add(assigned_var)
# Add name to call so that the runtime can check value
runtime = ast.Name(self.__runtime_arg, ast.Load())
attr = ast.Attribute(runtime, "get_output", ast.Load())
call = ast.Call(attr,
[ast.Str(assigned_var)] + node.value.args, [],
None, None)
node.value = call
return node
def visit_FunctionDef(self, node):
# Don't recurse into FunctionDef. Also, filter @Inline
if len(node.decorator_list) == 1 and \
isinstance(node.decorator_list[0].func, ast.Name) and \
node.decorator_list[0].func.id == "Inline":
return []
node.decorator_list = []
return node
def visit_Call(self, node):
# Do not rewrite lhs of a set_to to .get(), i.e., don't recurse:
if u.is_set_to_call(node):
node.args = [self.visit(arg) for arg in node.args]
else:
self.generic_visit(node)
return node
def visit_Name(self, node):
if node.id in self.__parameters:
return ast.Num(n=param_values[node.id])
if node.id in self.__declared:
# Rewrite "foo" to "foo.get()"
attr = ast.Attribute(node, "get", ast.Load())
call = ast.Call(attr, [], [], None, None)
return call
else:
return node
def visit_ImportFrom(self, node):
if node.module is "dummy":
return []
return node
def visit_Module(self, node):
self.generic_visit(node)
var_alias = ast.alias("Var", None)
runtime_alias = ast.alias("Runtime", None)
importStmt = ast.ImportFrom("terpret_run_runtime",
[var_alias, runtime_alias], 0)
fun_name = "__generated_%s" % str(uuid.uuid4()).replace('-', '')
arguments = ast.arguments(
[ast.Name(self.__runtime_arg, ast.Param())], None, None, [])
fun_def = ast.FunctionDef(name=fun_name,
args=arguments,
body=[importStmt] + node.body,
decorator_list=[])
return (fun_name, ast.Module([fun_def]))
# Load model, turn into something we can execute:
model = open(model_filename, 'r').read()
model_ast = ast.parse(model)
model_ast = u.replace_hypers(model_ast, hypers)
unrolled_model = unroll.unroll_and_flatten(model_ast,
do_checks=False,
print_info=False)
(fun_name, runnable_model) = Transformer().visit(unrolled_model)
ast.fix_missing_locations(runnable_model)
return (fun_name, runnable_model)