def compile_smt(model_filename, hypers_filename, data_filename, train_batch, out_dir): (parsed_model, data, hypers, out_name) = u.read_inputs(model_filename, hypers_filename, data_filename, train_batch) print("Unrolling execution model.") parsed_model = u.replace_hypers(parsed_model, hypers) unrolled_parsed_model = unroller.unroll_and_flatten(parsed_model, do_checks=False, print_info=False) input_dependents = tptv1.get_input_dependent_vars(unrolled_parsed_model) idx = 1 constraints = [] z3compilers = [] for i in data['instances']: print("Generating SMT constraint for I/O example %i." % idx) z3compiler = ToZ3ConstraintsVisitor(tag="__ex%i" % idx, variables_to_tag=input_dependents) constraints.extend(z3compiler.visit(unrolled_parsed_model)) for var_name, vals in i.iteritems(): if vals is None: pass elif isinstance(vals, list): for i, val in enumerate(vals): if val is not None: var_name_item = "%s_%s" % (var_name, i) constraints.append( z3compiler.get_expr(var_name_item) == IntVal(val)) else: constraints.append( z3compiler.get_expr(var_name) == IntVal(vals)) z3compilers.append(z3compiler) idx = idx + 1 # Unify things: z3compiler = z3compilers[0] for i in xrange(1, len(z3compilers)): z3compilerP = z3compilers[i] for param in z3compiler.get_params(): constraints.append( z3compiler.get_expr(param) == z3compilerP.get_expr(param)) out_file_name = os.path.join(out_dir, out_name + ".smt2") print "Writing SMTLIB2 benchmark info to '%s'." % out_file_name if not os.path.isdir(out_dir): os.makedirs(out_dir) solver = Solver() idx = 0 for c in constraints: # Debugging helper if things unexpectedly end up UNSAT: # solver.assert_and_track(c, "c%i" % idx) solver.add(c) idx = idx + 1 with open(out_file_name, 'w') as f: f.write(solver.to_smt2()) f.write("(get-model)")
def compile_smt(model_filename, hypers_filename, data_filename, train_batch, out_dir): (parsed_model, data, hypers, out_name) = u.read_inputs(model_filename, hypers_filename, data_filename, train_batch) print ("Unrolling execution model.") parsed_model = u.replace_hypers(parsed_model, hypers) unrolled_parsed_model = unroller.unroll_and_flatten(parsed_model, do_checks=False, print_info=False) input_dependents = tptv1.get_input_dependent_vars(unrolled_parsed_model) idx = 1 constraints = [] z3compilers = [] for i in data['instances']: print ("Generating SMT constraint for I/O example %i." % idx) z3compiler = ToZ3ConstraintsVisitor(tag="__ex%i" % idx, variables_to_tag=input_dependents) constraints.extend(z3compiler.visit(unrolled_parsed_model)) for var_name, vals in i.iteritems(): if isinstance(vals, list): for i, val in enumerate(vals): var_name_item = "%s_%s" % (var_name, i) constraints.append( z3compiler.get_expr(var_name_item) == IntVal(val)) else: constraints.append( z3compiler.get_expr(var_name) == IntVal(vals)) z3compilers.append(z3compiler) idx = idx + 1 # Unify things: z3compiler = z3compilers[0] for i in xrange(1, len(z3compilers)): z3compilerP = z3compilers[i] for param in z3compiler.get_params(): constraints.append( z3compiler.get_expr(param) == z3compilerP.get_expr(param)) out_file_name = os.path.join(out_dir, out_name + ".smt2") print "Writing SMTLIB2 benchmark info to '%s'." % out_file_name solver = Solver() idx = 0 for c in constraints: # Debugging helper if things unexpectedly end up UNSAT: # solver.assert_and_track(c, "c%i" % idx) solver.add(c) idx = idx + 1 with open(out_file_name, 'w') as f: f.write(solver.to_smt2()) f.write("(get-model)")
def compile_to_tensorflow(self, source_filename, hypers_filename, out_directory, VERBOSE=False): source, hypers_list, filename = self.load_source_and_hypers( source_filename, hypers_filename) result_filename = hypers_filename.split("/")[-1].replace(".json", "") module_nodes = {} step = 0 for hypers_name, hypers in hypers_list.iteritems(): module_node = ast.parse(source) module_node = u.replace_hypers(module_node, hypers) transforms = self.get_transforms() for transform in transforms: module_node = transform(module_node) if VERBOSE: print 80 * "*" print unparse(module_node) intermediate_result_filename = \ os.path.join(out_directory, "%s_compiled_step%s.py" % (filename, step)) with open(intermediate_result_filename, 'w') as f: f.write(unparse(module_node)) step += 1 module_node = self.class_formatting_xform(module_node, hypers_name) module_nodes[hypers_name] = module_node module_node = self.merge_hypers(module_nodes) module_node = self.add_get_hypers_method(module_node, hypers_list) if VERBOSE: print 80 * "*" print unparse(u.get_class_node(module_node)) filename = source_filename.split("/")[-1].replace(".py", "") if not os.path.isdir(out_directory): os.makedirs(out_directory) result_filename = os.path.join(out_directory, "%s_compiled.py" % result_filename) print "Outputting to %s" % result_filename with open(result_filename, 'w') as f: f.write(unparse(module_node)) return module_node
def run(source_filename, hypers_filename=None): source = open(source_filename, 'r').read() if hypers_filename is None: hypers_list = {"default": ""} else: with open(hypers_filename, 'r') as f: hypers_list = json.load(f) for hypers_name, hypers in hypers_list.iteritems(): module = ast.parse(source) if hypers != "": module = u.replace_hypers(module, hypers) unrolled_node = unroller.unroll_and_flatten(module, do_checks=True) print(astunparse.unparse(unrolled_node)) # Only do the first one... break return None
def translate_to_tptv1(parsed_model, data_batch, hypers): parsed_model = u.replace_hypers(parsed_model, hypers) input_dependents = get_input_dependent_vars(parsed_model) idx_var_name = "input_idx" idx_var = ast.Name(idx_var_name, ast.Load()) input_number = len(data_batch['instances']) range_expr = ast.Num(input_number) input_vars = set() output_vars = set() var_decls = [] input_stmts = [] general_stmts = [] output_stmts = [] for stmt in parsed_model.body: if isinstance(stmt, ast.Assign) and is_input_declaration(stmt.value): input_vars.add(get_var_name(stmt.targets[0])) var_decls.append(stmt) elif isinstance(stmt, ast.Assign) and is_output_declaration(stmt.value): output_vars.add(get_var_name(stmt.targets[0])) var_decls.append(stmt) elif isinstance(stmt, ast.Assign) and is_var_declaration(stmt.value): var_decls.append(stmt) elif ast_uses_varset(stmt, input_dependents): input_stmts.append(add_input_indices(stmt, input_dependents, idx_var)) elif ast_uses_varset(stmt, output_vars): output_stmts.append(stmt) else: general_stmts.append(stmt) input_init = [] output_observation = [] for input_idx, instance in enumerate(data_batch['instances']): for var_name, val in instance.iteritems(): if var_name in input_vars: input_init.extend(generate_io_stmt(input_idx, var_name, val, "set_to_constant")) elif var_name in output_vars: output_observation.extend(generate_io_stmt(input_idx, var_name, val, "observe_value")) extended_var_decls = [] for var_decl in var_decls: # If input-dependent, extend dimension by one if get_var_name(var_decl.targets[0]) in input_dependents: new_decl = copy.deepcopy(var_decl) if isinstance(new_decl.value, ast.Subscript): new_decl.value = extend_subscript_for_input(new_decl.value, ast.Num(input_number)) else: new_decl.value = ast.Subscript(new_decl.value, ast.Index(ast.Num(input_number)), ast.Load()) extended_var_decls.append(new_decl) else: extended_var_decls.append(var_decl) input_loop = ast.For(ast.Name(idx_var_name, ast.Store()), ast.Call(ast.Name("range", ast.Load()), [range_expr], [], None, None), input_stmts, []) parsed_model.body = general_stmts + extended_var_decls + input_init + [input_loop] + output_stmts + output_observation ast.fix_missing_locations(parsed_model) return parsed_model
def instantiate_model(model_filename, hypers, param_values): class Transformer(ast.NodeTransformer): def __init__(self): self.__declared = set() self.__parameters = set() self.__runtime_arg = "tptRuntime_%s" % str(uuid.uuid4()).replace('-', '')[0:8] def visit_Assign(self, node): assigned_var = node.targets[0].id if u.is_param_definition(node): self.__parameters.add(assigned_var) node.value = ast.Num(n=param_values[assigned_var]) elif u.is_var_definition(node): self.__declared.add(assigned_var) node.value.args.append(ast.Str(assigned_var)) elif u.is_input_definition(node): self.__declared.add(assigned_var) # Add name to call so that the runtime can choose value runtime = ast.Name(self.__runtime_arg, ast.Load()) attr = ast.Attribute(runtime, "get_input", ast.Load()) call = ast.Call(attr, [ast.Str(assigned_var)] + node.value.args, [], None, None) node.value = call elif u.is_output_definition(node): self.__declared.add(assigned_var) # Add name to call so that the runtime can check value runtime = ast.Name(self.__runtime_arg, ast.Load()) attr = ast.Attribute(runtime, "get_output", ast.Load()) call = ast.Call(attr, [ast.Str(assigned_var)] + node.value.args, [], None, None) node.value = call return node def visit_FunctionDef(self, node): # Don't recurse into FunctionDef. Also, filter @Inline if len(node.decorator_list) == 1 and \ isinstance(node.decorator_list[0].func, ast.Name) and \ node.decorator_list[0].func.id == "Inline": return [] node.decorator_list = [] return node def visit_Call(self, node): # Do not rewrite lhs of a set_to to .get(), i.e., don't recurse: if u.is_set_to_call(node): node.args = [self.visit(arg) for arg in node.args] else: self.generic_visit(node) return node def visit_Name(self, node): if node.id in self.__parameters: return ast.Num(n=param_values[node.id]) if node.id in self.__declared: # Rewrite "foo" to "foo.get()" attr = ast.Attribute(node, "get", ast.Load()) call = ast.Call(attr, [], [], None, None) return call else: return node def visit_ImportFrom(self, node): if node.module is "dummy": return [] return node def visit_Module(self, node): self.generic_visit(node) var_alias = ast.alias("Var", None) runtime_alias = ast.alias("Runtime", None) importStmt = ast.ImportFrom("terpret_run_runtime", [var_alias, runtime_alias], 0) fun_name = "__generated_%s" % str(uuid.uuid4()).replace('-', '') arguments = ast.arguments([ast.Name(self.__runtime_arg, ast.Param())], None, None, []) fun_def = ast.FunctionDef(name=fun_name, args=arguments, body=[importStmt] + node.body, decorator_list=[]) return (fun_name, ast.Module([fun_def])) # Load model, turn into something we can execute: model = open(model_filename, 'r').read() model_ast = ast.parse(model) model_ast = u.replace_hypers(model_ast, hypers) unrolled_model = unroll.unroll_and_flatten(model_ast, do_checks=False, print_info=False) (fun_name, runnable_model) = Transformer().visit(unrolled_model) ast.fix_missing_locations(runnable_model) return (fun_name, runnable_model)
def instantiate_model(model_filename, hypers, param_values): class Transformer(ast.NodeTransformer): def __init__(self): self.__declared = set() self.__parameters = set() self.__runtime_arg = "tptRuntime_%s" % str(uuid.uuid4()).replace( '-', '')[0:8] def visit_Assign(self, node): assigned_var = node.targets[0].id if u.is_param_definition(node): self.__parameters.add(assigned_var) node.value = ast.Num(n=param_values[assigned_var]) elif u.is_var_definition(node): self.__declared.add(assigned_var) node.value.args.append(ast.Str(assigned_var)) elif u.is_input_definition(node): self.__declared.add(assigned_var) # Add name to call so that the runtime can choose value runtime = ast.Name(self.__runtime_arg, ast.Load()) attr = ast.Attribute(runtime, "get_input", ast.Load()) call = ast.Call(attr, [ast.Str(assigned_var)] + node.value.args, [], None, None) node.value = call elif u.is_output_definition(node): self.__declared.add(assigned_var) # Add name to call so that the runtime can check value runtime = ast.Name(self.__runtime_arg, ast.Load()) attr = ast.Attribute(runtime, "get_output", ast.Load()) call = ast.Call(attr, [ast.Str(assigned_var)] + node.value.args, [], None, None) node.value = call return node def visit_FunctionDef(self, node): # Don't recurse into FunctionDef. Also, filter @Inline if len(node.decorator_list) == 1 and \ isinstance(node.decorator_list[0].func, ast.Name) and \ node.decorator_list[0].func.id == "Inline": return [] node.decorator_list = [] return node def visit_Call(self, node): # Do not rewrite lhs of a set_to to .get(), i.e., don't recurse: if u.is_set_to_call(node): node.args = [self.visit(arg) for arg in node.args] else: self.generic_visit(node) return node def visit_Name(self, node): if node.id in self.__parameters: return ast.Num(n=param_values[node.id]) if node.id in self.__declared: # Rewrite "foo" to "foo.get()" attr = ast.Attribute(node, "get", ast.Load()) call = ast.Call(attr, [], [], None, None) return call else: return node def visit_ImportFrom(self, node): if node.module is "dummy": return [] return node def visit_Module(self, node): self.generic_visit(node) var_alias = ast.alias("Var", None) runtime_alias = ast.alias("Runtime", None) importStmt = ast.ImportFrom("terpret_run_runtime", [var_alias, runtime_alias], 0) fun_name = "__generated_%s" % str(uuid.uuid4()).replace('-', '') arguments = ast.arguments( [ast.Name(self.__runtime_arg, ast.Param())], None, None, []) fun_def = ast.FunctionDef(name=fun_name, args=arguments, body=[importStmt] + node.body, decorator_list=[]) return (fun_name, ast.Module([fun_def])) # Load model, turn into something we can execute: model = open(model_filename, 'r').read() model_ast = ast.parse(model) model_ast = u.replace_hypers(model_ast, hypers) unrolled_model = unroll.unroll_and_flatten(model_ast, do_checks=False, print_info=False) (fun_name, runnable_model) = Transformer().visit(unrolled_model) ast.fix_missing_locations(runnable_model) return (fun_name, runnable_model)