def main(argv): try: action = argv[1] except IndexError: raise RuntimeError('Action required') try: schema_path = argv[2] except IndexError: raise RuntimeError('Schema path required') # To avoid circular dependencies, don't load Id for types.asdl. if os.path.basename(schema_path) == 'types.asdl': app_types = {} else: from core.meta import Id app_types = {'id': runtime.UserType(Id)} if action == 'c': # Generate C code for the lexer with open(schema_path) as f: schema_ast, _ = front_end.LoadSchema(f, app_types) v = gen_cpp.CEnumVisitor(sys.stdout) v.VisitModule(schema_ast) elif action == 'py': # Generate Python code so we don't depend on ASDL schemas pickle_out_path = argv[3] with open(schema_path) as f: schema_ast, type_lookup = front_end.LoadSchema(f, app_types) f = sys.stdout f.write("""\ from asdl import const # For const.NO_INTEGER from asdl import runtime from pylib import unpickle from core import util f = util.GetResourceLoader().open('%s') TYPE_LOOKUP = unpickle.load_v2_subset(f) f.close() """ % pickle_out_path) v = gen_python.GenClassesVisitor(f) v.VisitModule(schema_ast) if pickle_out_path: # Pickle version 2 is better. (Pickle version 0 uses # s.decode('string-escape')! ) # In version 2, now I have 16 opcodes + STOP. with open(pickle_out_path, 'w') as f: pickle.dump(type_lookup, f, protocol=2) from core.util import log log('Wrote %s', pickle_out_path) else: raise RuntimeError('Invalid action %r' % action)
def main(argv): try: action = argv[1] except IndexError: raise RuntimeError('Action required') if action == 'py': # Prints the module # Called by asdl/run.sh py-cpp schema_path = argv[2] app_types = {'id': asdl.UserType('id_kind_asdl', 'Id_t')} with open(schema_path) as f: schema_ast, type_lookup = front_end.LoadSchema(f, app_types) root = sys.modules[__name__] # NOTE: We shouldn't pass in app_types for arith.asdl, but this is just a # demo. py_meta.MakeTypes(schema_ast, root, type_lookup) log('AST for this ASDL schema:') schema_ast.Print(sys.stdout, 0) print() log('Dynamically created a Python module with these types:') for name in dir(root): print('\t' + name) if 1: # NOTE: It can be pickled, but not marshaled import marshal import cPickle print(dir(marshal)) out_path = schema_path + '.pickle' with open(out_path, 'w') as f: #marshal.dump(type_lookup, f) # Version 2 is the highest protocol for Python 2.7. cPickle.dump(type_lookup.runtime_type_lookup, f, protocol=2) print('runtime_type_lookup:') for name, desc in type_lookup.runtime_type_lookup.items(): print(name) print(desc) print() print('Wrote %s' % out_path) elif action == 'arith-format': # pretty printing expr = argv[2] obj = typed_arith_parse.ParseShell(expr) tree = obj.PrettyTree() #treee= ['hi', 'there', ['a', 'b'], 'c'] f = fmt.DetectConsoleOutput(sys.stdout) fmt.PrintTree(tree, f) print() # Might need to print the output? # out.WriteToFile? else: raise RuntimeError('Invalid action %r' % action)
def _assertResolveError(self, code_str): f = cStringIO.StringIO(code_str) try: schema_ast = front_end.LoadSchema(f, {}) except front_end.ASDLSyntaxError as e: print(e) else: self.fail("Expected name resolution error: %r" % code_str)
def testLoadSchema(self): with open('asdl/demo.asdl') as f: schema_ast, type_lookup = front_end.LoadSchema(f, {}) #print(type_lookup) # Test fully-qualified name self.assertTrue('bool_expr__LogicalNot' in type_lookup) self.assertTrue('op_id__Plus' in type_lookup)
def main(argv): try: action = argv[1] except IndexError: raise RuntimeError('Action required') try: schema_path = argv[2] except IndexError: raise RuntimeError('Schema path required') schema_filename = os.path.basename(schema_path) if schema_filename in ('syntax.asdl', 'runtime.asdl'): app_types = {'id': meta.UserType('id_kind_asdl', 'Id_t')} else: app_types = {} if action == 'c': # Generate C code for the lexer with open(schema_path) as f: schema_ast, _ = front_end.LoadSchema(f, app_types) v = gen_cpp.CEnumVisitor(sys.stdout) v.VisitModule(schema_ast) elif action == 'cpp': # Generate C++ code for ASDL schemas out_prefix = argv[3] pretty_print_methods = bool(os.getenv('PRETTY_PRINT_METHODS', 'yes')) with open(schema_path) as f: schema_ast, type_lookup = front_end.LoadSchema(f, app_types) # asdl/typed_arith.asdl -> typed_arith_asdl ns = os.path.basename(schema_path).replace('.', '_') with open(out_prefix + '.h', 'w') as f: guard = ns.upper() f.write("""\ #ifndef %s #define %s """ % (guard, guard)) f.write("""\ #include <cstdint> #include "mylib.h" // for Str, List, etc. """) if pretty_print_methods: f.write(""" #include "hnode_asdl.h" using hnode_asdl::hnode_t; """) f.write("""\ namespace %s { """ % ns) v = gen_cpp.ForwardDeclareVisitor(f) v.VisitModule(schema_ast) v2 = gen_cpp.ClassDefVisitor( f, type_lookup, pretty_print_methods=pretty_print_methods) v2.VisitModule(schema_ast) f.write(""" } // namespace %s #endif // %s """ % (ns, guard)) with open(out_prefix + '.cc', 'w') as f: # HACK until we support 'use' if schema_filename == 'syntax.asdl': f.write('#include "id_kind_asdl.h" // hack\n') f.write('using id_kind_asdl::Id_t; // hack\n') f.write(""" #include <assert.h> #include "hnode_asdl.h" using hnode_asdl::hnode__Record; using hnode_asdl::hnode__Array; using hnode_asdl::hnode__External; using hnode_asdl::hnode__Leaf; using hnode_asdl::field; using hnode_asdl::color_e; // TODO: Generate this asdl/runtime.h header and include it? namespace runtime { // declare hnode_asdl::hnode__Record* NewRecord(Str* node_type); hnode_asdl::hnode__Leaf* NewLeaf(Str* s, hnode_asdl::color_t e_color); extern Str* TRUE_STR; extern Str* FALSE_STR; } // declare namespace runtime """) f.write(""" #include "%s.h" namespace %s { """ % (ns, ns)) v3 = gen_cpp.MethodDefVisitor( f, type_lookup, pretty_print_methods=pretty_print_methods) v3.VisitModule(schema_ast) f.write(""" } // namespace %s """ % ns) elif action == 'mypy': # Generated typed MyPy code with open(schema_path) as f: schema_ast, type_lookup = front_end.LoadSchema(f, app_types) try: abbrev_module_name = argv[3] except IndexError: abbrev_mod = None else: # Weird Python rule for importing: fromlist needs to be non-empty. abbrev_mod = __import__(abbrev_module_name, fromlist=['.']) f = sys.stdout for typ in app_types.itervalues(): if isinstance(typ, meta.UserType): f.write('from _devbuild.gen.%s import %s\n' % (typ.mod_name, typ.type_name)) # HACK f.write('from _devbuild.gen.%s import Id_str\n' % typ.mod_name) f.write('\n') # NOTE: Dict, Any are for AssocArray with 'dict' type. f.write("""\ from asdl import pybase from typing import Optional, List, Tuple, Dict, Any, cast """) pretty_print_methods = bool(os.getenv('PRETTY_PRINT_METHODS', 'yes')) optional_fields = bool(os.getenv('OPTIONAL_FIELDS', 'yes')) if pretty_print_methods: f.write(""" from asdl import runtime # For runtime.NO_SPID from asdl.runtime import NewRecord, NewLeaf from _devbuild.gen.hnode_asdl import color_e, hnode, hnode_e, hnode_t, field """) abbrev_mod_entries = dir(abbrev_mod) if abbrev_mod else [] v = gen_python.GenMyPyVisitor( f, type_lookup, abbrev_mod_entries, pretty_print_methods=pretty_print_methods, optional_fields=optional_fields) v.VisitModule(schema_ast) if abbrev_mod: f.write("""\ # # CONCATENATED FILE # """) package, module = abbrev_module_name.split('.') path = os.path.join(package, module + '.py') with open(path) as in_f: f.write(in_f.read()) else: raise RuntimeError('Invalid action %r' % action)
def main(argv): try: action = argv[1] except IndexError: raise RuntimeError('Action required') # TODO: Also generate a switch/static_cast<> pretty printer in C++! For # debugging. Might need to detect cycles though. if action == 'cpp': schema_path = argv[2] # NOTE: This import can't be at the top level osh/asdl_gen.py depends on # this gen_cpp.py module. We should move all the main() functions out of # asdl/ and into command line tools. from core.meta import Id app_types = {'id': meta.UserType('id_kind_asdl', 'Id_t')} with open(schema_path) as input_f: module, type_lookup = front_end.LoadSchema(input_f, app_types) # TODO: gen_cpp.py should be a library and the application should add Id? # Or we should enable ASDL metaprogramming, and let Id be a metaprogrammed # simple sum type. f = sys.stdout # How do mutation of strings, arrays, etc. work? Are they like C++ # containers, or their own? I think they mirror the oil language # semantics. # Every node should have a mirror. MutableObj. MutableRef (pointer). # MutableArithVar -- has std::string. The mirrors are heap allocated. # All the mutable ones should support Dump()/Encode()? # You can just write more at the end... don't need to disturb existing # nodes? Rewrite pointers. alignment = 4 enc = encode.Params(alignment) d = {'pointer_type': enc.pointer_type} f.write("""\ #include <cstdint> class Obj { public: // Decode a 3 byte integer from little endian inline int Int(int n) const; inline const Obj& Ref(const %(pointer_type)s* base, int n) const; inline const Obj* Optional(const %(pointer_type)s* base, int n) const; // NUL-terminated inline const char* Str(const %(pointer_type)s* base, int n) const; protected: uint8_t bytes_[1]; // first is ID; rest are a payload }; """ % d) # Id should be treated as an enum. c = ChainOfVisitors( ForwardDeclareVisitor(f), ClassDefVisitor(f, enc, type_lookup, enum_types=['Id'])) c.VisitModule(module) f.write("""\ inline int Obj::Int(int n) const { return bytes_[n] + (bytes_[n+1] << 8) + (bytes_[n+2] << 16); } inline const Obj& Obj::Ref(const %(pointer_type)s* base, int n) const { int offset = Int(n); return reinterpret_cast<const Obj&>(base[offset]); } inline const Obj* Obj::Optional(const %(pointer_type)s* base, int n) const { int offset = Int(n); if (offset) { return reinterpret_cast<const Obj*>(base + offset); } else { return nullptr; } } inline const char* Obj::Str(const %(pointer_type)s* base, int n) const { int offset = Int(n); return reinterpret_cast<const char*>(base + offset); } """ % d) # uint32_t* and char*/Obj* aren't related, so we need to use # reinterpret_cast<>. # http://stackoverflow.com/questions/10151834/why-cant-i-static-cast-between-char-and-unsigned-char else: raise RuntimeError('Invalid action %r' % action)
def main(argv): try: action = argv[1] except IndexError: raise RuntimeError('Action required') try: schema_path = argv[2] except IndexError: raise RuntimeError('Schema path required') schema_filename = os.path.basename(schema_path) if schema_filename in ('syntax.asdl', 'runtime.asdl'): app_types = {'id': UserType('id_kind_asdl', 'Id_t')} else: app_types = {} if action == 'c': # Generate C code for the lexer with open(schema_path) as f: schema_ast = front_end.LoadSchema(f, app_types) v = gen_cpp.CEnumVisitor(sys.stdout) v.VisitModule(schema_ast) elif action == 'cpp': # Generate C++ code for ASDL schemas out_prefix = argv[3] pretty_print_methods = bool(os.getenv('PRETTY_PRINT_METHODS', 'yes')) with open(schema_path) as f: schema_ast = front_end.LoadSchema(f, app_types) # asdl/typed_arith.asdl -> typed_arith_asdl ns = os.path.basename(schema_path).replace('.', '_') with open(out_prefix + '.h', 'w') as f: guard = ns.upper() f.write("""\ // %s.h is generated by asdl/tool.py #ifndef %s #define %s """ % (out_prefix, guard, guard)) f.write("""\ #include <cstdint> #include "mylib.h" // for Str, List, etc. """) if pretty_print_methods: f.write("""\ #include "hnode_asdl.h" using hnode_asdl::hnode_t; """) if app_types: f.write("""\ #include "id_kind_asdl.h" using id_kind_asdl::Id_t; """) for use in schema_ast.uses: # Forward declarations in the header, like # namespace syntax_asdl { class command_t; } # must come BEFORE namespace, so it can't be in the visitor. # assume sum type for now! cpp_names = [ 'class %s;' % asdl_.TypeNameHeuristic(n) for n in use.type_names ] f.write('namespace %s_asdl { %s }\n' % (use.mod_name, ' '.join(cpp_names))) f.write('\n') f.write("""\ namespace %s { """ % ns) v = gen_cpp.ForwardDeclareVisitor(f) v.VisitModule(schema_ast) debug_info = {} v2 = gen_cpp.ClassDefVisitor( f, pretty_print_methods=pretty_print_methods, simple_int_sums=_SIMPLE, debug_info=debug_info) v2.VisitModule(schema_ast) f.write(""" } // namespace %s #endif // %s """ % (ns, guard)) try: debug_info_path = argv[4] except IndexError: pass else: with open(debug_info_path, 'w') as f: from pprint import pformat f.write('''\ cpp_namespace = %r tags_to_types = \\ %s ''' % (ns, pformat(debug_info))) with open(out_prefix + '.cc', 'w') as f: f.write("""\ // %s.cc is generated by asdl/tool.py #include "%s.h" #include <assert.h> #include "asdl_runtime.h" // generated code uses wrappers here """ % (out_prefix, ns)) # To call pretty-printing methods for use in schema_ast.uses: f.write('#include "%s_asdl.h" // ASDL use\n' % use.mod_name) f.write("""\ // Generated code uses these types using hnode_asdl::hnode__Record; using hnode_asdl::hnode__Array; using hnode_asdl::hnode__External; using hnode_asdl::hnode__Leaf; using hnode_asdl::field; using hnode_asdl::color_e; """) if app_types: f.write('using id_kind_asdl::Id_str;\n') f.write(""" namespace %s { """ % ns) v3 = gen_cpp.MethodDefVisitor( f, pretty_print_methods=pretty_print_methods, simple_int_sums=_SIMPLE) v3.VisitModule(schema_ast) f.write(""" } // namespace %s """ % ns) elif action == 'mypy': # Generated typed MyPy code with open(schema_path) as f: schema_ast = front_end.LoadSchema(f, app_types) try: abbrev_module_name = argv[3] except IndexError: abbrev_mod = None else: # Weird Python rule for importing: fromlist needs to be non-empty. abbrev_mod = __import__(abbrev_module_name, fromlist=['.']) f = sys.stdout # TODO: Remove Any once we stop using it f.write("""\ from asdl import pybase from typing import Optional, List, Tuple, Dict, Any, cast, TYPE_CHECKING """) if schema_ast.uses: f.write('\n') f.write('if TYPE_CHECKING:\n') for use in schema_ast.uses: py_names = [asdl_.TypeNameHeuristic(n) for n in use.type_names] # indented f.write(' from _devbuild.gen.%s_asdl import %s\n' % (use.mod_name, ', '.join(py_names))) f.write('\n') for typ in app_types.itervalues(): if isinstance(typ, UserType): f.write('from _devbuild.gen.%s import %s\n' % (typ.mod_name, typ.type_name)) # HACK f.write('from _devbuild.gen.%s import Id_str\n' % typ.mod_name) f.write('\n') pretty_print_methods = bool(os.getenv('PRETTY_PRINT_METHODS', 'yes')) optional_fields = bool(os.getenv('OPTIONAL_FIELDS', 'yes')) if pretty_print_methods: f.write(""" from asdl import runtime # For runtime.NO_SPID from asdl.runtime import NewRecord, NewLeaf from _devbuild.gen.hnode_asdl import color_e, hnode, hnode_e, hnode_t, field """) abbrev_mod_entries = dir(abbrev_mod) if abbrev_mod else [] v = gen_python.GenMyPyVisitor( f, abbrev_mod_entries, pretty_print_methods=pretty_print_methods, optional_fields=optional_fields, simple_int_sums=_SIMPLE) v.VisitModule(schema_ast) if abbrev_mod: f.write("""\ # # CONCATENATED FILE # """) package, module = abbrev_module_name.split('.') path = os.path.join(package, module + '.py') with open(path) as in_f: f.write(in_f.read()) else: raise RuntimeError('Invalid action %r' % action)
def main(argv): try: action = argv[1] except IndexError: raise RuntimeError('Action required') try: schema_path = argv[2] except IndexError: raise RuntimeError('Schema path required') if os.path.basename(schema_path) == 'types.asdl': app_types = {} else: app_types = {'id': meta.UserType('id_kind_asdl', 'Id_t')} if action == 'c': # Generate C code for the lexer with open(schema_path) as f: schema_ast, _ = front_end.LoadSchema(f, app_types) v = gen_cpp.CEnumVisitor(sys.stdout) v.VisitModule(schema_ast) elif action == 'mypy': # typed mypy with open(schema_path) as f: schema_ast, type_lookup = front_end.LoadSchema(f, app_types) try: abbrev_module_name = argv[3] except IndexError: abbrev_mod = None else: # Weird Python rule for importing: fromlist needs to be non-empty. abbrev_mod = __import__(abbrev_module_name, fromlist=['.']) f = sys.stdout for typ in app_types.itervalues(): if isinstance(typ, meta.UserType): f.write('from _devbuild.gen.%s import %s\n' % (typ.mod_name, typ.type_name)) f.write('\n') f.write("""\ from asdl import const # For const.NO_INTEGER from asdl import runtime PrettyLeaf = runtime.PrettyLeaf PrettyArray = runtime.PrettyArray PrettyNode = runtime.PrettyNode Color_TypeName = runtime.Color_TypeName Color_StringConst = runtime.Color_StringConst Color_OtherConst = runtime.Color_OtherConst Color_UserType = runtime.Color_UserType from typing import Optional, List, Tuple """) abbrev_mod_entries = dir(abbrev_mod) if abbrev_mod else [] v = gen_python.GenMyPyVisitor(f, type_lookup, abbrev_mod_entries) v.VisitModule(schema_ast) if abbrev_mod: f.write("""\ # # CONCATENATED FILE # """) package, module = abbrev_module_name.split('.') path = os.path.join(package, module + '.py') with open(path) as in_f: f.write(in_f.read()) else: raise RuntimeError('Invalid action %r' % action)
def _assertResolve(self, code_str): f = cStringIO.StringIO(code_str) schema_ast = front_end.LoadSchema(f, {}) print(schema_ast)
def testSharedVariant(self): with open('asdl/shared_variant.asdl') as f: schema_ast = front_end.LoadSchema(f, {}, verbose=False) print(schema_ast)
def testLoadSchema(self): with open('asdl/typed_demo.asdl') as f: schema_ast = front_end.LoadSchema(f, {}, verbose=True) print(schema_ast)
def main(argv): try: action = argv[1] except IndexError: raise RuntimeError('Action required') try: schema_path = argv[2] except IndexError: raise RuntimeError('Schema path required') if os.path.basename(schema_path) in ('syntax.asdl', 'runtime.asdl'): app_types = {'id': meta.UserType('id_kind_asdl', 'Id_t')} else: app_types = {} if action == 'c': # Generate C code for the lexer with open(schema_path) as f: schema_ast, _ = front_end.LoadSchema(f, app_types) v = gen_cpp.CEnumVisitor(sys.stdout) v.VisitModule(schema_ast) elif action == 'cpp': # Generate C++ code for ASDL schemas with open(schema_path) as f: schema_ast, type_lookup = front_end.LoadSchema(f, app_types) # asdl/typed_arith.asdl -> typed_arith_asdl ns = os.path.basename(schema_path).replace('.', '_') f = sys.stdout f.write("""\ #include <cstdint> #include "runtime.h" // for Str, List, etc. namespace %s { """ % ns) v = gen_cpp.ForwardDeclareVisitor(f) v.VisitModule(schema_ast) v2 = gen_cpp.ClassDefVisitor(f, type_lookup) v2.VisitModule(schema_ast) f.write('} // namespace %s\n' % ns) elif action == 'mypy': # Generated typed MyPy code with open(schema_path) as f: schema_ast, type_lookup = front_end.LoadSchema(f, app_types) try: abbrev_module_name = argv[3] except IndexError: abbrev_mod = None else: # Weird Python rule for importing: fromlist needs to be non-empty. abbrev_mod = __import__(abbrev_module_name, fromlist=['.']) f = sys.stdout for typ in app_types.itervalues(): if isinstance(typ, meta.UserType): f.write('from _devbuild.gen.%s import %s\n' % (typ.mod_name, typ.type_name)) f.write('\n') # NOTE: Dict, Any are for AssocArray with 'dict' type. f.write("""\ from asdl import const # For const.NO_INTEGER from asdl import runtime from asdl.runtime import ( PrettyLeaf, PrettyArray, PrettyNode, Color_TypeName, Color_StringConst, Color_OtherConst, Color_UserType, ) from typing import Optional, List, Tuple, Dict, Any """) abbrev_mod_entries = dir(abbrev_mod) if abbrev_mod else [] v = gen_python.GenMyPyVisitor(f, type_lookup, abbrev_mod_entries) v.VisitModule(schema_ast) if abbrev_mod: f.write("""\ # # CONCATENATED FILE # """) package, module = abbrev_module_name.split('.') path = os.path.join(package, module + '.py') with open(path) as in_f: f.write(in_f.read()) else: raise RuntimeError('Invalid action %r' % action)
#!/usr/bin/env python """ arith_ast.py """ import sys from asdl import front_end from asdl import py_meta from core import util f = util.GetResourceLoader().open('asdl/arith.asdl') _asdl_module, _type_lookup = front_end.LoadSchema(f, {}) # no app_types f.close() root = sys.modules[__name__] py_meta.MakeTypes(_asdl_module, root, _type_lookup)
def testSharedVariant(self): with open('asdl/shared_variant.asdl') as f: schema_ast, type_lookup = front_end.LoadSchema(f, {}, verbose=True)