def _ParseAndMakeTypes(schema_path, root): module = asdl.parse(schema_path) app_types = { 'identifier': asdl.UserType(Identifier), 'bytes': asdl.UserType(Bytes), 'object': asdl.UserType(PyObject), 'constant': asdl.UserType(Constant), 'singleton': asdl.UserType(Singleton), } # Check for type errors if not asdl.check(module, app_types): raise AssertionError('ASDL file is invalid') py_meta.MakeTypes(module, root, app_types)
def main(argv): try: action = argv[1] except IndexError: raise RuntimeError('Action required') try: schema_path = argv[2] except IndexError: raise RuntimeError('Schema path required') # To avoid circular dependencies, don't load Id for types.asdl. if os.path.basename(schema_path) == 'types.asdl': app_types = {} else: from osh.meta import Id app_types = {'id': asdl.UserType(Id)} if action == 'c': # Generate C code for the lexer with open(schema_path) as f: schema_ast, _ = front_end.LoadSchema(f, app_types) v = gen_cpp.CEnumVisitor(sys.stdout) v.VisitModule(schema_ast) elif action == 'py': # Generate Python code so we don't depend on ASDL schemas pickle_out_path = argv[3] with open(schema_path) as f: schema_ast, type_lookup = front_end.LoadSchema(f, app_types) f = sys.stdout f.write("""\ from asdl import asdl_ as asdl from asdl import const # For const.NO_INTEGER from asdl import py_meta from asdl import unpickle from core import util f = util.GetResourceLoader().open('%s') TYPE_LOOKUP = unpickle.load_v2_subset(f) f.close() """ % pickle_out_path) v = gen_python.GenClassesVisitor(f) v.VisitModule(schema_ast) if pickle_out_path: # Pickle version 2 is better. (Pickle version 0 uses # s.decode('string-escape')! ) # In version 2, now I have 16 opcodes + STOP. with open(pickle_out_path, 'w') as f: pickle.dump(type_lookup, f, protocol=2) from core.util import log log('Wrote %s', pickle_out_path) else: raise RuntimeError('Invalid action %r' % action)
def main(argv): try: action = argv[1] except IndexError: raise RuntimeError('Action required') if action == 'py': # Prints the module # Called by asdl/run.sh py-cpp schema_path = argv[2] app_types = {'id': asdl.UserType('id_kind_asdl', 'Id_t')} with open(schema_path) as f: schema_ast, type_lookup = front_end.LoadSchema(f, app_types) root = sys.modules[__name__] # NOTE: We shouldn't pass in app_types for arith.asdl, but this is just a # demo. py_meta.MakeTypes(schema_ast, root, type_lookup) log('AST for this ASDL schema:') schema_ast.Print(sys.stdout, 0) print() log('Dynamically created a Python module with these types:') for name in dir(root): print('\t' + name) if 1: # NOTE: It can be pickled, but not marshaled import marshal import cPickle print(dir(marshal)) out_path = schema_path + '.pickle' with open(out_path, 'w') as f: #marshal.dump(type_lookup, f) # Version 2 is the highest protocol for Python 2.7. cPickle.dump(type_lookup.runtime_type_lookup, f, protocol=2) print('runtime_type_lookup:') for name, desc in type_lookup.runtime_type_lookup.items(): print(name) print(desc) print() print('Wrote %s' % out_path) elif action == 'arith-format': # pretty printing expr = argv[2] obj = typed_arith_parse.ParseShell(expr) tree = obj.PrettyTree() #treee= ['hi', 'there', ['a', 'b'], 'c'] f = fmt.DetectConsoleOutput(sys.stdout) fmt.PrintTree(tree, f) print() # Might need to print the output? # out.WriteToFile? else: raise RuntimeError('Invalid action %r' % action)
def main(argv): try: action = argv[1] except IndexError: raise RuntimeError('Action required') if action == 'py': # Prints the module schema_path = argv[2] with open(schema_path) as f: module = asdl.parse(f) app_types = {'id': asdl.UserType(Id)} type_lookup = asdl.ResolveTypes(module, app_types) # Note this is a big tree. But we really want a graph of pointers to # instances. # Type(name, Product(...)) # Type(name, Sum([Constructor(...), ...])) #print(module) root = sys.modules[__name__] # NOTE: We shouldn't pass in app_types for arith.asdl, but this is just a # demo. py_meta.MakeTypes(module, root, type_lookup) print('Dynamically created a Python module with these types:') for name in dir(root): print('\t' + name) elif action == 'arith-encode': # oheap encoding expr = argv[2] out_path = argv[3] obj = arith_parse.ParseShell(expr) print('Encoding %r into binary:' % expr) print(obj) enc = encode.Params() with open(out_path, 'wb') as f: out = encode.BinOutput(f) encode.EncodeRoot(obj, enc, out) elif action == 'arith-format': # pretty printing expr = argv[2] obj = arith_parse.ParseShell(expr) #out = fmt.TextOutput(sys.stdout) tree = fmt.MakeTree(obj) #treee= ['hi', 'there', ['a', 'b'], 'c'] f = fmt.DetectConsoleOutput(sys.stdout) fmt.PrintTree(tree, f) print() # Might need to print the output? # out.WriteToFile? else: raise RuntimeError('Invalid action %r' % action)
def _ParseAndMakeTypes(f, root): module = asdl.parse(f) app_types = {'id': asdl.UserType(Id)} # Check for type errors if not asdl.check(module, app_types): raise AssertionError('ASDL file is invalid') py_meta.MakeTypes(module, root, app_types)
def _LoadSchema(f): module = asdl.parse(f) app_types = {'id': asdl.UserType(Id)} type_lookup = asdl.ResolveTypes(module, app_types) # Check for type errors if not asdl.check(module, app_types): raise AssertionError('ASDL file is invalid') return module, type_lookup
def main(argv): #print lex_mode_e #print dir(lex_mode_e) app_types = {'id': asdl.UserType(Id)} with open('osh/types.asdl') as f: asdl_module, _ = asdl.LoadSchema(f, app_types) v = gen_cpp.CEnumVisitor(sys.stdout) v.VisitModule(asdl_module)
def LoadSchema(f): app_types = {'id': asdl.UserType(Id)} asdl_module = asdl.parse(f) if not asdl.check(asdl_module, app_types): raise AssertionError('ASDL file is invalid') type_lookup = asdl.ResolveTypes(asdl_module, app_types) return asdl_module, type_lookup
def _ParseAndMakeTypes(f, root): # TODO: A better syntax for this might be: # # id = external # # in osh.asdl. Then we can show an error if it's not provided. app_types = {'id': asdl.UserType(Id)} module = asdl.parse(f) # Check for type errors if not asdl.check(module, app_types): raise AssertionError('ASDL file is invalid') py_meta.MakeTypes(module, root, app_types)
def main(argv): try: action = argv[1] except IndexError: raise RuntimeError('Action required') if action == 'py': schema_path = argv[2] module = asdl.parse(schema_path) root = sys.modules[__name__] # NOTE: We shouldn't pass in app_types for arith.asdl, but this is just a # demo. py_meta.MakeTypes(module, root, app_types={'id': asdl.UserType(Id)}) print(dir(root)) elif action == 'arith-encode': expr = argv[2] out_path = argv[3] obj = arith_parse.ParseShell(expr) print('Encoding %r into binary:' % expr) print(obj) enc = encode.Params() with open(out_path, 'wb') as f: out = encode.BinOutput(f) encode.EncodeRoot(obj, enc, out) elif action == 'arith-format': expr = argv[2] obj = arith_parse.ParseShell(expr) #out = fmt.TextOutput(sys.stdout) tree = fmt.MakeTree(obj) #treee= ['hi', 'there', ['a', 'b'], 'c'] fmt.PrintTree(tree, sys.stdout) # Might need to print the output? # out.WriteToFile? else: raise RuntimeError('Invalid action %r' % action)
def main(argv): try: action = argv[1] except IndexError: raise RuntimeError('Action required') # TODO: Also generate a switch/static_cast<> pretty printer in C++! For # debugging. Might need to detect cycles though. if action == 'cpp': schema_path = argv[2] # NOTE: This import can't be at the top level osh/asdl_gen.py depends on # this gen_cpp.py module. We should move all the main() functions out of # asdl/ and into command line tools. from osh.meta import Id app_types = {'id': asdl.UserType(Id)} with open(schema_path) as input_f: module, type_lookup = front_end.LoadSchema(input_f, app_types) # TODO: gen_cpp.py should be a library and the application should add Id? # Or we should enable ASDL metaprogramming, and let Id be a metaprogrammed # simple sum type. f = sys.stdout # How do mutation of strings, arrays, etc. work? Are they like C++ # containers, or their own? I think they mirror the oil language # semantics. # Every node should have a mirror. MutableObj. MutableRef (pointer). # MutableArithVar -- has std::string. The mirrors are heap allocated. # All the mutable ones should support Dump()/Encode()? # You can just write more at the end... don't need to disturb existing # nodes? Rewrite pointers. alignment = 4 enc = encode.Params(alignment) d = {'pointer_type': enc.pointer_type} f.write("""\ #include <cstdint> class Obj { public: // Decode a 3 byte integer from little endian inline int Int(int n) const; inline const Obj& Ref(const %(pointer_type)s* base, int n) const; inline const Obj* Optional(const %(pointer_type)s* base, int n) const; // NUL-terminated inline const char* Str(const %(pointer_type)s* base, int n) const; protected: uint8_t bytes_[1]; // first is ID; rest are a payload }; """ % d) # Id should be treated as an enum. c = ChainOfVisitors( ForwardDeclareVisitor(f), ClassDefVisitor(f, enc, type_lookup, enum_types=['Id'])) c.VisitModule(module) f.write("""\ inline int Obj::Int(int n) const { return bytes_[n] + (bytes_[n+1] << 8) + (bytes_[n+2] << 16); } inline const Obj& Obj::Ref(const %(pointer_type)s* base, int n) const { int offset = Int(n); return reinterpret_cast<const Obj&>(base[offset]); } inline const Obj* Obj::Optional(const %(pointer_type)s* base, int n) const { int offset = Int(n); if (offset) { return reinterpret_cast<const Obj*>(base + offset); } else { return nullptr; } } inline const char* Obj::Str(const %(pointer_type)s* base, int n) const { int offset = Int(n); return reinterpret_cast<const char*>(base + offset); } """ % d) # uint32_t* and char*/Obj* aren't related, so we need to use # reinterpret_cast<>. # http://stackoverflow.com/questions/10151834/why-cant-i-static-cast-between-char-and-unsigned-char else: raise RuntimeError('Invalid action %r' % action)
# Instantiate the spec # ID_SPEC = id_kind.IdSpec(Id, Kind, _ID_NAMES, _ID_INSTANCES, _ID_TO_KIND, BOOL_ARG_TYPES) id_kind.AddKinds(ID_SPEC) id_kind.AddBoolKinds(ID_SPEC, Id, types.bool_arg_type_e) # must come second id_kind.SetupTestBuiltin(Id, Kind, ID_SPEC, TEST_UNARY_LOOKUP, TEST_BINARY_LOOKUP, TEST_OTHER_LOOKUP, types.bool_arg_type_e) # Debug _kind_sizes = ID_SPEC.kind_sizes APP_TYPES = {'id': asdl.UserType(Id)} # # Instantiate osh/osh.asdl # f = util.GetResourceLoader().open('osh/osh.asdl') _asdl_module, _type_lookup = asdl.LoadSchema(f, APP_TYPES) ast = _AsdlModule() if 0: py_meta.MakeTypes(_asdl_module, ast, _type_lookup) else: # Exported for the generated code to use OSH_TYPE_LOOKUP = _type_lookup