def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
try:
schema_path = argv[2]
except IndexError:
raise RuntimeError('Schema path required')
# To avoid circular dependencies, don't load Id for types.asdl.
if os.path.basename(schema_path) == 'types.asdl':
app_types = {}
else:
from core.meta import Id
app_types = {'id': runtime.UserType(Id)}
if action == 'c': # Generate C code for the lexer
with open(schema_path) as f:
schema_ast, _ = front_end.LoadSchema(f, app_types)
v = gen_cpp.CEnumVisitor(sys.stdout)
v.VisitModule(schema_ast)
elif action == 'py': # Generate Python code so we don't depend on ASDL schemas
pickle_out_path = argv[3]
with open(schema_path) as f:
schema_ast, type_lookup = front_end.LoadSchema(f, app_types)
f = sys.stdout
f.write("""\
from asdl import const # For const.NO_INTEGER
from asdl import runtime
from pylib import unpickle
from core import util
f = util.GetResourceLoader().open('%s')
TYPE_LOOKUP = unpickle.load_v2_subset(f)
f.close()
""" % pickle_out_path)
v = gen_python.GenClassesVisitor(f)
v.VisitModule(schema_ast)
if pickle_out_path:
# Pickle version 2 is better. (Pickle version 0 uses
# s.decode('string-escape')! )
# In version 2, now I have 16 opcodes + STOP.
with open(pickle_out_path, 'w') as f:
pickle.dump(type_lookup, f, protocol=2)
from core.util import log
log('Wrote %s', pickle_out_path)
else:
raise RuntimeError('Invalid action %r' % action)
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
if action == 'py': # Prints the module
# Called by asdl/run.sh py-cpp
schema_path = argv[2]
app_types = {'id': asdl.UserType('id_kind_asdl', 'Id_t')}
with open(schema_path) as f:
schema_ast, type_lookup = front_end.LoadSchema(f, app_types)
root = sys.modules[__name__]
# NOTE: We shouldn't pass in app_types for arith.asdl, but this is just a
# demo.
py_meta.MakeTypes(schema_ast, root, type_lookup)
log('AST for this ASDL schema:')
schema_ast.Print(sys.stdout, 0)
print()
log('Dynamically created a Python module with these types:')
for name in dir(root):
print('\t' + name)
if 1:
# NOTE: It can be pickled, but not marshaled
import marshal
import cPickle
print(dir(marshal))
out_path = schema_path + '.pickle'
with open(out_path, 'w') as f:
#marshal.dump(type_lookup, f)
# Version 2 is the highest protocol for Python 2.7.
cPickle.dump(type_lookup.runtime_type_lookup, f, protocol=2)
print('runtime_type_lookup:')
for name, desc in type_lookup.runtime_type_lookup.items():
print(name)
print(desc)
print()
print('Wrote %s' % out_path)
elif action == 'arith-format': # pretty printing
expr = argv[2]
obj = typed_arith_parse.ParseShell(expr)
tree = obj.PrettyTree()
#treee= ['hi', 'there', ['a', 'b'], 'c']
f = fmt.DetectConsoleOutput(sys.stdout)
fmt.PrintTree(tree, f)
print()
# Might need to print the output?
# out.WriteToFile?
else:
raise RuntimeError('Invalid action %r' % action)
def _assertResolveError(self, code_str):
f = cStringIO.StringIO(code_str)
try:
schema_ast = front_end.LoadSchema(f, {})
except front_end.ASDLSyntaxError as e:
print(e)
else:
self.fail("Expected name resolution error: %r" % code_str)
def testLoadSchema(self):
with open('asdl/demo.asdl') as f:
schema_ast, type_lookup = front_end.LoadSchema(f, {})
#print(type_lookup)
# Test fully-qualified name
self.assertTrue('bool_expr__LogicalNot' in type_lookup)
self.assertTrue('op_id__Plus' in type_lookup)
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
try:
schema_path = argv[2]
except IndexError:
raise RuntimeError('Schema path required')
schema_filename = os.path.basename(schema_path)
if schema_filename in ('syntax.asdl', 'runtime.asdl'):
app_types = {'id': meta.UserType('id_kind_asdl', 'Id_t')}
else:
app_types = {}
if action == 'c': # Generate C code for the lexer
with open(schema_path) as f:
schema_ast, _ = front_end.LoadSchema(f, app_types)
v = gen_cpp.CEnumVisitor(sys.stdout)
v.VisitModule(schema_ast)
elif action == 'cpp': # Generate C++ code for ASDL schemas
out_prefix = argv[3]
pretty_print_methods = bool(os.getenv('PRETTY_PRINT_METHODS', 'yes'))
with open(schema_path) as f:
schema_ast, type_lookup = front_end.LoadSchema(f, app_types)
# asdl/typed_arith.asdl -> typed_arith_asdl
ns = os.path.basename(schema_path).replace('.', '_')
with open(out_prefix + '.h', 'w') as f:
guard = ns.upper()
f.write("""\
#ifndef %s
#define %s
""" % (guard, guard))
f.write("""\
#include <cstdint>
#include "mylib.h" // for Str, List, etc.
""")
if pretty_print_methods:
f.write("""
#include "hnode_asdl.h"
using hnode_asdl::hnode_t;
""")
f.write("""\
namespace %s {
""" % ns)
v = gen_cpp.ForwardDeclareVisitor(f)
v.VisitModule(schema_ast)
v2 = gen_cpp.ClassDefVisitor(
f, type_lookup, pretty_print_methods=pretty_print_methods)
v2.VisitModule(schema_ast)
f.write("""
} // namespace %s
#endif // %s
""" % (ns, guard))
with open(out_prefix + '.cc', 'w') as f:
# HACK until we support 'use'
if schema_filename == 'syntax.asdl':
f.write('#include "id_kind_asdl.h" // hack\n')
f.write('using id_kind_asdl::Id_t; // hack\n')
f.write("""
#include <assert.h>
#include "hnode_asdl.h"
using hnode_asdl::hnode__Record;
using hnode_asdl::hnode__Array;
using hnode_asdl::hnode__External;
using hnode_asdl::hnode__Leaf;
using hnode_asdl::field;
using hnode_asdl::color_e;
// TODO: Generate this asdl/runtime.h header and include it?
namespace runtime { // declare
hnode_asdl::hnode__Record* NewRecord(Str* node_type);
hnode_asdl::hnode__Leaf* NewLeaf(Str* s, hnode_asdl::color_t e_color);
extern Str* TRUE_STR;
extern Str* FALSE_STR;
} // declare namespace runtime
""")
f.write("""
#include "%s.h"
namespace %s {
""" % (ns, ns))
v3 = gen_cpp.MethodDefVisitor(
f, type_lookup, pretty_print_methods=pretty_print_methods)
v3.VisitModule(schema_ast)
f.write("""
} // namespace %s
""" % ns)
elif action == 'mypy': # Generated typed MyPy code
with open(schema_path) as f:
schema_ast, type_lookup = front_end.LoadSchema(f, app_types)
try:
abbrev_module_name = argv[3]
except IndexError:
abbrev_mod = None
else:
# Weird Python rule for importing: fromlist needs to be non-empty.
abbrev_mod = __import__(abbrev_module_name, fromlist=['.'])
f = sys.stdout
for typ in app_types.itervalues():
if isinstance(typ, meta.UserType):
f.write('from _devbuild.gen.%s import %s\n' %
(typ.mod_name, typ.type_name))
# HACK
f.write('from _devbuild.gen.%s import Id_str\n' % typ.mod_name)
f.write('\n')
# NOTE: Dict, Any are for AssocArray with 'dict' type.
f.write("""\
from asdl import pybase
from typing import Optional, List, Tuple, Dict, Any, cast
""")
pretty_print_methods = bool(os.getenv('PRETTY_PRINT_METHODS', 'yes'))
optional_fields = bool(os.getenv('OPTIONAL_FIELDS', 'yes'))
if pretty_print_methods:
f.write("""
from asdl import runtime # For runtime.NO_SPID
from asdl.runtime import NewRecord, NewLeaf
from _devbuild.gen.hnode_asdl import color_e, hnode, hnode_e, hnode_t, field
""")
abbrev_mod_entries = dir(abbrev_mod) if abbrev_mod else []
v = gen_python.GenMyPyVisitor(
f,
type_lookup,
abbrev_mod_entries,
pretty_print_methods=pretty_print_methods,
optional_fields=optional_fields)
v.VisitModule(schema_ast)
if abbrev_mod:
f.write("""\
#
# CONCATENATED FILE
#
""")
package, module = abbrev_module_name.split('.')
path = os.path.join(package, module + '.py')
with open(path) as in_f:
f.write(in_f.read())
else:
raise RuntimeError('Invalid action %r' % action)
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
# TODO: Also generate a switch/static_cast<> pretty printer in C++! For
# debugging. Might need to detect cycles though.
if action == 'cpp':
schema_path = argv[2]
# NOTE: This import can't be at the top level osh/asdl_gen.py depends on
# this gen_cpp.py module. We should move all the main() functions out of
# asdl/ and into command line tools.
from core.meta import Id
app_types = {'id': meta.UserType('id_kind_asdl', 'Id_t')}
with open(schema_path) as input_f:
module, type_lookup = front_end.LoadSchema(input_f, app_types)
# TODO: gen_cpp.py should be a library and the application should add Id?
# Or we should enable ASDL metaprogramming, and let Id be a metaprogrammed
# simple sum type.
f = sys.stdout
# How do mutation of strings, arrays, etc. work? Are they like C++
# containers, or their own? I think they mirror the oil language
# semantics.
# Every node should have a mirror. MutableObj. MutableRef (pointer).
# MutableArithVar -- has std::string. The mirrors are heap allocated.
# All the mutable ones should support Dump()/Encode()?
# You can just write more at the end... don't need to disturb existing
# nodes? Rewrite pointers.
alignment = 4
enc = encode.Params(alignment)
d = {'pointer_type': enc.pointer_type}
f.write("""\
#include <cstdint>
class Obj {
public:
// Decode a 3 byte integer from little endian
inline int Int(int n) const;
inline const Obj& Ref(const %(pointer_type)s* base, int n) const;
inline const Obj* Optional(const %(pointer_type)s* base, int n) const;
// NUL-terminated
inline const char* Str(const %(pointer_type)s* base, int n) const;
protected:
uint8_t bytes_[1]; // first is ID; rest are a payload
};
""" % d)
# Id should be treated as an enum.
c = ChainOfVisitors(
ForwardDeclareVisitor(f),
ClassDefVisitor(f, enc, type_lookup, enum_types=['Id']))
c.VisitModule(module)
f.write("""\
inline int Obj::Int(int n) const {
return bytes_[n] + (bytes_[n+1] << 8) + (bytes_[n+2] << 16);
}
inline const Obj& Obj::Ref(const %(pointer_type)s* base, int n) const {
int offset = Int(n);
return reinterpret_cast<const Obj&>(base[offset]);
}
inline const Obj* Obj::Optional(const %(pointer_type)s* base, int n) const {
int offset = Int(n);
if (offset) {
return reinterpret_cast<const Obj*>(base + offset);
} else {
return nullptr;
}
}
inline const char* Obj::Str(const %(pointer_type)s* base, int n) const {
int offset = Int(n);
return reinterpret_cast<const char*>(base + offset);
}
""" % d)
# uint32_t* and char*/Obj* aren't related, so we need to use
# reinterpret_cast<>.
# http://stackoverflow.com/questions/10151834/why-cant-i-static-cast-between-char-and-unsigned-char
else:
raise RuntimeError('Invalid action %r' % action)
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
try:
schema_path = argv[2]
except IndexError:
raise RuntimeError('Schema path required')
schema_filename = os.path.basename(schema_path)
if schema_filename in ('syntax.asdl', 'runtime.asdl'):
app_types = {'id': UserType('id_kind_asdl', 'Id_t')}
else:
app_types = {}
if action == 'c': # Generate C code for the lexer
with open(schema_path) as f:
schema_ast = front_end.LoadSchema(f, app_types)
v = gen_cpp.CEnumVisitor(sys.stdout)
v.VisitModule(schema_ast)
elif action == 'cpp': # Generate C++ code for ASDL schemas
out_prefix = argv[3]
pretty_print_methods = bool(os.getenv('PRETTY_PRINT_METHODS', 'yes'))
with open(schema_path) as f:
schema_ast = front_end.LoadSchema(f, app_types)
# asdl/typed_arith.asdl -> typed_arith_asdl
ns = os.path.basename(schema_path).replace('.', '_')
with open(out_prefix + '.h', 'w') as f:
guard = ns.upper()
f.write("""\
// %s.h is generated by asdl/tool.py
#ifndef %s
#define %s
""" % (out_prefix, guard, guard))
f.write("""\
#include <cstdint>
#include "mylib.h" // for Str, List, etc.
""")
if pretty_print_methods:
f.write("""\
#include "hnode_asdl.h"
using hnode_asdl::hnode_t;
""")
if app_types:
f.write("""\
#include "id_kind_asdl.h"
using id_kind_asdl::Id_t;
""")
for use in schema_ast.uses:
# Forward declarations in the header, like
# namespace syntax_asdl { class command_t; }
# must come BEFORE namespace, so it can't be in the visitor.
# assume sum type for now!
cpp_names = [
'class %s;' % asdl_.TypeNameHeuristic(n)
for n in use.type_names
]
f.write('namespace %s_asdl { %s }\n' %
(use.mod_name, ' '.join(cpp_names)))
f.write('\n')
f.write("""\
namespace %s {
""" % ns)
v = gen_cpp.ForwardDeclareVisitor(f)
v.VisitModule(schema_ast)
debug_info = {}
v2 = gen_cpp.ClassDefVisitor(
f,
pretty_print_methods=pretty_print_methods,
simple_int_sums=_SIMPLE,
debug_info=debug_info)
v2.VisitModule(schema_ast)
f.write("""
} // namespace %s
#endif // %s
""" % (ns, guard))
try:
debug_info_path = argv[4]
except IndexError:
pass
else:
with open(debug_info_path, 'w') as f:
from pprint import pformat
f.write('''\
cpp_namespace = %r
tags_to_types = \\
%s
''' % (ns, pformat(debug_info)))
with open(out_prefix + '.cc', 'w') as f:
f.write("""\
// %s.cc is generated by asdl/tool.py
#include "%s.h"
#include <assert.h>
#include "asdl_runtime.h" // generated code uses wrappers here
""" % (out_prefix, ns))
# To call pretty-printing methods
for use in schema_ast.uses:
f.write('#include "%s_asdl.h" // ASDL use\n' %
use.mod_name)
f.write("""\
// Generated code uses these types
using hnode_asdl::hnode__Record;
using hnode_asdl::hnode__Array;
using hnode_asdl::hnode__External;
using hnode_asdl::hnode__Leaf;
using hnode_asdl::field;
using hnode_asdl::color_e;
""")
if app_types:
f.write('using id_kind_asdl::Id_str;\n')
f.write("""
namespace %s {
""" % ns)
v3 = gen_cpp.MethodDefVisitor(
f,
pretty_print_methods=pretty_print_methods,
simple_int_sums=_SIMPLE)
v3.VisitModule(schema_ast)
f.write("""
} // namespace %s
""" % ns)
elif action == 'mypy': # Generated typed MyPy code
with open(schema_path) as f:
schema_ast = front_end.LoadSchema(f, app_types)
try:
abbrev_module_name = argv[3]
except IndexError:
abbrev_mod = None
else:
# Weird Python rule for importing: fromlist needs to be non-empty.
abbrev_mod = __import__(abbrev_module_name, fromlist=['.'])
f = sys.stdout
# TODO: Remove Any once we stop using it
f.write("""\
from asdl import pybase
from typing import Optional, List, Tuple, Dict, Any, cast, TYPE_CHECKING
""")
if schema_ast.uses:
f.write('\n')
f.write('if TYPE_CHECKING:\n')
for use in schema_ast.uses:
py_names = [asdl_.TypeNameHeuristic(n) for n in use.type_names]
# indented
f.write(' from _devbuild.gen.%s_asdl import %s\n' %
(use.mod_name, ', '.join(py_names)))
f.write('\n')
for typ in app_types.itervalues():
if isinstance(typ, UserType):
f.write('from _devbuild.gen.%s import %s\n' %
(typ.mod_name, typ.type_name))
# HACK
f.write('from _devbuild.gen.%s import Id_str\n' % typ.mod_name)
f.write('\n')
pretty_print_methods = bool(os.getenv('PRETTY_PRINT_METHODS', 'yes'))
optional_fields = bool(os.getenv('OPTIONAL_FIELDS', 'yes'))
if pretty_print_methods:
f.write("""
from asdl import runtime # For runtime.NO_SPID
from asdl.runtime import NewRecord, NewLeaf
from _devbuild.gen.hnode_asdl import color_e, hnode, hnode_e, hnode_t, field
""")
abbrev_mod_entries = dir(abbrev_mod) if abbrev_mod else []
v = gen_python.GenMyPyVisitor(
f,
abbrev_mod_entries,
pretty_print_methods=pretty_print_methods,
optional_fields=optional_fields,
simple_int_sums=_SIMPLE)
v.VisitModule(schema_ast)
if abbrev_mod:
f.write("""\
#
# CONCATENATED FILE
#
""")
package, module = abbrev_module_name.split('.')
path = os.path.join(package, module + '.py')
with open(path) as in_f:
f.write(in_f.read())
else:
raise RuntimeError('Invalid action %r' % action)
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
try:
schema_path = argv[2]
except IndexError:
raise RuntimeError('Schema path required')
if os.path.basename(schema_path) == 'types.asdl':
app_types = {}
else:
app_types = {'id': meta.UserType('id_kind_asdl', 'Id_t')}
if action == 'c': # Generate C code for the lexer
with open(schema_path) as f:
schema_ast, _ = front_end.LoadSchema(f, app_types)
v = gen_cpp.CEnumVisitor(sys.stdout)
v.VisitModule(schema_ast)
elif action == 'mypy': # typed mypy
with open(schema_path) as f:
schema_ast, type_lookup = front_end.LoadSchema(f, app_types)
try:
abbrev_module_name = argv[3]
except IndexError:
abbrev_mod = None
else:
# Weird Python rule for importing: fromlist needs to be non-empty.
abbrev_mod = __import__(abbrev_module_name, fromlist=['.'])
f = sys.stdout
for typ in app_types.itervalues():
if isinstance(typ, meta.UserType):
f.write('from _devbuild.gen.%s import %s\n' %
(typ.mod_name, typ.type_name))
f.write('\n')
f.write("""\
from asdl import const # For const.NO_INTEGER
from asdl import runtime
PrettyLeaf = runtime.PrettyLeaf
PrettyArray = runtime.PrettyArray
PrettyNode = runtime.PrettyNode
Color_TypeName = runtime.Color_TypeName
Color_StringConst = runtime.Color_StringConst
Color_OtherConst = runtime.Color_OtherConst
Color_UserType = runtime.Color_UserType
from typing import Optional, List, Tuple
""")
abbrev_mod_entries = dir(abbrev_mod) if abbrev_mod else []
v = gen_python.GenMyPyVisitor(f, type_lookup, abbrev_mod_entries)
v.VisitModule(schema_ast)
if abbrev_mod:
f.write("""\
#
# CONCATENATED FILE
#
""")
package, module = abbrev_module_name.split('.')
path = os.path.join(package, module + '.py')
with open(path) as in_f:
f.write(in_f.read())
else:
raise RuntimeError('Invalid action %r' % action)
def _assertResolve(self, code_str):
f = cStringIO.StringIO(code_str)
schema_ast = front_end.LoadSchema(f, {})
print(schema_ast)
def testSharedVariant(self):
with open('asdl/shared_variant.asdl') as f:
schema_ast = front_end.LoadSchema(f, {}, verbose=False)
print(schema_ast)
def testLoadSchema(self):
with open('asdl/typed_demo.asdl') as f:
schema_ast = front_end.LoadSchema(f, {}, verbose=True)
print(schema_ast)
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
try:
schema_path = argv[2]
except IndexError:
raise RuntimeError('Schema path required')
if os.path.basename(schema_path) in ('syntax.asdl', 'runtime.asdl'):
app_types = {'id': meta.UserType('id_kind_asdl', 'Id_t')}
else:
app_types = {}
if action == 'c': # Generate C code for the lexer
with open(schema_path) as f:
schema_ast, _ = front_end.LoadSchema(f, app_types)
v = gen_cpp.CEnumVisitor(sys.stdout)
v.VisitModule(schema_ast)
elif action == 'cpp': # Generate C++ code for ASDL schemas
with open(schema_path) as f:
schema_ast, type_lookup = front_end.LoadSchema(f, app_types)
# asdl/typed_arith.asdl -> typed_arith_asdl
ns = os.path.basename(schema_path).replace('.', '_')
f = sys.stdout
f.write("""\
#include <cstdint>
#include "runtime.h" // for Str, List, etc.
namespace %s {
""" % ns)
v = gen_cpp.ForwardDeclareVisitor(f)
v.VisitModule(schema_ast)
v2 = gen_cpp.ClassDefVisitor(f, type_lookup)
v2.VisitModule(schema_ast)
f.write('} // namespace %s\n' % ns)
elif action == 'mypy': # Generated typed MyPy code
with open(schema_path) as f:
schema_ast, type_lookup = front_end.LoadSchema(f, app_types)
try:
abbrev_module_name = argv[3]
except IndexError:
abbrev_mod = None
else:
# Weird Python rule for importing: fromlist needs to be non-empty.
abbrev_mod = __import__(abbrev_module_name, fromlist=['.'])
f = sys.stdout
for typ in app_types.itervalues():
if isinstance(typ, meta.UserType):
f.write('from _devbuild.gen.%s import %s\n' %
(typ.mod_name, typ.type_name))
f.write('\n')
# NOTE: Dict, Any are for AssocArray with 'dict' type.
f.write("""\
from asdl import const # For const.NO_INTEGER
from asdl import runtime
from asdl.runtime import (
PrettyLeaf, PrettyArray, PrettyNode,
Color_TypeName, Color_StringConst, Color_OtherConst, Color_UserType,
)
from typing import Optional, List, Tuple, Dict, Any
""")
abbrev_mod_entries = dir(abbrev_mod) if abbrev_mod else []
v = gen_python.GenMyPyVisitor(f, type_lookup, abbrev_mod_entries)
v.VisitModule(schema_ast)
if abbrev_mod:
f.write("""\
#
# CONCATENATED FILE
#
""")
package, module = abbrev_module_name.split('.')
path = os.path.join(package, module + '.py')
with open(path) as in_f:
f.write(in_f.read())
else:
raise RuntimeError('Invalid action %r' % action)
#!/usr/bin/env python
"""
arith_ast.py
"""
import sys
from asdl import front_end
from asdl import py_meta
from core import util
f = util.GetResourceLoader().open('asdl/arith.asdl')
_asdl_module, _type_lookup = front_end.LoadSchema(f, {}) # no app_types
f.close()
root = sys.modules[__name__]
py_meta.MakeTypes(_asdl_module, root, _type_lookup)
def testSharedVariant(self):
with open('asdl/shared_variant.asdl') as f:
schema_ast, type_lookup = front_end.LoadSchema(f, {}, verbose=True)
