def _ParseAndMakeTypes(schema_path, root):
module = asdl.parse(schema_path)
app_types = {
'identifier': asdl.UserType(Identifier),
'bytes': asdl.UserType(Bytes),
'object': asdl.UserType(PyObject),
'constant': asdl.UserType(Constant),
'singleton': asdl.UserType(Singleton),
}
# Check for type errors
if not asdl.check(module, app_types):
raise AssertionError('ASDL file is invalid')
py_meta.MakeTypes(module, root, app_types)
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
try:
schema_path = argv[2]
except IndexError:
raise RuntimeError('Schema path required')
# To avoid circular dependencies, don't load Id for types.asdl.
if os.path.basename(schema_path) == 'types.asdl':
app_types = {}
else:
from osh.meta import Id
app_types = {'id': asdl.UserType(Id)}
if action == 'c': # Generate C code for the lexer
with open(schema_path) as f:
schema_ast, _ = front_end.LoadSchema(f, app_types)
v = gen_cpp.CEnumVisitor(sys.stdout)
v.VisitModule(schema_ast)
elif action == 'py': # Generate Python code so we don't depend on ASDL schemas
pickle_out_path = argv[3]
with open(schema_path) as f:
schema_ast, type_lookup = front_end.LoadSchema(f, app_types)
f = sys.stdout
f.write("""\
from asdl import asdl_ as asdl
from asdl import const # For const.NO_INTEGER
from asdl import py_meta
from asdl import unpickle
from core import util
f = util.GetResourceLoader().open('%s')
TYPE_LOOKUP = unpickle.load_v2_subset(f)
f.close()
""" % pickle_out_path)
v = gen_python.GenClassesVisitor(f)
v.VisitModule(schema_ast)
if pickle_out_path:
# Pickle version 2 is better. (Pickle version 0 uses
# s.decode('string-escape')! )
# In version 2, now I have 16 opcodes + STOP.
with open(pickle_out_path, 'w') as f:
pickle.dump(type_lookup, f, protocol=2)
from core.util import log
log('Wrote %s', pickle_out_path)
else:
raise RuntimeError('Invalid action %r' % action)
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
if action == 'py': # Prints the module
# Called by asdl/run.sh py-cpp
schema_path = argv[2]
app_types = {'id': asdl.UserType('id_kind_asdl', 'Id_t')}
with open(schema_path) as f:
schema_ast, type_lookup = front_end.LoadSchema(f, app_types)
root = sys.modules[__name__]
# NOTE: We shouldn't pass in app_types for arith.asdl, but this is just a
# demo.
py_meta.MakeTypes(schema_ast, root, type_lookup)
log('AST for this ASDL schema:')
schema_ast.Print(sys.stdout, 0)
print()
log('Dynamically created a Python module with these types:')
for name in dir(root):
print('\t' + name)
if 1:
# NOTE: It can be pickled, but not marshaled
import marshal
import cPickle
print(dir(marshal))
out_path = schema_path + '.pickle'
with open(out_path, 'w') as f:
#marshal.dump(type_lookup, f)
# Version 2 is the highest protocol for Python 2.7.
cPickle.dump(type_lookup.runtime_type_lookup, f, protocol=2)
print('runtime_type_lookup:')
for name, desc in type_lookup.runtime_type_lookup.items():
print(name)
print(desc)
print()
print('Wrote %s' % out_path)
elif action == 'arith-format': # pretty printing
expr = argv[2]
obj = typed_arith_parse.ParseShell(expr)
tree = obj.PrettyTree()
#treee= ['hi', 'there', ['a', 'b'], 'c']
f = fmt.DetectConsoleOutput(sys.stdout)
fmt.PrintTree(tree, f)
print()
# Might need to print the output?
# out.WriteToFile?
else:
raise RuntimeError('Invalid action %r' % action)
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
if action == 'py': # Prints the module
schema_path = argv[2]
with open(schema_path) as f:
module = asdl.parse(f)
app_types = {'id': asdl.UserType(Id)}
type_lookup = asdl.ResolveTypes(module, app_types)
# Note this is a big tree. But we really want a graph of pointers to
# instances.
# Type(name, Product(...))
# Type(name, Sum([Constructor(...), ...]))
#print(module)
root = sys.modules[__name__]
# NOTE: We shouldn't pass in app_types for arith.asdl, but this is just a
# demo.
py_meta.MakeTypes(module, root, type_lookup)
print('Dynamically created a Python module with these types:')
for name in dir(root):
print('\t' + name)
elif action == 'arith-encode': # oheap encoding
expr = argv[2]
out_path = argv[3]
obj = arith_parse.ParseShell(expr)
print('Encoding %r into binary:' % expr)
print(obj)
enc = encode.Params()
with open(out_path, 'wb') as f:
out = encode.BinOutput(f)
encode.EncodeRoot(obj, enc, out)
elif action == 'arith-format': # pretty printing
expr = argv[2]
obj = arith_parse.ParseShell(expr)
#out = fmt.TextOutput(sys.stdout)
tree = fmt.MakeTree(obj)
#treee= ['hi', 'there', ['a', 'b'], 'c']
f = fmt.DetectConsoleOutput(sys.stdout)
fmt.PrintTree(tree, f)
print()
# Might need to print the output?
# out.WriteToFile?
else:
raise RuntimeError('Invalid action %r' % action)
def _ParseAndMakeTypes(f, root):
module = asdl.parse(f)
app_types = {'id': asdl.UserType(Id)}
# Check for type errors
if not asdl.check(module, app_types):
raise AssertionError('ASDL file is invalid')
py_meta.MakeTypes(module, root, app_types)
def _LoadSchema(f):
module = asdl.parse(f)
app_types = {'id': asdl.UserType(Id)}
type_lookup = asdl.ResolveTypes(module, app_types)
# Check for type errors
if not asdl.check(module, app_types):
raise AssertionError('ASDL file is invalid')
return module, type_lookup
def main(argv):
#print lex_mode_e
#print dir(lex_mode_e)
app_types = {'id': asdl.UserType(Id)}
with open('osh/types.asdl') as f:
asdl_module, _ = asdl.LoadSchema(f, app_types)
v = gen_cpp.CEnumVisitor(sys.stdout)
v.VisitModule(asdl_module)
def LoadSchema(f):
app_types = {'id': asdl.UserType(Id)}
asdl_module = asdl.parse(f)
if not asdl.check(asdl_module, app_types):
raise AssertionError('ASDL file is invalid')
type_lookup = asdl.ResolveTypes(asdl_module, app_types)
return asdl_module, type_lookup
def _ParseAndMakeTypes(f, root):
# TODO: A better syntax for this might be:
#
# id = external
#
# in osh.asdl. Then we can show an error if it's not provided.
app_types = {'id': asdl.UserType(Id)}
module = asdl.parse(f)
# Check for type errors
if not asdl.check(module, app_types):
raise AssertionError('ASDL file is invalid')
py_meta.MakeTypes(module, root, app_types)
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
if action == 'py':
schema_path = argv[2]
module = asdl.parse(schema_path)
root = sys.modules[__name__]
# NOTE: We shouldn't pass in app_types for arith.asdl, but this is just a
# demo.
py_meta.MakeTypes(module, root, app_types={'id': asdl.UserType(Id)})
print(dir(root))
elif action == 'arith-encode':
expr = argv[2]
out_path = argv[3]
obj = arith_parse.ParseShell(expr)
print('Encoding %r into binary:' % expr)
print(obj)
enc = encode.Params()
with open(out_path, 'wb') as f:
out = encode.BinOutput(f)
encode.EncodeRoot(obj, enc, out)
elif action == 'arith-format':
expr = argv[2]
obj = arith_parse.ParseShell(expr)
#out = fmt.TextOutput(sys.stdout)
tree = fmt.MakeTree(obj)
#treee= ['hi', 'there', ['a', 'b'], 'c']
fmt.PrintTree(tree, sys.stdout)
# Might need to print the output?
# out.WriteToFile?
else:
raise RuntimeError('Invalid action %r' % action)
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
# TODO: Also generate a switch/static_cast<> pretty printer in C++! For
# debugging. Might need to detect cycles though.
if action == 'cpp':
schema_path = argv[2]
# NOTE: This import can't be at the top level osh/asdl_gen.py depends on
# this gen_cpp.py module. We should move all the main() functions out of
# asdl/ and into command line tools.
from osh.meta import Id
app_types = {'id': asdl.UserType(Id)}
with open(schema_path) as input_f:
module, type_lookup = front_end.LoadSchema(input_f, app_types)
# TODO: gen_cpp.py should be a library and the application should add Id?
# Or we should enable ASDL metaprogramming, and let Id be a metaprogrammed
# simple sum type.
f = sys.stdout
# How do mutation of strings, arrays, etc. work? Are they like C++
# containers, or their own? I think they mirror the oil language
# semantics.
# Every node should have a mirror. MutableObj. MutableRef (pointer).
# MutableArithVar -- has std::string. The mirrors are heap allocated.
# All the mutable ones should support Dump()/Encode()?
# You can just write more at the end... don't need to disturb existing
# nodes? Rewrite pointers.
alignment = 4
enc = encode.Params(alignment)
d = {'pointer_type': enc.pointer_type}
f.write("""\
#include <cstdint>
class Obj {
public:
// Decode a 3 byte integer from little endian
inline int Int(int n) const;
inline const Obj& Ref(const %(pointer_type)s* base, int n) const;
inline const Obj* Optional(const %(pointer_type)s* base, int n) const;
// NUL-terminated
inline const char* Str(const %(pointer_type)s* base, int n) const;
protected:
uint8_t bytes_[1]; // first is ID; rest are a payload
};
""" % d)
# Id should be treated as an enum.
c = ChainOfVisitors(
ForwardDeclareVisitor(f),
ClassDefVisitor(f, enc, type_lookup, enum_types=['Id']))
c.VisitModule(module)
f.write("""\
inline int Obj::Int(int n) const {
return bytes_[n] + (bytes_[n+1] << 8) + (bytes_[n+2] << 16);
}
inline const Obj& Obj::Ref(const %(pointer_type)s* base, int n) const {
int offset = Int(n);
return reinterpret_cast<const Obj&>(base[offset]);
}
inline const Obj* Obj::Optional(const %(pointer_type)s* base, int n) const {
int offset = Int(n);
if (offset) {
return reinterpret_cast<const Obj*>(base + offset);
} else {
return nullptr;
}
}
inline const char* Obj::Str(const %(pointer_type)s* base, int n) const {
int offset = Int(n);
return reinterpret_cast<const char*>(base + offset);
}
""" % d)
# uint32_t* and char*/Obj* aren't related, so we need to use
# reinterpret_cast<>.
# http://stackoverflow.com/questions/10151834/why-cant-i-static-cast-between-char-and-unsigned-char
else:
raise RuntimeError('Invalid action %r' % action)
# Instantiate the spec
#
ID_SPEC = id_kind.IdSpec(Id, Kind, _ID_NAMES, _ID_INSTANCES, _ID_TO_KIND,
BOOL_ARG_TYPES)
id_kind.AddKinds(ID_SPEC)
id_kind.AddBoolKinds(ID_SPEC, Id, types.bool_arg_type_e) # must come second
id_kind.SetupTestBuiltin(Id, Kind, ID_SPEC, TEST_UNARY_LOOKUP,
TEST_BINARY_LOOKUP, TEST_OTHER_LOOKUP,
types.bool_arg_type_e)
# Debug
_kind_sizes = ID_SPEC.kind_sizes
APP_TYPES = {'id': asdl.UserType(Id)}
#
# Instantiate osh/osh.asdl
#
f = util.GetResourceLoader().open('osh/osh.asdl')
_asdl_module, _type_lookup = asdl.LoadSchema(f, APP_TYPES)
ast = _AsdlModule()
if 0:
py_meta.MakeTypes(_asdl_module, ast, _type_lookup)
else:
# Exported for the generated code to use
OSH_TYPE_LOOKUP = _type_lookup
