def _CheckType(value, expected_desc):
"""Is value of type expected_desc?
Args:
value: Obj or primitive type
expected_desc: instance of asdl.Product, asl.Sum, asdl.StrType,
asdl.IntType, ArrayType, MaybeType, etc.
"""
if isinstance(expected_desc, asdl.Constructor):
# This doesn't make sense because the descriptors are derived from the
# declared types. You can declare a field as arith_expr_e but not
# ArithBinary.
raise AssertionError("Invalid Constructor descriptor")
if isinstance(expected_desc, asdl.MaybeType):
if value is None:
return True
return _CheckType(value, expected_desc.desc)
if isinstance(expected_desc, asdl.ArrayType):
if not isinstance(value, list):
return False
# Now check all entries
for item in value:
if not _CheckType(item, expected_desc.desc):
return False
return True
if isinstance(expected_desc, asdl.StrType):
return isinstance(value, str)
if isinstance(expected_desc, asdl.IntType):
return isinstance(value, int)
if isinstance(expected_desc, asdl.BoolType):
return isinstance(value, bool)
if isinstance(expected_desc, asdl.UserType):
return isinstance(value, expected_desc.typ)
try:
actual_desc = value.__class__.ASDL_TYPE
except AttributeError:
return False # it's not of the right type
if isinstance(expected_desc, asdl.Product):
return actual_desc is expected_desc
if isinstance(expected_desc, asdl.Sum):
if asdl.is_simple(expected_desc):
return actual_desc is expected_desc
else:
for cons in expected_desc.types: # It has to be one of the alternatives
#log("CHECKING desc %s against %s" % (desc, cons))
if actual_desc is cons:
return True
return False
raise AssertionError('Invalid descriptor %r: %r' %
(expected_desc.__class__, expected_desc))
def _MakeReflection(module, app_types):
# Types that fields are declared with: int, id, word_part, etc.
# Fields are NOT declared with Constructor names.
type_lookup = dict(meta.BUILTIN_TYPES)
type_lookup.update(app_types)
# TODO: Need to resolve 'imports' to the right descriptor. Code generation
# relies on it:
# - To pick the method to call in AbbreviatedTree etc.
# - To generate 'value_t' instead of 'value' in type annotations.
for u in module.uses:
for type_name in u.type_names:
type_lookup[type_name] = None # Placeholder
# NOTE: We need two passes because types can be mutually recursive, e.g.
# asdl/arith.asdl.
# First pass: collect declared types and make entries for them.
for d in module.dfns:
ast_node = d.value
if isinstance(ast_node, asdl.Product):
type_lookup[d.name] = meta.CompoundType([])
elif isinstance(ast_node, asdl.Sum):
is_simple = asdl.is_simple(ast_node)
type_lookup[d.name] = meta.SumType(is_simple)
else:
raise AssertionError(ast_node)
# Second pass: resolve type declarations in Product and constructor.
for d in module.dfns:
ast_node = d.value
if isinstance(ast_node, asdl.Product):
runtime_type = type_lookup[d.name]
_AppendFields(ast_node.fields, type_lookup, runtime_type.fields)
elif isinstance(ast_node, asdl.Sum):
sum_type = type_lookup[d.name] # the one we just created
for cons in ast_node.types:
fields_out = []
# fully-qualified name. Use a _ so we can share strings with class
# name.
key = '%s__%s' % (d.name, cons.name)
cons_type = meta.CompoundType(fields_out)
type_lookup[key] = cons_type
_AppendFields(cons.fields, type_lookup, fields_out)
sum_type.cases.append(cons_type)
else:
raise AssertionError(ast_node)
return type_lookup
def EncodeArray(obj_list, item_desc, enc, out):
"""
Args:
obj_list: List of Obj values
Returns:
ref
"""
array_chunk = bytearray()
enc.Int(len(obj_list), array_chunk) # Length prefix
if isinstance(item_desc, asdl.IntType) or \
isinstance(item_desc, asdl.BoolType):
for item in obj_list:
enc.Int(item, array_chunk)
elif isinstance(item_desc, asdl.UserType):
# Assume Id for now
for item in obj_list:
enc.Int(item.enum_value, array_chunk)
elif isinstance(item_desc, asdl.StrType):
for item in obj_list:
ref = out.Write(enc.PaddedStr(item))
enc.Ref(ref, array_chunk)
elif isinstance(item_desc, asdl.Sum) and asdl.is_simple(item_desc):
for item in obj_list:
enc.Int(item.enum_id, array_chunk)
else:
# A simple value is either an int, enum, or pointer. (Later: Iter<Str>
# might be possible for locality.)
assert \
isinstance(item_desc, asdl.SumType) or \
isinstance(item_desc, asdl.CompoundType), item_desc
# This is like vector<T*>
# Later:
# - Product types can be put in line
# - Sum types can even be put in line, if you have List<T> rather than
# Array<T>. Array implies O(1) random access; List doesn't.
for item in obj_list:
try:
ref = EncodeObj(item, enc, out)
except EncodeError as e:
if not e.details_printed:
util.log("Error encoding array: %s (item %s)", e, item)
e.details_printed = True
raise
enc.Ref(ref, array_chunk)
this_ref = out.Write(enc.PaddedBlock(array_chunk))
return this_ref
def _MakeReflection(module, app_types):
# Types that fields are declared with: int, id, word_part, etc.
# Fields are NOT declared with Constructor names.
type_lookup = dict(runtime.BUILTIN_TYPES)
type_lookup.update(app_types)
# NOTE: We need two passes because types can be mutually recursive, e.g.
# asdl/arith.asdl.
# First pass: collect declared types and make entries for them.
for d in module.dfns:
ast_node = d.value
if isinstance(ast_node, asdl.Product):
type_lookup[d.name] = runtime.CompoundType([])
elif isinstance(ast_node, asdl.Sum):
is_simple = asdl.is_simple(ast_node)
type_lookup[d.name] = runtime.SumType(is_simple)
else:
raise AssertionError(ast_node)
# Second pass: resolve type declarations in Product and constructor.
for d in module.dfns:
ast_node = d.value
if isinstance(ast_node, asdl.Product):
runtime_type = type_lookup[d.name]
_AppendFields(ast_node.fields, type_lookup, runtime_type.fields)
elif isinstance(ast_node, asdl.Sum):
sum_type = type_lookup[d.name] # the one we just created
for cons in ast_node.types:
fields_out = []
# fully-qualified name. Use a _ so we can share strings with class
# name.
key = '%s__%s' % (d.name, cons.name)
cons_type = runtime.CompoundType(fields_out)
type_lookup[key] = cons_type
_AppendFields(cons.fields, type_lookup, fields_out)
sum_type.cases.append(cons_type)
else:
raise AssertionError(ast_node)
return type_lookup
def MakeFieldSubtree(obj, field_name, desc, abbrev_hook, omit_empty=True):
try:
field_val = getattr(obj, field_name)
except AttributeError:
# This happens when required fields are not initialized, e.g. FuncCall()
# without setting name.
raise AssertionError('%s is missing field %r' %
(obj.__class__, field_name))
if isinstance(desc, asdl.IntType):
out_val = _ColoredString(str(field_val), _OTHER_LITERAL)
elif isinstance(desc, asdl.BoolType):
out_val = _ColoredString('T' if field_val else 'F', _OTHER_LITERAL)
elif isinstance(desc, asdl.DictType):
raise AssertionError
elif isinstance(desc, asdl.Sum) and asdl.is_simple(desc):
out_val = field_val.name
elif isinstance(desc, asdl.StrType):
out_val = _ColoredString(field_val, _STRING_LITERAL)
elif isinstance(desc, asdl.ArrayType):
out_val = []
obj_list = field_val
for child_obj in obj_list:
t = MakeTree(child_obj, abbrev_hook)
out_val.append(t)
if omit_empty and not obj_list:
out_val = None
elif isinstance(desc, asdl.MaybeType):
if field_val is None:
out_val = None
else:
out_val = MakeTree(field_val, abbrev_hook)
else:
out_val = MakeTree(field_val, abbrev_hook)
return out_val
def _GetCppType(self, field):
"""Return a string for the C++ name of the type."""
type_name = field.type
cpp_type = _BUILTINS.get(type_name)
if cpp_type is not None:
return cpp_type
typ = self.type_lookup[type_name]
if isinstance(typ, asdl.Sum) and asdl.is_simple(typ):
# Use the enum instead of the class.
return "%s_e" % type_name
# - Pointer for optional type.
# - ints and strings should generally not be optional? We don't have them
# in osh yet, so leave it out for now.
if field.opt:
return "%s_t*" % type_name
return "%s_t&" % type_name
def EncodeArray(obj_list, item_desc, enc, out):
"""
Args:
obj_list: List of Obj values
Returns:
ref
"""
array_chunk = bytearray()
enc.Int(len(obj_list), array_chunk) # Length prefix
if isinstance(item_desc, asdl.IntType) or \
isinstance(item_desc, asdl.BoolType):
for item in obj_list:
enc.Int(item, array_chunk)
elif isinstance(item_desc, asdl.Sum) and asdl.is_simple(item_desc):
for item in obj_list:
enc.Int(item.enum_id, array_chunk)
else:
# A simple value is either an int, enum, or pointer. (Later: Iter<Str>
# might be possible for locality.)
assert isinstance(item_desc, asdl.Sum) or isinstance(
item_desc, asdl.Product), item_desc
# This is like vector<T*>
# Later:
# - Product types can be put in line
# - Sum types can even be put in line, if you have List<T> rather than
# Array<T>. Array implies O(1) random access; List doesn't.
for item in obj_list:
# Recursive call.
ref = EncodeObj(item, enc, out)
enc.Ref(ref, array_chunk)
this_ref = out.Write(enc.PaddedBlock(array_chunk))
return this_ref
def EncodeObj(obj, enc, out):
"""
Args:
obj: Obj to encode
enc: encoding params
out: output file
Returns:
ref: Reference to the last block
"""
# Algorithm: Depth first, post-order traversal. First obj is the first leaf.
# last obj is the root.
#
# Array is a homogeneous type.
this_chunk = bytearray()
assert isinstance(obj, py_meta.CompoundObj), \
'%r is not a compound obj (%r)' % (obj, obj.__class__)
# Constructor objects have a tag.
if isinstance(obj.ASDL_TYPE, asdl.Constructor):
enc.Tag(obj.tag, this_chunk)
for name, desc in obj.ASDL_TYPE.GetFields(): # encode in order
field_val = getattr(obj, name)
# TODO:
# - Float would be inline, etc.
# - Repeated value: write them all adjacent to each other?
is_maybe = False
if isinstance(desc, asdl.MaybeType):
is_maybe = True
desc = desc.desc # descent
#
# Now look at types
#
if isinstance(desc, asdl.IntType) or isinstance(desc, asdl.BoolType):
enc.Int(field_val, this_chunk)
elif isinstance(desc, asdl.Sum) and asdl.is_simple(desc):
# Encode enums as integers. TODO later: Don't use 3 bytes! Can use 1
# byte for most enums.
enc.Int(field_val.enum_id, this_chunk)
# Write variable length field first, assuming that it's a ref/pointer.
# TODO: allow one inline, hanging string or array per record.
elif isinstance(desc, asdl.StrType):
ref = out.Write(enc.PaddedStr(field_val))
enc.Ref(ref, this_chunk)
elif isinstance(desc, asdl.ArrayType):
item_desc = desc.desc
ref = EncodeArray(field_val, item_desc, enc, out)
enc.Ref(ref, this_chunk)
elif isinstance(desc, asdl.UserType):
if is_maybe and field_val is None: # e.g. id? prefix_op
enc.Ref(0, this_chunk)
else:
# Assume Id for now
enc.Int(field_val.enum_value, this_chunk)
else:
if is_maybe and field_val is None:
enc.Ref(0, this_chunk)
else:
try:
ref = EncodeObj(field_val, enc, out)
except EncodeError as e:
if not e.details_printed:
util.log("Error encoding %s : %s (val %s)", name, e,
field_val)
e.details_printed = True
raise
enc.Ref(ref, this_chunk)
# Write the parent record
this_ref = out.Write(enc.PaddedBlock(this_chunk))
return this_ref
def visitSum(self, sum, name):
for t in sum.types:
# Simple sum types can't conflict
if asdl.is_simple(sum):
continue
self.visit(t, name)
def VisitSum(self, sum, name, depth):
if asdl.is_simple(sum):
self.VisitSimpleSum(sum, name, depth)
else:
self.VisitCompoundSum(sum, name, depth)
def MakeTypes(module, root, type_lookup):
"""
Args:
module: asdl.Module
root: an object/package to add types to
"""
for defn in module.dfns:
typ = defn.value
#print('TYPE', defn.name, typ)
if isinstance(typ, asdl.Sum):
sum_type = typ
if asdl.is_simple(sum_type):
# An object without fields, which can be stored inline.
# Create a class called foo_e. Unlike the CompoundObj case, it doesn't
# have subtypes. Instead if has attributes foo_e.Bar, which Bar is an
# instance of foo_e.
#
# Problem: This means you have a dichotomy between:
# cflow_e.Break vs. cflow_e.Break()
# If you add a non-simple type like cflow_e.Return(5), the usage will
# change. I haven't run into this problem in practice yet.
class_name = defn.name + '_e'
class_attr = {'ASDL_TYPE': sum_type} # asdl.Sum
cls = type(class_name, (SimpleObj, ), class_attr)
setattr(root, class_name, cls)
# NOTE: Right now the ASDL_TYPE for for an enum value is the Sum type,
# not the Constructor type. We may want to change this if we need
# reflection.
for i, cons in enumerate(sum_type.types):
enum_id = i + 1
name = cons.name
val = cls(enum_id,
cons.name) # Instantiate SimpleObj subtype
# Set a static attribute like op_id.Plus, op_id.Minus.
setattr(cls, name, val)
else:
tag_num = {}
# e.g. for arith_expr
# Should this be arith_expr_t? It is in C++.
base_class = type(defn.name, (DebugCompoundObj, ), {})
setattr(root, defn.name, base_class)
# Make a type and a enum tag for each alternative.
for i, cons in enumerate(sum_type.types):
tag = i + 1 # zero reserved?
tag_num[cons.name] = tag # for enum
class_attr = {
'ASDL_TYPE': cons, # asdl.Constructor
'tag': tag, # Does this API change?
}
cls = type(cons.name, (base_class, ), class_attr)
setattr(root, cons.name, cls)
# e.g. arith_expr_e.Const == 1
enum_name = defn.name + '_e'
tag_enum = type(enum_name, (), tag_num)
setattr(root, enum_name, tag_enum)
elif isinstance(typ, asdl.Product):
class_attr = {'ASDL_TYPE': typ}
cls = type(defn.name, (DebugCompoundObj, ), class_attr)
setattr(root, defn.name, cls)
else:
raise AssertionError(typ)
def EncodeObj(obj, enc, out):
"""
Args:
obj: Obj to encode
enc: encoding params
out: output file
Returns:
ref: Reference to the last block
"""
# Algorithm: Depth first, post-order traversal. First obj is the first leaf.
# last obj is the root.
#
# Array is a homogeneous type.
this_chunk = bytearray()
assert isinstance(obj, py_meta.CompoundObj), \
'%r is not a compound obj (%r)' % (obj, obj.__class__)
# Constructor objects have a tag.
if isinstance(obj.DESCRIPTOR, asdl.Constructor):
enc.Tag(obj.tag, this_chunk)
for name in obj.FIELDS: # encode in order
desc = obj.DESCRIPTOR_LOOKUP[name]
#print('\n\n------------')
#print('field DESC', name, desc)
field_val = getattr(obj, name)
#print('VALUE', field_val)
# TODO:
# - Float would be inline, etc.
# - Repeated value: write them all adjacent to each other?
# INLINE
if isinstance(desc, asdl.IntType) or isinstance(desc, asdl.BoolType):
enc.Int(field_val, this_chunk)
elif isinstance(desc, asdl.Sum) and asdl.is_simple(desc):
# Encode enums as integers. TODO later: Don't use 3 bytes! Can use 1
# byte for most enums.
enc.Int(field_val.enum_id, this_chunk)
# Write variable length field first, assuming that it's a ref/pointer.
# TODO: allow one inline, hanging string or array per record.
elif isinstance(desc, asdl.StrType):
ref = out.Write(enc.PaddedStr(field_val))
enc.Ref(ref, this_chunk)
elif isinstance(desc, asdl.ArrayType):
item_desc = desc.desc
ref = EncodeArray(field_val, item_desc, enc, out)
enc.Ref(ref, this_chunk)
elif isinstance(desc, asdl.MaybeType):
item_desc = desc.desc
ok = False
if isinstance(item_desc, asdl.Sum):
if not asdl.is_simple(item_desc):
ok = True
elif isinstance(item_desc, asdl.Product):
ok = True
# TODO: Fix this for span_id. Need to extract a method.
if not ok:
raise AssertionError(
"Currently not encoding simple optional types: %s" % field_val)
if field_val is None:
enc.Ref(0, this_chunk)
else:
ref = EncodeObj(field_val, enc, out)
enc.Ref(ref, this_chunk)
elif isinstance(desc, asdl.UserType):
# Assume Id for now
enc.Int(field_val.enum_value, this_chunk)
else:
# Recursive call for child records. Write children before parents.
ref = EncodeObj(field_val, enc, out)
enc.Ref(ref, this_chunk)
# Write the parent record
this_ref = out.Write(enc.PaddedBlock(this_chunk))
return this_ref
def MakeTypes(module, root, app_types=None):
"""
Args:
module: asdl.Module
root: an object/package to add types to
"""
app_types = app_types or {}
for defn in module.dfns:
typ = defn.value
#print('TYPE', defn.name, typ)
if isinstance(typ, asdl.Sum):
sum_type = typ
if asdl.is_simple(sum_type):
# An object without fields, which can be stored inline.
class_attr = {'DESCRIPTOR': sum_type} # asdl.Sum
cls = type(defn.name, (SimpleObj, ), class_attr)
#print('CLASS', cls)
setattr(root, defn.name, cls)
for i, cons in enumerate(sum_type.types):
enum_id = i + 1
name = cons.name
val = cls(enum_id,
cons.name) # Instantiate SimpleObj subtype
# Set a static attribute like op_id.Plus, op_id.Minus.
setattr(cls, name, val)
else:
tag_num = {}
# e.g. for arith_expr
# Should this be arith_expr_t? It is in C++.
base_class = type(defn.name, (CompoundObj, ), {})
setattr(root, defn.name, base_class)
# Make a type and a enum tag for each alternative.
for i, cons in enumerate(sum_type.types):
tag = i + 1 # zero reserved?
tag_num[cons.name] = tag # for enum
# Add 'int* spids' to every constructor.
class_attr = _MakeFieldDescriptors(module, cons.fields,
app_types)
class_attr['DESCRIPTOR'] = cons # asdl.Constructor
class_attr['tag'] = tag
cls = type(cons.name, (base_class, ), class_attr)
setattr(root, cons.name, cls)
# e.g. arith_expr_e.Const == 1
enum_name = defn.name + '_e'
tag_enum = type(enum_name, (), tag_num)
setattr(root, enum_name, tag_enum)
elif isinstance(typ, asdl.Product):
class_attr = _MakeFieldDescriptors(module, typ.fields, app_types)
class_attr['DESCRIPTOR'] = typ
cls = type(defn.name, (CompoundObj, ), class_attr)
setattr(root, defn.name, cls)
else:
raise AssertionError(typ)
