def _CheckType(value, expected_desc): """Is value of type expected_desc? Args: value: Obj or primitive type expected_desc: instance of asdl.Product, asl.Sum, asdl.StrType, asdl.IntType, ArrayType, MaybeType, etc. """ if isinstance(expected_desc, asdl.Constructor): # This doesn't make sense because the descriptors are derived from the # declared types. You can declare a field as arith_expr_e but not # ArithBinary. raise AssertionError("Invalid Constructor descriptor") if isinstance(expected_desc, asdl.MaybeType): if value is None: return True return _CheckType(value, expected_desc.desc) if isinstance(expected_desc, asdl.ArrayType): if not isinstance(value, list): return False # Now check all entries for item in value: if not _CheckType(item, expected_desc.desc): return False return True if isinstance(expected_desc, asdl.StrType): return isinstance(value, str) if isinstance(expected_desc, asdl.IntType): return isinstance(value, int) if isinstance(expected_desc, asdl.BoolType): return isinstance(value, bool) if isinstance(expected_desc, asdl.UserType): return isinstance(value, expected_desc.typ) try: actual_desc = value.__class__.ASDL_TYPE except AttributeError: return False # it's not of the right type if isinstance(expected_desc, asdl.Product): return actual_desc is expected_desc if isinstance(expected_desc, asdl.Sum): if asdl.is_simple(expected_desc): return actual_desc is expected_desc else: for cons in expected_desc.types: # It has to be one of the alternatives #log("CHECKING desc %s against %s" % (desc, cons)) if actual_desc is cons: return True return False raise AssertionError('Invalid descriptor %r: %r' % (expected_desc.__class__, expected_desc))
def _MakeReflection(module, app_types): # Types that fields are declared with: int, id, word_part, etc. # Fields are NOT declared with Constructor names. type_lookup = dict(meta.BUILTIN_TYPES) type_lookup.update(app_types) # TODO: Need to resolve 'imports' to the right descriptor. Code generation # relies on it: # - To pick the method to call in AbbreviatedTree etc. # - To generate 'value_t' instead of 'value' in type annotations. for u in module.uses: for type_name in u.type_names: type_lookup[type_name] = None # Placeholder # NOTE: We need two passes because types can be mutually recursive, e.g. # asdl/arith.asdl. # First pass: collect declared types and make entries for them. for d in module.dfns: ast_node = d.value if isinstance(ast_node, asdl.Product): type_lookup[d.name] = meta.CompoundType([]) elif isinstance(ast_node, asdl.Sum): is_simple = asdl.is_simple(ast_node) type_lookup[d.name] = meta.SumType(is_simple) else: raise AssertionError(ast_node) # Second pass: resolve type declarations in Product and constructor. for d in module.dfns: ast_node = d.value if isinstance(ast_node, asdl.Product): runtime_type = type_lookup[d.name] _AppendFields(ast_node.fields, type_lookup, runtime_type.fields) elif isinstance(ast_node, asdl.Sum): sum_type = type_lookup[d.name] # the one we just created for cons in ast_node.types: fields_out = [] # fully-qualified name. Use a _ so we can share strings with class # name. key = '%s__%s' % (d.name, cons.name) cons_type = meta.CompoundType(fields_out) type_lookup[key] = cons_type _AppendFields(cons.fields, type_lookup, fields_out) sum_type.cases.append(cons_type) else: raise AssertionError(ast_node) return type_lookup
def EncodeArray(obj_list, item_desc, enc, out): """ Args: obj_list: List of Obj values Returns: ref """ array_chunk = bytearray() enc.Int(len(obj_list), array_chunk) # Length prefix if isinstance(item_desc, asdl.IntType) or \ isinstance(item_desc, asdl.BoolType): for item in obj_list: enc.Int(item, array_chunk) elif isinstance(item_desc, asdl.UserType): # Assume Id for now for item in obj_list: enc.Int(item.enum_value, array_chunk) elif isinstance(item_desc, asdl.StrType): for item in obj_list: ref = out.Write(enc.PaddedStr(item)) enc.Ref(ref, array_chunk) elif isinstance(item_desc, asdl.Sum) and asdl.is_simple(item_desc): for item in obj_list: enc.Int(item.enum_id, array_chunk) else: # A simple value is either an int, enum, or pointer. (Later: Iter<Str> # might be possible for locality.) assert \ isinstance(item_desc, asdl.SumType) or \ isinstance(item_desc, asdl.CompoundType), item_desc # This is like vector<T*> # Later: # - Product types can be put in line # - Sum types can even be put in line, if you have List<T> rather than # Array<T>. Array implies O(1) random access; List doesn't. for item in obj_list: try: ref = EncodeObj(item, enc, out) except EncodeError as e: if not e.details_printed: util.log("Error encoding array: %s (item %s)", e, item) e.details_printed = True raise enc.Ref(ref, array_chunk) this_ref = out.Write(enc.PaddedBlock(array_chunk)) return this_ref
def _MakeReflection(module, app_types): # Types that fields are declared with: int, id, word_part, etc. # Fields are NOT declared with Constructor names. type_lookup = dict(runtime.BUILTIN_TYPES) type_lookup.update(app_types) # NOTE: We need two passes because types can be mutually recursive, e.g. # asdl/arith.asdl. # First pass: collect declared types and make entries for them. for d in module.dfns: ast_node = d.value if isinstance(ast_node, asdl.Product): type_lookup[d.name] = runtime.CompoundType([]) elif isinstance(ast_node, asdl.Sum): is_simple = asdl.is_simple(ast_node) type_lookup[d.name] = runtime.SumType(is_simple) else: raise AssertionError(ast_node) # Second pass: resolve type declarations in Product and constructor. for d in module.dfns: ast_node = d.value if isinstance(ast_node, asdl.Product): runtime_type = type_lookup[d.name] _AppendFields(ast_node.fields, type_lookup, runtime_type.fields) elif isinstance(ast_node, asdl.Sum): sum_type = type_lookup[d.name] # the one we just created for cons in ast_node.types: fields_out = [] # fully-qualified name. Use a _ so we can share strings with class # name. key = '%s__%s' % (d.name, cons.name) cons_type = runtime.CompoundType(fields_out) type_lookup[key] = cons_type _AppendFields(cons.fields, type_lookup, fields_out) sum_type.cases.append(cons_type) else: raise AssertionError(ast_node) return type_lookup
def MakeFieldSubtree(obj, field_name, desc, abbrev_hook, omit_empty=True): try: field_val = getattr(obj, field_name) except AttributeError: # This happens when required fields are not initialized, e.g. FuncCall() # without setting name. raise AssertionError('%s is missing field %r' % (obj.__class__, field_name)) if isinstance(desc, asdl.IntType): out_val = _ColoredString(str(field_val), _OTHER_LITERAL) elif isinstance(desc, asdl.BoolType): out_val = _ColoredString('T' if field_val else 'F', _OTHER_LITERAL) elif isinstance(desc, asdl.DictType): raise AssertionError elif isinstance(desc, asdl.Sum) and asdl.is_simple(desc): out_val = field_val.name elif isinstance(desc, asdl.StrType): out_val = _ColoredString(field_val, _STRING_LITERAL) elif isinstance(desc, asdl.ArrayType): out_val = [] obj_list = field_val for child_obj in obj_list: t = MakeTree(child_obj, abbrev_hook) out_val.append(t) if omit_empty and not obj_list: out_val = None elif isinstance(desc, asdl.MaybeType): if field_val is None: out_val = None else: out_val = MakeTree(field_val, abbrev_hook) else: out_val = MakeTree(field_val, abbrev_hook) return out_val
def _GetCppType(self, field): """Return a string for the C++ name of the type.""" type_name = field.type cpp_type = _BUILTINS.get(type_name) if cpp_type is not None: return cpp_type typ = self.type_lookup[type_name] if isinstance(typ, asdl.Sum) and asdl.is_simple(typ): # Use the enum instead of the class. return "%s_e" % type_name # - Pointer for optional type. # - ints and strings should generally not be optional? We don't have them # in osh yet, so leave it out for now. if field.opt: return "%s_t*" % type_name return "%s_t&" % type_name
def EncodeArray(obj_list, item_desc, enc, out): """ Args: obj_list: List of Obj values Returns: ref """ array_chunk = bytearray() enc.Int(len(obj_list), array_chunk) # Length prefix if isinstance(item_desc, asdl.IntType) or \ isinstance(item_desc, asdl.BoolType): for item in obj_list: enc.Int(item, array_chunk) elif isinstance(item_desc, asdl.Sum) and asdl.is_simple(item_desc): for item in obj_list: enc.Int(item.enum_id, array_chunk) else: # A simple value is either an int, enum, or pointer. (Later: Iter<Str> # might be possible for locality.) assert isinstance(item_desc, asdl.Sum) or isinstance( item_desc, asdl.Product), item_desc # This is like vector<T*> # Later: # - Product types can be put in line # - Sum types can even be put in line, if you have List<T> rather than # Array<T>. Array implies O(1) random access; List doesn't. for item in obj_list: # Recursive call. ref = EncodeObj(item, enc, out) enc.Ref(ref, array_chunk) this_ref = out.Write(enc.PaddedBlock(array_chunk)) return this_ref
def EncodeObj(obj, enc, out): """ Args: obj: Obj to encode enc: encoding params out: output file Returns: ref: Reference to the last block """ # Algorithm: Depth first, post-order traversal. First obj is the first leaf. # last obj is the root. # # Array is a homogeneous type. this_chunk = bytearray() assert isinstance(obj, py_meta.CompoundObj), \ '%r is not a compound obj (%r)' % (obj, obj.__class__) # Constructor objects have a tag. if isinstance(obj.ASDL_TYPE, asdl.Constructor): enc.Tag(obj.tag, this_chunk) for name, desc in obj.ASDL_TYPE.GetFields(): # encode in order field_val = getattr(obj, name) # TODO: # - Float would be inline, etc. # - Repeated value: write them all adjacent to each other? is_maybe = False if isinstance(desc, asdl.MaybeType): is_maybe = True desc = desc.desc # descent # # Now look at types # if isinstance(desc, asdl.IntType) or isinstance(desc, asdl.BoolType): enc.Int(field_val, this_chunk) elif isinstance(desc, asdl.Sum) and asdl.is_simple(desc): # Encode enums as integers. TODO later: Don't use 3 bytes! Can use 1 # byte for most enums. enc.Int(field_val.enum_id, this_chunk) # Write variable length field first, assuming that it's a ref/pointer. # TODO: allow one inline, hanging string or array per record. elif isinstance(desc, asdl.StrType): ref = out.Write(enc.PaddedStr(field_val)) enc.Ref(ref, this_chunk) elif isinstance(desc, asdl.ArrayType): item_desc = desc.desc ref = EncodeArray(field_val, item_desc, enc, out) enc.Ref(ref, this_chunk) elif isinstance(desc, asdl.UserType): if is_maybe and field_val is None: # e.g. id? prefix_op enc.Ref(0, this_chunk) else: # Assume Id for now enc.Int(field_val.enum_value, this_chunk) else: if is_maybe and field_val is None: enc.Ref(0, this_chunk) else: try: ref = EncodeObj(field_val, enc, out) except EncodeError as e: if not e.details_printed: util.log("Error encoding %s : %s (val %s)", name, e, field_val) e.details_printed = True raise enc.Ref(ref, this_chunk) # Write the parent record this_ref = out.Write(enc.PaddedBlock(this_chunk)) return this_ref
def visitSum(self, sum, name): for t in sum.types: # Simple sum types can't conflict if asdl.is_simple(sum): continue self.visit(t, name)
def VisitSum(self, sum, name, depth): if asdl.is_simple(sum): self.VisitSimpleSum(sum, name, depth) else: self.VisitCompoundSum(sum, name, depth)
def MakeTypes(module, root, type_lookup): """ Args: module: asdl.Module root: an object/package to add types to """ for defn in module.dfns: typ = defn.value #print('TYPE', defn.name, typ) if isinstance(typ, asdl.Sum): sum_type = typ if asdl.is_simple(sum_type): # An object without fields, which can be stored inline. # Create a class called foo_e. Unlike the CompoundObj case, it doesn't # have subtypes. Instead if has attributes foo_e.Bar, which Bar is an # instance of foo_e. # # Problem: This means you have a dichotomy between: # cflow_e.Break vs. cflow_e.Break() # If you add a non-simple type like cflow_e.Return(5), the usage will # change. I haven't run into this problem in practice yet. class_name = defn.name + '_e' class_attr = {'ASDL_TYPE': sum_type} # asdl.Sum cls = type(class_name, (SimpleObj, ), class_attr) setattr(root, class_name, cls) # NOTE: Right now the ASDL_TYPE for for an enum value is the Sum type, # not the Constructor type. We may want to change this if we need # reflection. for i, cons in enumerate(sum_type.types): enum_id = i + 1 name = cons.name val = cls(enum_id, cons.name) # Instantiate SimpleObj subtype # Set a static attribute like op_id.Plus, op_id.Minus. setattr(cls, name, val) else: tag_num = {} # e.g. for arith_expr # Should this be arith_expr_t? It is in C++. base_class = type(defn.name, (DebugCompoundObj, ), {}) setattr(root, defn.name, base_class) # Make a type and a enum tag for each alternative. for i, cons in enumerate(sum_type.types): tag = i + 1 # zero reserved? tag_num[cons.name] = tag # for enum class_attr = { 'ASDL_TYPE': cons, # asdl.Constructor 'tag': tag, # Does this API change? } cls = type(cons.name, (base_class, ), class_attr) setattr(root, cons.name, cls) # e.g. arith_expr_e.Const == 1 enum_name = defn.name + '_e' tag_enum = type(enum_name, (), tag_num) setattr(root, enum_name, tag_enum) elif isinstance(typ, asdl.Product): class_attr = {'ASDL_TYPE': typ} cls = type(defn.name, (DebugCompoundObj, ), class_attr) setattr(root, defn.name, cls) else: raise AssertionError(typ)
def EncodeObj(obj, enc, out): """ Args: obj: Obj to encode enc: encoding params out: output file Returns: ref: Reference to the last block """ # Algorithm: Depth first, post-order traversal. First obj is the first leaf. # last obj is the root. # # Array is a homogeneous type. this_chunk = bytearray() assert isinstance(obj, py_meta.CompoundObj), \ '%r is not a compound obj (%r)' % (obj, obj.__class__) # Constructor objects have a tag. if isinstance(obj.DESCRIPTOR, asdl.Constructor): enc.Tag(obj.tag, this_chunk) for name in obj.FIELDS: # encode in order desc = obj.DESCRIPTOR_LOOKUP[name] #print('\n\n------------') #print('field DESC', name, desc) field_val = getattr(obj, name) #print('VALUE', field_val) # TODO: # - Float would be inline, etc. # - Repeated value: write them all adjacent to each other? # INLINE if isinstance(desc, asdl.IntType) or isinstance(desc, asdl.BoolType): enc.Int(field_val, this_chunk) elif isinstance(desc, asdl.Sum) and asdl.is_simple(desc): # Encode enums as integers. TODO later: Don't use 3 bytes! Can use 1 # byte for most enums. enc.Int(field_val.enum_id, this_chunk) # Write variable length field first, assuming that it's a ref/pointer. # TODO: allow one inline, hanging string or array per record. elif isinstance(desc, asdl.StrType): ref = out.Write(enc.PaddedStr(field_val)) enc.Ref(ref, this_chunk) elif isinstance(desc, asdl.ArrayType): item_desc = desc.desc ref = EncodeArray(field_val, item_desc, enc, out) enc.Ref(ref, this_chunk) elif isinstance(desc, asdl.MaybeType): item_desc = desc.desc ok = False if isinstance(item_desc, asdl.Sum): if not asdl.is_simple(item_desc): ok = True elif isinstance(item_desc, asdl.Product): ok = True # TODO: Fix this for span_id. Need to extract a method. if not ok: raise AssertionError( "Currently not encoding simple optional types: %s" % field_val) if field_val is None: enc.Ref(0, this_chunk) else: ref = EncodeObj(field_val, enc, out) enc.Ref(ref, this_chunk) elif isinstance(desc, asdl.UserType): # Assume Id for now enc.Int(field_val.enum_value, this_chunk) else: # Recursive call for child records. Write children before parents. ref = EncodeObj(field_val, enc, out) enc.Ref(ref, this_chunk) # Write the parent record this_ref = out.Write(enc.PaddedBlock(this_chunk)) return this_ref
def MakeTypes(module, root, app_types=None): """ Args: module: asdl.Module root: an object/package to add types to """ app_types = app_types or {} for defn in module.dfns: typ = defn.value #print('TYPE', defn.name, typ) if isinstance(typ, asdl.Sum): sum_type = typ if asdl.is_simple(sum_type): # An object without fields, which can be stored inline. class_attr = {'DESCRIPTOR': sum_type} # asdl.Sum cls = type(defn.name, (SimpleObj, ), class_attr) #print('CLASS', cls) setattr(root, defn.name, cls) for i, cons in enumerate(sum_type.types): enum_id = i + 1 name = cons.name val = cls(enum_id, cons.name) # Instantiate SimpleObj subtype # Set a static attribute like op_id.Plus, op_id.Minus. setattr(cls, name, val) else: tag_num = {} # e.g. for arith_expr # Should this be arith_expr_t? It is in C++. base_class = type(defn.name, (CompoundObj, ), {}) setattr(root, defn.name, base_class) # Make a type and a enum tag for each alternative. for i, cons in enumerate(sum_type.types): tag = i + 1 # zero reserved? tag_num[cons.name] = tag # for enum # Add 'int* spids' to every constructor. class_attr = _MakeFieldDescriptors(module, cons.fields, app_types) class_attr['DESCRIPTOR'] = cons # asdl.Constructor class_attr['tag'] = tag cls = type(cons.name, (base_class, ), class_attr) setattr(root, cons.name, cls) # e.g. arith_expr_e.Const == 1 enum_name = defn.name + '_e' tag_enum = type(enum_name, (), tag_num) setattr(root, enum_name, tag_enum) elif isinstance(typ, asdl.Product): class_attr = _MakeFieldDescriptors(module, typ.fields, app_types) class_attr['DESCRIPTOR'] = typ cls = type(defn.name, (CompoundObj, ), class_attr) setattr(root, defn.name, cls) else: raise AssertionError(typ)