def javaMethods(self, type, clsname, ctorname, fields, depth):
# The java ctors
token = asdl.Field('Token', 'token')
token.typedef = False
fpargs = ", ".join([self.fieldDef(f) for f in [token] + fields])
self.emit("public %s(%s) {" % (ctorname, fpargs), depth)
self.emit("super(token);", depth+1)
self.javaConstructorHelper(fields, depth)
self.emit("}", depth)
self.emit("", 0)
tree = asdl.Field('PythonTree', 'tree')
tree.typedef = False
fpargs = ", ".join([self.fieldDef(f) for f in [tree] + fields])
self.emit("public %s(%s) {" % (ctorname, fpargs), depth)
self.emit("super(tree);", depth+1)
self.javaConstructorHelper(fields, depth)
self.emit("}", depth)
self.emit("", 0)
if fpargs:
fpargs += ", "
# The toString() method
self.emit("public String toString() {", depth)
self.emit('return "%s";' % clsname, depth+1)
self.emit("}", depth)
self.emit("", 0)
# The pickle() method
#self.emit("public void pickle(DataOutputStream ostream) throws IOException {", depth)
#self.emit("pickleThis(%s, ostream);" % type.index, depth+1);
#for f in fields:
# self.emit("pickleThis(this.%s, ostream);" % f.name, depth+1)
#self.emit("}", depth)
#self.emit("", 0)
# The accept() method
self.emit("public <R> R accept(VisitorIF<R> visitor) throws Exception {", depth)
if clsname == ctorname:
self.emit('return visitor.visit%s(this);' % clsname, depth+1)
else:
self.emit('traverse(visitor);' % clsname, depth+1)
self.emit('return null;' % clsname, depth+1)
self.emit("}", depth)
self.emit("", 0)
# The visitChildren() method
self.emit("public void traverse(VisitorIF visitor) throws Exception {", depth)
for f in fields:
if self.bltinnames.has_key(str(f.type)):
continue
if f.typedef.simple:
continue
if f.seq:
self.emit('if (%s != null) {' % f.name, depth+1)
self.emit('for (int i = 0; i < %s.length; i++) {' % f.name,
depth+2)
self.emit('if (%s[i] != null)' % f.name, depth+3)
self.emit('%s[i].accept(visitor);' % f.name, depth+4)
self.emit('}', depth+2)
self.emit('}', depth+1)
else:
self.emit('if (%s != null)' % f.name, depth+1)
self.emit('%s.accept(visitor);' % f.name, depth+2)
self.emit('}', depth)
self.emit("", 0)
def copy_field(field):
return asdl.Field(field.type, field.name, field.seq, field.opt)
def javaConstructors(self, type, name, clsname, is_product, fields, depth):
self.emit("public %s(PyType subType) {" % (clsname), depth)
self.emit("super(subType);", depth + 1)
self.emit("}", depth)
if len(fields) > 0:
self.emit("public %s() {" % (clsname), depth)
self.emit("this(TYPE);", depth + 1)
self.emit("}", depth)
fnames = ['"%s"' % f.name for f in fields]
else:
fnames = []
if str(name) in ('stmt', 'expr', 'excepthandler'):
fnames.extend(['"lineno"', '"col_offset"'])
fpargs = ", ".join(fnames)
self.emit("@ExposedNew", depth)
self.emit("@ExposedMethod", depth)
self.emit(
"public void %s___init__(PyObject[] args, String[] keywords) {" %
clsname, depth)
self.emit(
'ArgParser ap = new ArgParser("%s", args, keywords, new String[]' %
clsname, depth + 1)
self.emit('{%s}, %s);' % (fpargs, len(fields)), depth + 2)
i = 0
for f in fields:
self.emit(
"set%s(ap.getPyObject(%s));" %
(self.processFieldName(f.name), str(i)), depth + 1)
i += 1
if str(name) in ('stmt', 'expr', 'excepthandler'):
self.emit("int lin = ap.getInt(%s, -1);" % str(i), depth + 1)
self.emit("if (lin != -1) {", depth + 1)
self.emit("setLineno(lin);", depth + 2)
self.emit("}", depth + 1)
self.emit("", 0)
self.emit("int col = ap.getInt(%s, -1);" % str(i + 1), depth + 1)
self.emit("if (col != -1) {", depth + 1)
self.emit("setLineno(col);", depth + 2)
self.emit("}", depth + 1)
self.emit("", 0)
self.emit("}", depth)
self.emit("", 0)
fpargs = ", ".join(["PyObject %s" % f.name for f in fields])
self.emit("public %s(%s) {" % (clsname, fpargs), depth)
for f in fields:
self.emit("set%s(%s);" % (self.processFieldName(f.name), f.name),
depth + 1)
self.emit("}", depth)
self.emit("", 0)
token = asdl.Field('Token', 'token')
token.typedef = False
fpargs = ", ".join([self.fieldDef(f) for f in [token] + fields])
self.emit("public %s(%s) {" % (clsname, fpargs), depth)
self.emit("super(token);", depth + 1)
self.javaConstructorHelper(fields, depth)
self.emit("}", depth)
self.emit("", 0)
ttype = asdl.Field('int', 'ttype')
ttype.typedef = False
fpargs = ", ".join([self.fieldDef(f) for f in [ttype, token] + fields])
self.emit("public %s(%s) {" % (clsname, fpargs), depth)
self.emit("super(ttype, token);", depth + 1)
self.javaConstructorHelper(fields, depth)
self.emit("}", depth)
self.emit("", 0)
tree = asdl.Field('PythonTree', 'tree')
tree.typedef = False
fpargs = ", ".join([self.fieldDef(f) for f in [tree] + fields])
self.emit("public %s(%s) {" % (clsname, fpargs), depth)
self.emit("super(tree);", depth + 1)
self.javaConstructorHelper(fields, depth)
self.emit("}", depth)
self.emit("", 0)
def extract_location(self, typename, depth):
row = self.decode_field(asdl.Field('int', 'lineno'), typename)
column = self.decode_field(asdl.Field('int', 'col_offset'), typename)
self.emit(f"let _location = ast::Location::new({row}, {column});",
depth)
def javaConstructors(self, type, name, clsname, is_product, fields, depth):
if len(fields) > 0:
self.emit("public %s() {" % (clsname), depth)
self.emit("super(TYPE);", depth + 1)
self.emit("}", depth)
fnames = ['"%s"' % f.name for f in fields]
else:
fnames = []
if str(name) in ('stmt', 'expr', 'excepthandler'):
fnames.extend(['"lineno"', '"col_offset"'])
fpargs = ", ".join(fnames)
self.emit("@ExposedNew", depth)
self.emit("@ExposedSlot(SlotFunc.NEW)", depth)
self.emit(
"public static PyObject %s_new(PyNewWrapper _new, boolean init, PyType subtype, PyObject[] args, String[] keywords) {"
% clsname, depth)
self.emit("return new %s(subtype);" % clsname, depth + 1)
self.emit("}", depth)
self.emit('@ExposedMethod(names={"__init__"})', depth)
self.emit(
"public void %s___init__(PyObject[] args, String[] keywords) {" %
clsname, depth)
self.emit(
'ArgParser ap = new ArgParser("%s", args, keywords, new String[]' %
clsname, depth + 1)
self.emit('{%s}, %s, true);' % (fpargs, len(fields)), depth + 2)
i = 0
for f in fields:
self.emit(
"set%s(ap.getPyObject(%s, Py.None));" %
(self.processFieldName(f.name), str(i)), depth + 1)
i += 1
if str(name) in ('stmt', 'expr', 'excepthandler'):
self.emit("PyObject lin = ap.getOptionalArg(%s);" % str(i),
depth + 1)
self.emit("if (lin != null) {", depth + 1)
self.emit("lineno = lin;", depth + 2)
self.emit("}", depth + 1)
self.emit("", 0)
self.emit("PyObject col = ap.getOptionalArg(%s);" % str(i + 1),
depth + 1)
self.emit("if (col != null) {", depth + 1)
self.emit("col_offset = col;", depth + 2)
self.emit("}", depth + 1)
self.emit("", 0)
self.emit("}", depth)
self.emit("", 0)
fpargs = ", ".join(["PyObject %s" % f.name for f in fields])
self.emit("public %s(%s) {" % (clsname, fpargs), depth)
self.emit("super(TYPE);", depth + 1)
for f in fields:
self.emit("set%s(%s);" % (self.processFieldName(f.name), f.name),
depth + 1)
self.emit("}", depth)
self.emit("", 0)
self.emit("// called from derived class", depth)
self.emit("public %s(PyType subtype) {" % clsname, depth)
self.emit("super(subtype);", depth + 1)
self.emit("}", depth)
self.emit("", 0)
node = asdl.Field('Node', 'token')
node.typedef = False
fpargs = ", ".join([self.fieldDef(f) for f in [node] + fields])
self.emit("public %s(%s) {" % (clsname, fpargs), depth)
self.emit("super(TYPE, token);", depth + 1)
self.javaConstructorHelper(fields, depth)
self.emit("}", depth)
self.emit("", 0)
token = asdl.Field('Token', 'token')
token.typedef = False
fpargs = ", ".join([self.fieldDef(f) for f in [token] + fields])
self.emit("public %s(%s) {" % (clsname, fpargs), depth)
self.emit("super(TYPE, token);", depth + 1)
self.javaConstructorHelper(fields, depth)
self.emit("}", depth)
self.emit("", 0)
tree = asdl.Field('PythonTree', 'tree')
tree.typedef = False
fpargs = ", ".join([self.fieldDef(f) for f in [tree] + fields])
self.emit("public %s(%s) {" % (clsname, fpargs), depth)
self.emit("super(TYPE, tree);", depth + 1)
self.javaConstructorHelper(fields, depth)
self.emit("}", depth)
self.emit("", 0)
token = asdl.Field('Token', 'getToken()')
fpargs = ", ".join([f"this.{f.name}" for f in [token] + fields])
self.emit(f"public {clsname} copy() {{", depth)
self.emit(f"return new {clsname}({fpargs});", depth + 1)
self.emit("}", depth)
self.emit("", 0)
def _template_to_constructor(self, template_dict, suffix, seq_field):
hole_node_types = {}
# Find where the holes occur
stack = [(None, template_dict['idiom'], None)]
while stack:
parent, node, child_index = stack.pop()
node_type, ref_symbols, hole_id, children = node
if hole_id is not None:
assert hole_id not in hole_node_types
# node_type could be:
# - name of field
# => hole type is same as field's type
# - name of type, if it only has one child
# - binarizer
hyphenated_node_type = None
unhyphenated_node_type = None
hole_type_str = template_dict['holes'][hole_id]['type']
if hole_type_str == 'AddChild':
node_type_for_field_type = node_type
elif hole_type_str == 'ReplaceSelf':
# Two types of ReplaceSelf
# 1. Node has a hyphen: should be a repeated field
# 2. Node lacks a hyphen, and
# 2a. node is same as parent: a repeated field
# 2b. node is not the same as parent: an elem
if '-' in node_type:
node_type_for_field_type = node_type
else:
node_type_for_field_type = parent[0]
#if '-' in parent_type:
# hyphenated_node_type = parent_type
#else:
# unhyphenated_node_type = parent_type
field_info = self._get_field_info_from_name(
node_type_for_field_type)
# Check for situations like
# Call-args
# | \
# List[0] Call-args[1]
#
# Tuple
# | \
# Tuple[0] Tuple[1]
# where hole 0 should not be a sequence.
if field_info.seq and hole_type_str == 'ReplaceSelf' and '-' not in node_type:
assert child_index in (0, 1)
seq = child_index == 1
else:
seq = field_info.seq
hole_node_types[hole_id] = (field_info.type, seq,
field_info.opt)
stack += [(node, child, i) for i, child in enumerate(children)]
# Create fields for the holes
fields = []
for hole in template_dict['holes']:
i = hole['id']
field_type, seq, opt = hole_node_types[i]
field = asdl.Field(type=field_type,
name='hole{}'.format(i),
seq=seq,
opt=opt)
field.hole_type = HoleType[hole['type']]
fields.append(field)
constructor = asdl.Constructor(
'Template{}{}'.format(template_dict['id'], suffix), fields)
constructor.template = self.convert_idiom_ast(
template_dict['idiom'],
template_id=template_dict['id'],
seq_field=seq_field)
return constructor
def run_iteration(self, trees):
triple_occurrences, node_type_counts = self.count_triples(trees)
self.pre_iteration_counts.append(node_type_counts)
# Most frequent
most_freq_triple, most_freq_occurrences = max(
triple_occurrences.items(), key=lambda kv: len(kv[1]))
if len(most_freq_occurrences) == 1:
raise Exception('No more work to do!')
existing_type_name, field_name, field_info = most_freq_triple
tuple_name = field_name if isinstance(field_name,
tuple) else (field_name, )
existing_type = self.type_infos[existing_type_name]
existing_field = existing_type.unset_fields[field_name]
promoted_fields = []
promoted_seq_elem_counts = collections.Counter()
promoted_preset_fields = {}
if isinstance(field_info, Primitive) or field_info is None:
pass
else:
# Figure out which fields of type `field_info` should be promoted
# Example:
# most_freq_triple = ('Call', 'func', 'Name')
# field_info = 'Name'
# type_infos['Name'].unset_fields = {'id': Field(identifier, id)}
for is_preset, (field_field_name, field_field) in itertools.chain(
zip(itertools.repeat(False),
self.type_infos[field_info].unset_fields.items()),
zip(itertools.repeat(True),
self.type_infos[field_info].preset_fields.items())):
if isinstance(field_field_name, tuple):
field_field_tuple_name = field_field_name
else:
field_field_tuple_name = (field_field_name, )
if existing_field.seq:
suffix = (existing_type.preset_seq_elem_counts[tuple_name],
) + field_field_tuple_name
else:
suffix = field_field_tuple_name
new_name = tuple_name + suffix
if isinstance(field_field, asdl.Field):
new_field = asdl.Field(type=field_field.type,
name=new_name,
seq=field_field.seq,
opt=field_field.opt)
else:
new_field = field_field
if is_preset:
promoted_preset_fields[new_name] = new_field
else:
promoted_fields.append((field_field, new_field))
seq_elem_count = self.type_infos[
field_info].preset_seq_elem_counts[field_field_tuple_name]
if seq_elem_count:
promoted_seq_elem_counts[new_name] = seq_elem_count
# Create a new type
new_preset_fields = {
**existing_type.preset_fields,
**promoted_preset_fields
}
new_preset_seq_elem_counts = existing_type.preset_seq_elem_counts + promoted_seq_elem_counts
if existing_field.seq and field_info is not None:
new_preset_fields[tuple_name + (
new_preset_seq_elem_counts[tuple_name], )] = field_info
new_preset_seq_elem_counts[tuple_name] += 1
else:
new_preset_fields[tuple_name] = field_info
new_unset_fields = {
**{f.name: f
for old_field, f in promoted_fields},
**existing_type.unset_fields
}
if field_info is None or not existing_field.seq:
# Only unset if...
# - field is not sequential
# - field has been set to None, meaning the end of a sequence
del new_unset_fields[field_name]
new_type = TypeInfo(name='Type{:04d}_{}'.format(
self.iterations_finished, existing_type.base_name),
base_name=existing_type.base_name,
predecessor_name=existing_type.name,
predecessor_triple=most_freq_triple,
unset_fields=new_unset_fields,
preset_fields=new_preset_fields,
preset_seq_elem_counts=new_preset_seq_elem_counts)
self.type_infos[new_type.name] = new_type
self.created_types.append(new_type)
self.type_graph.add_edge(new_type.name, existing_type.name)
self.iterations_finished += 1
# Tracks which occurrences have been removed due to promotion.
discarded = IdentitySet()
for occ in most_freq_occurrences:
if occ in discarded:
continue
occ['_type'] = new_type.name
def delete_obsoleted_field():
if existing_field.seq:
# todo: change 0 if we can promote other elements
del occ[field_name][0]
if not occ[field_name]:
del occ[field_name]
else:
del occ[field_name]
if isinstance(field_info, Primitive):
delete_obsoleted_field()
elif field_info is None:
pass
else:
if existing_field.seq:
# todo: change 0 if we can promote other elements
value_to_promote = occ[field_name][0]
else:
value_to_promote = occ[field_name]
delete_obsoleted_field()
discarded.add(value_to_promote)
for old_field, new_field in promoted_fields:
if old_field.name not in value_to_promote:
assert old_field.opt or old_field.seq
continue
occ[new_field.name] = value_to_promote[old_field.name]
assert occ[new_field.name]