def main(srcfile, dump_module=False):
argv0 = sys.argv[0]
components = argv0.split(os.sep)
argv0 = os.sep.join(components[-2:])
auto_gen_msg = common_msg % argv0
mod = asdl.parse(srcfile)
if dump_module:
print('Parsed Module:')
print(mod)
if not asdl.check(mod):
sys.exit(1)
if H_FILE:
with open(H_FILE, "w") as f:
f.write(auto_gen_msg)
f.write('#ifndef Py_PYTHON_AST_H\n')
f.write('#define Py_PYTHON_AST_H\n')
f.write('#ifdef __cplusplus\n')
f.write('extern "C" {\n')
f.write('#endif\n')
f.write('\n')
f.write('#include "asdl.h"\n')
f.write('\n')
f.write('#undef Yield /* undefine macro conflicting with <winbase.h> */\n')
f.write('\n')
c = ChainOfVisitors(TypeDefVisitor(f),
StructVisitor(f))
c.visit(mod)
f.write("// Note: these macros affect function definitions, not only call sites.\n")
PrototypeVisitor(f).visit(mod)
f.write("\n")
f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
f.write("int PyAST_Check(PyObject* obj);\n")
f.write('\n')
f.write('#ifdef __cplusplus\n')
f.write('}\n')
f.write('#endif\n')
f.write('#endif /* !Py_PYTHON_AST_H */\n')
if C_FILE:
with open(C_FILE, "w") as f:
f.write(auto_gen_msg)
f.write('#include <stddef.h>\n')
f.write('\n')
f.write('#include "Python.h"\n')
f.write('#include "%s-ast.h"\n' % mod.name)
f.write('\n')
f.write("static PyTypeObject AST_type;\n")
v = ChainOfVisitors(
PyTypesDeclareVisitor(f),
PyTypesVisitor(f),
Obj2ModPrototypeVisitor(f),
FunctionVisitor(f),
ObjVisitor(f),
Obj2ModVisitor(f),
ASTModuleVisitor(f),
PartingShots(f),
)
v.visit(mod)
def main(asdlfile, outputfile):
argv0 = sys.argv[0]
components = argv0.split(os.sep)
argv0 = os.sep.join(components[-2:])
auto_gen_msg = common_msg % argv0
mod = asdl.parse(asdlfile)
if not asdl.check(mod):
sys.exit(1)
f = open(outputfile, "wb")
f.write(auto_gen_msg)
c = ChainOfVisitors(TypeDefVisitor(f),
)
c.visit(mod)
f.write("\n"*5)
f.write("/* ---------------------- */\n")
f.write("/* constructors for nodes */\n")
f.write("/* ---------------------- */\n")
f.write("\n"*5)
v = ChainOfVisitors(
FunctionVisitor(f),
FieldNamesVisitor(f),
)
v.visit(mod)
f.close()
def main(outdir, grammar="Python.asdl"):
mod = asdl.parse(grammar)
if not asdl.check(mod):
sys.exit(1)
c = ChainOfVisitors(AnalyzeVisitor(outdir),
JavaVisitor(outdir),
VisitorVisitor(outdir))
c.visit(mod)
def main(srcfile):
argv0 = sys.argv[0]
components = argv0.split(os.sep)
argv0 = os.sep.join(components[-2:])
auto_gen_msg = common_msg % argv0
mod = asdl.parse(srcfile)
if not asdl.check(mod):
sys.exit(1)
if INC_DIR:
p = "%s/%s-ast.h" % (INC_DIR, mod.name)
f = open(p, "wb")
f.write(auto_gen_msg)
f.write('#include "asdl.h"\n\n')
f.write('#ifdef __cplusplus\n'
'extern "C" {\n'
'#endif\n\n')
c = ChainOfVisitors(TypeDefVisitor(f),
StructVisitor(f),
PrototypeVisitor(f),
)
c.visit(mod)
f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
f.write("int PyAST_Check(PyObject* obj);\n")
f.write('\n#ifdef __cplusplus\n'
'}\n' # To end the extern "C"
'#endif\n')
f.close()
if SRC_DIR:
p = os.path.join(SRC_DIR, str(mod.name) + "-ast.c")
f = open(p, "wb")
f.write(auto_gen_msg)
f.write(c_file_msg % parse_version(mod))
f.write('#include "Python.h"\n')
f.write('#include "%s-ast.h"\n' % mod.name)
f.write('\n')
f.write("static PyTypeObject AST_type;\n")
v = ChainOfVisitors(
PyTypesDeclareVisitor(f),
PyTypesVisitor(f),
Obj2ModPrototypeVisitor(f),
FunctionVisitor(f),
ObjVisitor(f),
Obj2ModVisitor(f),
ASTModuleVisitor(f),
PartingShots(f),
)
v.visit(mod)
f.close()
def main(input_file, c_file, h_file, dump_module=False):
auto_gen_msg = AUTOGEN_MESSAGE.format("/".join(Path(__file__).parts[-2:]))
mod = asdl.parse(input_file)
if dump_module:
print('Parsed Module:')
print(mod)
if not asdl.check(mod):
sys.exit(1)
for file, writer in (c_file, write_source), (h_file, write_header):
if file is not None:
with file.open("w") as f:
f.write(auto_gen_msg)
writer(f, mod)
print(file, "regenerated.")
def main(srcfile, dump_module=False):
argv0 = sys.argv[0]
components = argv0.split(os.sep)
argv0 = os.sep.join(components[-2:])
auto_gen_msg = common_msg % argv0
mod = asdl.parse(srcfile)
if dump_module:
print('Parsed Module:')
print(mod)
if not asdl.check(mod):
sys.exit(1)
if INC_DIR:
p = "%s/%s-ast.h" % (INC_DIR, mod.name)
f = open(p, "w")
f.write(auto_gen_msg)
f.write('#include "asdl.h"\n\n')
c = ChainOfVisitors(
TypeDefVisitor(f),
StructVisitor(f),
PrototypeVisitor(f),
)
c.visit(mod)
f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
f.write(
"mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
f.write("int PyAST_Check(PyObject* obj);\n")
f.close()
if SRC_DIR:
p = os.path.join(SRC_DIR, str(mod.name) + "-ast.c")
f = open(p, "w")
f.write(auto_gen_msg)
f.write('#include <stddef.h>\n')
f.write('\n')
f.write('#include "Python.h"\n')
f.write('#include "%s-ast.h"\n' % mod.name)
f.write('\n')
f.write("static PyTypeObject AST_type;\n")
v = ChainOfVisitors(
PyTypesDeclareVisitor(f),
PyTypesVisitor(f),
Obj2ModPrototypeVisitor(f),
FunctionVisitor(f),
ObjVisitor(f),
Obj2ModVisitor(f),
ASTModuleVisitor(f),
PartingShots(f),
)
v.visit(mod)
f.close()
def __init__(
self,
asdl_file,
output_from=True, #< changed
use_table_pointer=True, #< changed
include_literals=False, #< changed
include_columns=True,
end_with_from=True, #< changed
clause_order=None,
infer_from_conditions=True, #< changed
factorize_sketch=2):
# collect pointers and checkers
self.pointers = set(['table', 'column', 'value'])
custom_primitive_type_checkers = {}
custom_primitive_type_checkers['table'] = lambda x: isinstance(x, int)
custom_primitive_type_checkers['column'] = lambda x: isinstance(x, int)
custom_primitive_type_checkers['value'] = lambda x: isinstance(x, int)
self.include_columns = include_columns
# create ast wrapper
self.factorize_sketch = factorize_sketch
self.ast_wrapper = ast_util.ASTWrapper(
asdl.parse(asdl_file),
custom_primitive_type_checkers=custom_primitive_type_checkers)
# from field
self.output_from = output_from
self.end_with_from = end_with_from
self.clause_order = clause_order
self.infer_from_conditions = infer_from_conditions
if self.clause_order:
# clause order is prioritized over configurations like end_with_from
assert factorize_sketch == 2 # TODO support other grammars
sql_fields = self.ast_wrapper.product_types['sql'].fields
letter2field = {k: v for k, v in zip("SFWGOI", sql_fields)}
new_sql_fields = [letter2field[k] for k in self.clause_order]
self.ast_wrapper.product_types['sql'].fields = new_sql_fields
else:
if not self.output_from:
sql_fields = self.ast_wrapper.product_types['sql'].fields
assert sql_fields[1].name == 'from'
del sql_fields[1]
else:
sql_fields = self.ast_wrapper.product_types['sql'].fields
assert sql_fields[1].name == "from"
if self.end_with_from:
sql_fields.append(sql_fields[1])
del sql_fields[1]
def main(srcfile, output_header, output_file):
# determine "auto gen" message to appear at top of output
argv0 = sys.argv[0]
components = argv0.split(os.sep)
argv0 = os.sep.join(components[-2:])
common_msg = "/* File automatically generated by %s. */\n\n"
auto_gen_msg = common_msg % argv0
# parse the ASDL file
mod = asdl.parse(srcfile)
if not asdl.check(mod):
sys.exit(1)
# write the header
f = open(output_header, "w")
f.write(auto_gen_msg)
f.write("#pragma once\n\n")
f.write('#include <vector>\n')
f.write('#include <string>\n')
f.write('#include <any>\n')
c = ChainOfVisitors(
NamespaceVisitor(f),
TypeDefVisitor(f),
StructVisitor(f),
BaseVisitorVistor(f),
InheritanceVisitor(f),
PrototypeVisitor(f),
ToStringPrototypeVisitor(f),
)
c.visit(mod)
f.write("}\n")
f.close()
# write the output
f = open(output_file, "w")
f.write(auto_gen_msg)
f.write('#include <set>\n')
f.write('#include <sstream>\n\n')
f.write('#include "%s"\n\n' % output_header)
c = ChainOfVisitors(
NamespaceVisitor(f),
FunctionVisitor(f),
ToStringVisitorVisitor(f),
ToStringFunctionVisitor(f),
)
c.visit(mod)
f.write("}\n")
f.close()
def main(srcfile, dump_module=False):
argv0 = sys.argv[0]
components = argv0.split(os.sep)
argv0 = os.sep.join(components[-2:])
auto_gen_msg = common_msg % argv0
mod = asdl.parse(srcfile)
if dump_module:
print('Parsed Module:')
print(mod)
if not asdl.check(mod):
sys.exit(1)
if INC_DIR:
p = "%s/%s-ast.h" % (INC_DIR, mod.name)
f = open(p, "w")
f.write(auto_gen_msg)
f.write('#include "asdl.h"\n\n')
c = ChainOfVisitors(TypeDefVisitor(f),
StructVisitor(f),
PrototypeVisitor(f),
)
c.visit(mod)
f.write("PyObject* Ta35AST_mod2obj(mod_ty t);\n")
f.write("mod_ty Ta35AST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
f.write("int Ta35AST_Check(PyObject* obj);\n")
f.close()
if SRC_DIR:
p = os.path.join(SRC_DIR, str(mod.name) + "-ast.c")
f = open(p, "w")
f.write(auto_gen_msg)
f.write('#include <stddef.h>\n')
f.write('\n')
f.write('#include "Python.h"\n')
f.write('#include "%s-ast.h"\n' % mod.name)
f.write('\n')
f.write("static PyTypeObject AST_type;\n")
v = ChainOfVisitors(
PyTypesDeclareVisitor(f),
PyTypesVisitor(f),
Obj2ModPrototypeVisitor(f),
FunctionVisitor(f),
ObjVisitor(f),
Obj2ModVisitor(f),
ASTModuleVisitor(f),
PartingShots(f),
)
v.visit(mod)
f.close()
def __init__(
self,
output_from=False,
use_table_pointer=False,
include_literals=True,
include_columns=True,
):
custom_primitive_type_checkers = {}
self.pointers = set()
if use_table_pointer:
custom_primitive_type_checkers["table"] = ast_util.FilterType(int)
self.pointers.add("table")
if include_columns:
custom_primitive_type_checkers["column"] = ast_util.FilterType(int)
self.pointers.add("column")
self.ast_wrapper = ast_util.ASTWrapper(
asdl.parse(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
"Spider.asdl")),
custom_primitive_type_checkers=custom_primitive_type_checkers,
)
self.output_from = output_from
self.include_literals = include_literals
self.include_columns = include_columns
if not self.output_from:
sql_fields = self.ast_wrapper.product_types["sql"].fields
assert sql_fields[1].name == "from"
del sql_fields[1]
if not use_table_pointer:
self.ast_wrapper.singular_types["Table"].fields[0].type = "int"
if not include_literals:
sql_fields = self.ast_wrapper.singular_types["sql"].fields
for field in sql_fields:
if field.name == "limit":
field.opt = False
field.type = "singleton"
if not include_columns:
col_unit_fields = self.ast_wrapper.singular_types[
"col_unit"].fields
assert col_unit_fields[1].name == "col_id"
del col_unit_fields[1]
def __init__(self,
output_from=False,
use_table_pointer=False,
include_literals=True,
include_columns=True):
custom_primitive_type_checkers = {}
self.pointers = set()
if use_table_pointer:
custom_primitive_type_checkers['table'] = lambda x: isinstance(
x, int)
self.pointers.add('table')
if include_columns:
custom_primitive_type_checkers['column'] = lambda x: isinstance(
x, int)
self.pointers.add('column')
self.ast_wrapper = ast_util.ASTWrapper(
asdl.parse(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
'Spider.asdl')),
custom_primitive_type_checkers=custom_primitive_type_checkers)
self.output_from = output_from
self.include_literals = include_literals
self.include_columns = include_columns
if not self.output_from:
sql_fields = self.ast_wrapper.product_types['sql'].fields
assert sql_fields[1].name == 'from'
del sql_fields[1]
if not use_table_pointer:
self.ast_wrapper.singular_types['Table'].fields[0].type = 'int'
if not include_literals:
sql_fields = self.ast_wrapper.singular_types['sql'].fields
for field in sql_fields:
if field.name == 'limit':
field.opt = False
field.type = 'singleton'
if not include_columns:
col_unit_fields = self.ast_wrapper.singular_types[
'col_unit'].fields
assert col_unit_fields[1].name == 'col_id'
del col_unit_fields[1]
def main(srcfile):
argv0 = sys.argv[0]
components = argv0.split(os.sep)
argv0 = os.sep.join(components[-2:])
auto_gen_msg = '/* File automatically generated by %s */\n' % argv0
mod = asdl.parse(srcfile)
if not asdl.check(mod):
sys.exit(1)
if INC_DIR:
p = "%s/%s-ast.h" % (INC_DIR, mod.name)
f = open(p, "wb")
print >> f, auto_gen_msg
print >> f, '#ifndef %s_AST_H' % str(mod.name).upper()
print >> f, '#define %s_AST_H\n' % str(mod.name).upper()
print >> f, '#include "asdl.h"\n'
c = ChainOfVisitors(TypeDefVisitor(f), StructVisitor(f),
PrototypePreamble(f), PrototypeVisitor(f),
PrototypePostamble(f))
c.visit(mod)
print >> f, "PyObject* PyAST_mod2obj(mod_ty t);"
print >> f, "\n#endif"
f.close()
if SRC_DIR:
p = os.path.join(SRC_DIR, str(mod.name) + "-ast.c")
f = open(p, "wb")
print >> f, auto_gen_msg
print >> f, '#include "Python.h"'
print >> f, '#include "%s-ast.h"' % mod.name
print >> f
print >> f, "static PyTypeObject* AST_type;"
v = ChainOfVisitors(
PyTypesDeclareVisitor(f),
PyTypesVisitor(f),
FunctionVisitor(f),
ObjVisitor(f),
ASTModuleVisitor(f),
PartingShots(f),
)
v.visit(mod)
f.close()
def main(argv):
if len(argv) == 3:
def_file, out_file = argv[1:]
elif len(argv) == 1:
print "Assuming default values of Python.asdl and ast.py"
here = os.path.dirname(__file__)
def_file = os.path.join(here, "Python.asdl")
out_file = os.path.join(here, "..", "ast.py")
else:
print >> sys.stderr, "invalid arguments"
return 2
mod = asdl.parse(def_file)
data = ASDLData(mod)
fp = open(out_file, "w")
try:
fp.write(HEAD)
for visitor in visitors:
visitor(fp, data).visit(mod)
finally:
fp.close()
def main(argv):
if len(argv) == 3:
def_file, out_file = argv[1:]
elif len(argv) == 1:
print "Assuming default values of Python.asdl and ast.py"
here = os.path.dirname(__file__)
def_file = os.path.join(here, "Python.asdl")
out_file = os.path.join(here, "..", "ast.py")
else:
print >> sys.stderr, "invalid arguments"
return 2
mod = asdl.parse(def_file)
data = ASDLData(mod)
fp = open(out_file, "w")
try:
fp.write(HEAD)
for visitor in visitors:
visitor(fp, data).visit(mod)
finally:
fp.close()
def main(input_filename, ast_mod_filename, ast_def_filename, dump_module=False):
auto_gen_msg = AUTOGEN_MESSAGE.format("/".join(Path(__file__).parts[-2:]))
mod = asdl.parse(input_filename)
if dump_module:
print('Parsed Module:')
print(mod)
if not asdl.check(mod):
sys.exit(1)
typeinfo = {}
FindUserdataTypesVisitor(typeinfo).visit(mod)
with ast_def_filename.open("w") as def_file, \
ast_mod_filename.open("w") as mod_file:
def_file.write(auto_gen_msg)
write_ast_def(mod, typeinfo, def_file)
mod_file.write(auto_gen_msg)
write_ast_mod(mod, mod_file)
print(f"{ast_def_filename}, {ast_mod_filename} regenerated.")
def main(argv):
if len(argv) == 4:
def_file, out_file, util_import_part = argv[1:]
elif len(argv) == 1:
print("Assuming default values of AST.asdl and ast.py")
here = os.path.dirname(__file__)
def_file = os.path.join(here, "AST.asdl")
out_file = os.path.join(here, "..", "lfortran", "ast", "ast.py")
util_import_part = ".utils"
else:
print("invalid arguments")
return 2
mod = asdl.parse(def_file)
data = ASDLData(mod)
fp = open(out_file, "w")
try:
fp.write(HEAD.format(util_import_part))
for visitor in visitors:
visitor(fp, data).visit(mod)
finally:
fp.close()
def main(srcfile):
argv0 = sys.argv[0]
components = argv0.split(os.sep)
argv0 = os.sep.join(components[-2:])
auto_gen_msg = '/* File automatically generated by %s */\n' % argv0
mod = asdl.parse(srcfile)
if not asdl.check(mod):
sys.exit(1)
if INC_DIR:
p = "%s/%s-ast.h" % (INC_DIR, mod.name)
f = open(p, "wb")
print >> f, auto_gen_msg
print >> f, '#include "asdl.h"\n'
c = ChainOfVisitors(TypeDefVisitor(f),
StructVisitor(f),
PrototypeVisitor(f),
)
c.visit(mod)
print >>f, "PyObject* PyAST_mod2obj(mod_ty t);"
f.close()
if SRC_DIR:
p = os.path.join(SRC_DIR, str(mod.name) + "-ast.c")
f = open(p, "wb")
print >> f, auto_gen_msg
print >> f, '#include "Python.h"'
print >> f, '#include "%s-ast.h"' % mod.name
print >> f
print >>f, "static PyTypeObject* AST_type;"
v = ChainOfVisitors(
PyTypesDeclareVisitor(f),
PyTypesVisitor(f),
FunctionVisitor(f),
ObjVisitor(f),
ASTModuleVisitor(f),
PartingShots(f),
)
v.visit(mod)
f.close()
def main(srcfile):
argv0 = sys.argv[0]
components = argv0.split(os.sep)
argv0 = os.sep.join(components[-2:])
auto_gen_msg = common_msg % argv0
mod = asdl.parse(srcfile)
if not asdl.check(mod):
sys.exit(1)
p = "%s-ast.go" % mod.name
f = open(p, "w")
f.write(auto_gen_msg)
f.write('package ast\n')
f.write('import "github.com/go-python/gpython/py"\n')
c = ChainOfVisitors(TypeDefVisitor(f),
StructVisitor(f),
PrototypeVisitor(f),
)
c.visit(mod)
#f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
#f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
#f.write("int PyAST_Check(PyObject* obj);\n")
#f.write("PyTypeObject AST_type;\n")
v = ChainOfVisitors(
PyTypesDeclareVisitor(f),
PyTypesVisitor(f),
Obj2ModPrototypeVisitor(f),
FunctionVisitor(f),
### FIXME ObjVisitor(f),
### FIXME Obj2ModVisitor(f),
### FIXME ASTModuleVisitor(f),
### FIXME PartingShots(f),
)
v.visit(mod)
f.close()
subprocess.check_call(["gofmt", "-w", p])
def main(srcfile, dump_module=False):
argv0 = sys.argv[0]
components = argv0.split(os.sep)
# Always join with '/' so different OS does not keep changing the file
argv0 = '/'.join(components[-2:])
auto_gen_msg = common_msg % argv0
mod = asdl.parse(srcfile)
if dump_module:
print('Parsed Module:')
print(mod)
if not asdl.check(mod):
sys.exit(1)
if H_FILE:
with open(H_FILE, "w") as f:
f.write(auto_gen_msg)
f.write('#ifndef Py_PYTHON_AST_H\n')
f.write('#define Py_PYTHON_AST_H\n')
f.write('#ifdef __cplusplus\n')
f.write('extern "C" {\n')
f.write('#endif\n')
f.write('\n')
f.write('#ifndef Py_LIMITED_API\n')
f.write('#include "asdl.h"\n')
f.write('\n')
f.write('#undef Yield /* undefine macro conflicting with <winbase.h> */\n')
f.write('\n')
c = ChainOfVisitors(TypeDefVisitor(f),
StructVisitor(f))
c.visit(mod)
f.write("// Note: these macros affect function definitions, not only call sites.\n")
PrototypeVisitor(f).visit(mod)
f.write("\n")
f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
f.write("int PyAST_Check(PyObject* obj);\n")
f.write("#endif /* !Py_LIMITED_API */\n")
f.write('\n')
f.write('#ifdef __cplusplus\n')
f.write('}\n')
f.write('#endif\n')
f.write('#endif /* !Py_PYTHON_AST_H */\n')
if C_FILE:
with open(C_FILE, "w") as f:
f.write(auto_gen_msg)
f.write('#include <stddef.h>\n')
f.write('\n')
f.write('#include "Python.h"\n')
f.write('#include "%s-ast.h"\n' % mod.name)
f.write('#include "structmember.h"\n')
f.write('\n')
generate_module_def(f, mod)
v = ChainOfVisitors(
PyTypesDeclareVisitor(f),
PyTypesVisitor(f),
Obj2ModPrototypeVisitor(f),
FunctionVisitor(f),
ObjVisitor(f),
Obj2ModVisitor(f),
ASTModuleVisitor(f),
PartingShots(f),
)
v.visit(mod)
def visitSum(self, sum, name, depth):
if not sum.simple:
for t in sum.types:
self.visit(t, name, depth)
def visitProduct(self, product, name, depth):
pass
def visitConstructor(self, cons, name, depth):
self.ctors.append(cons.name)
class ChainOfVisitors:
def __init__(self, *visitors):
self.visitors = visitors
def visit(self, object):
for v in self.visitors:
v.visit(object)
if __name__ == "__main__":
if len(sys.argv) < 2:
sys.argv.append('Python.asdl')
mod = asdl.parse(sys.argv[1])
if not asdl.check(mod):
sys.exit(1)
c = ChainOfVisitors(AnalyzeVisitor(), JavaVisitor(), VisitorVisitor())
c.visit(mod)
def main(srcfile, dump_module=False):
argv0 = sys.argv[0]
components = argv0.split(os.sep)
argv0 = os.sep.join(components[-2:])
auto_gen_msg = common_msg % argv0
mod = asdl.parse(srcfile)
if dump_module:
print('Parsed Module:')
print(mod)
if not asdl.check(mod):
sys.exit(1)
if H_FILE:
with open(H_FILE, "w") as f:
f.write(auto_gen_msg)
f.write('#ifndef Py_PYTHON_AST_H\n')
f.write('#define Py_PYTHON_AST_H\n')
f.write('#ifdef __cplusplus\n')
f.write('extern "C" {\n')
f.write('#endif\n')
f.write('\n')
f.write('#include "asdl.h"\n')
f.write('\n')
f.write(
'#undef Yield /* undefine macro conflicting with <winbase.h> */\n'
)
f.write('\n')
c = ChainOfVisitors(
TypeDefVisitor(f),
StructVisitor(f),
PrototypeVisitor(f),
)
c.visit(mod)
f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
f.write(
"mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n"
)
f.write("int PyAST_Check(PyObject* obj);\n")
f.write('\n')
f.write('#ifdef __cplusplus\n')
f.write('}\n')
f.write('#endif\n')
f.write('#endif /* !Py_PYTHON_AST_H */\n')
if C_FILE:
with open(C_FILE, "w") as f:
f.write(auto_gen_msg)
f.write('#include <stddef.h>\n')
f.write('\n')
f.write('#include "Python.h"\n')
f.write('#include "%s-ast.h"\n' % mod.name)
f.write('\n')
f.write("static PyTypeObject AST_type;\n")
v = ChainOfVisitors(
PyTypesDeclareVisitor(f),
PyTypesVisitor(f),
Obj2ModPrototypeVisitor(f),
FunctionVisitor(f),
ObjVisitor(f),
Obj2ModVisitor(f),
ASTModuleVisitor(f),
PartingShots(f),
)
v.visit(mod)
class PythonGrammar:
ast_wrapper = ast_util.ASTWrapper(
asdl.parse(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
'Python.asdl')))
root_type = 'Module'
pointers = set()
@classmethod
def parse(cls, code, section):
try:
py_ast = ast.parse(code)
return cls.from_native_ast(py_ast)
except SyntaxError:
return None
@classmethod
def unparse(cls, tree, item):
ast_tree = cls.to_native_ast(tree)
return astor.to_source(ast_tree)
@classmethod
def tokenize_field_value(cls, field_value):
if isinstance(field_value, bytes):
field_value = field_value.encode('latin1')
else:
field_value = str(field_value)
return split_string_whitespace_and_camelcase(field_value)
@classmethod
def from_native_ast(cls, node):
if not isinstance(node, ast.AST):
return node
# type: (ast.AST) -> Dict[str, Any]
node_type = node.__class__.__name__
field_infos = {
f.name: f
for f in cls.ast_wrapper.singular_types[node_type].fields
}
result = {'_type': node_type} # type: Dict[str, Any]
for field, value in ast.iter_fields(node):
if field in PYTHON_AST_FIELD_BLACKLIST.get(node_type, set()):
continue
field_info = field_infos[field]
if field_info.opt and value is None:
continue
if isinstance(value, (list, tuple)):
assert field_info.seq
if value:
result[field] = [cls.from_native_ast(v) for v in value]
else:
result[field] = cls.from_native_ast(value)
return result
@classmethod
def to_native_ast(cls, node):
if isinstance(node, (list, tuple)):
return [cls.to_native_ast(item) for item in node]
elif not isinstance(node, dict):
return node
result = getattr(ast, node['_type'])()
# Add any missing fields
type_info = cls.ast_wrapper.singular_types[node['_type']]
for field_info in type_info.fields:
if field_info.seq:
value = node.get(field_info.name, [])
elif field_info.opt:
value = node.get(field_info.name, None)
else:
value = node[field_info.name]
setattr(result, field_info.name, cls.to_native_ast(value))
return result
def __init__(self,
output_from=False,
use_table_pointer=False,
include_literals=True,
include_columns=True,
end_with_from=False,
clause_order=None,
infer_from_conditions=False,
factorize_sketch=0):
# collect pointers and checkers
custom_primitive_type_checkers = {}
self.pointers = set()
if use_table_pointer:
custom_primitive_type_checkers['table'] = lambda x: isinstance(
x, int)
self.pointers.add('table')
self.include_columns = include_columns
if include_columns:
custom_primitive_type_checkers['column'] = lambda x: isinstance(
x, int)
self.pointers.add('column')
# create ast wrapper
self.factorize_sketch = factorize_sketch
if self.factorize_sketch == 0:
asdl_file = "Spider.asdl"
elif self.factorize_sketch == 1:
asdl_file = "Spider_f1.asdl"
elif self.factorize_sketch == 2:
asdl_file = "Spider_f2.asdl"
else:
raise NotImplementedError
self.ast_wrapper = ast_util.ASTWrapper(
asdl.parse(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
asdl_file)),
custom_primitive_type_checkers=custom_primitive_type_checkers)
if not use_table_pointer:
self.ast_wrapper.singular_types['Table'].fields[0].type = 'int'
if not include_columns:
col_unit_fields = self.ast_wrapper.singular_types[
'col_unit'].fields
assert col_unit_fields[1].name == 'col_id'
del col_unit_fields[1]
# literals of limit field
self.include_literals = include_literals
if not self.include_literals:
if self.factorize_sketch == 0:
limit_field = self.ast_wrapper.singular_types['sql'].fields[6]
else:
limit_field = self.ast_wrapper.singular_types[
'sql_orderby'].fields[1]
assert limit_field.name == 'limit'
limit_field.opt = False
limit_field.type = 'singleton'
# from field
self.output_from = output_from
self.end_with_from = end_with_from
self.clause_order = clause_order
self.infer_from_conditions = infer_from_conditions
if self.clause_order:
# clause order is prioritized over configurations like end_with_from
assert factorize_sketch == 2 # TODO support other grammars
sql_fields = self.ast_wrapper.product_types['sql'].fields
letter2field = {k: v for k, v in zip("SFWGOI", sql_fields)}
new_sql_fields = [letter2field[k] for k in self.clause_order]
self.ast_wrapper.product_types['sql'].fields = new_sql_fields
else:
if not self.output_from:
sql_fields = self.ast_wrapper.product_types['sql'].fields
assert sql_fields[1].name == 'from'
del sql_fields[1]
else:
sql_fields = self.ast_wrapper.product_types['sql'].fields
assert sql_fields[1].name == "from"
if self.end_with_from:
sql_fields.append(sql_fields[1])
del sql_fields[1]
def visitSum(self, sum, name, depth):
if not sum.simple:
for t in sum.types:
self.visit(t, name, depth)
def visitProduct(self, product, name, depth):
pass
def visitConstructor(self, cons, name, depth):
self.ctors.append(cons.name)
class ChainOfVisitors:
def __init__(self, *visitors):
self.visitors = visitors
def visit(self, object):
for v in self.visitors:
v.visit(object)
if __name__ == "__main__":
mod = asdl.parse(sys.argv[1])
if not asdl.check(mod):
sys.exit(1)
c = ChainOfVisitors(AnalyzeVisitor(),
JavaVisitor(),
VisitorVisitor())
c.visit(mod)
def main(srcfile, dump_module=False):
argv0 = sys.argv[0]
components = argv0.split(os.sep)
argv0 = os.sep.join(components[-2:])
mod = asdl.parse(srcfile)
if dump_module:
print('Parsed Module:')
print(mod)
if not asdl.check(mod):
sys.exit(1)
if H_FILE:
with open(H_FILE, "w") as f:
f.write("\
#ifndef COSMOPOLITAN_THIRD_PARTY_PYTHON_INCLUDE_PYTHON_AST_H_\n\
#define COSMOPOLITAN_THIRD_PARTY_PYTHON_INCLUDE_PYTHON_AST_H_\n\
#include \"third_party/python/Include/asdl.h\"\n\
#if !(__ASSEMBLER__ + __LINKER__ + 0)\n\
COSMOPOLITAN_C_START_\n\
/* clang-format off */\n\
/* File automatically generated by %s. */\n\
\n\
" % argv0)
c = ChainOfVisitors(
TypeDefVisitor(f),
StructVisitor(f),
PrototypeVisitor(f),
)
c.visit(mod)
f.write("\
PyObject* PyAST_mod2obj(mod_ty t);\n\
mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n\
int PyAST_Check(PyObject* obj);\n\
\n\
COSMOPOLITAN_C_END_\n\
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */\n\
#endif /* COSMOPOLITAN_THIRD_PARTY_PYTHON_INCLUDE_PYTHON_AST_H_ */\n")
if C_FILE:
with open(C_FILE, "w") as f:
f.write('\
#include "third_party/python/Include/%s-ast.h"\n\
#include "third_party/python/Include/abstract.h"\n\
#include "third_party/python/Include/boolobject.h"\n\
#include "third_party/python/Include/descrobject.h"\n\
#include "third_party/python/Include/dictobject.h"\n\
#include "third_party/python/Include/listobject.h"\n\
#include "third_party/python/Include/longobject.h"\n\
#include "third_party/python/Include/modsupport.h"\n\
#include "third_party/python/Include/object.h"\n\
#include "third_party/python/Include/objimpl.h"\n\
#include "third_party/python/Include/pyerrors.h"\n\
#include "third_party/python/Include/pythonrun.h"\n\
#include "third_party/python/Include/tupleobject.h"\n\
#include "third_party/python/Include/yoink.h"\n\
/* clang-format off */\n\
\n\
PYTHON_PROVIDE("_ast");\n\
\n\
/* File automatically generated by %s. */\n\
\n\
static PyTypeObject AST_type;\n\
' % (mod.name, argv0))
v = ChainOfVisitors(
PyTypesDeclareVisitor(f),
PyTypesVisitor(f),
Obj2ModPrototypeVisitor(f),
FunctionVisitor(f),
ObjVisitor(f),
Obj2ModVisitor(f),
ASTModuleVisitor(f),
PartingShots(f),
)
v.visit(mod)
def setUpClass(cls):
# Parse Python.asdl into a ast.Module and run the check on it.
# There's no need to do this for each test method, hence setUpClass.
cls.mod = asdl.parse('./Python.asdl')
cls.assertTrue(asdl.check(cls.mod), 'Module validation failed')
return f
def visitProduct(self, product, name):
self.file.write("typedef ");
self.file.write(str(name).title())
self.file.write(" = {\n")
fields = []
for field in product.fields:
fields.append("\t" + self.field_to_str(field))
self.file.write(",\n".join(fields))
self.file.write("\n}\n\n")
dir = os.path.dirname(__file__)
mod = asdl.parse(dir + "/Python.asdl")
if not asdl.check(mod):
raise Exception("wrong!")
file = open(dir + "/../src/Python.hx", "w")
file.write("// THIS FILE IS GENERATED FROM Python.asdl\n\n")
file.write("typedef Identifier = String;\n\n")
file.write("typedef Bytes = String;\n\n")
file.write("typedef Singleton = String;\n\n")
file.write("typedef Object = String; // used for Num O_o\n\n")
HaxeVisitor(file).visit(mod)
file.close()
def setUp(self):
self.mod = asdl.parse(os.path.join(test_dir, 'Python.asdl'))
self.assertTrue(asdl.check(self.mod), 'Module validation failed')
self.types = self.mod.types