def parse_intern_es6(data):
glob.g_lines = data.split("\n")
if glob.g_preprocess_code:
data = preprocess_text(data, glob.g_file)
if glob.g_print_tokens:
print("printing tokens")
plexer.input(data)
tok = plexer.token()
while tok != None:
print(tok)
tok = plexer.token()
plexer.input(data)
glob.g_lexer = plexer
result = parser.parse(data, lexer=plexer)
if result == None:
if glob.g_error_pre != None:
glob.g_error = True
result = StatementList()
if glob.g_error:
print_err(glob.g_error_pre)
if glob.g_print_nodes:
print(result)
typespace = JSTypeSpace()
if len(result) > 0 and type(
result[0]) == StrLitNode and result[0].val == '"use strict"':
glob.g_force_global_strict = True
elif len(result) > 0 and type(
result[0]) == StrLitNode and result[0].val == '"not_a_module"':
glob.g_es6_modules = False
if glob.g_force_global_strict:
kill_bad_globals(result, typespace)
if glob.g_write_manifest and glob.g_outfile != "":
buf = gen_manifest_file(result, typespace)
file = open(glob.g_outfile + ".manifest", "w")
file.write(buf)
file.close()
process_comments(result, typespace)
if glob.g_enable_static_vars:
process_static_vars(result, typespace)
from js_format import format_es6
buf = format_es6(result, typespace)
if glob.g_outfile == "":
print(buf)
return buf, result
def parse_intern_es6(data):
glob.g_lines = data.split("\n")
if glob.g_preprocess_code:
data = preprocess_text(data, glob.g_file)
if glob.g_print_tokens:
print("printing tokens")
plexer.input(data)
tok = plexer.token()
while tok != None:
print(tok)
tok = plexer.token()
plexer.input(data)
glob.g_lexer = plexer
result = parser.parse(data, lexer=plexer)
if result == None:
if glob.g_error_pre != None:
glob.g_error = True
result = StatementList()
if glob.g_error:
print_err(glob.g_error_pre)
if glob.g_print_nodes:
print(result)
typespace = JSTypeSpace()
if len(result) > 0 and type(result[0]) == StrLitNode and result[0].val == '"use strict"':
glob.g_force_global_strict = True
elif len(result) > 0 and type(result[0]) == StrLitNode and result[0].val == '"not_a_module"':
glob.g_es6_modules = False
if glob.g_force_global_strict:
kill_bad_globals(result, typespace)
if glob.g_write_manifest and glob.g_outfile != "":
buf = gen_manifest_file(result, typespace)
file = open(glob.g_outfile + ".manifest", "w")
file.write(buf)
file.close()
process_comments(result, typespace)
if glob.g_enable_static_vars:
process_static_vars(result, typespace)
from js_format import format_es6
buf = format_es6(result, typespace)
if glob.g_outfile == "":
print(buf)
return buf, result
def find_functions(node, depth):
if type(node.parent) == AssignNode:
node.name = build_name(node.parent.children[0])
#add function to global space if necassary
if type(node.parent.parent) in [int, None]:
typespace.globals[node.name] = node
#parse type string
typestr = lines[node.lineno - 1].strip()
typestr = typestr.replace("//", "").replace("/*",
"").replace("*/",
"").strip()
if not re.search(r"\#\s?\$", typestr):
print("Function %s lacks type information" % node.name)
return
typestr = typestr.replace("#", "").strip()
res = parser.parse(typestr)
try:
ret = res.children[0].children[1].val
except:
print("Could not get return type for function %s", node.name)
ret = "undefined"
try:
types = tuple([
x.val if type(x) != AssignNode else str(x.children[0].val)
for x in res.children[0].children[0].children[1].children
])
except:
print("Could not get parameter types for function %s", node.name)
return
print("%s %s%s" % (ret, node.name, str(types)))
real_arglen = len(node.children[0].children)
if real_arglen != len(types):
print("Error: wrong number of function arguments")
print("%d: %s %s%s" % (node.lineno, ret, node.name, types))
#build member dict
node.parameters = types
node.ret = ret
if func_is_class(node):
typespace.types[node.name] = node
node.members = {}
def find_functions(node, depth):
if type(node.parent) == AssignNode:
node.name = build_name(node.parent.children[0])
#add function to global space if necassary
if type(node.parent.parent) in [int, None]:
typespace.globals[node.name] = node
#parse type string
typestr = lines[node.lineno-1].strip()
typestr = typestr.replace("//", "").replace("/*", "").replace("*/", "").strip()
if not re.search(r"\#\s?\$", typestr):
print("Function %s lacks type information" % node.name)
return
typestr = typestr.replace("#", "").strip()
res = parser.parse(typestr)
try:
ret = res.children[0].children[1].val
except:
print("Could not get return type for function %s", node.name)
ret = "undefined"
try:
types = tuple([x.val if type(x) != AssignNode else str(x.children[0].val) for x in res.children[0].children[0].children[1].children])
except:
print("Could not get parameter types for function %s", node.name)
return
print("%s %s%s"%(ret, node.name, str(types)))
real_arglen = len(node.children[0].children)
if real_arglen != len(types):
print("Error: wrong number of function arguments")
print("%d: %s %s%s"%(node.lineno, ret, node.name, types))
#build member dict
node.parameters = types
node.ret = ret
if func_is_class(node):
typespace.types[node.name] = node
node.members = {}
def js_parse(data,
args=None,
file="",
flatten=True,
print_stack=True,
start_node=None,
print_warnings=False,
exit_on_err=True,
log_productions=False,
validate=False):
back = glob.copy()
def safe_get(data, i):
if i < 0: return ""
elif i >= len(data): return ""
return data[i]
if args != None:
if not isinstance(args, tuple) and not isinstance(args, list):
if caniter(args) and not isinstance(args, Node) \
and type(args) not in [str, bytes]:
args = list(args)
else:
args = (args, ) #encapsulate single arguments in a tuple
i = 0
ai = 0
while i < len(data) - 2:
if data[i] == "$" and safe_get(data, i - 1) != "$":
i1 = i
t = data[i + 1]
i += 2
arg, i = fetch_int(data, i)
if arg == None:
arg = ai
ai += 1
else:
arg -= 1
ai = max(arg, ai)
if arg >= len(args):
raise RuntimeError(
"Not enough args for format conversion in js_parse()")
if t == "n":
buf = args[arg].gen_js(0)
elif t == "s":
buf = str(args[arg])
elif t in ["d", "i", "u"]:
buf = str(int(args[arg]))
elif t in ["f", "lf"]:
buf = str(float(args[arg]))
elif t == "x":
buf = hex(int(args[arg]))
else:
buf = data[i1:i]
data = data[:i1] + buf + data[i:]
i = i1 + len(buf)
i += 1
glob.reset()
glob.g_exit_on_err = exit_on_err
glob.g_lexer = plexer
glob.g_production_debug = False
glob.g_file = file
glob.g_print_stack = print_stack
glob.g_print_warnings = print_warnings
glob.g_log_productions = log_productions
glob.g_validate_mode = validate
plexer.lineno = plexer.lexer.lineno = 0
plexer.input(data)
ret = parser.parse(data, lexer=plexer)
if glob.g_error:
print("------------LLLLLLLLLLLLLLLLLLL yeek!!!")
ret = None
if glob.g_clear_slashr:
print("\n")
def fix_parents(node, lastnode=None):
if node.parent in [0, None]:
node.parent = lastnode
for c in node.children:
fix_parents(c, node)
if ret != None:
fix_parents(ret)
if flatten:
ret = flatten_statementlists(ret, None)
if ret == None:
traceback.print_stack()
sys.stderr.write(
"error: internal parse error within js_parse\n")
sys.exit(-1)
if start_node != None and ret != None:
def visit(n):
if type(n) == start_node:
return n
for c in n.children:
c2 = visit(c)
if c2 != None:
return c2
ret = visit(ret)
if ret != None:
combine_try_nodes(ret)
glob.load(back)
return ret
def parse():
result = parser.parse(data, lexer=plexer)
#print(result)
print("\n")
def traverse(node, ntype, func, extra_arg=NoExtraArg, depth=0):
for c in node.children:
traverse(c, ntype, func, extra_arg, depth + 1)
if type(node) == ntype:
if extra_arg == NoExtraArg:
func(node, depth)
else:
func(node, depth, extra_arg)
def build_name(node):
if type(node) == IdentNode:
return node.val
elif type(node) == BinOpNode and node.op == ".":
return build_name(node.children[0]) + "." + build_name(
node.children[1])
else:
return "unknown"
lines = list(data.split("\n"))
typespace = JSTypeSpace()
def find_functions(node, depth):
if type(node.parent) == AssignNode:
node.name = build_name(node.parent.children[0])
#add function to global space if necassary
if type(node.parent.parent) in [int, None]:
typespace.globals[node.name] = node
#parse type string
typestr = lines[node.lineno - 1].strip()
typestr = typestr.replace("//", "").replace("/*",
"").replace("*/",
"").strip()
if not re.search(r"\#\s?\$", typestr):
print("Function %s lacks type information" % node.name)
return
typestr = typestr.replace("#", "").strip()
res = parser.parse(typestr)
try:
ret = res.children[0].children[1].val
except:
print("Could not get return type for function %s", node.name)
ret = "undefined"
try:
types = tuple([
x.val if type(x) != AssignNode else str(x.children[0].val)
for x in res.children[0].children[0].children[1].children
])
except:
print("Could not get parameter types for function %s", node.name)
return
print("%s %s%s" % (ret, node.name, str(types)))
real_arglen = len(node.children[0].children)
if real_arglen != len(types):
print("Error: wrong number of function arguments")
print("%d: %s %s%s" % (node.lineno, ret, node.name, types))
#build member dict
node.parameters = types
node.ret = ret
if func_is_class(node):
typespace.types[node.name] = node
node.members = {}
def build_globals():
for c in result.children:
if type(c) == AssignNode:
if type(c.children[0]) == IdentNode:
globname = c.children[0].val
gvtype = typespace.build_type(c.children[1], {}, None)
typespace.globals[globname] = gvtype
elif type(c.children[0]) == BinOpNode:
node = typespace.lookup(c.children[0].children[0])
if node != None:
node.type = typespace.build_type(
c.children[1], {}, None)
if typespace.member_lookup(
node, c.children[0].children[1]) == None:
if node.type == None:
typespace.member_add(node,
c.children[0].children[1],
UnkownTypeNode())
else:
typespace.member_add(node,
c.children[0].children[1],
node.type)
if type(node
) == ObjLitNode and node.is_prototype:
typespace.member_add(
node.parent, c.children[0].children[1],
node.type)
def propegate_types(node, depth, report_unknowns=True):
locals = {}
globals = typespace.globals
def trav(n, depth):
varname = build_name(n.children[0])
vtype = typespace.build_type(n.children[1], locals, node)
if vtype == None:
if report_unknowns:
print("Unknown type for variable " + varname +
", in function/class " + node.name)
return
if varname in locals or varname in node.members:
#print("TODO: check that object member types don't change")
return
#member assignment
#print("node:", node.name + ",", "is class:", func_is_class(node), node.ret)
if func_is_class(node) and "this." in varname and varname.count(
".") == 1:
node.members[varname] = vtype
if "this.prototype" not in node.members:
node.members["this.prototype"] = ObjLitNode()
node.add(node.members["this.prototype"])
varname = varname.replace("this.", "")
node.members["this.prototype"].add(
AssignNode(IdentNode(varname), vtype))
#make sure the prototype is marked as such
node.members["this.prototype"].name = "prototype"
node.members["this.prototype"].is_prototype = True
elif "." not in varname and "[" not in varname and "(" not in varname and varname != "unknown":
locals[varname] = vtype
traverse(node, AssignNode, trav)
traverse(result, FunctionNode, find_functions)
traverse(result, FunctionNode, propegate_types, False)
build_globals()
traverse(result, FunctionNode, propegate_types, True)
def parse_intern_es6(data):
glob.g_lines = data.split("\n")
glob.g_filedata = data
check_for_preprocess()
if glob.g_preprocess_code:
data = preprocess_text(data, glob.g_file)
if glob.g_print_tokens:
print("printing tokens")
plexer.input(data)
tok = plexer.token()
while tok != None:
print(tok)
tok = plexer.token()
plexer.input(data)
glob.g_lexer = plexer
result = parser.parse(data, lexer=plexer)
if result == None:
if glob.g_error_pre != None:
glob.g_error = True
result = StatementList()
if glob.g_error:
print_err(glob.g_error_pre)
if glob.g_print_nodes:
print(result)
flatten_var_decls_exprlists(result, typespace)
typespace = JSTypeSpace()
#handle some directives
for c in result:
if type(c) != StrLitNode: break
if c.val[1:-1] == "use strict":
glob.g_force_global_strict = True
elif c.val[1:-1] == "not_a_module":
glob.g_es6_modules = False
if glob.g_force_global_strict:
kill_bad_globals(result, typespace)
if glob.g_write_manifest and glob.g_outfile != "":
buf = gen_manifest_file(result, typespace)
file = open(glob.g_outfile + ".manifest", "w")
file.write(buf)
file.close()
if glob.g_include_comments:
process_comments(result, typespace)
if glob.g_enable_static_vars:
process_static_vars(result, typespace)
from js_format import format_es6
buf = format_es6(result, typespace)
if glob.g_outfile == "":
print(syntaxColor(buf))
return buf, result
def test_array(self):
js=parser.parse('json.json')
arr1=['glossary', 'GlossDiv', 'GlossList', 'GlossEntry', 'GlossDef:GlossSeeAlso']
arr2=['GlossDef', 'GlossSeeAlso']
arr=['GML', 'XML']
self.assertEqual (parser.arr(js, arr1, arr2), arr)
import sys
def args():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('filename', nargs='?', type=str, help="Json file name")
parser.add_argument('st', nargs='?',type=str, help='TODO')
return parser.parse_args()
if __name__ == "__main__":
argument=args()
st=argument.st.split('.')
name=argument.filename
st1=st[len(st)-1].split(':')
from js_parse import parser
js=parser.parse(name)
if len(st1)==1:
js=parser.obj_text(js, st)
if isinstance(js, list):
print(st[len(st)-1]+" is array, use : for array")
else:
print(js)
elif len(st1)==2:
js=parser.arr(js, st, st1)
if isinstance(js, dict):
print(st1[1]+" is object, use . for object")
elif isinstance(js, list):
print(js)
else:
print(st1[1]+" is text, use . for txt")
def parse():
result = parser.parse(data, lexer=plexer)
#print(result)
print("\n")
def traverse(node, ntype, func, extra_arg=NoExtraArg, depth=0):
for c in node.children:
traverse(c, ntype, func, extra_arg, depth+1)
if type(node) == ntype:
if extra_arg == NoExtraArg:
func(node, depth)
else:
func(node, depth, extra_arg)
def build_name(node):
if type(node) == IdentNode:
return node.val
elif type(node) == BinOpNode and node.op == ".":
return build_name(node.children[0])+"."+build_name(node.children[1])
else:
return "unknown"
lines = list(data.split("\n"))
typespace = JSTypeSpace()
def find_functions(node, depth):
if type(node.parent) == AssignNode:
node.name = build_name(node.parent.children[0])
#add function to global space if necassary
if type(node.parent.parent) in [int, None]:
typespace.globals[node.name] = node
#parse type string
typestr = lines[node.lineno-1].strip()
typestr = typestr.replace("//", "").replace("/*", "").replace("*/", "").strip()
if not re.search(r"\#\s?\$", typestr):
print("Function %s lacks type information" % node.name)
return
typestr = typestr.replace("#", "").strip()
res = parser.parse(typestr)
try:
ret = res.children[0].children[1].val
except:
print("Could not get return type for function %s", node.name)
ret = "undefined"
try:
types = tuple([x.val if type(x) != AssignNode else str(x.children[0].val) for x in res.children[0].children[0].children[1].children])
except:
print("Could not get parameter types for function %s", node.name)
return
print("%s %s%s"%(ret, node.name, str(types)))
real_arglen = len(node.children[0].children)
if real_arglen != len(types):
print("Error: wrong number of function arguments")
print("%d: %s %s%s"%(node.lineno, ret, node.name, types))
#build member dict
node.parameters = types
node.ret = ret
if func_is_class(node):
typespace.types[node.name] = node
node.members = {}
def build_globals():
for c in result.children:
if type(c) == AssignNode:
if type(c.children[0]) == IdentNode:
globname = c.children[0].val
gvtype = typespace.build_type(c.children[1], {}, None)
typespace.globals[globname] = gvtype
elif type(c.children[0]) == BinOpNode:
node = typespace.lookup(c.children[0].children[0])
if node != None:
node.type = typespace.build_type(c.children[1], {}, None)
if typespace.member_lookup(node, c.children[0].children[1]) == None:
if node.type == None:
typespace.member_add(node, c.children[0].children[1], UnkownTypeNode())
else:
typespace.member_add(node, c.children[0].children[1], node.type)
if type(node) == ObjLitNode and node.is_prototype:
typespace.member_add(node.parent, c.children[0].children[1], node.type)
def propegate_types(node, depth, report_unknowns=True):
locals = {}
globals = typespace.globals
def trav(n, depth):
varname = build_name(n.children[0])
vtype = typespace.build_type(n.children[1], locals, node)
if vtype == None:
if report_unknowns:
print("Unknown type for variable " + varname +", in function/class " + node.name)
return
if varname in locals or varname in node.members:
#print("TODO: check that object member types don't change")
return
#member assignment
#print("node:", node.name + ",", "is class:", func_is_class(node), node.ret)
if func_is_class(node) and "this." in varname and varname.count(".") == 1:
node.members[varname] = vtype
if "this.prototype" not in node.members:
node.members["this.prototype"] = ObjLitNode()
node.add(node.members["this.prototype"])
varname = varname.replace("this.", "")
node.members["this.prototype"].add(AssignNode(IdentNode(varname), vtype))
#make sure the prototype is marked as such
node.members["this.prototype"].name = "prototype"
node.members["this.prototype"].is_prototype = True
elif "." not in varname and "[" not in varname and "(" not in varname and varname != "unknown":
locals[varname] = vtype
traverse(node, AssignNode, trav)
traverse(result, FunctionNode, find_functions)
traverse(result, FunctionNode, propegate_types, False)
build_globals()
traverse(result, FunctionNode, propegate_types, True)
def parse_intern(data, create_logger=False, expand_loops=True, expand_generators=True):
glob.g_lines = data.split("\n")
if glob.g_preprocess_code:
data = preprocess_text(data, glob.g_file)
if glob.g_print_tokens:
plexer.input(data)
tok = plexer.token()
while tok != None:
print(tok)
tok = plexer.token()
plexer.input(data)
glob.g_lexer = plexer
result = parser.parse(data, lexer=plexer)
if result == None:
if glob.g_error_pre != None:
glob.g_error = True
result = StatementList()
if glob.g_error:
print_err(glob.g_error_pre)
typespace = JSTypeSpace()
if result != None:
if len(result) > 0 and type(result[0]) == StrLitNode and result[0].val == '"use strict"':
glob.g_force_global_strict = True
elif len(result) > 0 and type(result[0]) == StrLitNode and result[0].val == '"not_a_module"':
glob.g_es6_modules = False
if glob.g_compile_statics_only:
process_static_vars(result, typespace)
return result.gen_js(0), result
if glob.g_enable_let:
process_let(result, typespace)
if glob.g_force_global_strict:
kill_bad_globals(result, typespace)
#handle .? operator
transform_exisential_operators(result, typespace)
if glob.g_write_manifest and glob.g_outfile != "":
buf = gen_manifest_file(result, typespace);
file = open(glob.g_outfile+".manifest", "w")
file.write(buf)
file.close()
if glob.g_es6_modules:
module_transform(result, typespace)
if glob.g_require_js:
expand_requirejs_classes(result, typespace);
else:
expand_harmony_classes(result, typespace);
expand_typed_classes(result, typespace)
if glob.g_clear_slashr:
print("\n")
if result != None and len(result) == 0:
result = None
return "", None
#sys.stdout.write("Error: empty compilation\n");
#raise JSError("Empty compilation");
global f_id
f_id = [0]
flatten_statementlists(result, typespace)
has_generators = [False]
def has_generator(n):
if type(n) == YieldNode:
has_generators[0] = True
for c in n:
has_generator(c)
has_generator(result)
if expand_loops or has_generators[0]:
expand_of_loops(result, typespace)
#combine_try_nodes may have nested statementlists again, so better reflatten
flatten_statementlists(result, typespace)
if create_logger:
traverse(result, FunctionNode, create_type_logger)
process_arrow_function_this(result, typespace)
if expand_generators:
flatten_statementlists(result, typespace)
process_generators(result, typespace);
flatten_statementlists(result, typespace)
if glob.g_add_opt_initializers:
add_func_opt_code(result, typespace)
if glob.g_combine_ifelse_nodes:
combine_if_else_nodes(result)
if glob.g_print_nodes:
print("nodes: ", result)
pass
if glob.g_replace_instanceof and not glob.g_require_js:
replace_instanceof(result, typespace)
if glob.g_enable_static_vars:
process_static_vars(result, typespace)
if glob.g_do_docstrings:
process_docstrings(result, typespace)
if glob.g_gen_v7_bytecode:
from js_opcode_emit2 import v7_emit_bytecode
buf = v7_emit_bytecode(result, typespace)
elif not glob.g_minify:
buf = result.gen_js(0)
else:
buf, smap = js_minify(result)
if glob.g_outfile == "":
print(buf)
return buf, result
def process(buf):
glob.g_lexer = js_lex.plexer
glob.g_destructuring = False
ret = parser.parse(buf, lexer=js_lex.plexer)
marks = [-1 for x in range(len(buf))]
level = [0]
color = [0]
nodes = []
def marklexpos(n):
if len(n) == 0:
return
for c in n:
marklexpos(c)
n.lexpos = n[0].lexpos
n.lexpos2 = n[-1].lexpos2
def recurse(n):
if 1: #n.level == level[0]:
nodes.append(n)
for c in n:
recurse(c)
#return
mark(ret)
marklexpos(ret)
recurse(ret)
level = 1
c1 = 0
for ni in range(0, 15):
for n in nodes:
if n.level != ni: continue
if not isinstance(n, FunctionNode):
continue
found = 0
for i in range(n.lexpos, n.lexpos2):
if marks[i] < 0:
marks[i] = c1
found = 1
pass
if found:
c1 = (c1 + 1) % (len(colors) - 1)
#print(n.lexpos, n.lexpos2, n.get_line_str(), n.gen_js(0))
for i in range(len(buf)):
if marks[i] == -1:
c = 0
else:
c = marks[i] + 1
setcolor(colors[c])
sys.stdout.write(buf[i])
def test_parse(self):
self.assertEqual (parser.parse('json.json')['glossary']['title'], 'example glossary')
def parse_intern(data,
create_logger=False,
expand_loops=True,
expand_generators=True):
glob.g_lines = data.split("\n")
if glob.g_preprocess_code:
data = preprocess_text(data, glob.g_file)
if glob.g_print_tokens:
plexer.input(data)
tok = plexer.token()
while tok != None:
print(tok)
tok = plexer.token()
plexer.input(data)
glob.g_lexer = plexer
result = parser.parse(data, lexer=plexer)
if result == None:
if glob.g_error_pre != None:
glob.g_error = True
result = StatementList()
if glob.g_error:
print_err(glob.g_error_pre)
typespace = JSTypeSpace()
if result != None:
if len(result) > 0 and type(
result[0]) == StrLitNode and result[0].val == '"use strict"':
glob.g_force_global_strict = True
elif len(result) > 0 and type(
result[0]) == StrLitNode and result[0].val == '"not_a_module"':
glob.g_es6_modules = False
if glob.g_compile_statics_only:
process_static_vars(result, typespace)
return result.gen_js(0), result
if glob.g_enable_let:
process_let(result, typespace)
if glob.g_force_global_strict:
kill_bad_globals(result, typespace)
#handle .? operator
transform_exisential_operators(result, typespace)
if glob.g_write_manifest and glob.g_outfile != "":
buf = gen_manifest_file(result, typespace)
file = open(glob.g_outfile + ".manifest", "w")
file.write(buf)
file.close()
if glob.g_es6_modules:
module_transform(result, typespace)
if glob.g_require_js:
expand_requirejs_classes(result, typespace)
else:
expand_harmony_classes(result, typespace)
expand_typed_classes(result, typespace)
if glob.g_clear_slashr:
print("\n")
if result != None and len(result) == 0:
result = None
return "", None
#sys.stdout.write("Error: empty compilation\n");
#raise JSError("Empty compilation");
global f_id
f_id = [0]
flatten_statementlists(result, typespace)
has_generators = [False]
def has_generator(n):
if type(n) == YieldNode:
has_generators[0] = True
for c in n:
has_generator(c)
has_generator(result)
if expand_loops or has_generators[0]:
expand_of_loops(result, typespace)
#combine_try_nodes may have nested statementlists again, so better reflatten
flatten_statementlists(result, typespace)
if create_logger:
traverse(result, FunctionNode, create_type_logger)
process_arrow_function_this(result, typespace)
if expand_generators:
flatten_statementlists(result, typespace)
process_generators(result, typespace)
flatten_statementlists(result, typespace)
if glob.g_add_opt_initializers:
add_func_opt_code(result, typespace)
if glob.g_combine_ifelse_nodes:
combine_if_else_nodes(result)
if glob.g_print_nodes:
print("nodes: ", result)
pass
if glob.g_replace_instanceof and not glob.g_require_js:
replace_instanceof(result, typespace)
if glob.g_enable_static_vars:
process_static_vars(result, typespace)
if glob.g_do_docstrings:
process_docstrings(result, typespace)
if glob.g_gen_v7_bytecode:
from js_opcode_emit2 import v7_emit_bytecode
buf = v7_emit_bytecode(result, typespace)
elif not glob.g_minify:
buf = result.gen_js(0)
else:
buf, smap = js_minify(result)
if glob.g_outfile == "":
print(buf)
return buf, result
def parse_intern(data,
create_logger=False,
expand_loops=True,
expand_generators=True):
glob.g_lines = data.split("\n")
glob.g_filedata = data
check_for_preprocess()
if glob.g_preprocess_code:
have_cpp = False
for l in glob.g_lines:
l = l.strip()
if l.startswith("#"):
have_cpp = True
if have_cpp:
data = preprocess_text(data, glob.g_file)
if glob.g_print_tokens:
plexer.input(data)
tok = plexer.token()
while tok != None:
print(tok)
tok = plexer.token()
plexer.input(data)
glob.g_lexer = plexer
result = parser.parse(data, lexer=plexer, debug=glob.g_production_debug)
if result == None:
if glob.g_error_pre != None:
glob.g_error = True
result = StatementList()
if glob.g_error:
print_err(glob.g_error_pre)
typespace = JSTypeSpace()
glob.g_lexdata = data
if glob.g_infer_class_properties:
from js_process_ast import infer_class_properties
data = infer_class_properties(result, typespace, glob.g_filedata)
if glob.g_outfile == "":
print(data)
return data, result
if glob.g_raw_code:
buf = result.gen_js(0)
if glob.g_outfile == "":
print(syntaxColor(buf))
return buf, result
if 1: #glob.g_enable_let:
process_let(result, typespace)
pass
flatten_var_decls_exprlists(result, typespace)
if glob.g_type_file != "":
if not os.path.exists(glob.g_type_file):
sys.stderr.write("File does not exist: %s\n" % glob.g_type_file)
sys.exit(-1)
import json
import js_typelogger
file = open(glob.g_type_file, "r")
buf = file.read()
file.close()
data2 = json.loads(buf)
inserts = js_typelogger.load_types(result, typespace, data2)
js_typelogger.emit_dynamic_literals(result, typespace, data2)
if glob.g_apply_types:
data = js_typelogger.apply_inserts(result, typespace, inserts,
glob.g_filedata)
if glob.g_outfile == "":
print(data)
return data, result
if create_logger:
import js_typelogger
js_typelogger.create_type_logger(result, typespace)
#handle some directives
for c in result:
if type(c) != StrLitNode: break
if c.val[1:-1] == "use strict":
glob.g_force_global_strict = True
elif c.val[1:-1] == "not_a_module":
glob.g_es6_modules = False
if glob.g_profile_coverage:
from js_process_ast import coverage_profile
coverage_profile(result, typespace)
if glob.g_compile_statics_only:
process_static_vars(result, typespace)
return result.gen_js(0), result
if glob.g_force_global_strict:
kill_bad_globals(result, typespace)
pass
#handle .? operator
if glob.g_transform_exisential:
transform_exisential_operators(result, typespace)
if glob.g_write_manifest and glob.g_outfile != "":
buf = gen_manifest_file(result, typespace)
file = open(glob.g_outfile + ".manifest", "w")
file.write(buf)
file.close()
if glob.g_es6_modules:
module_transform(result, typespace)
pass
#"""
if glob.g_require_js:
expand_requirejs_classes(result, typespace)
elif glob.g_expand_classes:
expand_harmony_classes(result, typespace)
expand_typed_classes(result, typespace)
else:
create_class_list(result, typespace)
pass
#"""
if glob.g_clear_slashr:
print("\n")
if result != None and len(result) == 0:
result = None
return "", None
#sys.stdout.write("Error: empty compilation\n");
#raise JSError("Empty compilation");
global f_id
f_id = [0]
flatten_statementlists(result, typespace)
has_generators = [False]
def has_generator(n):
if not glob.g_expand_generators: return
if type(n) == YieldNode:
has_generators[0] = True
for c in n:
has_generator(c)
has_generator(result)
if expand_loops or has_generators[0]:
expand_of_loops(result, typespace)
#combine_try_nodes may have nested statementlists again, so better reflatten
flatten_statementlists(result, typespace)
#don't need to do this anymore, yay!
#process_arrow_function_this(result, typespace)
if expand_generators:
flatten_statementlists(result, typespace)
process_generators(result, typespace)
flatten_statementlists(result, typespace)
if glob.g_add_opt_initializers:
add_func_opt_code(result, typespace)
debug_forloop_expansion = False
if debug_forloop_expansion:
reset = 0
if reset or not os.path.exists("cur_d.txt"):
f = open("cur_d.txt", "w")
f.write("-1")
f.close()
f = open("cur_d.txt", "r")
d = int(f.read())
print("\n\nd: %d\n" % d)
traverse_i(result,
ForInNode,
expand_mozilla_forloops,
d,
use_scope=True)
f.close()
f = open("cur_d.txt", "w")
f.write(str(d + 1))
f.close()
if glob.g_combine_ifelse_nodes:
combine_if_else_nodes(result)
if glob.g_print_nodes:
print("nodes: ", result)
pass
if glob.g_replace_instanceof and not glob.g_require_js:
replace_instanceof(result, typespace)
if glob.g_enable_static_vars:
process_static_vars(result, typespace)
if glob.g_do_docstrings:
process_docstrings(result, typespace)
if glob.g_include_comments:
process_comments(result, typespace)
if glob.g_gen_source_map:
smap = SourceMap()
def set_smap(node, smap):
node.smap = smap
for n in node:
set_smap(n, smap)
set_smap(result, smap)
if not glob.g_minify:
buf = result.gen_js(0)
map = gen_source_map(data, buf, smap)
else:
buf, smap = js_minify(result)
map = gen_source_map(data, buf, smap)
if glob.g_add_srcmap_ref:
spath = glob.g_outfile
if os.path.sep in spath: spath = os.path.split(spath)[1]
buf += "\n//# sourceMappingURL=/content/%s\n" % (spath + ".map")
if glob.g_gen_smap_orig:
f = open(glob.g_outfile + ".origsrc", "w")
f.write(data)
f.close()
else:
if not glob.g_minify:
buf = result.gen_js(0)
else:
buf, smap = js_minify(result)
if glob.g_outfile == "":
sys.stdout.write(syntaxColor(buf) + "\n")
if 0:
file = open("generator_test.html", "w")
file.write("""
<html><head><title>Generator Test</title></head>
<script>
function arr_iter(keys)
{
this.keys = keys;
this.cur = 0;
this.next = function() {
if (this.cur >= this.keys.length) {
throw StopIteration;
}
return this.keys[this.cur++];
}
}
__use_Iterator = true;
function __get_iter(obj)
{
if (obj.__proto__.hasOwnProperty("__iterator__") || obj.hasOwnProperty("__iterator__")) {
return obj.__iterator__();
} else {
if (__use_Iterator) {
return Iterator(obj);
} else {
keys = []
for (var k in obj) {
keys.push(k)
}
return new arr_iter(keys);
}
}
}
function __get_iter2(obj)
{
if (obj.__proto__.hasOwnProperty("__iterator__") || obj.hasOwnProperty("__iterator__")) {
return obj.__iterator__();
} else {
keys = []
for (var k in obj) {
keys.push([k, obj[k]])
}
return new arr_iter(keys);
}
}
try {
_tst = Iterator({});
} catch (Error) {
__use_Iterator = false;
Iterator = __get_iter2;
}
FrameContinue = {1:1};
FrameBreak = {2:2};
""")
file.write(buf.replace("yield", "return"))
file.write("""
j = 0;
for (var tst in new range2(2, 8)) {
if (_do_frame_debug) console.log(tst);
if (j > 10)
break;
j++;
}
</script>
</html>
""")
file.close()
pass
return buf, result
def js_parse(
data,
args=None,
file="",
flatten=True,
print_stack=True,
start_node=None,
print_warnings=False,
exit_on_err=True,
log_productions=False,
validate=False,
):
back = glob.copy()
def safe_get(data, i):
if i < 0:
return ""
elif i >= len(data):
return ""
return data[i]
if args != None:
if not isinstance(args, tuple) and not isinstance(args, list):
if caniter(args) and not isinstance(args, Node) and type(args) not in [str, bytes]:
args = list(args)
else:
args = (args,) # encapsulate single arguments in a tuple
i = 0
ai = 0
while i < len(data) - 2:
if data[i] == "$" and safe_get(data, i - 1) != "$":
i1 = i
t = data[i + 1]
i += 2
arg, i = fetch_int(data, i)
if arg == None:
arg = ai
ai += 1
else:
arg -= 1
ai = max(arg, ai)
if arg >= len(args):
raise RuntimeError("Not enough args for format conversion in js_parse()")
if t == "n":
buf = args[arg].gen_js(0)
elif t == "s":
buf = str(args[arg])
elif t in ["d", "i", "u"]:
buf = str(int(args[arg]))
elif t in ["f", "lf"]:
buf = str(float(args[arg]))
elif t == "x":
buf = hex(int(args[arg]))
else:
buf = data[i1:i]
data = data[:i1] + buf + data[i:]
i = i1 + len(buf)
i += 1
glob.reset()
glob.g_exit_on_err = exit_on_err
glob.g_lexer = plexer
glob.g_production_debug = False
glob.g_file = file
glob.g_print_stack = print_stack
glob.g_print_warnings = print_warnings
glob.g_log_productions = log_productions
glob.g_validate_mode = validate
plexer.lineno = plexer.lexer.lineno = 0
plexer.input(data)
ret = parser.parse(data, lexer=plexer)
if glob.g_error:
print("------------LLLLLLLLLLLLLLLLLLL yeek!!!")
ret = None
if glob.g_clear_slashr:
print("\n")
def fix_parents(node, lastnode=None):
if node.parent in [0, None]:
node.parent = lastnode
for c in node.children:
fix_parents(c, node)
if ret != None:
fix_parents(ret)
if flatten:
ret = flatten_statementlists(ret, None)
if ret == None:
traceback.print_stack()
sys.stderr.write("error: internal parse error within js_parse\n")
sys.exit(-1)
if start_node != None and ret != None:
def visit(n):
if type(n) == start_node:
return n
for c in n.children:
c2 = visit(c)
if c2 != None:
return c2
ret = visit(ret)
if ret != None:
combine_try_nodes(ret)
glob.load(back)
return ret
def js_parse(data,
args=None,
file="",
flatten=True,
print_stack=True,
start_node=None,
print_warnings=False,
exit_on_err=True,
log_productions=False,
validate=False):
back = glob.copy()
def safe_get(data, i):
if i < 0: return ""
elif i >= len(data): return ""
return data[i]
if args != None:
if not isinstance(args, tuple) and not isinstance(args, list):
if caniter(args) and not isinstance(args, Node) \
and type(args) not in [str, bytes]:
args = list(args)
else:
args = (args, ) #encapsulate single arguments in a tuple
i = 0
ai = 0
while i < len(data) - 2:
if data[i] == "$":
if safe_get(data, i + 1) == "$": #handle '$$'
data = data[:i] + data[i + 1:]
i += 1
continue
i1 = i
t = data[i + 1]
i += 2
arg, i = fetch_int(data, i)
if arg == None:
arg = ai
ai += 1
else:
arg -= 1
ai = max(arg, ai)
if arg >= len(args):
raise RuntimeError(
"Not enough args for format conversion in js_parse()")
if t == "n":
buf = args[arg].gen_js(0)
elif t == "s":
buf = str(args[arg])
elif t in ["d", "i", "u"]:
buf = str(int(args[arg]))
elif t in ["f", "lf"]:
buf = str(float(args[arg]))
elif t == "x":
buf = hex(int(args[arg]))
else:
buf = data[i1:i]
data = data[:i1] + buf + data[i:]
i = i1 + len(buf)
i += 1
glob.reset()
glob.g_exit_on_err = exit_on_err
glob.g_lexer = plexer
linemap = glob.g_lexer.linemap
glob.g_lexdata = data
glob.g_production_debug = False
glob.g_file = file
glob.g_print_stack = print_stack
glob.g_print_warnings = print_warnings
glob.g_log_productions = log_productions
glob.g_validate_mode = validate
glob.g_inside_js_parse = True
plexer.lineno = plexer.lexer.lineno = 0
plexer.input(data)
ret = None
try:
ret = parser.parse(data, lexer=plexer)
except:
pass
glob.g_inside_js_parse = False
glob.g_lexer.linemap = linemap
if glob.g_error:
ret = None
if ret is None:
ls = data.split("\n")
s2 = ""
for i in range(len(ls)):
s2 += str(i + 1) + ": " + ls[i] + "\n"
print(s2)
col = glob.g_lexpos
i = min(col, len(data) - 1)
while i >= 0 and i < len(data) and data[i] != "\n":
i -= 1
if i > 0:
col -= i
pass
i = max(i, 0)
i2 = i + 1
while i2 < len(data) and data[i2] != "\n":
i2 += 1
#line = data.split("\n")[glob.g_line]
if data[i] == "\n":
line = data[i + 1:i2]
else:
line = data[i:i2]
while col < len(line) and col >= 0 and line[col] in [
"\n", "\r", "\t", " "
]:
col += 1
if col > 0 and col < len(line):
line = line[:col] + termColor(line[col], 41) + line[col + 1:]
raise SyntaxError("(js_parse intern):" + str(glob.g_line + 1) + ":" +
str(col + 1) + "\n\t" + line)
if ret:
flatten_statementlists(ret, None)
if glob.g_clear_slashr:
print("\n")
def fix_parents(node, lastnode=None):
if node.parent in [0, None]:
node.parent = lastnode
for c in node.children:
fix_parents(c, node)
if ret != None:
fix_parents(ret)
if flatten:
ret = flatten_statementlists(ret, None)
if ret == None:
traceback.print_stack()
sys.stderr.write(
"error: internal parse error within js_parse\n")
sys.exit(-1)
if start_node != None and ret != None:
def visit(n):
if type(n) == start_node:
return n
for c in n.children:
c2 = visit(c)
if c2 != None:
return c2
ret = visit(ret)
if ret != None:
combine_try_nodes(ret)
glob.load(back)
return ret
def parse_intern(data, create_logger=False, expand_loops=True, expand_generators=True):
glob.g_lines = data.split("\n")
if glob.g_preprocess_code:
data = preprocess_text(data, glob.g_file)
if glob.g_print_tokens:
plexer.input(data)
tok = plexer.token()
while tok != None:
print(tok)
tok = plexer.token()
plexer.input(data)
glob.g_lexer = plexer
result = parser.parse(data, lexer=plexer)
if result == None:
if glob.g_error_pre != None:
glob.g_error = True
result = StatementList()
if glob.g_error:
print_err(glob.g_error_pre)
typespace = JSTypeSpace()
if result != None:
if len(result) > 0 and type(result[0]) == StrLitNode and result[0].val == '"use strict"':
glob.g_force_global_strict = True
elif len(result) > 0 and type(result[0]) == StrLitNode and result[0].val == '"not_a_module"':
glob.g_es6_modules = False
if glob.g_force_global_strict:
kill_bad_globals(result, typespace)
# handle .? operator
transform_exisential_operators(result, typespace)
if glob.g_write_manifest and glob.g_outfile != "":
buf = gen_manifest_file(result, typespace)
file = open(glob.g_outfile + ".manifest", "w")
file.write(buf)
file.close()
if glob.g_es6_modules:
module_transform(result, typespace)
if glob.g_require_js:
expand_requirejs_classes(result, typespace)
else:
expand_harmony_classes(result, typespace)
expand_typed_classes(result, typespace)
if glob.g_clear_slashr:
print("\n")
if result != None and len(result) == 0:
result = None
return "", None
# sys.stdout.write("Error: empty compilation\n");
# raise JSError("Empty compilation");
global f_id
f_id = [0]
flatten_statementlists(result, typespace)
if expand_loops:
expand_of_loops(result, typespace)
# combine_try_nodes may have nested statementlists again, so better reflatten
flatten_statementlists(result, typespace)
if create_logger:
traverse(result, FunctionNode, create_type_logger)
if expand_generators:
flatten_statementlists(result, typespace)
process_generators(result, typespace)
flatten_statementlists(result, typespace)
if glob.g_add_opt_initializers:
add_func_opt_code(result, typespace)
debug_forloop_expansion = False
if debug_forloop_expansion:
reset = 0
if reset or not os.path.exists("cur_d.txt"):
f = open("cur_d.txt", "w")
f.write("-1")
f.close()
f = open("cur_d.txt", "r")
d = int(f.read())
print("\n\nd: %d\n" % d)
traverse_i(result, ForInNode, expand_mozilla_forloops, d, use_scope=True)
f.close()
f = open("cur_d.txt", "w")
f.write(str(d + 1))
f.close()
if glob.g_combine_ifelse_nodes:
combine_if_else_nodes(result)
if glob.g_print_nodes:
print("nodes: ", result)
pass
if glob.g_replace_instanceof and not glob.g_require_js:
replace_instanceof(result, typespace)
if glob.g_enable_static_vars:
process_static_vars(result, typespace)
if glob.g_do_docstrings:
process_docstrings(result, typespace)
if glob.g_gen_source_map:
smap = SourceMap()
def set_smap(node, smap):
node.smap = smap
for n in node:
set_smap(n, smap)
set_smap(result, smap)
if not glob.g_minify:
buf = result.gen_js(0)
map = gen_source_map(data, buf, smap)
else:
buf, smap = js_minify(result)
map = gen_source_map(data, buf, smap)
if glob.g_add_srcmap_ref:
spath = glob.g_outfile
if os.path.sep in spath:
spath = os.path.split(spath)[1]
buf += "\n//# sourceMappingURL=/content/%s\n" % (spath + ".map")
if glob.g_gen_smap_orig:
f = open(glob.g_outfile + ".origsrc", "w")
f.write(data)
f.close()
else:
if not glob.g_minify:
buf = result.gen_js(0)
else:
buf, smap = js_minify(result)
if glob.g_outfile == "":
print(buf)
if 0:
file = open("generator_test.html", "w")
file.write(
"""
<html><head><title>Generator Test</title></head>
<script>
function arr_iter(keys)
{
this.keys = keys;
this.cur = 0;
this.next = function() {
if (this.cur >= this.keys.length) {
throw StopIteration;
}
return this.keys[this.cur++];
}
}
__use_Iterator = true;
function __get_iter(obj)
{
if (obj.__proto__.hasOwnProperty("__iterator__") || obj.hasOwnProperty("__iterator__")) {
return obj.__iterator__();
} else {
if (__use_Iterator) {
return Iterator(obj);
} else {
keys = []
for (var k in obj) {
keys.push(k)
}
return new arr_iter(keys);
}
}
}
function __get_iter2(obj)
{
if (obj.__proto__.hasOwnProperty("__iterator__") || obj.hasOwnProperty("__iterator__")) {
return obj.__iterator__();
} else {
keys = []
for (var k in obj) {
keys.push([k, obj[k]])
}
return new arr_iter(keys);
}
}
try {
_tst = Iterator({});
} catch (Error) {
__use_Iterator = false;
Iterator = __get_iter2;
}
FrameContinue = {1:1};
FrameBreak = {2:2};
"""
)
file.write(buf.replace("yield", "return"))
file.write(
"""
j = 0;
for (var tst in new range2(2, 8)) {
if (_do_frame_debug) console.log(tst);
if (j > 10)
break;
j++;
}
</script>
</html>
"""
)
file.close()
pass
return buf, result
def test_text(self):
js=parser.parse('json.json')
arr=['glossary', 'title']
self.assertEqual (parser.obj_text(js, arr), 'example glossary')
