def __init__(self, lex_optimize=True, lextab=lextab,
yacc_optimize=True, yacctab=yacctab, yacc_debug=False):
self.lex_optimize = lex_optimize
self.lextab = lextab
self.yacc_optimize = yacc_optimize
self.yacctab = yacctab
self.yacc_debug = yacc_debug
self.lexer = Lexer()
self.lexer.build(optimize=lex_optimize, lextab=lextab)
self.tokens = self.lexer.tokens
self.parser = ply.yacc.yacc(
module=self, optimize=yacc_optimize,
debug=yacc_debug, tabmodule=yacctab, start='program')
# https://github.com/rspivak/slimit/issues/29
# lexer.auto_semi can cause a loop in a parser
# when a parser error happens on a token right after
# a newline.
# We keep record of the tokens that caused p_error
# and if the token has already been seen - we raise
# a SyntaxError exception to avoid looping over and
# over again.
self._error_tokens = {}
def extract(self, node):
lexer = JsLexer()
lexer.input(node.text)
regions = []
if not self._parse_succeeds(node.text):
return regions
while True:
try:
tok = lexer.token()
if not tok:
break
if tok.type == "REGEX":
start_char = tok.lexpos + 1
regex_parts = tok.value.split('/')
string = regex_parts[1]
flags = regex_parts[2]
if not self._are_flags_valid(flags):
continue
end_char = start_char + len(string) - 1
r = RegexRegion(string, node, start_char, end_char, string)
regions.append(r)
except (TypeError, AttributeError):
logging.warn("Failed to parse text: %s...", node.text[:100])
break
return regions
def LexingofFunction(str):
lexer = Lexer()
lexer.input(str)
lhs = ""
rhs = ""
flag = 0
for token in lexer:
# print token
tokenTemp = token.__str__()
if '=' not in tokenTemp and flag == 0:
tokenTemp = trim(tokenTemp)
lhs = lhs + tokenTemp
elif '=' in tokenTemp or ';' in tokenTemp:
flag = 1;
continue
else:
rhs = rhs + tokenTemp
print "Printing lhs", lhs
print "Printing rhs", rhs
addinHeap(lhs, rhs)
# CompareLHSandRHS(lhs,rhs)
return token
def custom_lex(text):
# lexes string, stops when it starts making TypeErrors
# input: f.read()
out_list = []
id_list = []
tok_list = []
str_list = []
num_list = []
lexer = Lexer()
lexer.input(text)
while True:
try:
token = lexer.token()
if not token:
break # break if end of token
tok_type = token.type
if tok_type=='ID':
id_list.append(token.value)
elif tok_type=='STRING':
str_list.append(token.value)
elif tok_type=='NUMBER':
num_list.append(token.value)
else:
tok_list.append(token.value)
out_list.append(token.value)
except TypeError:
break
except AttributeError:
break
return out_list,id_list,tok_list, str_list, num_list
def train_from_js_tokens(corpus):
lexer = Lexer()
tokens_corpus = []
for t in corpus:
lexer.input(t)
tokens_corpus.append(' '.join([token.type for token in lexer]))
return train_tfidf(tokens_corpus)
def tokenize_js(code, need_type_info=False):
# with open(code, 'r') as f:
# code = f.read()
lexer = Lexer()
lexer.input(code)
tokens = []
types = []
pos = []
while True:
token = lexer.token()
if not token:
break
tokens.append(token.value)
types.append(token.type)
pos.append([token.lineno, token.lexpos])
if need_type_info:
return tokens, types, pos
else:
return tokens, pos
# if __name__=="__main__":
# print("No code/function passed in, function below is used to show you a case:")
# print()
# CodeExample = open('def.js','r',encoding='utf-8').read()
# print()
# tokens, types, pos = tokenize_js(CodeExample, True)
# print(tokens)
# print(types)
# print(pos)
def lex(text, output_type='all'):
# lexes string, stops when it starts making TypeErrors
# input: f.read()
out_list = []
lexer = Lexer()
lexer.input(text)
while True:
try:
token = lexer.token()
if not token:
break # break if end of token
if output_type == 'value':
try:
out_list.append(token.value)
except AttributeError:
break
elif output_type == 'type':
try:
out_list.append(token.type)
except AttributeError:
break
else:
out_list.append(token)
except TypeError:
break
except AttributeError:
break
return out_list
def tokenize(self, code):
try:
script = code.decode('ascii', 'ignore')
lexer = Lexer()
lexer.input(script)
return lexer
except:
return None
def variables_functions_in_script(url):
js = extract_javascript_content(url)
identifiers = []
lexer = Lexer()
lexer.input(js)
for token in lexer:
if token.type == 'ID':
identifiers.append(
unicodedata.normalize('NFKD',
token.value).encode('ascii', 'ignore'))
return identifiers
def numberOfUnescapeAndEscape(script):
lexer = Lexer()
lexer.input(script)
counter = 0
while True:
token = lexer.token()
if not token:
break
if token.type == 'ID' and token.value in functions:
counter += 1
return counter
def numberDOModificationFunctions(script):
lexer = Lexer()
lexer.input(script)
counter = 0
while True:
token = lexer.token()
if not token:
break
if token.type == 'ID' and token.value in DOModificationFunctions:
counter += 1
return counter
def numberOfStringsContainingSubstring(script,substring='iframe'):
lexer = Lexer()
lexer.input(script)
counter = 0
while True:
token = lexer.token()
if not token:
break
if token.type == 'STRING' and substring in token.value:
counter+=1
return counter
def numberEval(script):
counter=0
lexer = Lexer()
lexer.input(script)
while True:
token = lexer.token()
if not token:
break
if token.value == 'eval':
counter+=1
return counter
def strExtractParse(s):
'''Extracts all the substrings from the script using a lexer. '''
l=[]
lexer = Lexer()
lexer.input(s)
while True:
token = lexer.token()
if not token:
break
if token.type == 'STRING':
l+=[token.value]
return l
def feature101(str):
#f = open(path)
#script = f.read()
lexer = Lexer()
lexer.input(str)
counter = 0
while True:
token = lexer.token()
if not token:
break
if token.type == 'STRING':
counter = token.value.count('\\x')
return counter
def tokenize(code, need_type_info=False):
lexer = Lexer()
lexer.input(code)
tokens = []
types = []
for token in lexer:
#print(token)
tokens.append(token.value)
types.append(token.type)
if need_type_info:
return tokens, types
else:
return tokens
def __init__(self, lex_optimize=True, lextab=lextab,
yacc_optimize=True, yacctab=yacctab, yacc_debug=False):
self.lex_optimize = lex_optimize
self.lextab = lextab
self.yacc_optimize = yacc_optimize
self.yacctab = yacctab
self.yacc_debug = yacc_debug
self.lexer = Lexer()
self.lexer.build(optimize=lex_optimize, lextab=lextab)
self.tokens = self.lexer.tokens
self.parser = ply.yacc.yacc(
module=self, optimize=yacc_optimize,
debug=yacc_debug, tabmodule=yacctab, start='program')
# https://github.com/rspivak/slimit/issues/29
# lexer.auto_semi can cause a loop in a parser
# when a parser error happens on a token right after
# a newline.
# We keep record of the tokens that caused p_error
# and if the token has already been seen - we raise
# a SyntaxError exception to avoid looping over and
# over again.
self._error_tokens = {}
def averageLengthOfStrings(script):
lexer = Lexer()
lexer.input(script)
strings = set()
totalStringsLength = 0
while True:
token = lexer.token()
if not token:
break
if token.type == 'STRING':
strings.add(token.value)
for string in strings:
totalStringsLength += len(string)-2
if len(strings) > 0:
return round(totalStringsLength/len(strings),2)
else:
return 0
def tokensUsedSlimit(inputFile):
'''
Given an input JavaScript file, create a list containing the SlimIt lexical tokens present in the file.
-------
Parameter:
- inputFile: File
Should it be malformed or no JS file, then an exception will be raised (see JsDetection.isJsFile).
-------
Returns:
- List
Contains the SlimIt lexical tokens present in the input file.
- or None if the file either is no JS or malformed.
'''
if JsDetection.isJsFile(
inputFile
) == 0: # Only if the current file is a well-formed JS sample
with open(inputFile, 'r') as inF:
s = ''
try:
for line in inF:
s += str(
line
) # Store the content of the JS file in a string, because far more quicker than using SlimIt minifier.
except UnicodeDecodeError:
print('Exception handling')
lexer = Lexer()
lexer.input(s)
l = []
try:
for token in lexer:
# Structure of a token: "LexToken(VAR,'var',1,0)"
tokenPart = str(token).split('(')
tokenComplete = tokenPart[1].split(
',') # Keyword as used in JS
l += [tokenComplete[0]]
return l # Lexical tokens
except TypeError:
print('Exception handling')
def __init__(self,
lex_optimize=True,
lextab=lextab,
yacc_optimize=True,
yacctab=yacctab,
yacc_debug=False):
self.lex_optimize = lex_optimize
self.lextab = lextab
self.yacc_optimize = yacc_optimize
self.yacctab = yacctab
self.yacc_debug = yacc_debug
self.lexer = Lexer()
self.lexer.build(optimize=lex_optimize, lextab=lextab)
self.tokens = self.lexer.tokens
self.parser = ply.yacc.yacc(module=self,
optimize=yacc_optimize,
debug=yacc_debug,
tabmodule=yacctab,
start='program')
def extract(self, node):
lexer = JsLexer()
lexer.input(node.text)
regions = []
while True:
try:
tok = lexer.token()
if not tok:
break
if tok.type == "STRING":
start_char = tok.lexpos + 1
string = tok.value[1:-1]
end_char = start_char + len(string) - 1
r = Region(node, start_char, end_char, string)
regions.append(r)
except (TypeError, AttributeError):
logging.warn("Failed to parse text: %s...", node.text[:100])
break
return regions
def __init__(self, lex_optimize=True, lextab=lextab,
yacc_optimize=True, yacctab=yacctab, yacc_debug=False):
self.lex_optimize = lex_optimize
self.lextab = lextab
self.yacc_optimize = yacc_optimize
self.yacctab = yacctab
self.yacc_debug = yacc_debug
self.lexer = Lexer()
self.lexer.build(optimize=lex_optimize, lextab=lextab)
self.tokens = self.lexer.tokens
self.parser = ply.yacc.yacc(
module=self, optimize=yacc_optimize,
debug=yacc_debug, tabmodule=yacctab, start='program')
__author__ = 'khushboomandlecha'
# Trying out the lexer function
from slimit.lexer import Lexer
lexer = Lexer()
lexer.input('o = {};o.p1 = 3;y = prompt(\'Which property do you want?\');if (o[y]) {l = 0;}')
file = open('outputfile.txt','w')
for token in lexer:
file.write(str(token))
file.write('\n')
print token
file.close()
class Parser(object):
"""JavaScript parser(ECMA-262 5th edition grammar).
The '*noin' variants are needed to avoid confusing the `in` operator in
a relational expression with the `in` operator in a `for` statement.
'*nobf' stands for 'no brace or function'
"""
def __init__(self, lex_optimize=True, lextab=lextab,
yacc_optimize=True, yacctab=yacctab, yacc_debug=False):
self.lex_optimize = lex_optimize
self.lextab = lextab
self.yacc_optimize = yacc_optimize
self.yacctab = yacctab
self.yacc_debug = yacc_debug
self.lexer = Lexer()
self.lexer.build(optimize=lex_optimize, lextab=lextab)
self.tokens = self.lexer.tokens
self.parser = ply.yacc.yacc(
module=self, optimize=yacc_optimize,
debug=yacc_debug, tabmodule=yacctab, start='program')
def parse(self, text, debug=False):
return self.parser.parse(text, lexer=self.lexer, debug=debug)
def p_empty(self, p):
"""empty :"""
pass
def p_auto_semi(self, p):
"""auto_semi : error"""
pass
def p_error(self, token):
if token is None or token.type != 'SEMI':
next_token = self.lexer.auto_semi(token)
if next_token is not None:
self.parser.errok()
return next_token
elif token is not None and token.type == 'LINE_TERMINATOR':
# handle last LINE_TERMINATOR at the end of file
return self.lexer.token()
raise SyntaxError(
'Unexpected token (%s, %r) at %s:%s between %s and %s' % (
token.type, token.value, token.lineno, token.lexpos,
self.lexer.prev_token, self.lexer.token())
)
# Comment rules
# def p_single_line_comment(self, p):
# """single_line_comment : LINE_COMMENT"""
# pass
# def p_multi_line_comment(self, p):
# """multi_line_comment : BLOCK_COMMENT"""
# pass
# Main rules
def p_program(self, p):
"""program : source_elements"""
p[0] = ast.Program(p[1])
def p_source_elements(self, p):
"""source_elements : empty
| source_element_list
"""
p[0] = p[1]
def p_source_element_list(self, p):
"""source_element_list : source_element
| source_element_list source_element
"""
if len(p) == 2: # single source element
p[0] = [p[1]]
else:
p[1].append(p[2])
p[0] = p[1]
def p_source_element(self, p):
"""source_element : statement
| function_declaration
"""
p[0] = p[1]
def p_statement(self, p):
"""statement : block
| variable_statement
| empty_statement
| expr_statement
| if_statement
| iteration_statement
| continue_statement
| break_statement
| return_statement
| with_statement
| switch_statement
| labelled_statement
| throw_statement
| try_statement
| debugger_statement
"""
p[0] = p[1]
# By having source_elements in the production we support
# also function_declaration inside blocks
def p_block(self, p):
"""block : LBRACE source_elements RBRACE"""
p[0] = ast.Block(p[2])
def p_literal(self, p):
"""literal : null_literal
| boolean_literal
| numeric_literal
| string_literal
| regex_literal
"""
p[0] = p[1]
def p_boolean_literal(self, p):
"""boolean_literal : TRUE
| FALSE
"""
p[0] = ast.Boolean(p[1])
def p_null_literal(self, p):
"""null_literal : NULL"""
p[0] = ast.Null(p[1])
def p_numeric_literal(self, p):
"""numeric_literal : NUMBER"""
p[0] = ast.Number(p[1])
def p_string_literal(self, p):
"""string_literal : STRING"""
p[0] = ast.String(p[1])
def p_regex_literal(self, p):
"""regex_literal : REGEX"""
p[0] = ast.Regex(p[1])
def p_identifier(self, p):
"""identifier : ID"""
p[0] = ast.Identifier(p[1])
###########################################
# Expressions
###########################################
def p_primary_expr(self, p):
"""primary_expr : primary_expr_no_brace
| object_literal
"""
p[0] = p[1]
def p_primary_expr_no_brace(self, p):
"""primary_expr_no_brace : THIS
| identifier
| literal
| array_literal
| LPAREN expr RPAREN
"""
if p[1] == 'this':
p[0] = ast.This()
elif len(p) == 2:
p[0] = p[1]
else:
p[2]._parens = True
p[0] = p[2]
def p_array_literal_1(self, p):
"""array_literal : LBRACKET elision_opt RBRACKET"""
p[0] = ast.Array(items=p[2])
def p_array_literal_2(self, p):
"""array_literal : LBRACKET element_list RBRACKET
| LBRACKET element_list COMMA elision_opt RBRACKET
"""
items = p[2]
if len(p) == 6:
items.extend(p[4])
p[0] = ast.Array(items=items)
def p_element_list(self, p):
"""element_list : elision_opt assignment_expr
| element_list COMMA elision_opt assignment_expr
"""
if len(p) == 3:
p[0] = p[1] + [p[2]]
else:
p[1].extend(p[3])
p[1].append(p[4])
p[0] = p[1]
def p_elision_opt_1(self, p):
"""elision_opt : empty"""
p[0] = []
def p_elision_opt_2(self, p):
"""elision_opt : elision"""
p[0] = p[1]
def p_elision(self, p):
"""elision : COMMA
| elision COMMA
"""
if len(p) == 2:
p[0] = [ast.Elision(p[1])]
else:
p[1].append(ast.Elision(p[2]))
p[0] = p[1]
def p_object_literal(self, p):
"""object_literal : LBRACE RBRACE
| LBRACE property_list RBRACE
| LBRACE property_list COMMA RBRACE
"""
if len(p) == 3:
p[0] = ast.Object()
else:
p[0] = ast.Object(properties=p[2])
def p_property_list(self, p):
"""property_list : property_assignment
| property_list COMMA property_assignment
"""
if len(p) == 2:
p[0] = [p[1]]
else:
p[1].append(p[3])
p[0] = p[1]
# XXX: GET / SET
def p_property_assignment(self, p):
"""property_assignment : property_name COLON assignment_expr"""
if len(p) == 4:
p[0] = ast.Assign(left=p[1], op=p[2], right=p[3])
def p_property_name(self, p):
"""property_name : identifier
| string_literal
| numeric_literal
"""
p[0] = p[1]
# 11.2 Left-Hand-Side Expressions
def p_member_expr(self, p):
"""member_expr : primary_expr
| function_expr
| member_expr LBRACKET expr RBRACKET
| member_expr PERIOD identifier
| NEW member_expr arguments
"""
if len(p) == 2:
p[0] = p[1]
elif p[1] == 'new':
p[0] = ast.NewExpr(p[2], p[3])
elif p[2] == '.':
p[0] = ast.DotAccessor(p[1], p[3])
else:
p[0] = ast.BracketAccessor(p[1], p[3])
def p_member_expr_nobf(self, p):
"""member_expr_nobf : primary_expr_no_brace
| member_expr_nobf LBRACKET expr RBRACKET
| member_expr_nobf PERIOD identifier
| NEW member_expr arguments
"""
if len(p) == 2:
p[0] = p[1]
elif p[1] == 'new':
p[0] = ast.NewExpr(p[2], p[3])
elif p[2] == '.':
p[0] = ast.DotAccessor(p[1], p[3])
else:
p[0] = ast.BracketAccessor(p[1], p[3])
def p_new_expr(self, p):
"""new_expr : member_expr
| NEW new_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.NewExpr(p[2])
def p_new_expr_nobf(self, p):
"""new_expr_nobf : member_expr_nobf
| NEW new_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.NewExpr(p[2])
def p_call_expr(self, p):
"""call_expr : member_expr arguments
| call_expr arguments
| call_expr LBRACKET expr RBRACKET
| call_expr PERIOD identifier
"""
if len(p) == 3:
p[0] = ast.FunctionCall(p[1], p[2])
elif len(p) == 4:
p[0] = ast.DotAccessor(p[1], p[3])
else:
p[0] = ast.BracketAccessor(p[1], p[3])
def p_call_expr_nobf(self, p):
"""call_expr_nobf : member_expr_nobf arguments
| call_expr_nobf arguments
| call_expr_nobf LBRACKET expr RBRACKET
| call_expr_nobf PERIOD identifier
"""
if len(p) == 3:
p[0] = ast.FunctionCall(p[1], p[2])
elif len(p) == 4:
p[0] = ast.DotAccessor(p[1], p[3])
else:
p[0] = ast.BracketAccessor(p[1], p[3])
def p_arguments(self, p):
"""arguments : LPAREN RPAREN
| LPAREN argument_list RPAREN
"""
if len(p) == 4:
p[0] = p[2]
def p_argument_list(self, p):
"""argument_list : assignment_expr
| argument_list COMMA assignment_expr
"""
if len(p) == 2:
p[0] = [p[1]]
else:
p[1].append(p[3])
p[0] = p[1]
def p_lef_hand_side_expr(self, p):
"""left_hand_side_expr : new_expr
| call_expr
"""
p[0] = p[1]
def p_lef_hand_side_expr_nobf(self, p):
"""left_hand_side_expr_nobf : new_expr_nobf
| call_expr_nobf
"""
p[0] = p[1]
# 11.3 Postfix Expressions
def p_postfix_expr(self, p):
"""postfix_expr : left_hand_side_expr
| left_hand_side_expr PLUSPLUS
| left_hand_side_expr MINUSMINUS
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.UnaryOp(op=p[2], value=p[1], postfix=True)
def p_postfix_expr_nobf(self, p):
"""postfix_expr_nobf : left_hand_side_expr_nobf
| left_hand_side_expr_nobf PLUSPLUS
| left_hand_side_expr_nobf MINUSMINUS
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.UnaryOp(op=p[2], value=p[1], postfix=True)
# 11.4 Unary Operators
def p_unary_expr(self, p):
"""unary_expr : postfix_expr
| unary_expr_common
"""
p[0] = p[1]
def p_unary_expr_nobf(self, p):
"""unary_expr_nobf : postfix_expr_nobf
| unary_expr_common
"""
p[0] = p[1]
def p_unary_expr_common(self, p):
"""unary_expr_common : DELETE unary_expr
| VOID unary_expr
| TYPEOF unary_expr
| PLUSPLUS unary_expr
| MINUSMINUS unary_expr
| PLUS unary_expr
| MINUS unary_expr
| BNOT unary_expr
| NOT unary_expr
"""
p[0] = ast.UnaryOp(p[1], p[2])
# 11.5 Multiplicative Operators
def p_multiplicative_expr(self, p):
"""multiplicative_expr : unary_expr
| multiplicative_expr MULT unary_expr
| multiplicative_expr DIV unary_expr
| multiplicative_expr MOD unary_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
def p_multiplicative_expr_nobf(self, p):
"""multiplicative_expr_nobf : unary_expr_nobf
| multiplicative_expr_nobf MULT unary_expr
| multiplicative_expr_nobf DIV unary_expr
| multiplicative_expr_nobf MOD unary_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
# 11.6 Additive Operators
def p_additive_expr(self, p):
"""additive_expr : multiplicative_expr
| additive_expr PLUS multiplicative_expr
| additive_expr MINUS multiplicative_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
def p_additive_expr_nobf(self, p):
"""additive_expr_nobf : multiplicative_expr_nobf
| additive_expr_nobf PLUS multiplicative_expr
| additive_expr_nobf MINUS multiplicative_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
# 11.7 Bitwise Shift Operators
def p_shift_expr(self, p):
"""shift_expr : additive_expr
| shift_expr LSHIFT additive_expr
| shift_expr RSHIFT additive_expr
| shift_expr URSHIFT additive_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
def p_shift_expr_nobf(self, p):
"""shift_expr_nobf : additive_expr_nobf
| shift_expr_nobf LSHIFT additive_expr
| shift_expr_nobf RSHIFT additive_expr
| shift_expr_nobf URSHIFT additive_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
# 11.8 Relational Operators
def p_relational_expr(self, p):
"""relational_expr : shift_expr
| relational_expr LT shift_expr
| relational_expr GT shift_expr
| relational_expr LE shift_expr
| relational_expr GE shift_expr
| relational_expr INSTANCEOF shift_expr
| relational_expr IN shift_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
def p_relational_expr_noin(self, p):
"""relational_expr_noin : shift_expr
| relational_expr_noin LT shift_expr
| relational_expr_noin GT shift_expr
| relational_expr_noin LE shift_expr
| relational_expr_noin GE shift_expr
| relational_expr_noin INSTANCEOF shift_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
def p_relational_expr_nobf(self, p):
"""relational_expr_nobf : shift_expr_nobf
| relational_expr_nobf LT shift_expr
| relational_expr_nobf GT shift_expr
| relational_expr_nobf LE shift_expr
| relational_expr_nobf GE shift_expr
| relational_expr_nobf INSTANCEOF shift_expr
| relational_expr_nobf IN shift_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
# 11.9 Equality Operators
def p_equality_expr(self, p):
"""equality_expr : relational_expr
| equality_expr EQEQ relational_expr
| equality_expr NE relational_expr
| equality_expr STREQ relational_expr
| equality_expr STRNEQ relational_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
def p_equality_expr_noin(self, p):
"""equality_expr_noin : relational_expr_noin
| equality_expr_noin EQEQ relational_expr
| equality_expr_noin NE relational_expr
| equality_expr_noin STREQ relational_expr
| equality_expr_noin STRNEQ relational_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
def p_equality_expr_nobf(self, p):
"""equality_expr_nobf : relational_expr_nobf
| equality_expr_nobf EQEQ relational_expr
| equality_expr_nobf NE relational_expr
| equality_expr_nobf STREQ relational_expr
| equality_expr_nobf STRNEQ relational_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
# 11.10 Binary Bitwise Operators
def p_bitwise_and_expr(self, p):
"""bitwise_and_expr : equality_expr
| bitwise_and_expr BAND equality_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
def p_bitwise_and_expr_noin(self, p):
"""bitwise_and_expr_noin \
: equality_expr_noin
| bitwise_and_expr_noin BAND equality_expr_noin
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
def p_bitwise_and_expr_nobf(self, p):
"""bitwise_and_expr_nobf \
: equality_expr_nobf
| bitwise_and_expr_nobf BAND equality_expr_nobf
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
def p_bitwise_xor_expr(self, p):
"""bitwise_xor_expr : bitwise_and_expr
| bitwise_xor_expr BXOR bitwise_and_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
def p_bitwise_xor_expr_noin(self, p):
"""
bitwise_xor_expr_noin \
: bitwise_and_expr_noin
| bitwise_xor_expr_noin BXOR bitwise_and_expr_noin
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
def p_bitwise_xor_expr_nobf(self, p):
"""
bitwise_xor_expr_nobf \
: bitwise_and_expr_nobf
| bitwise_xor_expr_nobf BXOR bitwise_and_expr_nobf
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
def p_bitwise_or_expr(self, p):
"""bitwise_or_expr : bitwise_xor_expr
| bitwise_or_expr BOR bitwise_xor_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
def p_bitwise_or_expr_noin(self, p):
"""
bitwise_or_expr_noin \
: bitwise_xor_expr_noin
| bitwise_or_expr_noin BOR bitwise_xor_expr_noin
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
def p_bitwise_or_expr_nobf(self, p):
"""
bitwise_or_expr_nobf \
: bitwise_xor_expr_nobf
| bitwise_or_expr_nobf BOR bitwise_xor_expr_nobf
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
# 11.11 Binary Logical Operators
def p_logical_and_expr(self, p):
"""logical_and_expr : bitwise_or_expr
| logical_and_expr AND bitwise_or_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
def p_logical_and_expr_noin(self, p):
"""
logical_and_expr_noin : bitwise_or_expr_noin
| logical_and_expr_noin AND bitwise_or_expr_noin
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
def p_logical_and_expr_nobf(self, p):
"""
logical_and_expr_nobf : bitwise_or_expr_nobf
| logical_and_expr_nobf AND bitwise_or_expr_nobf
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
def p_logical_or_expr(self, p):
"""logical_or_expr : logical_and_expr
| logical_or_expr OR logical_and_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
def p_logical_or_expr_noin(self, p):
"""logical_or_expr_noin : logical_and_expr_noin
| logical_or_expr_noin OR logical_and_expr_noin
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
def p_logical_or_expr_nobf(self, p):
"""logical_or_expr_nobf : logical_and_expr_nobf
| logical_or_expr_nobf OR logical_and_expr_nobf
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3])
# 11.12 Conditional Operator ( ? : )
def p_conditional_expr(self, p):
"""
conditional_expr \
: logical_or_expr
| logical_or_expr CONDOP assignment_expr COLON assignment_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.Conditional(
predicate=p[1], consequent=p[3], alternative=p[5])
def p_conditional_expr_noin(self, p):
"""
conditional_expr_noin \
: logical_or_expr_noin
| logical_or_expr_noin CONDOP assignment_expr_noin COLON \
assignment_expr_noin
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.Conditional(
predicate=p[1], consequent=p[3], alternative=p[5])
def p_conditional_expr_nobf(self, p):
"""
conditional_expr_nobf \
: logical_or_expr_nobf
| logical_or_expr_nobf CONDOP assignment_expr COLON assignment_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.Conditional(
predicate=p[1], consequent=p[3], alternative=p[5])
# 11.13 Assignment Operators
def p_assignment_expr(self, p):
"""
assignment_expr \
: conditional_expr
| left_hand_side_expr assignment_operator assignment_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.Assign(left=p[1], op=p[2], right=p[3])
def p_assignment_expr_noin(self, p):
"""
assignment_expr_noin \
: conditional_expr_noin
| left_hand_side_expr assignment_operator assignment_expr_noin
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.Assign(left=p[1], op=p[2], right=p[3])
def p_assignment_expr_nobf(self, p):
"""
assignment_expr_nobf \
: conditional_expr_nobf
| left_hand_side_expr_nobf assignment_operator assignment_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.Assign(left=p[1], op=p[2], right=p[3])
def p_assignment_operator(self, p):
"""assignment_operator : EQ
| MULTEQUAL
| DIVEQUAL
| MODEQUAL
| PLUSEQUAL
| MINUSEQUAL
| LSHIFTEQUAL
| RSHIFTEQUAL
| URSHIFTEQUAL
| ANDEQUAL
| XOREQUAL
| OREQUAL
"""
p[0] = p[1]
# 11.4 Comma Operator
def p_expr(self, p):
"""expr : assignment_expr
| expr COMMA assignment_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.Comma(left=p[1], right=p[3])
def p_expr_noin(self, p):
"""expr_noin : assignment_expr_noin
| expr_noin COMMA assignment_expr_noin
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.Comma(left=p[1], right=p[3])
def p_expr_nobf(self, p):
"""expr_nobf : assignment_expr_nobf
| expr_nobf COMMA assignment_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.Comma(left=p[1], right=p[3])
# 12.2 Variable Statement
def p_variable_statement(self, p):
"""variable_statement : VAR variable_declaration_list SEMI
| VAR variable_declaration_list auto_semi
"""
p[0] = ast.VarStatement(p[2])
def p_variable_declaration_list(self, p):
"""
variable_declaration_list \
: variable_declaration
| variable_declaration_list COMMA variable_declaration
"""
if len(p) == 2:
p[0] = [p[1]]
else:
p[1].append(p[3])
p[0] = p[1]
def p_variable_declaration_list_noin(self, p):
"""
variable_declaration_list_noin \
: variable_declaration_noin
| variable_declaration_list_noin COMMA variable_declaration_noin
"""
if len(p) == 2:
p[0] = [p[1]]
else:
p[1].append(p[3])
p[0] = p[1]
def p_variable_declaration(self, p):
"""variable_declaration : identifier
| identifier initializer
"""
if len(p) == 2:
p[0] = ast.VarDecl(p[1])
else:
p[0] = ast.VarDecl(p[1], p[2])
def p_variable_declaration_noin(self, p):
"""variable_declaration_noin : identifier
| identifier initializer_noin
"""
if len(p) == 2:
p[0] = ast.VarDecl(p[1])
else:
p[0] = ast.VarDecl(p[1], p[2])
def p_initializer(self, p):
"""initializer : EQ assignment_expr"""
p[0] = p[2]
def p_initializer_noin(self, p):
"""initializer_noin : EQ assignment_expr_noin"""
p[0] = p[2]
# 12.3 Empty Statement
def p_empty_statement(self, p):
"""empty_statement : SEMI"""
p[0] = ast.EmptyStatement(p[1])
# 12.4 Expression Statement
def p_expr_statement(self, p):
"""expr_statement : expr_nobf SEMI
| expr_nobf auto_semi
"""
p[0] = ast.ExprStatement(p[1])
# 12.5 The if Statement
def p_if_statement_1(self, p):
"""if_statement : IF LPAREN expr RPAREN statement"""
p[0] = ast.If(predicate=p[3], consequent=p[5])
def p_if_statement_2(self, p):
"""if_statement : IF LPAREN expr RPAREN statement ELSE statement"""
p[0] = ast.If(predicate=p[3], consequent=p[5], alternative=p[7])
# 12.6 Iteration Statements
def p_iteration_statement_1(self, p):
"""
iteration_statement \
: DO statement WHILE LPAREN expr RPAREN SEMI
| DO statement WHILE LPAREN expr RPAREN auto_semi
"""
p[0] = ast.DoWhile(predicate=p[5], statement=p[2])
def p_iteration_statement_2(self, p):
"""iteration_statement : WHILE LPAREN expr RPAREN statement"""
p[0] = ast.While(predicate=p[3], statement=p[5])
def p_iteration_statement_3(self, p):
"""
iteration_statement \
: FOR LPAREN expr_noin_opt SEMI expr_opt SEMI expr_opt RPAREN \
statement
| FOR LPAREN VAR variable_declaration_list_noin SEMI expr_opt SEMI\
expr_opt RPAREN statement
"""
if len(p) == 10:
p[0] = ast.For(init=p[3], cond=p[5], count=p[7], statement=p[9])
else:
init = ast.VarStatement(p[4])
p[0] = ast.For(init=init, cond=p[6], count=p[8], statement=p[10])
def p_iteration_statement_4(self, p):
"""
iteration_statement \
: FOR LPAREN left_hand_side_expr IN expr RPAREN statement
"""
p[0] = ast.ForIn(item=p[3], iterable=p[5], statement=p[7])
def p_iteration_statement_5(self, p):
"""
iteration_statement : \
FOR LPAREN VAR identifier IN expr RPAREN statement
"""
p[0] = ast.ForIn(item=ast.VarDecl(p[4]), iterable=p[6], statement=p[8])
def p_iteration_statement_6(self, p):
"""
iteration_statement \
: FOR LPAREN VAR identifier initializer_noin IN expr RPAREN statement
"""
p[0] = ast.ForIn(item=ast.VarDecl(identifier=p[4], initializer=p[5]),
iterable=p[7], statement=p[9])
def p_expr_opt(self, p):
"""expr_opt : empty
| expr
"""
p[0] = p[1]
def p_expr_noin_opt(self, p):
"""expr_noin_opt : empty
| expr_noin
"""
p[0] = p[1]
# 12.7 The continue Statement
def p_continue_statement_1(self, p):
"""continue_statement : CONTINUE SEMI
| CONTINUE auto_semi
"""
p[0] = ast.Continue()
def p_continue_statement_2(self, p):
"""continue_statement : CONTINUE identifier SEMI
| CONTINUE identifier auto_semi
"""
p[0] = ast.Continue(p[2])
# 12.8 The break Statement
def p_break_statement_1(self, p):
"""break_statement : BREAK SEMI
| BREAK auto_semi
"""
p[0] = ast.Break()
def p_break_statement_2(self, p):
"""break_statement : BREAK identifier SEMI
| BREAK identifier auto_semi
"""
p[0] = ast.Break(p[2])
# 12.9 The return Statement
def p_return_statement_1(self, p):
"""return_statement : RETURN SEMI
| RETURN auto_semi
"""
p[0] = ast.Return()
def p_return_statement_2(self, p):
"""return_statement : RETURN expr SEMI
| RETURN expr auto_semi
"""
p[0] = ast.Return(expr=p[2])
# 12.10 The with Statement
def p_with_statement(self, p):
"""with_statement : WITH LPAREN expr RPAREN statement"""
p[0] = ast.With(expr=p[3], statement=p[5])
# 12.11 The switch Statement
def p_switch_statement(self, p):
"""switch_statement : SWITCH LPAREN expr RPAREN case_block"""
cases = []
default = None
# iterate over return values from case_block
for item in p[5]:
if isinstance(item, ast.Default):
default = item
elif isinstance(item, list):
cases.extend(item)
p[0] = ast.Switch(expr=p[3], cases=cases, default=default)
def p_case_block(self, p):
"""
case_block \
: LBRACE case_clauses_opt RBRACE
| LBRACE case_clauses_opt default_clause case_clauses_opt RBRACE
"""
p[0] = p[2:-1]
def p_case_clauses_opt(self, p):
"""case_clauses_opt : empty
| case_clauses
"""
p[0] = p[1]
def p_case_clauses(self, p):
"""case_clauses : case_clause
| case_clauses case_clause
"""
if len(p) == 2:
p[0] = [p[1]]
else:
p[1].append(p[2])
p[0] = p[1]
def p_case_clause(self, p):
"""case_clause : CASE expr COLON source_elements"""
p[0] = ast.Case(expr=p[2], elements=p[4])
def p_default_clause(self, p):
"""default_clause : DEFAULT COLON source_elements"""
p[0] = ast.Default(elements=p[3])
# 12.12 Labelled Statements
def p_labelled_statement(self, p):
"""labelled_statement : identifier COLON statement"""
p[0] = ast.Label(identifier=p[1], statement=p[3])
# 12.13 The throw Statement
def p_throw_statement(self, p):
"""throw_statement : THROW expr SEMI
| THROW expr auto_semi
"""
p[0] = ast.Throw(expr=p[2])
# 12.14 The try Statement
def p_try_statement_1(self, p):
"""try_statement : TRY block catch"""
p[0] = ast.Try(statements=p[2], catch=p[3])
def p_try_statement_2(self, p):
"""try_statement : TRY block finally"""
p[0] = ast.Try(statements=p[2], fin=p[3])
def p_try_statement_3(self, p):
"""try_statement : TRY block catch finally"""
p[0] = ast.Try(statements=p[2], catch=p[3], fin=p[4])
def p_catch(self, p):
"""catch : CATCH LPAREN identifier RPAREN block"""
p[0] = ast.Catch(identifier=p[3], elements=p[5])
def p_finally(self, p):
"""finally : FINALLY block"""
p[0] = ast.Finally(elements=p[2])
# 12.15 The debugger statement
def p_debugger_statement(self, p):
"""debugger_statement : DEBUGGER SEMI
| DEBUGGER auto_semi
"""
p[0] = ast.Debugger(p[1])
# 13 Function Definition
def p_function_declaration(self, p):
"""
function_declaration \
: FUNCTION identifier LPAREN RPAREN LBRACE function_body RBRACE
| FUNCTION identifier LPAREN formal_parameter_list RPAREN LBRACE \
function_body RBRACE
"""
if len(p) == 8:
p[0] = ast.FuncDecl(
identifier=p[2], parameters=None, elements=p[6])
else:
p[0] = ast.FuncDecl(
identifier=p[2], parameters=p[4], elements=p[7])
def p_function_expr_1(self, p):
"""
function_expr \
: FUNCTION LPAREN RPAREN LBRACE function_body RBRACE
| FUNCTION LPAREN formal_parameter_list RPAREN \
LBRACE function_body RBRACE
"""
if len(p) == 7:
p[0] = ast.FuncExpr(
identifier=None, parameters=None, elements=p[5])
else:
p[0] = ast.FuncExpr(
identifier=None, parameters=p[3], elements=p[6])
def p_function_expr_2(self, p):
"""
function_expr \
: FUNCTION identifier LPAREN RPAREN LBRACE function_body RBRACE
| FUNCTION identifier LPAREN formal_parameter_list RPAREN \
LBRACE function_body RBRACE
"""
if len(p) == 8:
p[0] = ast.FuncExpr(
identifier=p[2], parameters=None, elements=p[6])
else:
p[0] = ast.FuncExpr(
identifier=p[2], parameters=p[4], elements=p[7])
def p_formal_parameter_list(self, p):
"""formal_parameter_list : identifier
| formal_parameter_list COMMA identifier
"""
if len(p) == 2:
p[0] = [p[1]]
else:
p[1].append(p[3])
p[0] = p[1]
def p_function_body(self, p):
"""function_body : source_elements"""
p[0] = p[1]
from __future__ import division
from slimit.lexer import Lexer
from slimit.parser import Parser
from slimit.visitors.nodevisitor import ASTVisitor
text = open('test.js').read();
linecount = len(text.splitlines())
tokenList = []
tokens = {}
lexer = Lexer()
lexer.input(text)
for token in lexer:
tokens[token.lexpos] = dict(
t = token.type[:],
v = token.value[:],
prior = [t.value for t in tokenList[-5:]] + [''] * (5 - len(tokenList[-5:]))
)
tokenList.append(token)
parser = Parser(yacc_tracking=True)
tree = parser.parse(text)
class TrainingSample(object):
def __init__(self, structure, lineno, prior, match, completion):
self._structure = structure
self._slno = lineno
class Parser(object):
"""JavaScript parser(ECMA-262 5th edition grammar).
The '*noin' variants are needed to avoid confusing the `in` operator in
a relational expression with the `in` operator in a `for` statement.
'*nobf' stands for 'no brace or function'
"""
def __init__(self, lex_optimize=True, lextab=lextab,
yacc_optimize=True, yacctab=yacctab, yacc_debug=False):
self.lex_optimize = lex_optimize
self.lextab = lextab
self.yacc_optimize = yacc_optimize
self.yacctab = yacctab
self.yacc_debug = yacc_debug
self.lexer = Lexer()
self.lexer.build(optimize=lex_optimize, lextab=lextab)
self.tokens = self.lexer.tokens
self.parser = ply.yacc.yacc(
module=self, optimize=yacc_optimize,
debug=yacc_debug, tabmodule=yacctab, start='program')
# https://github.com/rspivak/slimit/issues/29
# lexer.auto_semi can cause a loop in a parser
# when a parser error happens on a token right after
# a newline.
# We keep record of the tokens that caused p_error
# and if the token has already been seen - we raise
# a SyntaxError exception to avoid looping over and
# over again.
self._error_tokens = {}
def _has_been_seen_before(self, token):
if token is None:
return False
key = token.type, token.value, token.lineno, token.lexpos
return key in self._error_tokens
def _mark_as_seen(self, token):
if token is None:
return
key = token.type, token.value, token.lineno, token.lexpos
self._error_tokens[key] = True
def _raise_syntax_error(self, token):
raise SyntaxError(
'Unexpected token (%s, %r) at %s:%s between %s and %s' % (
token.type, token.value, token.lineno, token.lexpos,
self.lexer.prev_token, self.lexer.token())
)
def parse(self, text, debug=False):
return self.parser.parse(
text,
lexer = self.lexer,
tracking = True,
debug = debug,
)
def p_empty(self, p):
"""empty :"""
pass
def p_auto_semi(self, p):
"""auto_semi : error"""
pass
def p_error(self, token):
# https://github.com/rspivak/slimit/issues/29
if self._has_been_seen_before(token):
self._raise_syntax_error(token)
if token is None or token.type != 'SEMI':
next_token = self.lexer.auto_semi(token)
if next_token is not None:
# https://github.com/rspivak/slimit/issues/29
self._mark_as_seen(token)
self.parser.errok()
return next_token
self._raise_syntax_error(token)
# Comment rules
# def p_single_line_comment(self, p):
# """single_line_comment : LINE_COMMENT"""
# pass
# def p_multi_line_comment(self, p):
# """multi_line_comment : BLOCK_COMMENT"""
# pass
# Main rules
def p_program(self, p):
"""program : source_elements"""
p[0] = ast.Program(p[1], lineno=p.lineno(1))
def p_source_elements(self, p):
"""source_elements : empty
| source_element_list
"""
p[0] = p[1]
def p_source_element_list(self, p):
"""source_element_list : source_element
| source_element_list source_element
"""
if len(p) == 2: # single source element
p[0] = [p[1]]
else:
p[1].append(p[2])
p[0] = p[1]
def p_source_element(self, p):
"""source_element : statement
| function_declaration
"""
p[0] = p[1]
def p_statement(self, p):
"""statement : block
| variable_statement
| empty_statement
| expr_statement
| if_statement
| iteration_statement
| continue_statement
| break_statement
| return_statement
| with_statement
| switch_statement
| labelled_statement
| throw_statement
| try_statement
| debugger_statement
| function_declaration
"""
p[0] = p[1]
# By having source_elements in the production we support
# also function_declaration inside blocks
def p_block(self, p):
"""block : LBRACE source_elements RBRACE"""
p[0] = ast.Block(p[2], lineno=p.lineno(2))
def p_literal(self, p):
"""literal : null_literal
| boolean_literal
| numeric_literal
| string_literal
| regex_literal
"""
p[0] = p[1]
def p_boolean_literal(self, p):
"""boolean_literal : TRUE
| FALSE
"""
p[0] = ast.Boolean(p[1], lineno=p.lineno(1))
def p_null_literal(self, p):
"""null_literal : NULL"""
p[0] = ast.Null(p[1], lineno=p.lineno(1))
def p_numeric_literal(self, p):
"""numeric_literal : NUMBER"""
p[0] = ast.Number(p[1], lineno=p.lineno(1))
def p_string_literal(self, p):
"""string_literal : STRING"""
p[0] = ast.String(p[1], lineno=p.lineno(1))
def p_regex_literal(self, p):
"""regex_literal : REGEX"""
p[0] = ast.Regex(p[1], lineno=p.lineno(1))
def p_identifier(self, p):
"""identifier : ID"""
p[0] = ast.Identifier(p[1], lineno=p.lineno(1))
###########################################
# Expressions
###########################################
def p_primary_expr(self, p):
"""primary_expr : primary_expr_no_brace
| object_literal
"""
p[0] = p[1]
def p_primary_expr_no_brace_1(self, p):
"""primary_expr_no_brace : identifier"""
p[1]._mangle_candidate = True
p[1]._in_expression = True
p[0] = p[1]
def p_primary_expr_no_brace_2(self, p):
"""primary_expr_no_brace : THIS"""
p[0] = ast.This(lineno=p.lineno(0))
def p_primary_expr_no_brace_3(self, p):
"""primary_expr_no_brace : literal
| array_literal
"""
p[0] = p[1]
def p_primary_expr_no_brace_4(self, p):
"""primary_expr_no_brace : LPAREN expr RPAREN"""
p[2]._parens = True
p[0] = p[2]
def p_array_literal_1(self, p):
"""array_literal : LBRACKET elision_opt RBRACKET"""
p[0] = ast.Array(items=p[2], lineno=p.lineno(2))
def p_array_literal_2(self, p):
"""array_literal : LBRACKET element_list RBRACKET
| LBRACKET element_list COMMA elision_opt RBRACKET
"""
items = p[2]
if len(p) == 6:
items.extend(p[4])
p[0] = ast.Array(items=items, lineno=p.lineno(0))
def p_element_list(self, p):
"""element_list : elision_opt assignment_expr
| element_list COMMA elision_opt assignment_expr
"""
if len(p) == 3:
p[0] = p[1] + [p[2]]
else:
p[1].extend(p[3])
p[1].append(p[4])
p[0] = p[1]
def p_elision_opt_1(self, p):
"""elision_opt : empty"""
p[0] = []
def p_elision_opt_2(self, p):
"""elision_opt : elision"""
p[0] = p[1]
def p_elision(self, p):
"""elision : COMMA
| elision COMMA
"""
if len(p) == 2:
p[0] = [ast.Elision(p[1], lineno=p.lineno(1))]
else:
p[1].append(ast.Elision(p[2], lineno=p.lineno(2)))
p[0] = p[1]
def p_object_literal(self, p):
"""object_literal : LBRACE RBRACE
| LBRACE property_list RBRACE
| LBRACE property_list COMMA RBRACE
"""
if len(p) == 3:
p[0] = ast.Object(lineno = p.lineno(0))
else:
p[0] = ast.Object(properties=p[2], lineno=p.lineno(2))
def p_property_list(self, p):
"""property_list : property_assignment
| property_list COMMA property_assignment
"""
if len(p) == 2:
p[0] = [p[1]]
else:
p[1].append(p[3])
p[0] = p[1]
# XXX: GET / SET
def p_property_assignment(self, p):
"""property_assignment \
: property_name COLON assignment_expr
| GETPROP property_name LPAREN RPAREN LBRACE function_body RBRACE
| SETPROP property_name LPAREN formal_parameter_list RPAREN \
LBRACE function_body RBRACE
"""
if len(p) == 4:
p[0] = ast.Assign(left=p[1], op=p[2], right=p[3],
lineno=p.lineno(1))
elif len(p) == 8:
p[0] = ast.GetPropAssign(prop_name=p[2], elements=p[6],
lineno=p.lineno(2))
else:
p[0] = ast.SetPropAssign(
prop_name=p[2], parameters=p[4], elements=p[7],
lineno=p.lineno(2))
def p_property_name(self, p):
"""property_name : identifier
| string_literal
| numeric_literal
| reserved_keyword
"""
p[0] = p[1]
# 11.2 Left-Hand-Side Expressions
def p_member_expr(self, p):
"""member_expr : primary_expr
| function_expr
| member_expr LBRACKET expr RBRACKET
| member_expr PERIOD identifier
| member_expr PERIOD reserved_keyword
| NEW member_expr arguments
"""
if len(p) == 2:
p[0] = p[1]
elif p[1] == 'new':
p[0] = ast.NewExpr(p[2], p[3], lineno=p.lineno(2))
elif p[2] == '.':
p[0] = ast.DotAccessor(p[1], p[3], lineno=p.lineno(1))
else:
p[0] = ast.BracketAccessor(p[1], p[3], lineno=p.lineno(1))
def p_member_expr_nobf(self, p):
"""member_expr_nobf : primary_expr_no_brace
| function_expr
| member_expr_nobf LBRACKET expr RBRACKET
| member_expr_nobf PERIOD identifier
| member_expr_nobf PERIOD reserved_keyword
| NEW member_expr arguments
"""
if len(p) == 2:
p[0] = p[1]
elif p[1] == 'new':
p[0] = ast.NewExpr(p[2], p[3],lineno=p.lineno(2))
elif p[2] == '.':
p[0] = ast.DotAccessor(p[1], p[3],lineno=p.lineno(1))
else:
p[0] = ast.BracketAccessor(p[1], p[3],lineno=p.lineno(1))
def p_new_expr(self, p):
"""new_expr : member_expr
| NEW new_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.NewExpr(p[2],lineno=p.lineno(2))
def p_new_expr_nobf(self, p):
"""new_expr_nobf : member_expr_nobf
| NEW new_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.NewExpr(p[2],lineno=p.lineno(2))
def p_call_expr(self, p):
"""call_expr : member_expr arguments
| call_expr arguments
| call_expr LBRACKET expr RBRACKET
| call_expr PERIOD identifier
| call_expr PERIOD reserved_keyword
"""
if len(p) == 3:
p[0] = ast.FunctionCall(p[1], p[2], lineno=p.lineno(1))
elif len(p) == 4:
p[0] = ast.DotAccessor(p[1], p[3],lineno=p.lineno(1))
else:
p[0] = ast.BracketAccessor(p[1], p[3], lineno=p.lineno(1))
def p_call_expr_nobf(self, p):
"""call_expr_nobf : member_expr_nobf arguments
| call_expr_nobf arguments
| call_expr_nobf LBRACKET expr RBRACKET
| call_expr_nobf PERIOD identifier
| call_expr_nobf PERIOD reserved_keyword
"""
if len(p) == 3:
p[0] = ast.FunctionCall(p[1], p[2], lineno=p.lineno(1))
elif len(p) == 4:
p[0] = ast.DotAccessor(p[1], p[3], lineno=p.lineno(1))
else:
p[0] = ast.BracketAccessor(p[1], p[3],lineno=p.lineno(1))
def p_arguments(self, p):
"""arguments : LPAREN RPAREN
| LPAREN argument_list RPAREN
"""
if len(p) == 4:
p[0] = p[2]
def p_argument_list(self, p):
"""argument_list : assignment_expr
| argument_list COMMA assignment_expr
"""
if len(p) == 2:
p[0] = [p[1]]
else:
p[1].append(p[3])
p[0] = p[1]
def p_lef_hand_side_expr(self, p):
"""left_hand_side_expr : new_expr
| call_expr
"""
p[0] = p[1]
def p_lef_hand_side_expr_nobf(self, p):
"""left_hand_side_expr_nobf : new_expr_nobf
| call_expr_nobf
"""
p[0] = p[1]
# 11.3 Postfix Expressions
def p_postfix_expr(self, p):
"""postfix_expr : left_hand_side_expr
| left_hand_side_expr PLUSPLUS
| left_hand_side_expr MINUSMINUS
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.UnaryOp(op=p[2], value=p[1], postfix=True,
lineno=p.lineno(1))
def p_postfix_expr_nobf(self, p):
"""postfix_expr_nobf : left_hand_side_expr_nobf
| left_hand_side_expr_nobf PLUSPLUS
| left_hand_side_expr_nobf MINUSMINUS
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.UnaryOp(op=p[2], value=p[1], postfix=True,
lineno=p.lineno(1))
# 11.4 Unary Operators
def p_unary_expr(self, p):
"""unary_expr : postfix_expr
| unary_expr_common
"""
p[0] = p[1]
def p_unary_expr_nobf(self, p):
"""unary_expr_nobf : postfix_expr_nobf
| unary_expr_common
"""
p[0] = p[1]
def p_unary_expr_common(self, p):
"""unary_expr_common : DELETE unary_expr
| VOID unary_expr
| TYPEOF unary_expr
| PLUSPLUS unary_expr
| MINUSMINUS unary_expr
| PLUS unary_expr
| MINUS unary_expr
| BNOT unary_expr
| NOT unary_expr
"""
p[0] = ast.UnaryOp(p[1], p[2], lineno=p.lineno(1))
# 11.5 Multiplicative Operators
def p_multiplicative_expr(self, p):
"""multiplicative_expr : unary_expr
| multiplicative_expr MULT unary_expr
| multiplicative_expr DIV unary_expr
| multiplicative_expr MOD unary_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
def p_multiplicative_expr_nobf(self, p):
"""multiplicative_expr_nobf : unary_expr_nobf
| multiplicative_expr_nobf MULT unary_expr
| multiplicative_expr_nobf DIV unary_expr
| multiplicative_expr_nobf MOD unary_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
# 11.6 Additive Operators
def p_additive_expr(self, p):
"""additive_expr : multiplicative_expr
| additive_expr PLUS multiplicative_expr
| additive_expr MINUS multiplicative_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
def p_additive_expr_nobf(self, p):
"""additive_expr_nobf : multiplicative_expr_nobf
| additive_expr_nobf PLUS multiplicative_expr
| additive_expr_nobf MINUS multiplicative_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
# 11.7 Bitwise Shift Operators
def p_shift_expr(self, p):
"""shift_expr : additive_expr
| shift_expr LSHIFT additive_expr
| shift_expr RSHIFT additive_expr
| shift_expr URSHIFT additive_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
def p_shift_expr_nobf(self, p):
"""shift_expr_nobf : additive_expr_nobf
| shift_expr_nobf LSHIFT additive_expr
| shift_expr_nobf RSHIFT additive_expr
| shift_expr_nobf URSHIFT additive_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
# 11.8 Relational Operators
def p_relational_expr(self, p):
"""relational_expr : shift_expr
| relational_expr LT shift_expr
| relational_expr GT shift_expr
| relational_expr LE shift_expr
| relational_expr GE shift_expr
| relational_expr INSTANCEOF shift_expr
| relational_expr IN shift_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
def p_relational_expr_noin(self, p):
"""relational_expr_noin : shift_expr
| relational_expr_noin LT shift_expr
| relational_expr_noin GT shift_expr
| relational_expr_noin LE shift_expr
| relational_expr_noin GE shift_expr
| relational_expr_noin INSTANCEOF shift_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
def p_relational_expr_nobf(self, p):
"""relational_expr_nobf : shift_expr_nobf
| relational_expr_nobf LT shift_expr
| relational_expr_nobf GT shift_expr
| relational_expr_nobf LE shift_expr
| relational_expr_nobf GE shift_expr
| relational_expr_nobf INSTANCEOF shift_expr
| relational_expr_nobf IN shift_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
# 11.9 Equality Operators
def p_equality_expr(self, p):
"""equality_expr : relational_expr
| equality_expr EQEQ relational_expr
| equality_expr NE relational_expr
| equality_expr STREQ relational_expr
| equality_expr STRNEQ relational_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno = p.lineno(1))
def p_equality_expr_noin(self, p):
"""equality_expr_noin : relational_expr_noin
| equality_expr_noin EQEQ relational_expr
| equality_expr_noin NE relational_expr
| equality_expr_noin STREQ relational_expr
| equality_expr_noin STRNEQ relational_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
def p_equality_expr_nobf(self, p):
"""equality_expr_nobf : relational_expr_nobf
| equality_expr_nobf EQEQ relational_expr
| equality_expr_nobf NE relational_expr
| equality_expr_nobf STREQ relational_expr
| equality_expr_nobf STRNEQ relational_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
# 11.10 Binary Bitwise Operators
def p_bitwise_and_expr(self, p):
"""bitwise_and_expr : equality_expr
| bitwise_and_expr BAND equality_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
def p_bitwise_and_expr_noin(self, p):
"""bitwise_and_expr_noin \
: equality_expr_noin
| bitwise_and_expr_noin BAND equality_expr_noin
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
def p_bitwise_and_expr_nobf(self, p):
"""bitwise_and_expr_nobf \
: equality_expr_nobf
| bitwise_and_expr_nobf BAND equality_expr_nobf
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
def p_bitwise_xor_expr(self, p):
"""bitwise_xor_expr : bitwise_and_expr
| bitwise_xor_expr BXOR bitwise_and_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
def p_bitwise_xor_expr_noin(self, p):
"""
bitwise_xor_expr_noin \
: bitwise_and_expr_noin
| bitwise_xor_expr_noin BXOR bitwise_and_expr_noin
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
def p_bitwise_xor_expr_nobf(self, p):
"""
bitwise_xor_expr_nobf \
: bitwise_and_expr_nobf
| bitwise_xor_expr_nobf BXOR bitwise_and_expr_nobf
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
def p_bitwise_or_expr(self, p):
"""bitwise_or_expr : bitwise_xor_expr
| bitwise_or_expr BOR bitwise_xor_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
def p_bitwise_or_expr_noin(self, p):
"""
bitwise_or_expr_noin \
: bitwise_xor_expr_noin
| bitwise_or_expr_noin BOR bitwise_xor_expr_noin
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
def p_bitwise_or_expr_nobf(self, p):
"""
bitwise_or_expr_nobf \
: bitwise_xor_expr_nobf
| bitwise_or_expr_nobf BOR bitwise_xor_expr_nobf
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
# 11.11 Binary Logical Operators
def p_logical_and_expr(self, p):
"""logical_and_expr : bitwise_or_expr
| logical_and_expr AND bitwise_or_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
def p_logical_and_expr_noin(self, p):
"""
logical_and_expr_noin : bitwise_or_expr_noin
| logical_and_expr_noin AND bitwise_or_expr_noin
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
def p_logical_and_expr_nobf(self, p):
"""
logical_and_expr_nobf : bitwise_or_expr_nobf
| logical_and_expr_nobf AND bitwise_or_expr_nobf
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
def p_logical_or_expr(self, p):
"""logical_or_expr : logical_and_expr
| logical_or_expr OR logical_and_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
def p_logical_or_expr_noin(self, p):
"""logical_or_expr_noin : logical_and_expr_noin
| logical_or_expr_noin OR logical_and_expr_noin
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
def p_logical_or_expr_nobf(self, p):
"""logical_or_expr_nobf : logical_and_expr_nobf
| logical_or_expr_nobf OR logical_and_expr_nobf
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.BinOp(op=p[2], left=p[1], right=p[3],
lineno=p.lineno(1))
# 11.12 Conditional Operator ( ? : )
def p_conditional_expr(self, p):
"""
conditional_expr \
: logical_or_expr
| logical_or_expr CONDOP assignment_expr COLON assignment_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.Conditional(
predicate=p[1], consequent=p[3], alternative=p[5],
lineno=p.lineno(1))
def p_conditional_expr_noin(self, p):
"""
conditional_expr_noin \
: logical_or_expr_noin
| logical_or_expr_noin CONDOP assignment_expr_noin COLON \
assignment_expr_noin
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.Conditional(
predicate=p[1], consequent=p[3], alternative=p[5],
lineno=p.lineno(1))
def p_conditional_expr_nobf(self, p):
"""
conditional_expr_nobf \
: logical_or_expr_nobf
| logical_or_expr_nobf CONDOP assignment_expr COLON assignment_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.Conditional(
predicate=p[1], consequent=p[3], alternative=p[5],
lineno=p.lineno(1))
# 11.13 Assignment Operators
def p_assignment_expr(self, p):
"""
assignment_expr \
: conditional_expr
| left_hand_side_expr assignment_operator assignment_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.Assign(left=p[1], op=p[2], right=p[3],
lineno=p.lineno(1))
def p_assignment_expr_noin(self, p):
"""
assignment_expr_noin \
: conditional_expr_noin
| left_hand_side_expr assignment_operator assignment_expr_noin
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.Assign(left=p[1], op=p[2], right=p[3],
lineno=p.lineno(1))
def p_assignment_expr_nobf(self, p):
"""
assignment_expr_nobf \
: conditional_expr_nobf
| left_hand_side_expr_nobf assignment_operator assignment_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.Assign(left=p[1], op=p[2], right=p[3],
lineno=p.lineno(1))
def p_assignment_operator(self, p):
"""assignment_operator : EQ
| MULTEQUAL
| DIVEQUAL
| MODEQUAL
| PLUSEQUAL
| MINUSEQUAL
| LSHIFTEQUAL
| RSHIFTEQUAL
| URSHIFTEQUAL
| ANDEQUAL
| XOREQUAL
| OREQUAL
"""
p[0] = p[1]
# 11.4 Comma Operator
def p_expr(self, p):
"""expr : assignment_expr
| expr COMMA assignment_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.Comma(left=p[1], right=p[3],lineno=p.lineno(1))
def p_expr_noin(self, p):
"""expr_noin : assignment_expr_noin
| expr_noin COMMA assignment_expr_noin
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.Comma(left=p[1], right=p[3],lineno=p.lineno(1))
def p_expr_nobf(self, p):
"""expr_nobf : assignment_expr_nobf
| expr_nobf COMMA assignment_expr
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = ast.Comma(left=p[1], right=p[3],lineno=p.lineno(1))
# 12.2 Variable Statement
def p_variable_statement(self, p):
"""variable_statement : VAR variable_declaration_list SEMI
| VAR variable_declaration_list auto_semi
"""
p[0] = ast.VarStatement(p[2],lineno=p.lineno(2))
def p_variable_declaration_list(self, p):
"""
variable_declaration_list \
: variable_declaration
| variable_declaration_list COMMA variable_declaration
"""
if len(p) == 2:
p[0] = [p[1]]
else:
p[1].append(p[3])
p[0] = p[1]
def p_variable_declaration_list_noin(self, p):
"""
variable_declaration_list_noin \
: variable_declaration_noin
| variable_declaration_list_noin COMMA variable_declaration_noin
"""
if len(p) == 2:
p[0] = [p[1]]
else:
p[1].append(p[3])
p[0] = p[1]
def p_variable_declaration(self, p):
"""variable_declaration : identifier
| identifier initializer
"""
if len(p) == 2:
p[0] = ast.VarDecl(p[1],lineno=p.lineno(1))
else:
p[0] = ast.VarDecl(p[1], p[2], lineno=p.lineno(1))
def p_variable_declaration_noin(self, p):
"""variable_declaration_noin : identifier
| identifier initializer_noin
"""
if len(p) == 2:
p[0] = ast.VarDecl(p[1],lineno=p.lineno(1))
else:
p[0] = ast.VarDecl(p[1], p[2],lineno=p.lineno(1))
def p_initializer(self, p):
"""initializer : EQ assignment_expr"""
p[0] = p[2]
def p_initializer_noin(self, p):
"""initializer_noin : EQ assignment_expr_noin"""
p[0] = p[2]
# 12.3 Empty Statement
def p_empty_statement(self, p):
"""empty_statement : SEMI"""
p[0] = ast.EmptyStatement(p[1],lineno=p.lineno(1))
# 12.4 Expression Statement
def p_expr_statement(self, p):
"""expr_statement : expr_nobf SEMI
| expr_nobf auto_semi
"""
p[0] = ast.ExprStatement(p[1],lineno=p.lineno(1))
# 12.5 The if Statement
def p_if_statement_1(self, p):
"""if_statement : IF LPAREN expr RPAREN statement"""
p[0] = ast.If(predicate=p[3], consequent=p[5], lineno=p.lineno(3))
def p_if_statement_2(self, p):
"""if_statement : IF LPAREN expr RPAREN statement ELSE statement"""
p[0] = ast.If(predicate=p[3], consequent=p[5], alternative=p[7],
lineno=p.lineno(3))
# 12.6 Iteration Statements
def p_iteration_statement_1(self, p):
"""
iteration_statement \
: DO statement WHILE LPAREN expr RPAREN SEMI
| DO statement WHILE LPAREN expr RPAREN auto_semi
"""
p[0] = ast.DoWhile(predicate=p[5], statement=p[2],
lineno=p.lineno(5))
def p_iteration_statement_2(self, p):
"""iteration_statement : WHILE LPAREN expr RPAREN statement"""
p[0] = ast.While(predicate=p[3], statement=p[5],
lineno=p.lineno(3))
def p_iteration_statement_3(self, p):
"""
iteration_statement \
: FOR LPAREN expr_noin_opt SEMI expr_opt SEMI expr_opt RPAREN \
statement
| FOR LPAREN VAR variable_declaration_list_noin SEMI expr_opt SEMI\
expr_opt RPAREN statement
"""
if len(p) == 10:
p[0] = ast.For(init=p[3], cond=p[5], count=p[7], statement=p[9],
lineno=p.lineno(3))
else:
init = ast.VarStatement(p[4],lineno=p.lineno(4))
p[0] = ast.For(init=init, cond=p[6], count=p[8], statement=p[10],
lineno=p.lineno(6))
def p_iteration_statement_4(self, p):
"""
iteration_statement \
: FOR LPAREN left_hand_side_expr IN expr RPAREN statement
"""
p[0] = ast.ForIn(item=p[3], iterable=p[5], statement=p[7],
lineno=p.lineno(3))
def p_iteration_statement_5(self, p):
"""
iteration_statement : \
FOR LPAREN VAR identifier IN expr RPAREN statement
"""
p[0] = ast.ForIn(
item=ast.VarDecl(p[4],lineno=p.lineno(4)),
iterable=p[6], statement=p[8],lineno=p.lineno(4))
def p_iteration_statement_6(self, p):
"""
iteration_statement \
: FOR LPAREN VAR identifier initializer_noin IN expr RPAREN statement
"""
p[0] = ast.ForIn(
item=ast.VarDecl(
identifier=p[4], initializer=p[5], lineno=p.lineno(4)
),
iterable=p[7], statement=p[9],lineno=p.lineno(4)
)
def p_expr_opt(self, p):
"""expr_opt : empty
| expr
"""
p[0] = p[1]
def p_expr_noin_opt(self, p):
"""expr_noin_opt : empty
| expr_noin
"""
p[0] = p[1]
# 12.7 The continue Statement
def p_continue_statement_1(self, p):
"""continue_statement : CONTINUE SEMI
| CONTINUE auto_semi
"""
p[0] = ast.Continue(lineno=p.lineno(0))
def p_continue_statement_2(self, p):
"""continue_statement : CONTINUE identifier SEMI
| CONTINUE identifier auto_semi
"""
p[0] = ast.Continue(p[2],lineno=p.lineno(2))
# 12.8 The break Statement
def p_break_statement_1(self, p):
"""break_statement : BREAK SEMI
| BREAK auto_semi
"""
p[0] = ast.Break(lineno=p.lineno(0))
def p_break_statement_2(self, p):
"""break_statement : BREAK identifier SEMI
| BREAK identifier auto_semi
"""
p[0] = ast.Break(p[2],lineno=p.lineno(2))
# 12.9 The return Statement
def p_return_statement_1(self, p):
"""return_statement : RETURN SEMI
| RETURN auto_semi
"""
p[0] = ast.Return(lineno=p.lineno(0))
def p_return_statement_2(self, p):
"""return_statement : RETURN expr SEMI
| RETURN expr auto_semi
"""
p[0] = ast.Return(expr=p[2], lineno=p.lineno(2))
# 12.10 The with Statement
def p_with_statement(self, p):
"""with_statement : WITH LPAREN expr RPAREN statement"""
p[0] = ast.With(expr=p[3], statement=p[5], lineno=p.lineno(3))
# 12.11 The switch Statement
def p_switch_statement(self, p):
"""switch_statement : SWITCH LPAREN expr RPAREN case_block"""
cases = []
default = None
# iterate over return values from case_block
for item in p[5]:
if isinstance(item, ast.Default):
default = item
elif isinstance(item, list):
cases.extend(item)
p[0] = ast.Switch(expr=p[3], cases=cases, default=default,
lineno=p.lineno(3))
def p_case_block(self, p):
"""
case_block \
: LBRACE case_clauses_opt RBRACE
| LBRACE case_clauses_opt default_clause case_clauses_opt RBRACE
"""
p[0] = p[2:-1]
def p_case_clauses_opt(self, p):
"""case_clauses_opt : empty
| case_clauses
"""
p[0] = p[1]
def p_case_clauses(self, p):
"""case_clauses : case_clause
| case_clauses case_clause
"""
if len(p) == 2:
p[0] = [p[1]]
else:
p[1].append(p[2])
p[0] = p[1]
def p_case_clause(self, p):
"""case_clause : CASE expr COLON source_elements"""
p[0] = ast.Case(expr=p[2], elements=p[4], lineno=p.lineno(2))
def p_default_clause(self, p):
"""default_clause : DEFAULT COLON source_elements"""
p[0] = ast.Default(elements=p[3],lineno=p.lineno(3))
# 12.12 Labelled Statements
def p_labelled_statement(self, p):
"""labelled_statement : identifier COLON statement"""
p[0] = ast.Label(identifier=p[1], statement=p[3],lineno=p.lineno(1))
# 12.13 The throw Statement
def p_throw_statement(self, p):
"""throw_statement : THROW expr SEMI
| THROW expr auto_semi
"""
p[0] = ast.Throw(expr=p[2],lineno=p.lineno(2))
# 12.14 The try Statement
def p_try_statement_1(self, p):
"""try_statement : TRY block catch"""
p[0] = ast.Try(statements=p[2], catch=p[3],lineno=p.lineno(2))
def p_try_statement_2(self, p):
"""try_statement : TRY block finally"""
p[0] = ast.Try(statements=p[2], fin=p[3],lineno=p.lineno(2))
def p_try_statement_3(self, p):
"""try_statement : TRY block catch finally"""
p[0] = ast.Try(statements=p[2], catch=p[3], fin=p[4],
lineno=p.lineno(2))
def p_catch(self, p):
"""catch : CATCH LPAREN identifier RPAREN block"""
p[0] = ast.Catch(identifier=p[3], elements=p[5],lineno=p.lineno(3))
def p_finally(self, p):
"""finally : FINALLY block"""
p[0] = ast.Finally(elements=p[2],lineno=p.lineno(2))
# 12.15 The debugger statement
def p_debugger_statement(self, p):
"""debugger_statement : DEBUGGER SEMI
| DEBUGGER auto_semi
"""
p[0] = ast.Debugger(p[1],lineno=p.lineno(1))
# 13 Function Definition
def p_function_declaration(self, p):
"""
function_declaration \
: FUNCTION identifier LPAREN RPAREN LBRACE function_body RBRACE
| FUNCTION identifier LPAREN formal_parameter_list RPAREN LBRACE \
function_body RBRACE
"""
if len(p) == 8:
p[0] = ast.FuncDecl(
identifier=p[2], parameters=None, elements=p[6],
lineno=p.lineno(2))
else:
p[0] = ast.FuncDecl(
identifier=p[2], parameters=p[4], elements=p[7],
lineno=p.lineno(2))
def p_function_expr_1(self, p):
"""
function_expr \
: FUNCTION LPAREN RPAREN LBRACE function_body RBRACE
| FUNCTION LPAREN formal_parameter_list RPAREN \
LBRACE function_body RBRACE
"""
if len(p) == 7:
p[0] = ast.FuncExpr(
identifier=None, parameters=None, elements=p[5],
lineno=p.lineno(5))
else:
p[0] = ast.FuncExpr(
identifier=None, parameters=p[3], elements=p[6],
lineno=p.lineno(3))
def p_reserved_keyword(self, p):
"""reserved_keyword : CASE
| DEFAULT
| SWITCH
| CATCH
| BREAK
| CONTINUE
| DEBUGGER
| DELETE
| DO
| ELSE
| FINALLY
| FOR
| FUNCTION
| IF
| IN
| INSTANCEOF
| NEW
| RETURN
| THIS
| THROW
| TRY
| TYPEOF
| VAR
| VOID
| WHILE
| WITH
| NULL
| TRUE
| FALSE
| CLASS
| CONST
| ENUM
| EXPORT
| EXTENDS
| IMPORT
| SUPER
"""
p[0] = ast.Identifier(p[1])
def p_function_expr_2(self, p):
"""
function_expr \
: FUNCTION identifier LPAREN RPAREN LBRACE function_body RBRACE
| FUNCTION identifier LPAREN formal_parameter_list RPAREN \
LBRACE function_body RBRACE
"""
if len(p) == 8:
p[0] = ast.FuncExpr(
identifier=p[2], parameters=None, elements=p[6],
lineno=p.lineno(2))
else:
p[0] = ast.FuncExpr(
identifier=p[2], parameters=p[4], elements=p[7],
lineno=p.lineno(2))
def p_formal_parameter_list(self, p):
"""formal_parameter_list : identifier
| formal_parameter_list COMMA identifier
"""
if len(p) == 2:
p[0] = [p[1]]
else:
p[1].append(p[3])
p[0] = p[1]
def p_function_body(self, p):
"""function_body : source_elements"""
p[0] = p[1]
return out_list,id_list,tok_list, str_list, num_list
# arguments related to the dataset
parser = argparse.ArgumentParser()
parser.add_argument("--mode",type=str, help='whether "train" or "test" data')
parser.add_argument("--dataset_name",type=str, help='name of preprocessed dataset')
parser.add_argument("--startfrom",type=int, default=0, help='whether starting from middle')
args = parser.parse_args()
# _actual is the result of removing all missing and undecodable files
file_dir = '/home/irteam/users/data/150kJavaScript/'
vocab_dir = '/home/irteam/users/mjchoi/github/JavaScriptAutoComplete/vocab/'
lexer = Lexer()
save_dir = os.path.join(file_dir,args.mode,args.dataset_name)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
if args.mode=='train':
list_name = 'programs_training_actual.txt'
elif args.mode=='test':
list_name = 'programs_eval_actual.txt'
else:
print("Insufficient mode !")
import sys
sys.exit()
vocab_dir = vocab_dir+args.dataset_name
"key2": "value2"
};
"""
class MyVisitor(ASTVisitor):
def visit(self, node):
for prop in node:
left, right = prop.left, prop.right
print('Property key=%s, value=%s' % (left.value, right.value))
self.visit(prop)
try:
parser = Parser()
tree = parser.parser(text)
visitor = MyVisitor()
visitor.visit(tree)
except Exception as e:
logging.exception(e)
# 4 Using lexer in your project
print('4 Using lexer in your project')
lexer = Lexer()
lexer.input('a=1;x=1+3;')
for token in lexer:
print(token)
def _get_lexer(self):
lexer = Lexer()
return lexer
def get_type_idx(self):
from slimit.lexer import Lexer
lexer = Lexer()
self.type2idx = dict()
for i, tok in enumerate(lexer.tokens):
self.type2idx[tok] = i
