def broken_string_token_handler(lexer, token):
match = PATT_BROKEN_STRING.match(token.value)
if not match:
return
# update the error token value to only include what was matched here
# as this will be the actual token that "failed"
token.value = match.group()
# calculate colno for current token colno before...
colno = lexer._get_colno(token)
# updating the newline indexes for the error reporting for raw
# lexpos
lexer._update_newline_idx(token)
# probe for the next values (which no valid rules will match)
position = lexer.lexer.lexpos + len(token.value)
failure = lexer.lexer.lexdata[position:position + 2]
if failure and failure[0] == '\\':
type_ = {'x': 'hexadecimal', 'u': 'unicode'}[failure[1]]
seq = re.match(r'\\[xu][0-9-a-f-A-F]*',
lexer.lexer.lexdata[position:]).group()
raise ECMASyntaxError(
"Invalid %s escape sequence '%s' at %s:%s" %
(type_, seq, lexer.lineno, lexer._get_colno_lexpos(position)))
tl = 16 # truncate length
raise ECMASyntaxError(
'Unterminated string literal %s at %s:%s' %
(repr_compat(token.value[:tl].strip() +
(token.value[tl:] and '...')), token.lineno, colno))
def t_error(self, token):
if self.cur_token:
# TODO make use of the extended calling signature when done
raise ECMASyntaxError('Illegal character %s at %s:%s after %s' % (
repr_compat(token.value[0]),
token.lineno,
self._get_colno(token),
format_lex_token(self.cur_token),
))
else:
raise ECMASyntaxError('Illegal character %s at %s:%s' % (
repr_compat(token.value[0]),
token.lineno,
self._get_colno(token),
))
def t_BROKEN_STRING(self, token):
# calculate colno for current token colno before...
colno = self._get_colno(token)
# updating the newline indexes for the error reporting for raw
# lexpos
self._update_newline_idx(token)
# probe for the next values (which no valid rules will match)
failure = self.lexer.lexdata[self.lexer.lexpos:self.lexer.lexpos + 2]
if failure and failure[0] == '\\':
type_ = {'x': 'hexadecimal', 'u': 'unicode'}[failure[1]]
seq = re.match(r'\\[xu][0-9-a-f-A-F]*',
self.lexer.lexdata[self.lexer.lexpos:]).group()
raise ECMASyntaxError("Invalid %s escape sequence '%s' at %s:%s" %
(type_, seq, self.lineno,
self._get_colno_lexpos(self.lexer.lexpos)))
tl = 16 # truncate length
raise ECMASyntaxError(
'Unterminated string literal %s at %s:%s' %
(repr_compat(token.value[:tl].strip() +
(token.value[tl:] and '...')), token.lineno, colno))
def _get_update_token(self):
self._set_tokens(self.get_lexer_token())
if self.cur_token is not None:
if self.cur_token.type in ('LPAREN', ):
# if we encounter a FOR, IF, WHILE, then whatever in
# the parentheses are marked. Otherwise just push
# into the inner marker list.
if (self.prev_token
and self.prev_token.type in IMPLIED_BLOCK_IDENTIFIER):
self.token_stack.append([self.cur_token, []])
else:
self.token_stack[-1][1].append(self.cur_token)
if self.cur_token.type in ('RPAREN', ):
# likewise, pop the inner marker first.
if self.token_stack[-1][1]:
self.token_stack[-1][1].pop()
else:
self.token_stack.pop()
if not self.token_stack:
# TODO actually give up earlier than this with the first
# mismatch.
raise ECMASyntaxError("Mismatched '%s' at %d:%d" % (
self.cur_token.value,
self.cur_token.lineno,
self.cur_token.colno,
))
# insert semicolon before restricted tokens
# See section 7.9.1 ECMA262
if (self.cur_token is not None
and self.cur_token.type == 'LINE_TERMINATOR'
and self.prev_token is not None and self.prev_token.type
in ['BREAK', 'CONTINUE', 'RETURN', 'THROW']):
return self._create_semi_token(self.cur_token)
return self.cur_token
def parser(text):
raise ECMASyntaxError('Illegal input')