def tokenize(self, s):
'''Tokenize comments, strings, identifiers, whitespace and operators.'''
i, result = 0, []
while i < len(s):
# Loop invariant: at end: j > i and s[i:j] is the new token.
j = i
ch = s[i]
if ch in '@\n': # Make *sure* these are separate tokens.
j += 1
elif ch == '#': # Preprocessor directive.
j = g.skip_to_end_of_line(s, i)
elif ch in ' \t':
j = g.skip_ws(s, i)
elif ch.isalpha() or ch == '_':
j = g.skip_c_id(s, i)
elif g.match(s, i, '//'):
j = g.skip_line(s, i)
elif g.match(s, i, '/*'):
j = self.skip_block_comment(s, i)
elif ch in "'\"":
j = g.skip_string(s, i)
else:
j += 1
assert j > i
result.append(''.join(s[i:j]))
i = j # Advance.
return result
def tokenize(self, s):
'''Tokenize comments, strings, identifiers, whitespace and operators.'''
i, result = 0, []
while i < len(s):
# Loop invariant: at end: j > i and s[i:j] is the new token.
j = i
ch = s[i]
if ch in '@\n': # Make *sure* these are separate tokens.
j += 1
elif ch == '#': # Preprocessor directive.
j = g.skip_to_end_of_line(s, i)
elif ch in ' \t':
j = g.skip_ws(s, i)
elif ch.isalpha() or ch == '_':
j = g.skip_c_id(s, i)
elif g.match(s, i, '//'):
j = g.skip_line(s, i)
elif g.match(s, i, '/*'):
j = self.skip_block_comment(s, i)
elif ch in "'\"":
j = g.skip_string(s, i)
else:
j += 1
assert j > i
result.append(''.join(s[i: j]))
i = j # Advance.
return result
def skipString(self, s, i):
'''
Skip a JavaScript string.
Return len(s) on unterminated string.
'''
if i < len(s) and s[i] in ('"', "'"):
return g.skip_string(s, i, verbose=False)
else:
# Match a regexp pattern.
delim = '/'
assert(s[i] == delim)
i += 1
n = len(s)
while i < n:
if s[i] == delim and s[i - 1] != '\\':
# This ignores flags, but does that matter?
return i + 1
else:
i += 1
return i
def skipString(self, s, i):
'''
Skip a JavaScript string.
Return len(s) on unterminated string.
'''
if i < len(s) and s[i] in ('"', "'"):
return g.skip_string(s, i, verbose=False)
else:
# Match a regexp pattern.
delim = '/'
assert (s[i] == delim)
i += 1
n = len(s)
while i < n:
if s[i] == delim and s[i - 1] != '\\':
# This ignores flags, but does that matter?
return i + 1
else:
i += 1
return i
def skipString(self, s, i):
if g.match(s, i, '"') or g.match(s, i, "'"):
return g.skip_string(s, i)
else:
return g.skip_heredoc_string(s, i)
def skipString(self, s, i):
# Returns len(s) on unterminated string.
if s.startswith('?', i):
return min(len(s), i + 3)
else:
return g.skip_string(s, i, verbose=False)
def skipString (self,s,i):
if g.match(s,i,'"') or g.match(s,i,"'"):
return g.skip_string(s,i)
else:
return g.skip_heredoc_string(s,i)
def skipString(self, s, i):
# Returns len(s) on unterminated string.
if s.startswith('?', i):
return min(len(s), i + 3)
else:
return g.skip_string(s, i, verbose=False)
