def tokenize(self, s): '''Tokenize comments, strings, identifiers, whitespace and operators.''' i, result = 0, [] while i < len(s): # Loop invariant: at end: j > i and s[i:j] is the new token. j = i ch = s[i] if ch in '@\n': # Make *sure* these are separate tokens. j += 1 elif ch == '#': # Preprocessor directive. j = g.skip_to_end_of_line(s, i) elif ch in ' \t': j = g.skip_ws(s, i) elif ch.isalpha() or ch == '_': j = g.skip_c_id(s, i) elif g.match(s, i, '//'): j = g.skip_line(s, i) elif g.match(s, i, '/*'): j = self.skip_block_comment(s, i) elif ch in "'\"": j = g.skip_string(s, i) else: j += 1 assert j > i result.append(''.join(s[i:j])) i = j # Advance. return result
def tokenize(self, s): '''Tokenize comments, strings, identifiers, whitespace and operators.''' i, result = 0, [] while i < len(s): # Loop invariant: at end: j > i and s[i:j] is the new token. j = i ch = s[i] if ch in '@\n': # Make *sure* these are separate tokens. j += 1 elif ch == '#': # Preprocessor directive. j = g.skip_to_end_of_line(s, i) elif ch in ' \t': j = g.skip_ws(s, i) elif ch.isalpha() or ch == '_': j = g.skip_c_id(s, i) elif g.match(s, i, '//'): j = g.skip_line(s, i) elif g.match(s, i, '/*'): j = self.skip_block_comment(s, i) elif ch in "'\"": j = g.skip_string(s, i) else: j += 1 assert j > i result.append(''.join(s[i: j])) i = j # Advance. return result
def skipString(self, s, i): ''' Skip a JavaScript string. Return len(s) on unterminated string. ''' if i < len(s) and s[i] in ('"', "'"): return g.skip_string(s, i, verbose=False) else: # Match a regexp pattern. delim = '/' assert(s[i] == delim) i += 1 n = len(s) while i < n: if s[i] == delim and s[i - 1] != '\\': # This ignores flags, but does that matter? return i + 1 else: i += 1 return i
def skipString(self, s, i): ''' Skip a JavaScript string. Return len(s) on unterminated string. ''' if i < len(s) and s[i] in ('"', "'"): return g.skip_string(s, i, verbose=False) else: # Match a regexp pattern. delim = '/' assert (s[i] == delim) i += 1 n = len(s) while i < n: if s[i] == delim and s[i - 1] != '\\': # This ignores flags, but does that matter? return i + 1 else: i += 1 return i
def skipString(self, s, i): if g.match(s, i, '"') or g.match(s, i, "'"): return g.skip_string(s, i) else: return g.skip_heredoc_string(s, i)
def skipString(self, s, i): # Returns len(s) on unterminated string. if s.startswith('?', i): return min(len(s), i + 3) else: return g.skip_string(s, i, verbose=False)
def skipString (self,s,i): if g.match(s,i,'"') or g.match(s,i,"'"): return g.skip_string(s,i) else: return g.skip_heredoc_string(s,i)