def tokenize(self, s):
'''Tokenize comments, strings, identifiers, whitespace and operators.'''
i, result = 0, []
while i < len(s):
# Loop invariant: at end: j > i and s[i:j] is the new token.
j = i
ch = s[i]
if ch in '@\n': # Make *sure* these are separate tokens.
j += 1
elif ch == '#': # Preprocessor directive.
j = g.skip_to_end_of_line(s, i)
elif ch in ' \t':
j = g.skip_ws(s, i)
elif ch.isalpha() or ch == '_':
j = g.skip_c_id(s, i)
elif g.match(s, i, '//'):
j = g.skip_line(s, i)
elif g.match(s, i, '/*'):
j = self.skip_block_comment(s, i)
elif ch in "'\"":
j = g.skip_string(s, i)
else:
j += 1
assert j > i
result.append(''.join(s[i:j]))
i = j # Advance.
return result
def tokenize(self, s):
'''Tokenize comments, strings, identifiers, whitespace and operators.'''
i, result = 0, []
while i < len(s):
# Loop invariant: at end: j > i and s[i:j] is the new token.
j = i
ch = s[i]
if ch in '@\n': # Make *sure* these are separate tokens.
j += 1
elif ch == '#': # Preprocessor directive.
j = g.skip_to_end_of_line(s, i)
elif ch in ' \t':
j = g.skip_ws(s, i)
elif ch.isalpha() or ch == '_':
j = g.skip_c_id(s, i)
elif g.match(s, i, '//'):
j = g.skip_line(s, i)
elif g.match(s, i, '/*'):
j = self.skip_block_comment(s, i)
elif ch in "'\"":
j = g.skip_string(s, i)
else:
j += 1
assert j > i
result.append(''.join(s[i: j]))
i = j # Advance.
return result
def context_names(self):
'''Return the present context name.'''
if self.context_stack:
result = []
for stack_i in -1, -2:
try:
fn, kind, indent, s = self.context_stack[stack_i]
except IndexError:
result.append('')
break
s = s.strip()
assert kind in ('class', 'def'), kind
i = g.skip_ws(s, 0)
i += len(kind)
i = g.skip_ws(s, i)
j = g.skip_c_id(s, i)
result.append(s[i: j])
return reversed(result)
else:
return ['', '']
def context_names(self):
"""Return the present context names."""
result = []
n = len(self.context_stack)
for i in n - 1, n - 2:
if i >= 0:
fn, kind, s = self.context_stack[i]
assert kind in ("class", "def", "module"), kind
if kind == "module":
result.append(s.strip())
else:
# Append the name following the class or def.
i = g.skip_ws(s, 0)
i += len(kind)
i = g.skip_ws(s, i)
j = g.skip_c_id(s, i)
result.append(s[i:j])
else:
result.append("")
break
# g.trace(list(reversed(result)))
return reversed(result)
def context_names(self):
'''Return the present context names.'''
result = []
n = len(self.context_stack)
for i in n - 1, n - 2:
if i >= 0:
fn, kind, s = self.context_stack[i]
assert kind in ('class', 'def', 'module'), kind
if kind == 'module':
result.append(s.strip())
else:
# Append the name following the class or def.
i = g.skip_ws(s, 0)
i += len(kind)
i = g.skip_ws(s, i)
j = g.skip_c_id(s, i)
result.append(s[i: j])
else:
result.append('')
break
# g.trace(list(reversed(result)))
return reversed(result)
def context_names(self):
'''Return the present context names.'''
result = []
n = len(self.context_stack)
for i in n - 1, n - 2:
if i >= 0:
fn, kind, s = self.context_stack[i]
assert kind in ('class', 'def', 'module'), kind
if kind == 'module':
result.append(s.strip())
else:
# Append the name following the class or def.
i = g.skip_ws(s, 0)
i += len(kind)
i = g.skip_ws(s, i)
j = g.skip_c_id(s, i)
result.append(s[i:j])
else:
result.append('')
break
# g.trace(list(reversed(result)))
return reversed(result)
