def parse_assignment(self, tok_text, start_line, isLocal=True):
# Don't bother trying to type it yet.
# Figure out whether we're in a proc or not
if isLocal:
collectionA = self.containers[VAR_KIND_LOCAL]
else:
collectionA = self.containers[VAR_KIND_GLOBAL]
if len(collectionA) == 0 or collectionA[-1] is None:
return
possibleType = self._finishVarAssignment(collectionA, tok_text, start_line)
update_collection(collectionA[-1], tok_text, start_line, possibleType)
def parse_assignment(self, tok_text, start_line, isLocal=True):
# Don't bother trying to type it yet.
# Figure out whether we're in a proc or not
if isLocal:
collectionA = self.containers[VAR_KIND_LOCAL]
else:
collectionA = self.containers[VAR_KIND_GLOBAL]
if len(collectionA) == 0 or collectionA[-1] is None:
return
possibleType = self._finishVarAssignment(collectionA, tok_text, start_line)
update_collection(collectionA[-1], tok_text, start_line, possibleType)
def parse_aux(self, curr_node, braceCount=0):
init_indentation = curr_node.indentation
tok_count = 0
tok_lim = 1000
cmdStart = True
curr_globals = {}
while 1:
tok = self.tokenizer.get_next_token()
if tok['style'] == shared_lexer.EOF_STYLE:
break
# style, text, start_column, start_line, end_column, end_line = tok
style, text = tok['style'], tok['text']
if style == self.classifier.style_word and \
(cmdStart or tok['start_column'] == self.tokenizer.get_curr_indentation()):
cmdStart = False
if text in ["namespace", "proc"]:
curr_indent = self.tokenizer.get_curr_indentation()
if text == "namespace":
tok1 = self.tokenizer.get_next_token()
if not (self.classifier.is_identifier(tok1, True) and tok1['text'] == "eval"):
continue
node_class, node_parser = self.get_parsing_objects(text)
if node_class is None:
sys.stderr.write("Couldn't get parsing objects for type %s\n" % text)
break
# Get the comments before further parsing.
comment_lines = remove_hashes(self.tokenizer.curr_comment())
nm_token = self.get_fully_qualified_name()
fqname = nm_token[0]
if not fqname:
break
# Handle only local names for now
if fqname.startswith("::") and text == "namespace":
fqname = fqname[2:]
new_node = node_class(fqname, tok['start_line'])
new_node.doc_lines = comment_lines
new_node.indentation = curr_indent
self.block_stack.append(new_node)
curr_node.append_node(new_node)
# Push new containers on the symbol table
self.containers[VAR_KIND_LOCAL].append(new_node.local_vars)
node_parser(new_node) # Has self bound to it
self.block_stack.pop()
self.containers[VAR_KIND_LOCAL].pop()
# Clear any comment that's hanging around
self.tokenizer.clear_comments()
elif text == "package":
tok1 = self.tokenizer.get_next_token()
if self.classifier.is_identifier(tok1, True):
if tok1['text'] == "require":
tok2 = self.tokenizer.get_next_token()
if self.classifier.is_identifier(tok2, True) and tok2['text'] != "Tcl":
curr_node.imports.append(Name_LineNum(tok2['text'], tok['start_line']))
elif text == "global":
# XXX: all tokens following 'global' should be declared vars
tok = self.tokenizer.get_next_token()
if self.classifier.is_identifier(tok, True):
curr_globals[tok['text']] = None
elif text == "set":
# XXX: Needs to handle lappend, append, incr, variable
# XXX: possibly dict set, array set, upvar, lassign,
# XXX: foreach, regsub (non-inline)
isLocal = isinstance(curr_node, MethodNode)
tok = self.tokenizer.get_next_token()
if self.classifier.is_operator(tok, '::'):
# Possibly a global variable. Check the next two tokens.
ident = self.tokenizer.get_next_token()
afterIdent = self.tokenizer.get_next_token()
if not self.classifier.is_operator(afterIdent, '::'):
# Valid global variable. Put back second token.
tok = ident
isLocal = False
self.tokenizer.put_back(afterIdent)
else:
# Not a valid global variable
# (e.g. "set ::foo::bar ..."). Put back both tokens.
self.tokenizer.put_back(afterIdent)
self.tokenizer.put_back(ident)
if self.classifier.is_identifier(tok, True):
if curr_globals.has_key(tok['text']):
pass
else:
self.parse_assignment(tok['text'], tok['start_line'], isLocal)
elif text == "variable" and isinstance(curr_node, ModuleNode):
# "variable" declarations in namespaces are considered
# namespace variables.
tok = self.tokenizer.get_next_token()
if self.classifier.is_identifier(tok, True):
self.parse_assignment(tok['text'], tok['start_line'], True)
elif text == "lassign":
tok = self.tokenizer.get_next_token()
if self.classifier.is_operator(tok, "{"):
start_line = tok['start_line'];
isLocal = isinstance(curr_node, MethodNode)
if isLocal:
collectionA = self.containers[VAR_KIND_LOCAL]
else:
collectionA = self.containers[VAR_KIND_GLOBAL]
vars = self._parse_name_list()
for v in vars:
update_collection(collectionA[-1], v, start_line)
elif self.classifier.is_any_operator(tok):
cmdStart = False
if text == "{":
braceCount += 1
elif text == "}":
braceCount -= 1
if braceCount <= 0:
break
elif text in (";", "["):
cmdStart = True
elif text == "\\":
# Skip the next token, whatever it is - bug 74850
tok = self.tokenizer.get_next_token()
else:
cmdStart = False
# Sanity check to make sure we haven't gone too far.
tok_count += 1
if tok_count > tok_lim and tok['start_column'] < init_indentation:
break
# end while
curr_node.set_line_end_num(self.tokenizer.curr_line_no())
return tok['style'] != shared_lexer.EOF_STYLE
def parse_aux(self, curr_node, braceCount=0):
init_indentation = curr_node.indentation
tok_count = 0
tok_lim = 1000
cmdStart = True
curr_globals = {}
while 1:
tok = self.tokenizer.get_next_token()
if tok['style'] == shared_lexer.EOF_STYLE:
break
# style, text, start_column, start_line, end_column, end_line = tok
style, text = tok['style'], tok['text']
if style == self.classifier.style_word and \
(cmdStart or tok['start_column'] == self.tokenizer.get_curr_indentation()):
cmdStart = False
if text in ["namespace", "proc"]:
curr_indent = self.tokenizer.get_curr_indentation()
if text == "namespace":
tok1 = self.tokenizer.get_next_token()
if not (self.classifier.is_identifier(tok1, True)
and tok1['text'] == "eval"):
continue
node_class, node_parser = self.get_parsing_objects(text)
if node_class is None:
sys.stderr.write(
"Couldn't get parsing objects for type %s\n" %
text)
break
# Get the comments before further parsing.
comment_lines = remove_hashes(
self.tokenizer.curr_comment())
nm_token = self.get_fully_qualified_name()
fqname = nm_token[0]
if not fqname:
break
# Handle only local names for now
if fqname.startswith("::") and text == "namespace":
fqname = fqname[2:]
new_node = node_class(fqname, tok['start_line'])
new_node.doc_lines = comment_lines
new_node.indentation = curr_indent
self.block_stack.append(new_node)
curr_node.append_node(new_node)
# Push new containers on the symbol table
self.containers[VAR_KIND_LOCAL].append(new_node.local_vars)
node_parser(new_node) # Has self bound to it
self.block_stack.pop()
self.containers[VAR_KIND_LOCAL].pop()
# Clear any comment that's hanging around
self.tokenizer.clear_comments()
elif text == "package":
tok1 = self.tokenizer.get_next_token()
if self.classifier.is_identifier(tok1, True):
if tok1['text'] == "require":
tok2 = self.tokenizer.get_next_token()
if self.classifier.is_identifier(
tok2, True) and tok2['text'] != "Tcl":
curr_node.imports.append(
Name_LineNum(tok2['text'],
tok['start_line']))
elif text == "global":
# XXX: all tokens following 'global' should be declared vars
tok = self.tokenizer.get_next_token()
if self.classifier.is_identifier(tok, True):
curr_globals[tok['text']] = None
elif text == "set":
# XXX: Needs to handle lappend, append, incr, variable
# XXX: possibly dict set, array set, upvar, lassign,
# XXX: foreach, regsub (non-inline)
tok = self.tokenizer.get_next_token()
if self.classifier.is_identifier(tok, True):
if curr_globals.has_key(tok['text']):
pass
else:
self.parse_assignment(
tok['text'], tok['start_line'],
isinstance(curr_node, MethodNode))
elif text == "lassign":
tok = self.tokenizer.get_next_token()
if self.classifier.is_operator(tok, "{"):
start_line = tok['start_line']
isLocal = isinstance(curr_node, MethodNode)
if isLocal:
collectionA = self.containers[VAR_KIND_LOCAL]
else:
collectionA = self.containers[VAR_KIND_GLOBAL]
vars = self._parse_name_list()
for v in vars:
update_collection(collectionA[-1], v, start_line)
elif self.classifier.is_any_operator(tok):
cmdStart = False
if text == "{":
braceCount += 1
elif text == "}":
braceCount -= 1
if braceCount <= 0:
break
elif text in (";", "["):
cmdStart = True
elif text == "\\":
# Skip the next token, whatever it is - bug 74850
tok = self.tokenizer.get_next_token()
else:
cmdStart = False
# Sanity check to make sure we haven't gone too far.
tok_count += 1
if tok_count > tok_lim and tok['start_column'] < init_indentation:
break
# end while
curr_node.set_line_end_num(self.tokenizer.curr_line_no())
return tok['style'] != shared_lexer.EOF_STYLE
def parse_aux(self, curr_node, braceCount=0):
init_indentation = curr_node.indentation
tok_count = 0
tok_lim = 1000
cmdStart = True
curr_globals = {}
while 1:
tok = self.tokenizer.get_next_token()
if tok['style'] == shared_lexer.EOF_STYLE:
break
if tok['text'] == 'oo':
# Lookahead for oo::class and oo::define.
next_tok = self.tokenizer.get_next_token()
if self.classifier.is_operator(next_tok, '::'):
next_tok = self.tokenizer.get_next_token()
if next_tok['text'] in ["class", "define"]:
tok = next_tok
oo_token = True
else:
self.tokenizer.put_back(next_tok)
else:
self.tokenizer.put_back(next_tok)
else:
oo_token = False
# style, text, start_column, start_line, end_column, end_line = tok
style, text = tok['style'], tok['text']
if style == self.classifier.style_word and \
(cmdStart or tok['start_column'] == self.tokenizer.get_curr_indentation()) or \
oo_token:
cmdStart = False
if text in ["namespace", "proc",
# Either oo::class or oo::define.
"class", "define",
# Instance methods.
"method", "constructor", "destructor"]:
curr_indent = self.tokenizer.get_curr_indentation()
if text == "namespace":
tok1 = self.tokenizer.get_next_token()
if not (self.classifier.is_identifier(tok1, True) and tok1['text'] == "eval"):
continue
node_class, node_parser = self.get_parsing_objects(text)
if node_class is None:
sys.stderr.write("Couldn't get parsing objects for type %s\n" % text)
break
# Get the comments before further parsing.
comment_lines = remove_hashes(self.tokenizer.curr_comment())
if text == "class":
# Since syntax is oo::class create ClassName, skip over
# the "create" token such that the name is the next
# token.
tok = self.tokenizer.get_next_token()
if tok['text'] != "create":
break
if text not in ["constructor", "destructor"]:
nm_token = self.get_fully_qualified_name()
fqname = nm_token[0]
else:
fqname = text
if not fqname:
break
# Handle only local names for now
if fqname.startswith("::") and text == "namespace":
fqname = fqname[2:]
# Create a new node for the namespace, proc, class, method,
# etc.
# However, for oo::define, the subsequent body should be
# added to an existing class. Find the previously created
# node and use it as the "new" node.
existing_node_found = False
if text == "define":
for node in curr_node.children:
if node.name == fqname:
new_node = node
existing_node_found = True
break
if not existing_node_found:
# Create the new parent node.
new_node = node_class(fqname, tok['start_line'])
new_node.doc_lines = comment_lines
new_node.indentation = curr_indent
curr_node.append_node(new_node)
# TODO: instance method names with lowercase are public
# while others are private. However, "export" can make
# private members public.
# For oo::class, determine if there's a body after it.
class_body = False
if text == "class":
next_tok = self.tokenizer.get_next_token()
class_body = self.classifier.is_operator(next_tok, '{')
self.tokenizer.put_back(next_tok)
# Anything in the subsequent body belongs to the parent
# node. Set this up.
if text != "class" or class_body:
self.block_stack.append(new_node)
# Push new containers on the symbol table
self.containers[VAR_KIND_LOCAL].append(new_node.local_vars)
node_parser(new_node) # Has self bound to it
self.block_stack.pop()
self.containers[VAR_KIND_LOCAL].pop()
# Clear any comment that's hanging around
self.tokenizer.clear_comments()
elif text == "package":
tok1 = self.tokenizer.get_next_token()
if self.classifier.is_identifier(tok1, True):
if tok1['text'] == "require":
tok2 = self.tokenizer.get_next_token()
if self.classifier.is_identifier(tok2, True) and tok2['text'] != "Tcl":
curr_node.imports.append(Name_LineNum(tok2['text'], tok['start_line']))
elif text == "global":
# XXX: all tokens following 'global' should be declared vars
tok = self.tokenizer.get_next_token()
if self.classifier.is_identifier(tok, True):
curr_globals[tok['text']] = None
elif text == "set":
# XXX: Needs to handle lappend, append, incr, variable
# XXX: possibly dict set, array set, upvar, lassign,
# XXX: foreach, regsub (non-inline)
isLocal = isinstance(curr_node, MethodNode)
tok = self.tokenizer.get_next_token()
if self.classifier.is_operator(tok, '::'):
# Possibly a global variable. Check the next two tokens.
ident = self.tokenizer.get_next_token()
afterIdent = self.tokenizer.get_next_token()
if not self.classifier.is_operator(afterIdent, '::'):
# Valid global variable. Put back second token.
tok = ident
isLocal = False
self.tokenizer.put_back(afterIdent)
else:
# Not a valid global variable
# (e.g. "set ::foo::bar ..."). Put back both tokens.
self.tokenizer.put_back(afterIdent)
self.tokenizer.put_back(ident)
if self.classifier.is_identifier(tok, True):
if tok['text'] in curr_globals:
pass
elif len(self.block_stack) > 1 and \
isinstance(self.block_stack[-2], ClassNode) and \
tok['text'] in self.block_stack[-2].local_vars:
# Declared instance variable. Move along.
pass
else:
self.parse_assignment(tok['text'], tok['start_line'], isLocal)
elif text == "variable" and isinstance(curr_node, ModuleNode):
# "variable" declarations in namespaces are considered
# namespace variables.
tok = self.tokenizer.get_next_token()
if self.classifier.is_identifier(tok, True):
self.parse_assignment(tok['text'], tok['start_line'], True)
elif text == "variable" and isinstance(curr_node, ClassNode):
# "variable" declarations in classes are considered instance
# variables.
line_num = tok['start_line']
tok = self.tokenizer.get_next_token()
while self.classifier.is_identifier(tok, True) and \
tok['start_line'] == line_num:
self.parse_assignment(tok['text'], tok['start_line'], True)
tok = self.tokenizer.get_next_token()
self.tokenizer.put_back(tok)
elif text == "lassign":
tok = self.tokenizer.get_next_token()
if self.classifier.is_operator(tok, "{"):
start_line = tok['start_line'];
isLocal = isinstance(curr_node, MethodNode)
if isLocal:
collectionA = self.containers[VAR_KIND_LOCAL]
else:
collectionA = self.containers[VAR_KIND_GLOBAL]
vars = self._parse_name_list()
for v in vars:
update_collection(collectionA[-1], v, start_line)
elif self.classifier.is_any_operator(tok):
cmdStart = False
if text == "{":
braceCount += 1
elif text == "}":
braceCount -= 1
if braceCount <= 0:
break
elif text in (";", "["):
cmdStart = True
elif text == "\\":
# Skip the next token, whatever it is - bug 74850
tok = self.tokenizer.get_next_token()
else:
cmdStart = False
# Sanity check to make sure we haven't gone too far.
tok_count += 1
if tok_count > tok_lim and tok['start_column'] < init_indentation:
break
# end while
curr_node.set_line_end_num(self.tokenizer.curr_line_no())
return tok['style'] != shared_lexer.EOF_STYLE
def parse_aux(self, curr_node, braceCount=0):
init_indentation = curr_node.indentation
tok_count = 0
tok_lim = 1000
cmdStart = True
curr_globals = {}
while 1:
tok = self.tokenizer.get_next_token()
if tok['style'] == shared_lexer.EOF_STYLE:
break
if tok['text'] == 'oo':
# Lookahead for oo::class and oo::define.
next_tok = self.tokenizer.get_next_token()
if self.classifier.is_operator(next_tok, '::'):
next_tok = self.tokenizer.get_next_token()
if next_tok['text'] in ["class", "define"]:
tok = next_tok
oo_token = True
else:
self.tokenizer.put_back(next_tok)
else:
self.tokenizer.put_back(next_tok)
else:
oo_token = False
# style, text, start_column, start_line, end_column, end_line = tok
style, text = tok['style'], tok['text']
if style == self.classifier.style_word and \
(cmdStart or tok['start_column'] == self.tokenizer.get_curr_indentation()) or \
oo_token:
cmdStart = False
if text in [
"namespace",
"proc",
# Either oo::class or oo::define.
"class",
"define",
# Instance methods.
"method",
"constructor",
"destructor"
]:
curr_indent = self.tokenizer.get_curr_indentation()
if text == "namespace":
tok1 = self.tokenizer.get_next_token()
if not (self.classifier.is_identifier(tok1, True)
and tok1['text'] == "eval"):
continue
node_class, node_parser = self.get_parsing_objects(text)
if node_class is None:
sys.stderr.write(
"Couldn't get parsing objects for type %s\n" %
text)
break
# Get the comments before further parsing.
comment_lines = remove_hashes(
self.tokenizer.curr_comment())
if text == "class":
# Since syntax is oo::class create ClassName, skip over
# the "create" token such that the name is the next
# token.
tok = self.tokenizer.get_next_token()
if tok['text'] != "create":
break
if text not in ["constructor", "destructor"]:
nm_token = self.get_fully_qualified_name()
fqname = nm_token[0]
else:
fqname = text
if not fqname:
break
# Handle only local names for now
if fqname.startswith("::") and text == "namespace":
fqname = fqname[2:]
# Create a new node for the namespace, proc, class, method,
# etc.
# However, for oo::define, the subsequent body should be
# added to an existing class. Find the previously created
# node and use it as the "new" node.
existing_node_found = False
if text == "define":
for node in curr_node.children:
if node.name == fqname:
new_node = node
existing_node_found = True
break
if not existing_node_found:
# Create the new parent node.
new_node = node_class(fqname, tok['start_line'])
new_node.doc_lines = comment_lines
new_node.indentation = curr_indent
curr_node.append_node(new_node)
# TODO: instance method names with lowercase are public
# while others are private. However, "export" can make
# private members public.
# For oo::class, determine if there's a body after it.
class_body = False
if text == "class":
next_tok = self.tokenizer.get_next_token()
class_body = self.classifier.is_operator(next_tok, '{')
self.tokenizer.put_back(next_tok)
# Anything in the subsequent body belongs to the parent
# node. Set this up.
if text != "class" or class_body:
self.block_stack.append(new_node)
# Push new containers on the symbol table
self.containers[VAR_KIND_LOCAL].append(
new_node.local_vars)
node_parser(new_node) # Has self bound to it
self.block_stack.pop()
self.containers[VAR_KIND_LOCAL].pop()
# Clear any comment that's hanging around
self.tokenizer.clear_comments()
elif text == "package":
tok1 = self.tokenizer.get_next_token()
if self.classifier.is_identifier(tok1, True):
if tok1['text'] == "require":
tok2 = self.tokenizer.get_next_token()
if self.classifier.is_identifier(
tok2, True) and tok2['text'] != "Tcl":
curr_node.imports.append(
Name_LineNum(tok2['text'],
tok['start_line']))
elif text == "global":
# XXX: all tokens following 'global' should be declared vars
tok = self.tokenizer.get_next_token()
if self.classifier.is_identifier(tok, True):
curr_globals[tok['text']] = None
elif text == "set":
# XXX: Needs to handle lappend, append, incr, variable
# XXX: possibly dict set, array set, upvar, lassign,
# XXX: foreach, regsub (non-inline)
isLocal = isinstance(curr_node, MethodNode)
tok = self.tokenizer.get_next_token()
if self.classifier.is_operator(tok, '::'):
# Possibly a global variable. Check the next two tokens.
ident = self.tokenizer.get_next_token()
afterIdent = self.tokenizer.get_next_token()
if not self.classifier.is_operator(afterIdent, '::'):
# Valid global variable. Put back second token.
tok = ident
isLocal = False
self.tokenizer.put_back(afterIdent)
else:
# Not a valid global variable
# (e.g. "set ::foo::bar ..."). Put back both tokens.
self.tokenizer.put_back(afterIdent)
self.tokenizer.put_back(ident)
if self.classifier.is_identifier(tok, True):
if tok['text'] in curr_globals:
pass
elif len(self.block_stack) > 1 and \
isinstance(self.block_stack[-2], ClassNode) and \
tok['text'] in self.block_stack[-2].local_vars:
# Declared instance variable. Move along.
pass
else:
self.parse_assignment(tok['text'],
tok['start_line'], isLocal)
elif text == "variable" and isinstance(curr_node, ModuleNode):
# "variable" declarations in namespaces are considered
# namespace variables.
tok = self.tokenizer.get_next_token()
if self.classifier.is_identifier(tok, True):
self.parse_assignment(tok['text'], tok['start_line'],
True)
elif text == "variable" and isinstance(curr_node, ClassNode):
# "variable" declarations in classes are considered instance
# variables.
line_num = tok['start_line']
tok = self.tokenizer.get_next_token()
while self.classifier.is_identifier(tok, True) and \
tok['start_line'] == line_num:
self.parse_assignment(tok['text'], tok['start_line'],
True)
tok = self.tokenizer.get_next_token()
self.tokenizer.put_back(tok)
elif text == "lassign":
tok = self.tokenizer.get_next_token()
if self.classifier.is_operator(tok, "{"):
start_line = tok['start_line']
isLocal = isinstance(curr_node, MethodNode)
if isLocal:
collectionA = self.containers[VAR_KIND_LOCAL]
else:
collectionA = self.containers[VAR_KIND_GLOBAL]
vars = self._parse_name_list()
for v in vars:
update_collection(collectionA[-1], v, start_line)
elif self.classifier.is_any_operator(tok):
cmdStart = False
if text == "{":
braceCount += 1
elif text == "}":
braceCount -= 1
if braceCount <= 0:
break
elif text in (";", "["):
cmdStart = True
elif text == "\\":
# Skip the next token, whatever it is - bug 74850
tok = self.tokenizer.get_next_token()
else:
cmdStart = False
# Sanity check to make sure we haven't gone too far.
tok_count += 1
if tok_count > tok_lim and tok['start_column'] < init_indentation:
break
# end while
curr_node.set_line_end_num(self.tokenizer.curr_line_no())
return tok['style'] != shared_lexer.EOF_STYLE
