def parse_string_expr(self, string_expr_node):
""" Parse a string node content. """
string_expr_node_value = string_expr_node['value']
string_expr_str = string_expr_node_value[1:-1]
# Care escaped string literals
if string_expr_node_value[0] == "'":
string_expr_str = string_expr_str.replace("''", "'")
else:
string_expr_str = string_expr_str.replace('\\"', '"')
# NOTE: This is a hack to parse expr1. See :help expr1
raw_ast = self.parse_string('echo ' + string_expr_str)
# We need the left node of ECHO node
parsed_string_expr_nodes = raw_ast['body'][0]['list']
start_pos = string_expr_node['pos']
def adjust_position(node):
pos = node['pos']
# Care 1-based index and the length of "echo ".
pos['col'] += start_pos['col'] - 1 - 5
# Care the length of "echo ".
pos['i'] += start_pos['i'] - 5
# Care 1-based index
pos['lnum'] += start_pos['lnum'] - 1
for parsed_string_expr_node in parsed_string_expr_nodes:
traverse(parsed_string_expr_node, on_enter=adjust_position)
return parsed_string_expr_nodes
def lint_file(self, path):
self._log_file_path_to_lint(path)
try:
root_ast = self._parser.parse_file(path)
except vimlparser.VimLParserException as exception:
parse_error = self._create_parse_error(path, str(exception))
return [parse_error]
except EncodingDetectionError as exception:
decoding_error = self._create_decoding_error(path, str(exception))
return [decoding_error]
self._violations = []
self._update_listeners_map()
# Given root AST to makepolicy flexibility
lint_context = {
'path': path,
'root_node': root_ast,
'stack_trace': [],
'plugins': self._plugins,
'config': self._config.get_config_dict(),
}
traverse(root_ast,
on_enter=lambda node: self._handle_enter(node, lint_context),
on_leave=lambda node: self._handle_leave(node, lint_context))
return self._violations
def attach_identifier_attributes(self, ast):
""" Attach 5 flags to the AST.
- is dynamic: True if the identifier name can be determined by static analysis.
- is member: True if the identifier is a member of a subscription/dot/slice node.
- is declaring: True if the identifier is used to declare.
- is autoload: True if the identifier is declared with autoload.
- is function: True if the identifier is a function. Vim distinguish
between function identifiers and variable identifiers.
- is declarative paramter: True if the identifier is a declarative
parameter. For example, the identifier "param" in Func(param) is a
declarative paramter.
- is on string expression context: True if the variable is on the
string expression context. The string expression context is the
string content on the 2nd argument of the map or filter function.
"""
redir_assignment_parser = RedirAssignmentParser()
ast_with_parsed_redir = redir_assignment_parser.process(ast)
map_and_filter_parser = MapAndFilterParser()
ast_with_parse_map_and_filter_and_redir = \
map_and_filter_parser.process(ast_with_parsed_redir)
traverse(ast_with_parse_map_and_filter_and_redir,
on_enter=self._enter_handler)
return ast
def traverse_string_expr_content(node, on_enter=None, on_leave=None):
string_expr_content_nodes = get_string_expr_content(node)
if string_expr_content_nodes is None:
return
for child_node in string_expr_content_nodes:
traverse(child_node, on_enter=on_enter, on_leave=on_leave)
def parse_string_expr(self, string_expr_node):
""" Parse a command :redir content. """
string_expr_node_value = string_expr_node['value']
string_expr_str = string_expr_node_value[1:-1]
# Care escaped string literals
if string_expr_node_value[0] == "'":
string_expr_str = string_expr_str.replace("''", "'")
else:
string_expr_str = string_expr_str.replace('\\"', '"')
# NOTE: This is a hack to parse expr1. See :help expr1
raw_ast = self.parse('echo ' + string_expr_str)
# We need the left node of ECHO node
parsed_string_expr_nodes = raw_ast['body'][0]['list']
start_pos = string_expr_node['pos']
def adjust_position(node):
pos = node['pos']
# Care 1-based index and the length of "echo ".
pos['col'] += start_pos['col'] - 1 - 5
# Care the length of "echo ".
pos['i'] += start_pos['i'] - 5
# Care 1-based index
pos['lnum'] += start_pos['lnum'] - 1
for parsed_string_expr_node in parsed_string_expr_nodes:
traverse(parsed_string_expr_node, on_enter=adjust_position)
return parsed_string_expr_nodes
def process(self, ast):
def enter_handler(node):
node_type = NodeType(node['type'])
if node_type is not NodeType.CALL:
return
called_function_identifier = node['left']
# Name node of the "map" or "filter" functions are always IDENTIFIER.
if NodeType(called_function_identifier['type']) is not NodeType.IDENTIFIER:
return
is_map_or_function_call = called_function_identifier.get('value') in {
'map': True,
'filter': True,
}
if not is_map_or_function_call:
return
string_expr_node = node['rlist'][1]
# We can analyze only STRING nodes by static analyzing.
if NodeType(string_expr_node['type']) is not NodeType.STRING:
return
parser = Parser()
string_expr_content_nodes = parser.parse_string_expr(string_expr_node)
node[STRING_EXPR_CONTENT] = string_expr_content_nodes
traverse(ast, on_enter=enter_handler)
return ast
def _enter_lambda_node(self, lambda_node, is_on_lambda_body, is_on_lambda_str):
# Function parameter names are in the r_list.
lambda_argument_nodes = lambda_node['rlist']
for lambda_argument_node in lambda_argument_nodes:
self._enter_identifier_terminate_node(
lambda_argument_node,
is_declarative=True,
is_lambda_argument=True,
is_on_lambda_str=is_on_lambda_str,
is_on_lambda_body=is_on_lambda_body,
)
# Traversing on lambda body context.
traverse(
lambda_node['left'],
on_enter=lambda node: self._enter_handler(
node,
is_on_lambda_body=True,
is_on_lambda_str=is_on_lambda_str,
)
)
# NOTE: Traversing to the lambda args and children was continued by the above traversing.
return SKIP_CHILDREN
def test_traverse_ignoring_while_children(self):
expected_order_of_events = [
{'node_type': NodeType.TOPLEVEL, 'handler': 'enter'},
{'node_type': NodeType.LET, 'handler': 'enter'},
{'node_type': NodeType.IDENTIFIER, 'handler': 'enter'},
{'node_type': NodeType.IDENTIFIER, 'handler': 'leave'},
{'node_type': NodeType.NUMBER, 'handler': 'enter'},
{'node_type': NodeType.NUMBER, 'handler': 'leave'},
{'node_type': NodeType.LET, 'handler': 'leave'},
{'node_type': NodeType.WHILE, 'handler': 'enter'},
{'node_type': NodeType.WHILE, 'handler': 'leave'},
{'node_type': NodeType.TOPLEVEL, 'handler': 'leave'},
]
def on_enter(node):
actual_order_of_events.append({
'node_type': NodeType(node['type']),
'handler': 'enter',
})
if NodeType(node['type']) is NodeType.WHILE:
return SKIP_CHILDREN
# Records visit node type name in order
actual_order_of_events = []
traverse(self.ast,
on_enter=on_enter,
on_leave=lambda node: actual_order_of_events.append({
'node_type': NodeType(node['type']),
'handler': 'leave',
}))
self.maxDiff = 2048
self.assertEqual(actual_order_of_events, expected_order_of_events)
def process(self, ast):
self.current_scope = []
self.root_scope = None
traverse(ast, on_enter=self._handle_enter, on_leave=self._handle_leave)
ast[ScopePlugin.SCOPE_TREE_KEY] = self.root_scope
def attach_identifier_attributes(self, ast): # type: (Dict[str, Any]) -> Dict[str, Any]
""" Attach 5 flags to the AST.
- is dynamic: True if the identifier name can be determined by static analysis.
- is member: True if the identifier is a member of a subscription/dot/slice node.
- is declaring: True if the identifier is used to declare.
- is autoload: True if the identifier is declared with autoload.
- is function: True if the identifier is a function. Vim distinguish
between function identifiers and variable identifiers.
- is declarative parameter: True if the identifier is a declarative
parameter. For example, the identifier "param" in Func(param) is a
declarative parameter.
- is on string expression context: True if the variable is on the
string expression context. The string expression context is the
string content on the 2nd argument of the map or filter function.
- is lambda argument: True if the identifier is a lambda argument.
"""
redir_assignment_parser = RedirAssignmentParser()
ast_with_parsed_redir = redir_assignment_parser.process(ast)
map_and_filter_parser = CallNodeParser()
ast_with_parse_map_and_filter_and_redir = \
map_and_filter_parser.process(ast_with_parsed_redir)
traverse(
ast_with_parse_map_and_filter_and_redir,
on_enter=lambda node: self._enter_handler(
node,
is_on_lambda_str=None,
is_on_lambda_body=None,
)
)
return ast
def traverse_string_expr_content(node, on_enter=None, on_leave=None):
string_expr_content_nodes = get_string_expr_content(node)
if string_expr_content_nodes is None:
return
for child_node in string_expr_content_nodes:
traverse(child_node, on_enter=on_enter, on_leave=on_leave)
def lint_file(self, path):
self._log_file_path_to_lint(path)
try:
root_ast = self._parser.parse_file(path)
except vimlparser.VimLParserException as exception:
parse_error = self._create_parse_error(path, str(exception))
return [parse_error]
except EncodingDetectionError as exception:
decoding_error = self._create_decoding_error(path, str(exception))
return [decoding_error]
self._violations = []
self._update_listeners_map()
# Given root AST to makepolicy flexibility
lint_context = {
'path': path,
'root_node': root_ast,
'stack_trace': [],
'plugins': self._plugins,
'config': self._config.get_config_dict(),
}
traverse(root_ast,
on_enter=lambda node: self._handle_enter(node, lint_context),
on_leave=lambda node: self._handle_leave(node, lint_context))
return self._violations
def _enter_lambda_node(self, lambda_node, is_on_lambda_body, is_on_lambda_str):
# Function parameter names are in the r_list.
lambda_argument_nodes = lambda_node['rlist']
for lambda_argument_node in lambda_argument_nodes:
self._enter_identifier_terminate_node(
lambda_argument_node,
is_declarative=True,
is_lambda_argument=True,
is_on_lambda_str=is_on_lambda_str,
is_on_lambda_body=is_on_lambda_body,
)
# Traversing on lambda body context.
traverse(
lambda_node['left'],
on_enter=lambda node: self._enter_handler(
node,
is_on_lambda_body=True,
is_on_lambda_str=is_on_lambda_str,
)
)
# NOTE: Traversing to the lambda args and children was continued by the above traversing.
return SKIP_CHILDREN
def _enter_str_expr_content_node(self, call_node):
string_expr_content_nodes = get_string_expr_content(call_node)
if not string_expr_content_nodes:
return
def enter_handler(node):
self._enter_identifier_like_node(node, is_on_str_expr_context=True)
for string_expr_content_node in string_expr_content_nodes:
traverse(string_expr_content_node, on_enter=enter_handler)
def _enter_str_expr_content_node(self, call_node):
string_expr_content_nodes = get_string_expr_content(call_node)
if not string_expr_content_nodes:
return
def enter_handler(node):
self._enter_identifier_like_node(node, is_on_str_expr_context=True)
for string_expr_content_node in string_expr_content_nodes:
traverse(string_expr_content_node, on_enter=enter_handler)
def collect_identifiers(
self, ast): # type: (Dict[str, Any]) -> CollectedIdentifiers
self._static_referencing_identifiers = []
self._static_declaring_identifiers = []
# TODO: Make more performance efficiency.
traverse(ast, on_enter=self._enter_handler)
return CollectedIdentifiers(self._static_declaring_identifiers,
self._static_referencing_identifiers)
def _enter_lambda_str_expr_content_node(self, lambda_string_expr_content_nodes, is_on_lambda_body):
for string_expr_content_node in lambda_string_expr_content_nodes:
traverse(
string_expr_content_node,
on_enter=lambda node: self._enter_handler(
node,
is_on_lambda_str=True,
is_on_lambda_body=is_on_lambda_body,
)
)
def traverse_string_expr_content(node, on_enter=None, on_leave=None):
lambda_string_expr_content_nodes = get_lambda_string_expr_content(node)
if lambda_string_expr_content_nodes is not None:
for child_node in lambda_string_expr_content_nodes:
traverse(child_node, on_enter=on_enter, on_leave=on_leave)
func_ref_string_expr_content_nodes = get_function_reference_string_expr_content(node)
if func_ref_string_expr_content_nodes is not None:
for child_node in func_ref_string_expr_content_nodes:
traverse(child_node, on_enter=on_enter, on_leave=on_leave)
def _enter_lambda_str_expr_content_node(self, lambda_string_expr_content_nodes, is_on_lambda_body):
for string_expr_content_node in lambda_string_expr_content_nodes:
traverse(
string_expr_content_node,
on_enter=lambda node: self._enter_handler(
node,
is_on_lambda_str=True,
is_on_lambda_body=is_on_lambda_body,
)
)
def is_valid(self, node, lint_context):
""" Whether the specified node is valid.
This policy prohibit scriptencoding missing when multibyte char exists.
"""
traverse(node, on_enter=self._check_scriptencoding)
if self.has_scriptencoding:
return True
return not _has_multibyte_char(lint_context)
def collect_identifiers(self, ast): # type: (Dict[str, Any]) -> CollectedIdentifiers
self._static_referencing_identifiers = []
self._static_declaring_identifiers = []
# TODO: Make more performance efficiency.
traverse(ast, on_enter=self._enter_handler)
return CollectedIdentifiers(
self._static_declaring_identifiers,
self._static_referencing_identifiers
)
def is_valid(self, node, lint_context):
""" Whether the specified node is valid.
This policy prohibit scriptencoding missing when multibyte char exists.
"""
traverse(node, on_enter=self._check_scriptencoding)
if self.has_scriptencoding:
return True
return not self._check_script_has_multibyte_char(lint_context)
def collect_identifiers(self, ast):
self.static_referencing_identifiers = []
self.static_declaring_identifiers = []
# TODO: Make more performance efficiency.
traverse(ast, on_enter=self._enter_handler)
return {
'static_declaring_identifiers': self.static_declaring_identifiers,
'static_referencing_identifiers': self.static_referencing_identifiers,
}
def collect_identifiers(self, ast):
self.static_referencing_identifiers = []
self.static_declaring_identifiers = []
# TODO: Make more performance efficiency.
traverse(ast, on_enter=self._enter_handler)
return {
'static_declaring_identifiers':
self.static_declaring_identifiers,
'static_referencing_identifiers':
self.static_referencing_identifiers,
}
def _set_string_expr_context_flag(cls, string_expr_content_nodes):
def enter_handler(node):
# NOTE: We need this flag only string nodes, because this flag is only for
# ProhibitUnnecessaryDoubleQuote.
if NodeType(node['type']) is NodeType.STRING:
node[STRING_EXPR_CONTEXT] = {
STRING_EXPR_CONTEXT_FLAG: True,
}
for string_expr_content_node in string_expr_content_nodes:
traverse(string_expr_content_node, on_enter=enter_handler)
return string_expr_content_nodes
def _traverse(self, root_ast, path):
self._prepare_for_traversal()
lint_context = {
'path': path,
'root_node': root_ast,
'stack_trace': [],
'plugins': self._plugins,
'config': self._config.get_config_dict(),
}
traverse(root_ast,
on_enter=lambda node: self._handle_enter(node, lint_context),
on_leave=lambda node: self._handle_leave(node, lint_context))
def test_traverse(self):
expected_order_of_events = [
{'node_type': NodeType.TOPLEVEL, 'handler': 'enter'},
{'node_type': NodeType.LET, 'handler': 'enter'},
{'node_type': NodeType.IDENTIFIER, 'handler': 'enter'},
{'node_type': NodeType.IDENTIFIER, 'handler': 'leave'},
{'node_type': NodeType.NUMBER, 'handler': 'enter'},
{'node_type': NodeType.NUMBER, 'handler': 'leave'},
{'node_type': NodeType.LET, 'handler': 'leave'},
{'node_type': NodeType.WHILE, 'handler': 'enter'},
{'node_type': NodeType.SMALLER, 'handler': 'enter'},
{'node_type': NodeType.IDENTIFIER, 'handler': 'enter'},
{'node_type': NodeType.IDENTIFIER, 'handler': 'leave'},
{'node_type': NodeType.NUMBER, 'handler': 'enter'},
{'node_type': NodeType.NUMBER, 'handler': 'leave'},
{'node_type': NodeType.SMALLER, 'handler': 'leave'},
{'node_type': NodeType.ECHO, 'handler': 'enter'},
{'node_type': NodeType.STRING, 'handler': 'enter'},
{'node_type': NodeType.STRING, 'handler': 'leave'},
{'node_type': NodeType.IDENTIFIER, 'handler': 'enter'},
{'node_type': NodeType.IDENTIFIER, 'handler': 'leave'},
{'node_type': NodeType.ECHO, 'handler': 'leave'},
{'node_type': NodeType.LET, 'handler': 'enter'},
{'node_type': NodeType.IDENTIFIER, 'handler': 'enter'},
{'node_type': NodeType.IDENTIFIER, 'handler': 'leave'},
{'node_type': NodeType.NUMBER, 'handler': 'enter'},
{'node_type': NodeType.NUMBER, 'handler': 'leave'},
{'node_type': NodeType.LET, 'handler': 'leave'},
{'node_type': NodeType.ENDWHILE, 'handler': 'enter'},
{'node_type': NodeType.ENDWHILE, 'handler': 'leave'},
{'node_type': NodeType.WHILE, 'handler': 'leave'},
{'node_type': NodeType.TOPLEVEL, 'handler': 'leave'},
]
# Records visit node type name in order
actual_order_of_events = []
traverse(self.ast,
on_enter=lambda node: actual_order_of_events.append({
'node_type': NodeType(node['type']),
'handler': 'enter',
}),
on_leave=lambda node: actual_order_of_events.append({
'node_type': NodeType(node['type']),
'handler': 'leave',
}))
self.maxDiff = 2048
self.assertEqual(actual_order_of_events, expected_order_of_events)
def _parse_string_expr_content_nodes(cls, string_expr_node):
parser = Parser()
string_expr_content_nodes = parser.parse_string_expr(string_expr_node)
def enter_handler(node):
# NOTE: We need this flag only string nodes, because this flag is only for
# ProhibitUnnecessaryDoubleQuote.
if NodeType(node['type']) is NodeType.STRING:
node[STRING_EXPR_CONTEXT] = {
'is_on_str_expr_context': True,
}
for string_expr_content_node in string_expr_content_nodes:
traverse(string_expr_content_node, on_enter=enter_handler)
return string_expr_content_nodes
def assertVariablesUnused(self, expected_variables_unused, scope_plugin, ast):
dec_id_footstamp_map = {id_name: False for id_name
in expected_variables_unused.values()}
def enter_handler(node):
if is_declarative_identifier(node):
id_name = node['value']
pprint(node)
self.assertEqual(expected_variables_unused[id_name],
scope_plugin.is_unused_declarative_identifier(node))
dec_id_footstamp_map[id_name] = True
traverse(ast, on_enter=enter_handler)
self.assertTrue(dec_id_footstamp_map.values())
def process(self, ast): # type: (Dict[str, Any]) -> None
""" Build a scope tree and links between scopes and identifiers by the
specified ast. You can access the built scope tree and the built links
by .scope_tree and .link_registry.
"""
id_classifier = IdentifierClassifier()
attached_ast = id_classifier.attach_identifier_attributes(ast)
# We are already in script local scope.
self._scope_tree_builder.enter_new_scope(ScopeVisibility.SCRIPT_LOCAL)
traverse(attached_ast,
on_enter=self._enter_handler,
on_leave=self._leave_handler)
self.scope_tree = self._scope_tree_builder.get_global_scope()
self.link_registry = self._scope_tree_builder.link_registry
def process(self, ast): # type: (Dict[str, Any]) -> None
""" Build a scope tree and links between scopes and identifiers by the
specified ast. You can access the built scope tree and the built links
by .scope_tree and .link_registry.
"""
id_classifier = IdentifierClassifier()
attached_ast = id_classifier.attach_identifier_attributes(ast)
# We are already in script local scope.
self._scope_tree_builder.enter_new_scope(ScopeVisibility.SCRIPT_LOCAL)
traverse(attached_ast,
on_enter=self._enter_handler,
on_leave=self._leave_handler)
self.scope_tree = self._scope_tree_builder.get_global_scope()
self.link_registry = self._scope_tree_builder.link_registry
def assertVariablesUndeclared(self, expected_variables_undeclared, scope_plugin, ast):
ref_id_footstamp_map = {id_name: False for id_name
in expected_variables_undeclared.values()}
def enter_handler(node):
if is_reference_identifier(node):
id_name = node['value']
pprint(node)
self.assertEqual(expected_variables_undeclared[id_name],
scope_plugin.is_unreachable_reference_identifier(node))
ref_id_footstamp_map[id_name] = True
traverse(ast, on_enter=enter_handler)
# Check all exlected identifiers were tested
self.assertTrue(ref_id_footstamp_map.values())
def test_traverse(self):
ast = self.create_ast(Fixtures.REDIR_VARIABLE)
parser = RedirAssignmentParser()
got_ast = parser.process(ast)
is_redir_content_visited = {
'g:var': False,
}
def enter_handler(node):
if NodeType(node['type']) is not NodeType.IDENTIFIER:
return
is_redir_content_visited[node['value']] = True
traverse(got_ast, on_enter=enter_handler)
self.assertTrue(all(is_redir_content_visited.values()))
def test_traverse(self):
ast = self.create_ast(Fixtures.REDIR_VARIABLE)
parser = RedirAssignmentParser()
got_ast = parser.process(ast)
is_redir_content_visited = {
'g:var': False,
}
def enter_handler(node):
if NodeType(node['type']) is not NodeType.IDENTIFIER:
return
is_redir_content_visited[node['value']] = True
traverse(got_ast, on_enter=enter_handler)
self.assertTrue(all(is_redir_content_visited.values()))
def test_traverse(self):
ast = self.create_ast(Fixtures.MAP_AND_FILTER_VARIABLE)
parser = CallNodeParser()
got_ast = parser.process(ast)
is_map_and_filter_content_visited = {
'v:val': False,
'v:key': False,
}
def enter_handler(node):
if NodeType(node['type']) is not NodeType.IDENTIFIER:
return
is_map_and_filter_content_visited[node['value']] = True
traverse(got_ast, on_enter=enter_handler)
self.assertTrue(all(is_map_and_filter_content_visited.values()))
def _handle_lambda_node(
self, lambda_node): # type: (Dict[str, Any]) -> Optional[str]
# This method do the following 4 steps:
# 1. Create a new scope of the lambda
# 2. The current scope point to the new scope
# 3. Add parameters to the new scope
# 4. Add variables in the function body to the new scope
# 1. Create a new scope of the function
# 2. The current scope point to the new scope
self._scope_tree_builder.enter_new_scope(ScopeVisibility.LAMBDA)
# 3. Add parameters to the new scope
has_variadic_symbol = False
param_nodes = lambda_node['rlist']
for param_node in param_nodes:
if param_node['value'] == '...':
has_variadic_symbol = True
else:
# the param_node type is always NodeType.IDENTIFIER
self._scope_tree_builder.handle_new_parameter_found(
param_node, is_lambda_argument=True)
# We can access a:0 and a:000 when the number of arguments is less than actual parameters.
self._scope_tree_builder.handle_new_parameters_list_and_length_found()
# In the context of lambda, we can access a:1 ... a:n when the number of arguments is less than actual parameters.
# XXX: We can not know what a:N we can access by static analysis, so we assume it is 20.
if has_variadic_symbol:
lambda_args_len = len(param_nodes) - 1
else:
lambda_args_len = len(param_nodes)
self._scope_tree_builder.handle_new_index_parameters_found(
lambda_args_len)
# 4. Add variables in the function body to the new scope
traverse(lambda_node['left'],
on_enter=self._enter_handler,
on_leave=self._leave_handler)
# Skip child nodes traversing
return SKIP_CHILDREN
def test_traverse(self):
ast = self.create_ast(Fixtures.MAP_AND_FILTER_VARIABLE)
parser = MapAndFilterParser()
got_ast = parser.process(ast)
is_map_and_filter_content_visited = {
'v:val': False,
'v:key': False,
}
def enter_handler(node):
if NodeType(node['type']) is not NodeType.IDENTIFIER:
return
is_map_and_filter_content_visited[node['value']] = True
traverse(got_ast, on_enter=enter_handler)
self.assertTrue(all(is_map_and_filter_content_visited.values()))
def test_process_with_builtin(self):
parser = Parser()
ast = parser.parse_file(Fixtures['BUILTIN'])
plugin = ScopePlugin()
plugin.process(ast)
expected_builtin_flags = {
'abs': True,
'sin': True,
'strlen': True,
'g:MyFunction': False,
}
# Keep identifier name that traverser visited
identifiers_checking_map = {
'abs': False,
'sin': False,
'strlen': False,
'g:MyFunction': False,
}
def test_identifier(node):
if NodeType(node['type']) is not NodeType.IDENTIFIER:
return
identifier = node
# We focus to non-definition identifier
if identifier[ScopePlugin.DEFINITION_IDENTIFIER_FLAG_KEY]:
return
identifier_name = identifier['value']
identifiers_checking_map[identifier_name] = True
is_builtin_identifier = identifier[
ScopePlugin.BUILTIN_IDENTIFIER_FLAG_KEY]
expected_builtin_flag = expected_builtin_flags[identifier_name]
self.assertEqual(is_builtin_identifier, expected_builtin_flag)
traverse(ast, on_enter=test_identifier)
self.assertTrue(all(identifiers_checking_map.values()))
def test_process_with_builtin(self):
parser = Parser()
ast = parser.parse_file(Fixtures['BUILTIN'])
plugin = ScopePlugin()
plugin.process(ast)
expected_builtin_flags = {
'abs': True,
'sin': True,
'strlen': True,
'g:MyFunction': False,
}
# Keep identifier name that traverser visited
identifiers_checking_map = {
'abs': False,
'sin': False,
'strlen': False,
'g:MyFunction': False,
}
def test_identifier(node):
if NodeType(node['type']) is not NodeType.IDENTIFIER:
return
identifier = node
# We focus to non-definition identifier
if identifier[ScopePlugin.DEFINITION_IDENTIFIER_FLAG_KEY]:
return
identifier_name = identifier['value']
identifiers_checking_map[identifier_name] = True
is_builtin_identifier = identifier[ScopePlugin.BUILTIN_IDENTIFIER_FLAG_KEY]
expected_builtin_flag = expected_builtin_flags[identifier_name]
self.assertEqual(is_builtin_identifier, expected_builtin_flag)
traverse(ast, on_enter=test_identifier)
self.assertTrue(all(identifiers_checking_map.values()))
def assertVariablesUnused(self, expected_variables_unused, scope_plugin,
ast):
dec_id_footstamp_map = {
id_name: False
for id_name in expected_variables_unused.values()
}
def enter_handler(node):
if is_declarative_identifier(node):
id_name = node['value']
pprint(node)
self.assertEqual(
expected_variables_unused[id_name],
scope_plugin.is_unused_declarative_identifier(node))
dec_id_footstamp_map[id_name] = True
traverse(ast, on_enter=enter_handler)
self.assertTrue(dec_id_footstamp_map.values())
def process(self, ast):
def enter_handler(node):
node_type = NodeType(node['type'])
if node_type is not NodeType.CALL:
return
called_function_identifier = node['left']
# Name node of the "map" or "filter" functions are always IDENTIFIER.
if NodeType(called_function_identifier['type']
) is not NodeType.IDENTIFIER:
return
is_map_or_function_call = called_function_identifier.get(
'value') in {
'map': True,
'filter': True,
}
if not is_map_or_function_call:
return
args = node['rlist']
# Prevent crash. See https://github.com/Kuniwak/vint/issues/256.
if len(args) < 2:
return
string_expr_node = args[1]
# We can analyze only STRING nodes by static analyzing.
if NodeType(string_expr_node['type']) is not NodeType.STRING:
return
string_expr_content_nodes = MapAndFilterParser._parse_string_expr_content_nodes(
string_expr_node)
node[STRING_EXPR_CONTENT] = string_expr_content_nodes
traverse(ast, on_enter=enter_handler)
return ast
def assertVariablesUndeclared(self, expected_variables_undeclared,
scope_plugin, ast):
ref_id_footstamp_map = {
id_name: False
for id_name in expected_variables_undeclared.values()
}
def enter_handler(node):
if is_reference_identifier(node):
id_name = node['value']
pprint(node)
self.assertEqual(
expected_variables_undeclared[id_name],
scope_plugin.is_unreachable_reference_identifier(node))
ref_id_footstamp_map[id_name] = True
traverse(ast, on_enter=enter_handler)
# Check all exlected identifiers were tested
self.assertTrue(ref_id_footstamp_map.values())
def parse_redir(self, redir_cmd):
""" Parse a command :redir content. """
redir_cmd_str = redir_cmd['str']
matched = re.match(r'redir?!?\s*(=>>?\s*)(\S+)', redir_cmd_str)
if matched:
redir_cmd_op = matched.group(1)
redir_cmd_body = matched.group(2)
arg_pos = redir_cmd['ea']['argpos']
# Position of the "redir_cmd_body"
start_pos = {
'col': arg_pos['col'] + len(redir_cmd_op),
'i': arg_pos['i'] + len(redir_cmd_op),
'lnum': arg_pos['lnum'],
}
# NOTE: This is a hack to parse variable node.
raw_ast = self.parse_string('echo ' + redir_cmd_body)
# We need the left node of ECHO node
redir_cmd_ast = raw_ast['body'][0]['list'][0]
def adjust_position(node):
pos = node['pos']
# Care 1-based index and the length of "echo ".
pos['col'] += start_pos['col'] - 1 - 5
# Care the length of "echo ".
pos['i'] += start_pos['i'] - 5
# Care 1-based index
pos['lnum'] += start_pos['lnum'] - 1
traverse(redir_cmd_ast, on_enter=adjust_position)
return redir_cmd_ast
return None
def _handle_lambda_node(self, lambda_node): # type: (Dict[str, Any]) -> Optional[str]
# This method do the following 4 steps:
# 1. Create a new scope of the lambda
# 2. The current scope point to the new scope
# 3. Add parameters to the new scope
# 4. Add variables in the function body to the new scope
# 1. Create a new scope of the function
# 2. The current scope point to the new scope
self._scope_tree_builder.enter_new_scope(ScopeVisibility.LAMBDA)
# 3. Add parameters to the new scope
has_variadic_symbol = False
param_nodes = lambda_node['rlist']
for param_node in param_nodes:
if param_node['value'] == '...':
has_variadic_symbol = True
else:
# the param_node type is always NodeType.IDENTIFIER
self._scope_tree_builder.handle_new_parameter_found(param_node, is_lambda_argument=True)
# We can access a:0 and a:000 when the number of arguments is less than actual parameters.
self._scope_tree_builder.handle_new_parameters_list_and_length_found()
# In the context of lambda, we can access a:1 ... a:n when the number of arguments is less than actual parameters.
# XXX: We can not know what a:N we can access by static analysis, so we assume it is 20.
if has_variadic_symbol:
lambda_args_len = len(param_nodes) - 1
else:
lambda_args_len = len(param_nodes)
self._scope_tree_builder.handle_new_index_parameters_found(lambda_args_len)
# 4. Add variables in the function body to the new scope
traverse(lambda_node['left'],
on_enter=self._enter_handler,
on_leave=self._leave_handler)
# Skip child nodes traversing
return SKIP_CHILDREN
def parse_redir(self, redir_cmd):
""" Parse a command :redir content. """
redir_cmd_str = redir_cmd['str']
matched = re.match(r'redir?!?\s*(=>>?\s*)(\S+)', redir_cmd_str)
if matched:
redir_cmd_op = matched.group(1)
redir_cmd_body = matched.group(2)
arg_pos = redir_cmd['ea']['argpos']
# Position of the "redir_cmd_body"
start_pos = {
'col': arg_pos['col'] + len(redir_cmd_op),
'i': arg_pos['i'] + len(redir_cmd_op),
'lnum': arg_pos['lnum'],
}
# NOTE: This is a hack to parse variable node.
raw_ast = self.parse('echo ' + redir_cmd_body)
# We need the left node of ECHO node
redir_cmd_ast = raw_ast['body'][0]['list'][0]
def adjust_position(node):
pos = node['pos']
# Care 1-based index and the length of "echo ".
pos['col'] += start_pos['col'] - 1 - 5
# Care the length of "echo ".
pos['i'] += start_pos['i'] - 5
# Care 1-based index
pos['lnum'] += start_pos['lnum'] - 1
traverse(redir_cmd_ast, on_enter=adjust_position)
return redir_cmd_ast
return None
def process(self, ast):
def enter_handler(node):
node_type = NodeType(node['type'])
if node_type is not NodeType.EXCMD:
return
is_redir_command = node['ea']['cmd'].get('name') == 'redir'
if not is_redir_command:
return
redir_cmd_str = node['str']
is_redir_assignment = '=>' in redir_cmd_str
if not is_redir_assignment:
return
parser = Parser()
redir_content_node = parser.parse_redir(node)
node[REDIR_CONTENT] = redir_content_node
traverse(ast, on_enter=enter_handler)
return ast
def process(self, ast):
def enter_handler(node):
node_type = NodeType(node['type'])
if node_type is not NodeType.EXCMD:
return
is_redir_command = node['ea']['cmd'].get('name') == 'redir'
if not is_redir_command:
return
redir_cmd_str = node['str']
is_redir_assignment = '=>' in redir_cmd_str
if not is_redir_assignment:
return
parser = Parser()
redir_content_node = parser.parse_redir(node)
node[REDIR_CONTENT] = redir_content_node
traverse(ast, on_enter=enter_handler)
return ast
def lint_file(self, path):
try:
root_ast = self._parser.parse_file(path)
except VimLParserException as exception:
parse_error = self._create_parse_error(path, str(exception))
return [parse_error]
except UnicodeDecodeError as exception:
# TODO: Is this a good way? I think a vint error is better.
unicode_decode_error = self._create_unicode_error(
path, str(exception))
return [unicode_decode_error]
self._violations = []
self._update_listeners_map()
# Given root AST to makepolicy flexibility
lint_context = {'path': path, 'root_node': root_ast, 'stack_trace': []}
traverse(root_ast,
on_enter=lambda node: self._handle_enter(node, lint_context),
on_leave=lambda node: self._handle_leave(node, lint_context))
return self._violations
def lint_file(self, path):
try:
root_ast = self._parser.parse_file(path)
except VimLParserException as exception:
parse_error = self._create_parse_error(path, str(exception))
return [parse_error]
except UnicodeDecodeError as exception:
# TODO: Is this a good way? I think a vint error is better.
unicode_decode_error = self._create_unicode_error(path, str(exception))
return [unicode_decode_error]
self._violations = []
self._update_listeners_map()
# Given root AST to makepolicy flexibility
lint_context = {'path': path, 'root_node': root_ast, 'stack_trace': []}
traverse(root_ast,
on_enter=lambda node: self._handle_enter(node, lint_context),
on_leave=lambda node: self._handle_leave(node, lint_context))
return self._violations
def process(self, ast):
def enter_handler(node):
node_type = NodeType(node['type'])
if node_type is not NodeType.CALL:
return
called_function_identifier = node['left']
# The name node type of "map" or "filter" or "call" are always IDENTIFIER.
if NodeType(called_function_identifier['type']) is not NodeType.IDENTIFIER:
return
called_function_identifier_value = called_function_identifier.get('value')
if called_function_identifier_value in ['map', 'filter']:
# Analyze second argument of "map" or "filter" if the node type is STRING.
self._attach_string_expr_content_to_map_or_func(node)
elif called_function_identifier_value in ['call', 'function']:
# Analyze first argument of "call" or "function" if the node type is STRING.
self._attach_string_expr_content_to_call_or_function(node)
traverse(ast, on_enter=enter_handler)
return ast
def process(self, ast):
def enter_handler(node):
node_type = NodeType(node['type'])
if node_type is not NodeType.CALL:
return
called_function_identifier = node['left']
# Name node of the "map" or "filter" functions are always IDENTIFIER.
if NodeType(called_function_identifier['type']
) is not NodeType.IDENTIFIER:
return
is_map_or_function_call = called_function_identifier.get(
'value') in {
'map': True,
'filter': True,
}
if not is_map_or_function_call:
return
string_expr_node = node['rlist'][1]
# We can analyze only STRING nodes by static analyzing.
if NodeType(string_expr_node['type']) is not NodeType.STRING:
return
parser = Parser()
string_expr_content_nodes = parser.parse_string_expr(
string_expr_node)
node[STRING_EXPR_CONTENT] = string_expr_content_nodes
traverse(ast, on_enter=enter_handler)
return ast
def _traverse(self, root_ast, lint_target):
if self._is_debug:
logging.debug('{cls}: checking `{file_path}`'.format(
cls=self.__class__.__name__,
file_path=lint_target.path)
)
logging.debug('{cls}: using config as {config_dict}'.format(
cls=self.__class__.__name__,
config_dict=self._config_dict_global
))
self._prepare_for_traversal()
lint_context = {
'lint_target': lint_target,
'root_node': root_ast,
'stack_trace': [],
'plugins': self._plugins,
'config': self._config.get_config_dict(),
}
traverse(root_ast,
on_enter=lambda node: self._handle_enter(node, lint_context),
on_leave=lambda node: self._handle_leave(node, lint_context))
def _handle_function_node(self, func_node): # type: (Dict[str, Any]) -> None
# We should interrupt traversing, because a node of the function
# name should be added to the parent scope before the current
# scope switched to a new scope of the function.
# We approach to it by the following 5 steps.
# 1. Add the function to the current scope
# 2. Create a new scope of the function
# 3. The current scope point to the new scope
# 4. Add parameters to the new scope
# 5. Add variables in the function body to the new scope
# 1. Add the function to the current scope
func_name_node = func_node['left']
traverse(func_name_node, on_enter=self._find_variable_like_nodes)
# 2. Create a new scope of the function
# 3. The current scope point to the new scope
self._scope_tree_builder.enter_new_scope(ScopeVisibility.FUNCTION_LOCAL)
has_variadic = False
# 4. Add parameters to the new scope
param_nodes = func_node['rlist']
for param_node in param_nodes:
if param_node['value'] == '...':
has_variadic = True
else:
# the param_node type is always NodeType.IDENTIFIER
self._scope_tree_builder.handle_new_parameter_found(param_node, is_lambda_argument=False)
# We can always access a:0, a:000
self._scope_tree_builder.handle_new_parameters_list_and_length_found()
# In a variadic function, we can access a:1 ... a:n
# (n = 20 - explicit parameters length). See :help a:0
if has_variadic:
# -1 means ignore '...'
self._scope_tree_builder.handle_new_index_parameters_found(len(param_nodes) - 1)
# We can access "a:firstline" and "a:lastline" if the function is
# declared with an attribute "range". See :func-range
attr = func_node['attr']
is_declared_with_range = attr['range'] is not 0
if is_declared_with_range:
self._scope_tree_builder.handle_new_range_parameters_found()
# We can access "l:self" is declared with an attribute "dict" or
# the function is a member of a dict. See :help self
is_declared_with_dict = attr['dict'] is not 0 \
or NodeType(func_name_node['type']) in FunctionNameNodesDeclaringVariableSelf
if is_declared_with_dict:
self._scope_tree_builder.handle_new_dict_parameter_found()
# 5. Add variables in the function body to the new scope
func_body_nodes = func_node['body']
for func_body_node in func_body_nodes:
traverse(func_body_node,
on_enter=self._enter_handler,
on_leave=self._leave_handler)
# Skip child nodes traversing
return SKIP_CHILDREN
def _handle_function_node(self, func_node):
# We should interrupt traversing, because a node of the function
# name should be added to the parent scope before the current
# scope switched to a new scope of the function.
# We approach to it by the following 5 steps.
# 1. Add the function to the current scope
# 2. Create a new scope of the function
# 3. The current scope point to the new scope
# 4. Add parameters to the new scope
# 5. Add variables in the function body to the new scope
# 1. Add the function to the current scope
func_name_node = func_node['left']
traverse(func_name_node, on_enter=self._find_variable_like_nodes)
# 2. Create a new scope of the function
# 3. The current scope point to the new scope
self._scope_tree_builder.enter_new_scope(
ScopeVisibility.FUNCTION_LOCAL)
has_variadic = False
# 4. Add parameters to the new scope
param_nodes = func_node['rlist']
for param_node in param_nodes:
if param_node['value'] == '...':
has_variadic = True
else:
# the param_node type is always NodeType.IDENTIFIER
self._scope_tree_builder.handle_new_parameter_found(param_node)
# We can always access a:0, a:000
self._scope_tree_builder.handle_new_parameters_list_and_length_found()
# In a variadic function, we can access a:1 ... a:n
# (n = 20 - explicit parameters length). See :help a:0
if has_variadic:
# -1 means ignore '...'
self._scope_tree_builder.handle_new_index_parameters_found(
len(param_nodes) - 1)
# We can access "a:firstline" and "a:lastline" if the function is
# declared with an attribute "range". See :func-range
attr = func_node['attr']
is_declared_with_range = attr['range'] is not 0
if is_declared_with_range:
self._scope_tree_builder.handle_new_range_parameters_found()
# We can access "l:self" is declared with an attribute "dict" or
# the function is a member of a dict. See :help self
is_declared_with_dict = attr['dict'] is not 0 \
or NodeType(func_name_node['type']) in FunctionNameNodesDeclaringVariableSelf
if is_declared_with_dict:
self._scope_tree_builder.handle_new_dict_parameter_found()
# 5. Add variables in the function body to the new scope
func_body_nodes = func_node['body']
for func_body_node in func_body_nodes:
traverse(func_body_node,
on_enter=self._enter_handler,
on_leave=self._leave_handler)
# Skip child nodes traversing
return SKIP_CHILDREN
def traverse_redir_content(node, on_enter=None, on_leave=None):
if REDIR_CONTENT not in node:
return
traverse(node[REDIR_CONTENT], on_enter=on_enter, on_leave=on_leave)
def test_traverse_ignoring_while_children(self):
expected_order_of_events = [
{
'node_type': NodeType.TOPLEVEL,
'handler': 'enter'
},
{
'node_type': NodeType.LET,
'handler': 'enter'
},
{
'node_type': NodeType.IDENTIFIER,
'handler': 'enter'
},
{
'node_type': NodeType.IDENTIFIER,
'handler': 'leave'
},
{
'node_type': NodeType.NUMBER,
'handler': 'enter'
},
{
'node_type': NodeType.NUMBER,
'handler': 'leave'
},
{
'node_type': NodeType.LET,
'handler': 'leave'
},
{
'node_type': NodeType.WHILE,
'handler': 'enter'
},
{
'node_type': NodeType.WHILE,
'handler': 'leave'
},
{
'node_type': NodeType.TOPLEVEL,
'handler': 'leave'
},
]
def on_enter(node):
actual_order_of_events.append({
'node_type': NodeType(node['type']),
'handler': 'enter',
})
if NodeType(node['type']) is NodeType.WHILE:
return SKIP_CHILDREN
# Records visit node type name in order
actual_order_of_events = []
traverse(self.ast,
on_enter=on_enter,
on_leave=lambda node: actual_order_of_events.append(
{
'node_type': NodeType(node['type']),
'handler': 'leave',
}))
self.maxDiff = 2048
self.assertEqual(actual_order_of_events, expected_order_of_events)
from vint.ast.plugin.scope_plugin import ScopePlugin
def prettify_node_type(node):
node['type'] = NodeType(node['type'])
if __name__ == '__main__':
arg_parser = ArgumentParser(prog='show_ast', description='Show AST')
arg_parser.add_argument('--enable-neovim', action='store_true', help='Enable Neovim syntax')
arg_parser.add_argument('files', nargs='*', help='File to parse')
namespace = vars(arg_parser.parse_args(sys.argv[1:]))
filepaths = map(Path, namespace['files'])
enable_neovim = namespace['enable_neovim']
scope_plugin = ScopePlugin()
parser = Parser(plugins=[scope_plugin], enable_neovim=enable_neovim)
for filepath in filepaths:
ast = parser.parse_file(filepath)
traverse(ast, on_enter=prettify_node_type)
print("////////// SCOPE TREE //////////\n")
pprint(scope_plugin._ref_tester._scope_linker.scope_tree)
print("\n\n")
print("////////// LINK REGISTRY //////////\n")
pprint(scope_plugin._ref_tester._scope_linker.link_registry._vars_to_declarative_ids_map)
pprint(scope_plugin._ref_tester._scope_linker.link_registry._ids_to_scopes_map)
