def parse_string_expr(self, string_expr_node): """ Parse a string node content. """ string_expr_node_value = string_expr_node['value'] string_expr_str = string_expr_node_value[1:-1] # Care escaped string literals if string_expr_node_value[0] == "'": string_expr_str = string_expr_str.replace("''", "'") else: string_expr_str = string_expr_str.replace('\\"', '"') # NOTE: This is a hack to parse expr1. See :help expr1 raw_ast = self.parse_string('echo ' + string_expr_str) # We need the left node of ECHO node parsed_string_expr_nodes = raw_ast['body'][0]['list'] start_pos = string_expr_node['pos'] def adjust_position(node): pos = node['pos'] # Care 1-based index and the length of "echo ". pos['col'] += start_pos['col'] - 1 - 5 # Care the length of "echo ". pos['i'] += start_pos['i'] - 5 # Care 1-based index pos['lnum'] += start_pos['lnum'] - 1 for parsed_string_expr_node in parsed_string_expr_nodes: traverse(parsed_string_expr_node, on_enter=adjust_position) return parsed_string_expr_nodes
def lint_file(self, path): self._log_file_path_to_lint(path) try: root_ast = self._parser.parse_file(path) except vimlparser.VimLParserException as exception: parse_error = self._create_parse_error(path, str(exception)) return [parse_error] except EncodingDetectionError as exception: decoding_error = self._create_decoding_error(path, str(exception)) return [decoding_error] self._violations = [] self._update_listeners_map() # Given root AST to makepolicy flexibility lint_context = { 'path': path, 'root_node': root_ast, 'stack_trace': [], 'plugins': self._plugins, 'config': self._config.get_config_dict(), } traverse(root_ast, on_enter=lambda node: self._handle_enter(node, lint_context), on_leave=lambda node: self._handle_leave(node, lint_context)) return self._violations
def attach_identifier_attributes(self, ast): """ Attach 5 flags to the AST. - is dynamic: True if the identifier name can be determined by static analysis. - is member: True if the identifier is a member of a subscription/dot/slice node. - is declaring: True if the identifier is used to declare. - is autoload: True if the identifier is declared with autoload. - is function: True if the identifier is a function. Vim distinguish between function identifiers and variable identifiers. - is declarative paramter: True if the identifier is a declarative parameter. For example, the identifier "param" in Func(param) is a declarative paramter. - is on string expression context: True if the variable is on the string expression context. The string expression context is the string content on the 2nd argument of the map or filter function. """ redir_assignment_parser = RedirAssignmentParser() ast_with_parsed_redir = redir_assignment_parser.process(ast) map_and_filter_parser = MapAndFilterParser() ast_with_parse_map_and_filter_and_redir = \ map_and_filter_parser.process(ast_with_parsed_redir) traverse(ast_with_parse_map_and_filter_and_redir, on_enter=self._enter_handler) return ast
def traverse_string_expr_content(node, on_enter=None, on_leave=None): string_expr_content_nodes = get_string_expr_content(node) if string_expr_content_nodes is None: return for child_node in string_expr_content_nodes: traverse(child_node, on_enter=on_enter, on_leave=on_leave)
def parse_string_expr(self, string_expr_node): """ Parse a command :redir content. """ string_expr_node_value = string_expr_node['value'] string_expr_str = string_expr_node_value[1:-1] # Care escaped string literals if string_expr_node_value[0] == "'": string_expr_str = string_expr_str.replace("''", "'") else: string_expr_str = string_expr_str.replace('\\"', '"') # NOTE: This is a hack to parse expr1. See :help expr1 raw_ast = self.parse('echo ' + string_expr_str) # We need the left node of ECHO node parsed_string_expr_nodes = raw_ast['body'][0]['list'] start_pos = string_expr_node['pos'] def adjust_position(node): pos = node['pos'] # Care 1-based index and the length of "echo ". pos['col'] += start_pos['col'] - 1 - 5 # Care the length of "echo ". pos['i'] += start_pos['i'] - 5 # Care 1-based index pos['lnum'] += start_pos['lnum'] - 1 for parsed_string_expr_node in parsed_string_expr_nodes: traverse(parsed_string_expr_node, on_enter=adjust_position) return parsed_string_expr_nodes
def process(self, ast): def enter_handler(node): node_type = NodeType(node['type']) if node_type is not NodeType.CALL: return called_function_identifier = node['left'] # Name node of the "map" or "filter" functions are always IDENTIFIER. if NodeType(called_function_identifier['type']) is not NodeType.IDENTIFIER: return is_map_or_function_call = called_function_identifier.get('value') in { 'map': True, 'filter': True, } if not is_map_or_function_call: return string_expr_node = node['rlist'][1] # We can analyze only STRING nodes by static analyzing. if NodeType(string_expr_node['type']) is not NodeType.STRING: return parser = Parser() string_expr_content_nodes = parser.parse_string_expr(string_expr_node) node[STRING_EXPR_CONTENT] = string_expr_content_nodes traverse(ast, on_enter=enter_handler) return ast
def _enter_lambda_node(self, lambda_node, is_on_lambda_body, is_on_lambda_str): # Function parameter names are in the r_list. lambda_argument_nodes = lambda_node['rlist'] for lambda_argument_node in lambda_argument_nodes: self._enter_identifier_terminate_node( lambda_argument_node, is_declarative=True, is_lambda_argument=True, is_on_lambda_str=is_on_lambda_str, is_on_lambda_body=is_on_lambda_body, ) # Traversing on lambda body context. traverse( lambda_node['left'], on_enter=lambda node: self._enter_handler( node, is_on_lambda_body=True, is_on_lambda_str=is_on_lambda_str, ) ) # NOTE: Traversing to the lambda args and children was continued by the above traversing. return SKIP_CHILDREN
def test_traverse_ignoring_while_children(self): expected_order_of_events = [ {'node_type': NodeType.TOPLEVEL, 'handler': 'enter'}, {'node_type': NodeType.LET, 'handler': 'enter'}, {'node_type': NodeType.IDENTIFIER, 'handler': 'enter'}, {'node_type': NodeType.IDENTIFIER, 'handler': 'leave'}, {'node_type': NodeType.NUMBER, 'handler': 'enter'}, {'node_type': NodeType.NUMBER, 'handler': 'leave'}, {'node_type': NodeType.LET, 'handler': 'leave'}, {'node_type': NodeType.WHILE, 'handler': 'enter'}, {'node_type': NodeType.WHILE, 'handler': 'leave'}, {'node_type': NodeType.TOPLEVEL, 'handler': 'leave'}, ] def on_enter(node): actual_order_of_events.append({ 'node_type': NodeType(node['type']), 'handler': 'enter', }) if NodeType(node['type']) is NodeType.WHILE: return SKIP_CHILDREN # Records visit node type name in order actual_order_of_events = [] traverse(self.ast, on_enter=on_enter, on_leave=lambda node: actual_order_of_events.append({ 'node_type': NodeType(node['type']), 'handler': 'leave', })) self.maxDiff = 2048 self.assertEqual(actual_order_of_events, expected_order_of_events)
def process(self, ast): self.current_scope = [] self.root_scope = None traverse(ast, on_enter=self._handle_enter, on_leave=self._handle_leave) ast[ScopePlugin.SCOPE_TREE_KEY] = self.root_scope
def attach_identifier_attributes(self, ast): # type: (Dict[str, Any]) -> Dict[str, Any] """ Attach 5 flags to the AST. - is dynamic: True if the identifier name can be determined by static analysis. - is member: True if the identifier is a member of a subscription/dot/slice node. - is declaring: True if the identifier is used to declare. - is autoload: True if the identifier is declared with autoload. - is function: True if the identifier is a function. Vim distinguish between function identifiers and variable identifiers. - is declarative parameter: True if the identifier is a declarative parameter. For example, the identifier "param" in Func(param) is a declarative parameter. - is on string expression context: True if the variable is on the string expression context. The string expression context is the string content on the 2nd argument of the map or filter function. - is lambda argument: True if the identifier is a lambda argument. """ redir_assignment_parser = RedirAssignmentParser() ast_with_parsed_redir = redir_assignment_parser.process(ast) map_and_filter_parser = CallNodeParser() ast_with_parse_map_and_filter_and_redir = \ map_and_filter_parser.process(ast_with_parsed_redir) traverse( ast_with_parse_map_and_filter_and_redir, on_enter=lambda node: self._enter_handler( node, is_on_lambda_str=None, is_on_lambda_body=None, ) ) return ast
def _enter_str_expr_content_node(self, call_node): string_expr_content_nodes = get_string_expr_content(call_node) if not string_expr_content_nodes: return def enter_handler(node): self._enter_identifier_like_node(node, is_on_str_expr_context=True) for string_expr_content_node in string_expr_content_nodes: traverse(string_expr_content_node, on_enter=enter_handler)
def collect_identifiers( self, ast): # type: (Dict[str, Any]) -> CollectedIdentifiers self._static_referencing_identifiers = [] self._static_declaring_identifiers = [] # TODO: Make more performance efficiency. traverse(ast, on_enter=self._enter_handler) return CollectedIdentifiers(self._static_declaring_identifiers, self._static_referencing_identifiers)
def _enter_lambda_str_expr_content_node(self, lambda_string_expr_content_nodes, is_on_lambda_body): for string_expr_content_node in lambda_string_expr_content_nodes: traverse( string_expr_content_node, on_enter=lambda node: self._enter_handler( node, is_on_lambda_str=True, is_on_lambda_body=is_on_lambda_body, ) )
def traverse_string_expr_content(node, on_enter=None, on_leave=None): lambda_string_expr_content_nodes = get_lambda_string_expr_content(node) if lambda_string_expr_content_nodes is not None: for child_node in lambda_string_expr_content_nodes: traverse(child_node, on_enter=on_enter, on_leave=on_leave) func_ref_string_expr_content_nodes = get_function_reference_string_expr_content(node) if func_ref_string_expr_content_nodes is not None: for child_node in func_ref_string_expr_content_nodes: traverse(child_node, on_enter=on_enter, on_leave=on_leave)
def is_valid(self, node, lint_context): """ Whether the specified node is valid. This policy prohibit scriptencoding missing when multibyte char exists. """ traverse(node, on_enter=self._check_scriptencoding) if self.has_scriptencoding: return True return not _has_multibyte_char(lint_context)
def collect_identifiers(self, ast): # type: (Dict[str, Any]) -> CollectedIdentifiers self._static_referencing_identifiers = [] self._static_declaring_identifiers = [] # TODO: Make more performance efficiency. traverse(ast, on_enter=self._enter_handler) return CollectedIdentifiers( self._static_declaring_identifiers, self._static_referencing_identifiers )
def is_valid(self, node, lint_context): """ Whether the specified node is valid. This policy prohibit scriptencoding missing when multibyte char exists. """ traverse(node, on_enter=self._check_scriptencoding) if self.has_scriptencoding: return True return not self._check_script_has_multibyte_char(lint_context)
def collect_identifiers(self, ast): self.static_referencing_identifiers = [] self.static_declaring_identifiers = [] # TODO: Make more performance efficiency. traverse(ast, on_enter=self._enter_handler) return { 'static_declaring_identifiers': self.static_declaring_identifiers, 'static_referencing_identifiers': self.static_referencing_identifiers, }
def _set_string_expr_context_flag(cls, string_expr_content_nodes): def enter_handler(node): # NOTE: We need this flag only string nodes, because this flag is only for # ProhibitUnnecessaryDoubleQuote. if NodeType(node['type']) is NodeType.STRING: node[STRING_EXPR_CONTEXT] = { STRING_EXPR_CONTEXT_FLAG: True, } for string_expr_content_node in string_expr_content_nodes: traverse(string_expr_content_node, on_enter=enter_handler) return string_expr_content_nodes
def _traverse(self, root_ast, path): self._prepare_for_traversal() lint_context = { 'path': path, 'root_node': root_ast, 'stack_trace': [], 'plugins': self._plugins, 'config': self._config.get_config_dict(), } traverse(root_ast, on_enter=lambda node: self._handle_enter(node, lint_context), on_leave=lambda node: self._handle_leave(node, lint_context))
def test_traverse(self): expected_order_of_events = [ {'node_type': NodeType.TOPLEVEL, 'handler': 'enter'}, {'node_type': NodeType.LET, 'handler': 'enter'}, {'node_type': NodeType.IDENTIFIER, 'handler': 'enter'}, {'node_type': NodeType.IDENTIFIER, 'handler': 'leave'}, {'node_type': NodeType.NUMBER, 'handler': 'enter'}, {'node_type': NodeType.NUMBER, 'handler': 'leave'}, {'node_type': NodeType.LET, 'handler': 'leave'}, {'node_type': NodeType.WHILE, 'handler': 'enter'}, {'node_type': NodeType.SMALLER, 'handler': 'enter'}, {'node_type': NodeType.IDENTIFIER, 'handler': 'enter'}, {'node_type': NodeType.IDENTIFIER, 'handler': 'leave'}, {'node_type': NodeType.NUMBER, 'handler': 'enter'}, {'node_type': NodeType.NUMBER, 'handler': 'leave'}, {'node_type': NodeType.SMALLER, 'handler': 'leave'}, {'node_type': NodeType.ECHO, 'handler': 'enter'}, {'node_type': NodeType.STRING, 'handler': 'enter'}, {'node_type': NodeType.STRING, 'handler': 'leave'}, {'node_type': NodeType.IDENTIFIER, 'handler': 'enter'}, {'node_type': NodeType.IDENTIFIER, 'handler': 'leave'}, {'node_type': NodeType.ECHO, 'handler': 'leave'}, {'node_type': NodeType.LET, 'handler': 'enter'}, {'node_type': NodeType.IDENTIFIER, 'handler': 'enter'}, {'node_type': NodeType.IDENTIFIER, 'handler': 'leave'}, {'node_type': NodeType.NUMBER, 'handler': 'enter'}, {'node_type': NodeType.NUMBER, 'handler': 'leave'}, {'node_type': NodeType.LET, 'handler': 'leave'}, {'node_type': NodeType.ENDWHILE, 'handler': 'enter'}, {'node_type': NodeType.ENDWHILE, 'handler': 'leave'}, {'node_type': NodeType.WHILE, 'handler': 'leave'}, {'node_type': NodeType.TOPLEVEL, 'handler': 'leave'}, ] # Records visit node type name in order actual_order_of_events = [] traverse(self.ast, on_enter=lambda node: actual_order_of_events.append({ 'node_type': NodeType(node['type']), 'handler': 'enter', }), on_leave=lambda node: actual_order_of_events.append({ 'node_type': NodeType(node['type']), 'handler': 'leave', })) self.maxDiff = 2048 self.assertEqual(actual_order_of_events, expected_order_of_events)
def _parse_string_expr_content_nodes(cls, string_expr_node): parser = Parser() string_expr_content_nodes = parser.parse_string_expr(string_expr_node) def enter_handler(node): # NOTE: We need this flag only string nodes, because this flag is only for # ProhibitUnnecessaryDoubleQuote. if NodeType(node['type']) is NodeType.STRING: node[STRING_EXPR_CONTEXT] = { 'is_on_str_expr_context': True, } for string_expr_content_node in string_expr_content_nodes: traverse(string_expr_content_node, on_enter=enter_handler) return string_expr_content_nodes
def assertVariablesUnused(self, expected_variables_unused, scope_plugin, ast): dec_id_footstamp_map = {id_name: False for id_name in expected_variables_unused.values()} def enter_handler(node): if is_declarative_identifier(node): id_name = node['value'] pprint(node) self.assertEqual(expected_variables_unused[id_name], scope_plugin.is_unused_declarative_identifier(node)) dec_id_footstamp_map[id_name] = True traverse(ast, on_enter=enter_handler) self.assertTrue(dec_id_footstamp_map.values())
def process(self, ast): # type: (Dict[str, Any]) -> None """ Build a scope tree and links between scopes and identifiers by the specified ast. You can access the built scope tree and the built links by .scope_tree and .link_registry. """ id_classifier = IdentifierClassifier() attached_ast = id_classifier.attach_identifier_attributes(ast) # We are already in script local scope. self._scope_tree_builder.enter_new_scope(ScopeVisibility.SCRIPT_LOCAL) traverse(attached_ast, on_enter=self._enter_handler, on_leave=self._leave_handler) self.scope_tree = self._scope_tree_builder.get_global_scope() self.link_registry = self._scope_tree_builder.link_registry
def assertVariablesUndeclared(self, expected_variables_undeclared, scope_plugin, ast): ref_id_footstamp_map = {id_name: False for id_name in expected_variables_undeclared.values()} def enter_handler(node): if is_reference_identifier(node): id_name = node['value'] pprint(node) self.assertEqual(expected_variables_undeclared[id_name], scope_plugin.is_unreachable_reference_identifier(node)) ref_id_footstamp_map[id_name] = True traverse(ast, on_enter=enter_handler) # Check all exlected identifiers were tested self.assertTrue(ref_id_footstamp_map.values())
def test_traverse(self): ast = self.create_ast(Fixtures.REDIR_VARIABLE) parser = RedirAssignmentParser() got_ast = parser.process(ast) is_redir_content_visited = { 'g:var': False, } def enter_handler(node): if NodeType(node['type']) is not NodeType.IDENTIFIER: return is_redir_content_visited[node['value']] = True traverse(got_ast, on_enter=enter_handler) self.assertTrue(all(is_redir_content_visited.values()))
def test_traverse(self): ast = self.create_ast(Fixtures.MAP_AND_FILTER_VARIABLE) parser = CallNodeParser() got_ast = parser.process(ast) is_map_and_filter_content_visited = { 'v:val': False, 'v:key': False, } def enter_handler(node): if NodeType(node['type']) is not NodeType.IDENTIFIER: return is_map_and_filter_content_visited[node['value']] = True traverse(got_ast, on_enter=enter_handler) self.assertTrue(all(is_map_and_filter_content_visited.values()))
def _handle_lambda_node( self, lambda_node): # type: (Dict[str, Any]) -> Optional[str] # This method do the following 4 steps: # 1. Create a new scope of the lambda # 2. The current scope point to the new scope # 3. Add parameters to the new scope # 4. Add variables in the function body to the new scope # 1. Create a new scope of the function # 2. The current scope point to the new scope self._scope_tree_builder.enter_new_scope(ScopeVisibility.LAMBDA) # 3. Add parameters to the new scope has_variadic_symbol = False param_nodes = lambda_node['rlist'] for param_node in param_nodes: if param_node['value'] == '...': has_variadic_symbol = True else: # the param_node type is always NodeType.IDENTIFIER self._scope_tree_builder.handle_new_parameter_found( param_node, is_lambda_argument=True) # We can access a:0 and a:000 when the number of arguments is less than actual parameters. self._scope_tree_builder.handle_new_parameters_list_and_length_found() # In the context of lambda, we can access a:1 ... a:n when the number of arguments is less than actual parameters. # XXX: We can not know what a:N we can access by static analysis, so we assume it is 20. if has_variadic_symbol: lambda_args_len = len(param_nodes) - 1 else: lambda_args_len = len(param_nodes) self._scope_tree_builder.handle_new_index_parameters_found( lambda_args_len) # 4. Add variables in the function body to the new scope traverse(lambda_node['left'], on_enter=self._enter_handler, on_leave=self._leave_handler) # Skip child nodes traversing return SKIP_CHILDREN
def test_traverse(self): ast = self.create_ast(Fixtures.MAP_AND_FILTER_VARIABLE) parser = MapAndFilterParser() got_ast = parser.process(ast) is_map_and_filter_content_visited = { 'v:val': False, 'v:key': False, } def enter_handler(node): if NodeType(node['type']) is not NodeType.IDENTIFIER: return is_map_and_filter_content_visited[node['value']] = True traverse(got_ast, on_enter=enter_handler) self.assertTrue(all(is_map_and_filter_content_visited.values()))
def test_process_with_builtin(self): parser = Parser() ast = parser.parse_file(Fixtures['BUILTIN']) plugin = ScopePlugin() plugin.process(ast) expected_builtin_flags = { 'abs': True, 'sin': True, 'strlen': True, 'g:MyFunction': False, } # Keep identifier name that traverser visited identifiers_checking_map = { 'abs': False, 'sin': False, 'strlen': False, 'g:MyFunction': False, } def test_identifier(node): if NodeType(node['type']) is not NodeType.IDENTIFIER: return identifier = node # We focus to non-definition identifier if identifier[ScopePlugin.DEFINITION_IDENTIFIER_FLAG_KEY]: return identifier_name = identifier['value'] identifiers_checking_map[identifier_name] = True is_builtin_identifier = identifier[ ScopePlugin.BUILTIN_IDENTIFIER_FLAG_KEY] expected_builtin_flag = expected_builtin_flags[identifier_name] self.assertEqual(is_builtin_identifier, expected_builtin_flag) traverse(ast, on_enter=test_identifier) self.assertTrue(all(identifiers_checking_map.values()))
def test_process_with_builtin(self): parser = Parser() ast = parser.parse_file(Fixtures['BUILTIN']) plugin = ScopePlugin() plugin.process(ast) expected_builtin_flags = { 'abs': True, 'sin': True, 'strlen': True, 'g:MyFunction': False, } # Keep identifier name that traverser visited identifiers_checking_map = { 'abs': False, 'sin': False, 'strlen': False, 'g:MyFunction': False, } def test_identifier(node): if NodeType(node['type']) is not NodeType.IDENTIFIER: return identifier = node # We focus to non-definition identifier if identifier[ScopePlugin.DEFINITION_IDENTIFIER_FLAG_KEY]: return identifier_name = identifier['value'] identifiers_checking_map[identifier_name] = True is_builtin_identifier = identifier[ScopePlugin.BUILTIN_IDENTIFIER_FLAG_KEY] expected_builtin_flag = expected_builtin_flags[identifier_name] self.assertEqual(is_builtin_identifier, expected_builtin_flag) traverse(ast, on_enter=test_identifier) self.assertTrue(all(identifiers_checking_map.values()))
def assertVariablesUnused(self, expected_variables_unused, scope_plugin, ast): dec_id_footstamp_map = { id_name: False for id_name in expected_variables_unused.values() } def enter_handler(node): if is_declarative_identifier(node): id_name = node['value'] pprint(node) self.assertEqual( expected_variables_unused[id_name], scope_plugin.is_unused_declarative_identifier(node)) dec_id_footstamp_map[id_name] = True traverse(ast, on_enter=enter_handler) self.assertTrue(dec_id_footstamp_map.values())
def process(self, ast): def enter_handler(node): node_type = NodeType(node['type']) if node_type is not NodeType.CALL: return called_function_identifier = node['left'] # Name node of the "map" or "filter" functions are always IDENTIFIER. if NodeType(called_function_identifier['type'] ) is not NodeType.IDENTIFIER: return is_map_or_function_call = called_function_identifier.get( 'value') in { 'map': True, 'filter': True, } if not is_map_or_function_call: return args = node['rlist'] # Prevent crash. See https://github.com/Kuniwak/vint/issues/256. if len(args) < 2: return string_expr_node = args[1] # We can analyze only STRING nodes by static analyzing. if NodeType(string_expr_node['type']) is not NodeType.STRING: return string_expr_content_nodes = MapAndFilterParser._parse_string_expr_content_nodes( string_expr_node) node[STRING_EXPR_CONTENT] = string_expr_content_nodes traverse(ast, on_enter=enter_handler) return ast
def assertVariablesUndeclared(self, expected_variables_undeclared, scope_plugin, ast): ref_id_footstamp_map = { id_name: False for id_name in expected_variables_undeclared.values() } def enter_handler(node): if is_reference_identifier(node): id_name = node['value'] pprint(node) self.assertEqual( expected_variables_undeclared[id_name], scope_plugin.is_unreachable_reference_identifier(node)) ref_id_footstamp_map[id_name] = True traverse(ast, on_enter=enter_handler) # Check all exlected identifiers were tested self.assertTrue(ref_id_footstamp_map.values())
def parse_redir(self, redir_cmd): """ Parse a command :redir content. """ redir_cmd_str = redir_cmd['str'] matched = re.match(r'redir?!?\s*(=>>?\s*)(\S+)', redir_cmd_str) if matched: redir_cmd_op = matched.group(1) redir_cmd_body = matched.group(2) arg_pos = redir_cmd['ea']['argpos'] # Position of the "redir_cmd_body" start_pos = { 'col': arg_pos['col'] + len(redir_cmd_op), 'i': arg_pos['i'] + len(redir_cmd_op), 'lnum': arg_pos['lnum'], } # NOTE: This is a hack to parse variable node. raw_ast = self.parse_string('echo ' + redir_cmd_body) # We need the left node of ECHO node redir_cmd_ast = raw_ast['body'][0]['list'][0] def adjust_position(node): pos = node['pos'] # Care 1-based index and the length of "echo ". pos['col'] += start_pos['col'] - 1 - 5 # Care the length of "echo ". pos['i'] += start_pos['i'] - 5 # Care 1-based index pos['lnum'] += start_pos['lnum'] - 1 traverse(redir_cmd_ast, on_enter=adjust_position) return redir_cmd_ast return None
def _handle_lambda_node(self, lambda_node): # type: (Dict[str, Any]) -> Optional[str] # This method do the following 4 steps: # 1. Create a new scope of the lambda # 2. The current scope point to the new scope # 3. Add parameters to the new scope # 4. Add variables in the function body to the new scope # 1. Create a new scope of the function # 2. The current scope point to the new scope self._scope_tree_builder.enter_new_scope(ScopeVisibility.LAMBDA) # 3. Add parameters to the new scope has_variadic_symbol = False param_nodes = lambda_node['rlist'] for param_node in param_nodes: if param_node['value'] == '...': has_variadic_symbol = True else: # the param_node type is always NodeType.IDENTIFIER self._scope_tree_builder.handle_new_parameter_found(param_node, is_lambda_argument=True) # We can access a:0 and a:000 when the number of arguments is less than actual parameters. self._scope_tree_builder.handle_new_parameters_list_and_length_found() # In the context of lambda, we can access a:1 ... a:n when the number of arguments is less than actual parameters. # XXX: We can not know what a:N we can access by static analysis, so we assume it is 20. if has_variadic_symbol: lambda_args_len = len(param_nodes) - 1 else: lambda_args_len = len(param_nodes) self._scope_tree_builder.handle_new_index_parameters_found(lambda_args_len) # 4. Add variables in the function body to the new scope traverse(lambda_node['left'], on_enter=self._enter_handler, on_leave=self._leave_handler) # Skip child nodes traversing return SKIP_CHILDREN
def parse_redir(self, redir_cmd): """ Parse a command :redir content. """ redir_cmd_str = redir_cmd['str'] matched = re.match(r'redir?!?\s*(=>>?\s*)(\S+)', redir_cmd_str) if matched: redir_cmd_op = matched.group(1) redir_cmd_body = matched.group(2) arg_pos = redir_cmd['ea']['argpos'] # Position of the "redir_cmd_body" start_pos = { 'col': arg_pos['col'] + len(redir_cmd_op), 'i': arg_pos['i'] + len(redir_cmd_op), 'lnum': arg_pos['lnum'], } # NOTE: This is a hack to parse variable node. raw_ast = self.parse('echo ' + redir_cmd_body) # We need the left node of ECHO node redir_cmd_ast = raw_ast['body'][0]['list'][0] def adjust_position(node): pos = node['pos'] # Care 1-based index and the length of "echo ". pos['col'] += start_pos['col'] - 1 - 5 # Care the length of "echo ". pos['i'] += start_pos['i'] - 5 # Care 1-based index pos['lnum'] += start_pos['lnum'] - 1 traverse(redir_cmd_ast, on_enter=adjust_position) return redir_cmd_ast return None
def process(self, ast): def enter_handler(node): node_type = NodeType(node['type']) if node_type is not NodeType.EXCMD: return is_redir_command = node['ea']['cmd'].get('name') == 'redir' if not is_redir_command: return redir_cmd_str = node['str'] is_redir_assignment = '=>' in redir_cmd_str if not is_redir_assignment: return parser = Parser() redir_content_node = parser.parse_redir(node) node[REDIR_CONTENT] = redir_content_node traverse(ast, on_enter=enter_handler) return ast
def lint_file(self, path): try: root_ast = self._parser.parse_file(path) except VimLParserException as exception: parse_error = self._create_parse_error(path, str(exception)) return [parse_error] except UnicodeDecodeError as exception: # TODO: Is this a good way? I think a vint error is better. unicode_decode_error = self._create_unicode_error( path, str(exception)) return [unicode_decode_error] self._violations = [] self._update_listeners_map() # Given root AST to makepolicy flexibility lint_context = {'path': path, 'root_node': root_ast, 'stack_trace': []} traverse(root_ast, on_enter=lambda node: self._handle_enter(node, lint_context), on_leave=lambda node: self._handle_leave(node, lint_context)) return self._violations
def lint_file(self, path): try: root_ast = self._parser.parse_file(path) except VimLParserException as exception: parse_error = self._create_parse_error(path, str(exception)) return [parse_error] except UnicodeDecodeError as exception: # TODO: Is this a good way? I think a vint error is better. unicode_decode_error = self._create_unicode_error(path, str(exception)) return [unicode_decode_error] self._violations = [] self._update_listeners_map() # Given root AST to makepolicy flexibility lint_context = {'path': path, 'root_node': root_ast, 'stack_trace': []} traverse(root_ast, on_enter=lambda node: self._handle_enter(node, lint_context), on_leave=lambda node: self._handle_leave(node, lint_context)) return self._violations
def process(self, ast): def enter_handler(node): node_type = NodeType(node['type']) if node_type is not NodeType.CALL: return called_function_identifier = node['left'] # The name node type of "map" or "filter" or "call" are always IDENTIFIER. if NodeType(called_function_identifier['type']) is not NodeType.IDENTIFIER: return called_function_identifier_value = called_function_identifier.get('value') if called_function_identifier_value in ['map', 'filter']: # Analyze second argument of "map" or "filter" if the node type is STRING. self._attach_string_expr_content_to_map_or_func(node) elif called_function_identifier_value in ['call', 'function']: # Analyze first argument of "call" or "function" if the node type is STRING. self._attach_string_expr_content_to_call_or_function(node) traverse(ast, on_enter=enter_handler) return ast
def process(self, ast): def enter_handler(node): node_type = NodeType(node['type']) if node_type is not NodeType.CALL: return called_function_identifier = node['left'] # Name node of the "map" or "filter" functions are always IDENTIFIER. if NodeType(called_function_identifier['type'] ) is not NodeType.IDENTIFIER: return is_map_or_function_call = called_function_identifier.get( 'value') in { 'map': True, 'filter': True, } if not is_map_or_function_call: return string_expr_node = node['rlist'][1] # We can analyze only STRING nodes by static analyzing. if NodeType(string_expr_node['type']) is not NodeType.STRING: return parser = Parser() string_expr_content_nodes = parser.parse_string_expr( string_expr_node) node[STRING_EXPR_CONTENT] = string_expr_content_nodes traverse(ast, on_enter=enter_handler) return ast
def _traverse(self, root_ast, lint_target): if self._is_debug: logging.debug('{cls}: checking `{file_path}`'.format( cls=self.__class__.__name__, file_path=lint_target.path) ) logging.debug('{cls}: using config as {config_dict}'.format( cls=self.__class__.__name__, config_dict=self._config_dict_global )) self._prepare_for_traversal() lint_context = { 'lint_target': lint_target, 'root_node': root_ast, 'stack_trace': [], 'plugins': self._plugins, 'config': self._config.get_config_dict(), } traverse(root_ast, on_enter=lambda node: self._handle_enter(node, lint_context), on_leave=lambda node: self._handle_leave(node, lint_context))
def _handle_function_node(self, func_node): # type: (Dict[str, Any]) -> None # We should interrupt traversing, because a node of the function # name should be added to the parent scope before the current # scope switched to a new scope of the function. # We approach to it by the following 5 steps. # 1. Add the function to the current scope # 2. Create a new scope of the function # 3. The current scope point to the new scope # 4. Add parameters to the new scope # 5. Add variables in the function body to the new scope # 1. Add the function to the current scope func_name_node = func_node['left'] traverse(func_name_node, on_enter=self._find_variable_like_nodes) # 2. Create a new scope of the function # 3. The current scope point to the new scope self._scope_tree_builder.enter_new_scope(ScopeVisibility.FUNCTION_LOCAL) has_variadic = False # 4. Add parameters to the new scope param_nodes = func_node['rlist'] for param_node in param_nodes: if param_node['value'] == '...': has_variadic = True else: # the param_node type is always NodeType.IDENTIFIER self._scope_tree_builder.handle_new_parameter_found(param_node, is_lambda_argument=False) # We can always access a:0, a:000 self._scope_tree_builder.handle_new_parameters_list_and_length_found() # In a variadic function, we can access a:1 ... a:n # (n = 20 - explicit parameters length). See :help a:0 if has_variadic: # -1 means ignore '...' self._scope_tree_builder.handle_new_index_parameters_found(len(param_nodes) - 1) # We can access "a:firstline" and "a:lastline" if the function is # declared with an attribute "range". See :func-range attr = func_node['attr'] is_declared_with_range = attr['range'] is not 0 if is_declared_with_range: self._scope_tree_builder.handle_new_range_parameters_found() # We can access "l:self" is declared with an attribute "dict" or # the function is a member of a dict. See :help self is_declared_with_dict = attr['dict'] is not 0 \ or NodeType(func_name_node['type']) in FunctionNameNodesDeclaringVariableSelf if is_declared_with_dict: self._scope_tree_builder.handle_new_dict_parameter_found() # 5. Add variables in the function body to the new scope func_body_nodes = func_node['body'] for func_body_node in func_body_nodes: traverse(func_body_node, on_enter=self._enter_handler, on_leave=self._leave_handler) # Skip child nodes traversing return SKIP_CHILDREN
def _handle_function_node(self, func_node): # We should interrupt traversing, because a node of the function # name should be added to the parent scope before the current # scope switched to a new scope of the function. # We approach to it by the following 5 steps. # 1. Add the function to the current scope # 2. Create a new scope of the function # 3. The current scope point to the new scope # 4. Add parameters to the new scope # 5. Add variables in the function body to the new scope # 1. Add the function to the current scope func_name_node = func_node['left'] traverse(func_name_node, on_enter=self._find_variable_like_nodes) # 2. Create a new scope of the function # 3. The current scope point to the new scope self._scope_tree_builder.enter_new_scope( ScopeVisibility.FUNCTION_LOCAL) has_variadic = False # 4. Add parameters to the new scope param_nodes = func_node['rlist'] for param_node in param_nodes: if param_node['value'] == '...': has_variadic = True else: # the param_node type is always NodeType.IDENTIFIER self._scope_tree_builder.handle_new_parameter_found(param_node) # We can always access a:0, a:000 self._scope_tree_builder.handle_new_parameters_list_and_length_found() # In a variadic function, we can access a:1 ... a:n # (n = 20 - explicit parameters length). See :help a:0 if has_variadic: # -1 means ignore '...' self._scope_tree_builder.handle_new_index_parameters_found( len(param_nodes) - 1) # We can access "a:firstline" and "a:lastline" if the function is # declared with an attribute "range". See :func-range attr = func_node['attr'] is_declared_with_range = attr['range'] is not 0 if is_declared_with_range: self._scope_tree_builder.handle_new_range_parameters_found() # We can access "l:self" is declared with an attribute "dict" or # the function is a member of a dict. See :help self is_declared_with_dict = attr['dict'] is not 0 \ or NodeType(func_name_node['type']) in FunctionNameNodesDeclaringVariableSelf if is_declared_with_dict: self._scope_tree_builder.handle_new_dict_parameter_found() # 5. Add variables in the function body to the new scope func_body_nodes = func_node['body'] for func_body_node in func_body_nodes: traverse(func_body_node, on_enter=self._enter_handler, on_leave=self._leave_handler) # Skip child nodes traversing return SKIP_CHILDREN
def traverse_redir_content(node, on_enter=None, on_leave=None): if REDIR_CONTENT not in node: return traverse(node[REDIR_CONTENT], on_enter=on_enter, on_leave=on_leave)
def test_traverse_ignoring_while_children(self): expected_order_of_events = [ { 'node_type': NodeType.TOPLEVEL, 'handler': 'enter' }, { 'node_type': NodeType.LET, 'handler': 'enter' }, { 'node_type': NodeType.IDENTIFIER, 'handler': 'enter' }, { 'node_type': NodeType.IDENTIFIER, 'handler': 'leave' }, { 'node_type': NodeType.NUMBER, 'handler': 'enter' }, { 'node_type': NodeType.NUMBER, 'handler': 'leave' }, { 'node_type': NodeType.LET, 'handler': 'leave' }, { 'node_type': NodeType.WHILE, 'handler': 'enter' }, { 'node_type': NodeType.WHILE, 'handler': 'leave' }, { 'node_type': NodeType.TOPLEVEL, 'handler': 'leave' }, ] def on_enter(node): actual_order_of_events.append({ 'node_type': NodeType(node['type']), 'handler': 'enter', }) if NodeType(node['type']) is NodeType.WHILE: return SKIP_CHILDREN # Records visit node type name in order actual_order_of_events = [] traverse(self.ast, on_enter=on_enter, on_leave=lambda node: actual_order_of_events.append( { 'node_type': NodeType(node['type']), 'handler': 'leave', })) self.maxDiff = 2048 self.assertEqual(actual_order_of_events, expected_order_of_events)
from vint.ast.plugin.scope_plugin import ScopePlugin def prettify_node_type(node): node['type'] = NodeType(node['type']) if __name__ == '__main__': arg_parser = ArgumentParser(prog='show_ast', description='Show AST') arg_parser.add_argument('--enable-neovim', action='store_true', help='Enable Neovim syntax') arg_parser.add_argument('files', nargs='*', help='File to parse') namespace = vars(arg_parser.parse_args(sys.argv[1:])) filepaths = map(Path, namespace['files']) enable_neovim = namespace['enable_neovim'] scope_plugin = ScopePlugin() parser = Parser(plugins=[scope_plugin], enable_neovim=enable_neovim) for filepath in filepaths: ast = parser.parse_file(filepath) traverse(ast, on_enter=prettify_node_type) print("////////// SCOPE TREE //////////\n") pprint(scope_plugin._ref_tester._scope_linker.scope_tree) print("\n\n") print("////////// LINK REGISTRY //////////\n") pprint(scope_plugin._ref_tester._scope_linker.link_registry._vars_to_declarative_ids_map) pprint(scope_plugin._ref_tester._scope_linker.link_registry._ids_to_scopes_map)