def _parse_file(src_file: ast.File, args: argparse.Namespace) -> Tuple[RawBaseType, Optional['CompilationError']]: parse_error: Optional['CompilationError'] = None parse_tree: Optional[RawBaseType] = None try: parse_tree = ast_raw.parse(src_file, args.python_version) parse_error = None except pgen2_tokenize.TokenError as exc: parse_error = ParseError(msg=str(exc), type='', context='', value='', srcpath=args.srcpath) except SyntaxError as exc: # TODO: This seems to sometimes be raised from an encoding error with # msg='unknown encoding: ...', exc.lineno=None, exc.offset=None parse_error = ParseError( msg=str(exc), type='', context=str(('', (exc.lineno, exc.offset))) if exc.lineno or exc.offset else '', value=exc.text or '', srcpath=args.srcpath) except UnicodeDecodeError as exc: parse_error = DecodeError(encoding=exc.encoding, start=exc.start, end=exc.end, reason=exc.reason, srcpath=args.srcpath) except pgen2_parse.ParseError as exc: parse_error = ParseError(msg=str(exc), type=pytree.type_repr(exc.type), value=exc.value or '', context=str(exc.context), srcpath=args.srcpath) return parse_tree, parse_error
def _process_parameters(parameters): if not _is_multi_line(parameters): return open_parenthesis, args_list, close_parenthesis = parameters.children elements = args_list.children if not elements: # TODO complain about multi-line argument list with nothing in it return first_element = elements[0] if open_parenthesis.lineno == first_element.get_lineno(): yield _error(first_element, ErrorCode.S100) last_element = elements[-1] # We only accept lack of trailing comma in case of the parameter # list containing any use of * or ** as adding the trailing comma # is a syntax error. no_variadic_arguments = all([ element.type not in (token.STAR, token.DOUBLESTAR) for element in elements ]) parent_nice_type = pytree.type_repr(parameters.parent.type) if last_element.type != token.COMMA and no_variadic_arguments: yield _error(last_element, ErrorCode.S101)
def _process_ast( src_file: ast.File, parse_tree: RawBaseType, args: argparse.Namespace ) -> Tuple[Optional[RawBaseType], Union['Crash', 'ParseError', ast_cooked.Base], Optional['ParseError']]: logging.getLogger('pykythe').debug('RAW= %r', parse_tree) new_parse_tree: Optional[RawBaseType] with_fqns: Union[ast_cooked.Base, 'ParseError', 'Crash'] parse_error: Optional[Union['ParseError', Exception]] try: cooked_nodes = ast_raw.cvt_parse_tree(parse_tree, args.python_version, src_file) with_fqns = ast_cooked.add_fqns(cooked_nodes, args.module, args.python_version) parse_error = None new_parse_tree = parse_tree except pgen2_parse.ParseError as exc: parse_error = ParseError(msg=str(exc), type=pytree.type_repr(exc.type), value=exc.value or '', context=str(exc.context), srcpath=args.srcpath) # This can happen with, e.g. function def struct unpacking new_parse_tree = None with_fqns = parse_error logging.getLogger('pykythe').error('pykythe.__main__._process_ast: Parse error: %s', parse_error) # DO NOT SUBMIT - error message form except Exception as exc: # pylint: disable=broad-except # DO NOT SUBMIT: the parse_error assignment is probably incorrect parse_error = exc # TODO: is this correct? we get this from an assertion check, for example new_parse_tree = parse_tree logging.getLogger('pykythe').error( 'pykythe.__main__._process_ast: Caught error in cvt_parse_tree/with_fqns: %s %r', exc, exc) traceback.print_exc() with_fqns = Crash(str=str(exc), repr=repr(exc), srcpath=args.srcpath) return new_parse_tree, with_fqns, parse_error
def _process_atom(atom): # The definition of atom node: # atom: ('(' [yield_expr|testlist_gexp] ')' | # '[' [listmaker] ']' | # '{' [dictsetmaker] '}' | # '`' testlist1 '`' | # NAME | NUMBER | STRING+ | '.' '.' '.') if len(atom.children) < 3 or not _is_multi_line(atom): return left = atom.children[0] if left.value not in {'{', '['}: return open_parenthesis, maker, close_parenthesis = atom.children if open_parenthesis.lineno == maker.get_lineno(): yield _error(maker, ErrorCode.S100) if maker.children: last_maker_element = maker.children[-1] else: # If we're dealing with a one element list we'll land here last_maker_element = maker # Enforcing trailing commas in list/dict/set comprehensions seems too strict # so we won't do it for now even if it is syntactically allowed. has_comprehension_inside = 'comp_for' in { pytree.type_repr(node.type) for node in maker.children } if last_maker_element.type != token.COMMA and not has_comprehension_inside: yield _error(last_maker_element, ErrorCode.S101)
def _process_atom(atom): # The definition of atom node: # atom: ('(' [yield_expr|testlist_gexp] ')' | # '[' [listmaker] ']' | # '{' [dictsetmaker] '}' | # '`' testlist1 '`' | # NAME | NUMBER | STRING+ | '.' '.' '.') if len(atom.children) < 3 or not _is_multi_line(atom): return left = atom.children[0] if left.value not in {'{', '['}: return open_parenthesis, maker, close_parenthesis = atom.children if open_parenthesis.lineno == maker.get_lineno(): yield _error(maker, ErrorCode.S100) last_maker_element = maker.children[-1] # Enforcing trailing commas in list/dict/set comprehensions seems too strict # so we won't do it for now even if it is syntactically allowed. has_comprehension_inside = 'comp_for' in { pytree.type_repr(node.type) for node in maker.children } if last_maker_element.type != token.COMMA and not has_comprehension_inside: yield _error(last_maker_element, ErrorCode.S101)
def transform(self, node, results): if type_repr(node.parent.type) != "file_input": return node name = results["name"].value if not name.startswith("_"): self._names.append(name) return node
def _process_import_from(import_from): # The definition of import_from node: # import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+) # 'import' ('*' | '(' import_as_names ')' | import_as_names)) last_element = import_from.children[-1] last_element_type = pytree.type_repr(last_element.type) if last_element_type == token.RPAR: import_from.children = import_from.children[-3:] return _process_parameters(import_from) return []
def node_fullname(node): typ = pytree.type_repr(node.type) if typ == "funcdef": return "{}[name={}]".format(node_name(node), repr(node.children[1].value)) elif typ == "classdef": return "{}[name={}]".format(node_name(node), repr(node.children[1].value)) elif typ == "typedargslist": return "{}[args={}]".format( node_name(node), " ".join([str(t.guess_type(c)) for c in node.children]) ) else: return node_name(node)
def _process_trailer(trailer): # The definition of trailer node: # trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME children = trailer.children if len(children) == 3: middle = children[1] if pytree.type_repr(middle.type) == 'atom': return _process_atom(middle) else: return _process_parameters(trailer) else: return []
def _process_tree(tree): iterables = [] nice_type = pytree.type_repr(tree.type) if nice_type == 'parameters': iterables.append(_process_parameters(tree)) elif nice_type == 'trailer': iterables.append(_process_trailer(tree)) elif nice_type == 'atom': iterables.append(_process_atom(tree)) iterables.extend(_process_tree(c) for c in tree.children) return itertools.chain.from_iterable(iterables)
def _IsNodeOfType(node, *types): """ Check if the node is of one of the given types. :param lib2to3.Node node: :param unicode types: The name of the node type. :return bool: """ from lib2to3.pytree import Node, type_repr return isinstance(node, Node) and type_repr(node.type) in types
def node_fullname(node): typ = pytree.type_repr(node.type) if typ == "funcdef": return "{}[name={}]".format(node_name(node), repr(node.children[1].value)) elif typ == "classdef": return "{}[name={}]".format(node_name(node), repr(node.children[1].value)) elif typ == "typedargslist": return "{}[args={}]".format( node_name(node), " ".join([repr(c.value) for c in node.children])) else: return node_name(node)
def _traverse(self, node): result = DictLikeList() for child in node.children: if isinstance(child, Node): result[type_repr(child.type)] = self._traverse(child) else: if self.options["show_prefix"]: value = f"{repr(child.prefix)} {repr(child.value)}" else: value = f"{repr(child.value)}" result[f"{token.tok_name[child.type]}: {value}"] = {} return result
def recur(context, indent): result = new_line(indent) result += "%s(%s, " % (context.__class__.__name__, type_repr(context.type)) #result += "%s, "% (context.prefix) if (len(context.children) == 0): result += '\'' if context.value == '\n': result += "\\n\'" elif context.value == '\r\n': result += "\\r\\n\'" else: result += context.value + '\'' else: result += "[\n" for i in context.children: result += recur(i, indent + 1) result += new_line(indent) + "]" result += "),\n" return result
def is_node_of_type(node, *types): return isinstance(node, Node) and pytree.type_repr(node.type) in types
def node_type(n) -> str: """Obtain the string type for the token""" if n.type <= token.NT_OFFSET: return token.tok_name[n.type] return pytree.type_repr(n.type)
def _is_unpacking_element(element): element_type = pytree.type_repr(element.type) if element_type == 'argument': return element.children[0].type in [token.STAR, token.DOUBLESTAR] return element_type == 'star_expr'
def type_name(node): if node.type in tok_name: return tok_name[node.type] return type_repr(node.type)