def _parse_file(src_file: ast.File,
args: argparse.Namespace) -> Tuple[RawBaseType, Optional['CompilationError']]:
parse_error: Optional['CompilationError'] = None
parse_tree: Optional[RawBaseType] = None
try:
parse_tree = ast_raw.parse(src_file, args.python_version)
parse_error = None
except pgen2_tokenize.TokenError as exc:
parse_error = ParseError(msg=str(exc), type='', context='', value='', srcpath=args.srcpath)
except SyntaxError as exc:
# TODO: This seems to sometimes be raised from an encoding error with
# msg='unknown encoding: ...', exc.lineno=None, exc.offset=None
parse_error = ParseError(
msg=str(exc),
type='',
context=str(('', (exc.lineno, exc.offset))) if exc.lineno or exc.offset else '',
value=exc.text or '',
srcpath=args.srcpath)
except UnicodeDecodeError as exc:
parse_error = DecodeError(encoding=exc.encoding,
start=exc.start,
end=exc.end,
reason=exc.reason,
srcpath=args.srcpath)
except pgen2_parse.ParseError as exc:
parse_error = ParseError(msg=str(exc),
type=pytree.type_repr(exc.type),
value=exc.value or '',
context=str(exc.context),
srcpath=args.srcpath)
return parse_tree, parse_error
def _process_parameters(parameters):
if not _is_multi_line(parameters):
return
open_parenthesis, args_list, close_parenthesis = parameters.children
elements = args_list.children
if not elements:
# TODO complain about multi-line argument list with nothing in it
return
first_element = elements[0]
if open_parenthesis.lineno == first_element.get_lineno():
yield _error(first_element, ErrorCode.S100)
last_element = elements[-1]
# We only accept lack of trailing comma in case of the parameter
# list containing any use of * or ** as adding the trailing comma
# is a syntax error.
no_variadic_arguments = all([
element.type not in (token.STAR, token.DOUBLESTAR)
for element in elements
])
parent_nice_type = pytree.type_repr(parameters.parent.type)
if last_element.type != token.COMMA and no_variadic_arguments:
yield _error(last_element, ErrorCode.S101)
def _process_ast(
src_file: ast.File, parse_tree: RawBaseType, args: argparse.Namespace
) -> Tuple[Optional[RawBaseType], Union['Crash', 'ParseError', ast_cooked.Base],
Optional['ParseError']]:
logging.getLogger('pykythe').debug('RAW= %r', parse_tree)
new_parse_tree: Optional[RawBaseType]
with_fqns: Union[ast_cooked.Base, 'ParseError', 'Crash']
parse_error: Optional[Union['ParseError', Exception]]
try:
cooked_nodes = ast_raw.cvt_parse_tree(parse_tree, args.python_version, src_file)
with_fqns = ast_cooked.add_fqns(cooked_nodes, args.module, args.python_version)
parse_error = None
new_parse_tree = parse_tree
except pgen2_parse.ParseError as exc:
parse_error = ParseError(msg=str(exc),
type=pytree.type_repr(exc.type),
value=exc.value or '',
context=str(exc.context),
srcpath=args.srcpath)
# This can happen with, e.g. function def struct unpacking
new_parse_tree = None
with_fqns = parse_error
logging.getLogger('pykythe').error('pykythe.__main__._process_ast: Parse error: %s',
parse_error) # DO NOT SUBMIT - error message form
except Exception as exc: # pylint: disable=broad-except
# DO NOT SUBMIT: the parse_error assignment is probably incorrect
parse_error = exc # TODO: is this correct? we get this from an assertion check, for example
new_parse_tree = parse_tree
logging.getLogger('pykythe').error(
'pykythe.__main__._process_ast: Caught error in cvt_parse_tree/with_fqns: %s %r',
exc, exc)
traceback.print_exc()
with_fqns = Crash(str=str(exc), repr=repr(exc), srcpath=args.srcpath)
return new_parse_tree, with_fqns, parse_error
def _process_atom(atom):
# The definition of atom node:
# atom: ('(' [yield_expr|testlist_gexp] ')' |
# '[' [listmaker] ']' |
# '{' [dictsetmaker] '}' |
# '`' testlist1 '`' |
# NAME | NUMBER | STRING+ | '.' '.' '.')
if len(atom.children) < 3 or not _is_multi_line(atom):
return
left = atom.children[0]
if left.value not in {'{', '['}:
return
open_parenthesis, maker, close_parenthesis = atom.children
if open_parenthesis.lineno == maker.get_lineno():
yield _error(maker, ErrorCode.S100)
if maker.children:
last_maker_element = maker.children[-1]
else:
# If we're dealing with a one element list we'll land here
last_maker_element = maker
# Enforcing trailing commas in list/dict/set comprehensions seems too strict
# so we won't do it for now even if it is syntactically allowed.
has_comprehension_inside = 'comp_for' in {
pytree.type_repr(node.type) for node in maker.children
}
if last_maker_element.type != token.COMMA and not has_comprehension_inside:
yield _error(last_maker_element, ErrorCode.S101)
def _process_atom(atom):
# The definition of atom node:
# atom: ('(' [yield_expr|testlist_gexp] ')' |
# '[' [listmaker] ']' |
# '{' [dictsetmaker] '}' |
# '`' testlist1 '`' |
# NAME | NUMBER | STRING+ | '.' '.' '.')
if len(atom.children) < 3 or not _is_multi_line(atom):
return
left = atom.children[0]
if left.value not in {'{', '['}:
return
open_parenthesis, maker, close_parenthesis = atom.children
if open_parenthesis.lineno == maker.get_lineno():
yield _error(maker, ErrorCode.S100)
last_maker_element = maker.children[-1]
# Enforcing trailing commas in list/dict/set comprehensions seems too strict
# so we won't do it for now even if it is syntactically allowed.
has_comprehension_inside = 'comp_for' in {
pytree.type_repr(node.type)
for node in maker.children
}
if last_maker_element.type != token.COMMA and not has_comprehension_inside:
yield _error(last_maker_element, ErrorCode.S101)
def transform(self, node, results):
if type_repr(node.parent.type) != "file_input":
return node
name = results["name"].value
if not name.startswith("_"):
self._names.append(name)
return node
def _process_import_from(import_from):
# The definition of import_from node:
# import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+)
# 'import' ('*' | '(' import_as_names ')' | import_as_names))
last_element = import_from.children[-1]
last_element_type = pytree.type_repr(last_element.type)
if last_element_type == token.RPAR:
import_from.children = import_from.children[-3:]
return _process_parameters(import_from)
return []
def node_fullname(node):
typ = pytree.type_repr(node.type)
if typ == "funcdef":
return "{}[name={}]".format(node_name(node), repr(node.children[1].value))
elif typ == "classdef":
return "{}[name={}]".format(node_name(node), repr(node.children[1].value))
elif typ == "typedargslist":
return "{}[args={}]".format(
node_name(node), " ".join([str(t.guess_type(c)) for c in node.children])
)
else:
return node_name(node)
def _process_trailer(trailer):
# The definition of trailer node:
# trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
children = trailer.children
if len(children) == 3:
middle = children[1]
if pytree.type_repr(middle.type) == 'atom':
return _process_atom(middle)
else:
return _process_parameters(trailer)
else:
return []
def _process_trailer(trailer):
# The definition of trailer node:
# trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
children = trailer.children
if len(children) == 3:
middle = children[1]
if pytree.type_repr(middle.type) == 'atom':
return _process_atom(middle)
else:
return _process_parameters(trailer)
else:
return []
def _process_tree(tree):
iterables = []
nice_type = pytree.type_repr(tree.type)
if nice_type == 'parameters':
iterables.append(_process_parameters(tree))
elif nice_type == 'trailer':
iterables.append(_process_trailer(tree))
elif nice_type == 'atom':
iterables.append(_process_atom(tree))
iterables.extend(_process_tree(c) for c in tree.children)
return itertools.chain.from_iterable(iterables)
def _IsNodeOfType(node, *types):
"""
Check if the node is of one of the given types.
:param lib2to3.Node node:
:param unicode types:
The name of the node type.
:return bool:
"""
from lib2to3.pytree import Node, type_repr
return isinstance(node, Node) and type_repr(node.type) in types
def _process_tree(tree):
iterables = []
nice_type = pytree.type_repr(tree.type)
if nice_type == 'parameters':
iterables.append(_process_parameters(tree))
elif nice_type == 'trailer':
iterables.append(_process_trailer(tree))
elif nice_type == 'atom':
iterables.append(_process_atom(tree))
iterables.extend(_process_tree(c) for c in tree.children)
return itertools.chain.from_iterable(iterables)
def node_fullname(node):
typ = pytree.type_repr(node.type)
if typ == "funcdef":
return "{}[name={}]".format(node_name(node),
repr(node.children[1].value))
elif typ == "classdef":
return "{}[name={}]".format(node_name(node),
repr(node.children[1].value))
elif typ == "typedargslist":
return "{}[args={}]".format(
node_name(node), " ".join([repr(c.value) for c in node.children]))
else:
return node_name(node)
def _traverse(self, node):
result = DictLikeList()
for child in node.children:
if isinstance(child, Node):
result[type_repr(child.type)] = self._traverse(child)
else:
if self.options["show_prefix"]:
value = f"{repr(child.prefix)} {repr(child.value)}"
else:
value = f"{repr(child.value)}"
result[f"{token.tok_name[child.type]}: {value}"] = {}
return result
def _IsNodeOfType(node, *types):
"""
Check if the node is of one of the given types.
:param lib2to3.Node node:
:param unicode types:
The name of the node type.
:return bool:
"""
from lib2to3.pytree import Node, type_repr
return isinstance(node, Node) and type_repr(node.type) in types
def recur(context, indent):
result = new_line(indent)
result += "%s(%s, " % (context.__class__.__name__, type_repr(context.type))
#result += "%s, "% (context.prefix)
if (len(context.children) == 0):
result += '\''
if context.value == '\n':
result += "\\n\'"
elif context.value == '\r\n':
result += "\\r\\n\'"
else:
result += context.value + '\''
else:
result += "[\n"
for i in context.children:
result += recur(i, indent + 1)
result += new_line(indent) + "]"
result += "),\n"
return result
def is_node_of_type(node, *types):
return isinstance(node, Node) and pytree.type_repr(node.type) in types
def node_type(n) -> str:
"""Obtain the string type for the token"""
if n.type <= token.NT_OFFSET:
return token.tok_name[n.type]
return pytree.type_repr(n.type)
def _is_unpacking_element(element):
element_type = pytree.type_repr(element.type)
if element_type == 'argument':
return element.children[0].type in [token.STAR, token.DOUBLESTAR]
return element_type == 'star_expr'
def type_name(node):
if node.type in tok_name:
return tok_name[node.type]
return type_repr(node.type)
def is_node_of_type(node, *types):
return isinstance(node, Node) and pytree.type_repr(node.type) in types
