def _api_rename(node: LN, capture: Capture, filename: Filename):
code = ''
for leaf in node.leaves():
code = code + leaf.value
found_rename = False
found_warning = False
api = None
for _api in rename_map.keys():
if utils.startswith(code, _api):
found_rename = True
api = _api
break
for _api in warning_map.keys():
if utils.startswith(code, _api):
found_warning = True
api = _api
break
if not found_rename and not found_warning:
return
# if found rename, replace old_api with new_api
if found_rename:
utils.replace_module_path(node, api, rename_map[api])
# if not found rename and found warning, print warning
elif found_warning:
log_warning(filename, node.get_lineno(), warning_map[api])
def _full_module_path(node: LN, capture: Capture, filename: Filename):
if not (isinstance(node, Leaf) and node.type == token.NAME):
return
if filename not in imports_map:
return
logger.debug("{} [{}]: {}".format(filename, list(capture), node))
# skip import statement
if utils.is_import_node(node):
return
# skip left operand in argument list
if utils.is_argument_node(node) and utils.is_left_operand(node):
return
# skip if it's already a full module path
if node.prev_sibling is not None and node.prev_sibling.type == token.DOT:
return
rename_dict = imports_map[filename]
if node.value in rename_dict:
# find old_name and new_name
old_name = node.value
new_name = rename_dict[old_name]
if node.parent is not None:
_node = utils.code_repr(new_name).children[0].children[0]
_node.parent = None
new_node = _node
new_node.children[0].prefix = node.prefix
if node.parent.type == python_symbols.power:
node.replace(new_node.children)
else:
node.replace(new_node)
log_info(
filename, node.get_lineno(),
"{} -> {}".format(utils.node2code(node),
utils.node2code(new_node)))
def _add_import(node: LN, capture: Capture, filename: Filename):
if node.type != python_symbols.file_input:
return
if filename in paddle_imported:
return
if filename in paddle_found:
touch_import(None, 'paddle', node, force=True)
log_info(filename, node.get_lineno(), 'add "import paddle"')
paddle_imported.add(filename)
def _norm(node: LN, capture: Capture, filename: Filename):
code = ''
for leaf in node.leaves():
code = code + leaf.value
found_alias = False
alias = None
for _alias in alias_map.keys():
if utils.startswith(code, _alias):
found_alias = True
alias = _alias
break
if not found_alias:
return
main_alias = alias_map[alias]
update_to = change_spec[main_alias].get('update_to', None)
# if main_alias contains "update_to" field, rename alias to "update_to" directly
utils.replace_module_path(node, alias, main_alias)
log_info(filename, node.get_lineno(),
'{} -> {}'.format(alias, main_alias))
def print_node(node: LN,
max_depth: int = 1000,
indent: str = "",
last: bool = True,
capture={}):
"""Debugging function to print node tree.
Arguments:
node: The node to print
max_depth: The maximum recursion depth to walk children
"""
if last:
first_i = "└─"
second_i = " "
else:
first_i = "├─"
second_i = "│ "
prefix = indent + first_i
name = ""
if node in capture.values():
name = ("\033[32m" + next(k for k, v in capture.items() if v == node) +
"\033[0m= ")
if type(node) is Node:
print(prefix + name + "Node[{}] prefix={} suffix={}".format(
type_repr(node.type), repr(node.prefix), repr(node.get_suffix())))
elif type(node) is Leaf:
print(indent + first_i + name + "Leaf({}, {}, col={}{})".format(
token.tok_name[node.type],
repr(node.value),
node.column,
", prefix={}".format(repr(node.prefix)) if node.prefix else "",
))
else:
raise RuntimeError("Unknown node type")
indent = indent + second_i
children = list(node.children)
if max_depth == 0 and children:
print(indent + f"└─...{len(children)} children")
else:
for i, child in enumerate(node.children):
print_node(
child,
indent=indent,
last=(i + 1) == len(children),
max_depth=max_depth - 1,
capture=capture,
)
def _remove_with_dygraph_guard(node: LN, capture: Capture,
filename: Filename):
# index of with_node, with_node will be replaced with simple_stmt node
with_node = capture['with']
parent = with_node.parent
idx = None
for i, child in enumerate(parent.children):
if child is with_node:
idx = i
break
# create simple_stmt node for "paddle.disable_static"
arg_list_nodes = capture['arg_list']
simple_stmt_disable_static = Node(python_symbols.simple_stmt,
[utils.newline_node(node)])
_node = utils.code_repr('paddle.disable_static' +
str(arg_list_nodes)).children[0].children[0]
_node.parent = None
simple_stmt_disable_static.insert_child(0, _node)
simple_stmt_disable_static.prefix = with_node.prefix
# create simple_stmt node for "paddle.enable_static"
simple_stmt_enable_static = Node(python_symbols.simple_stmt,
[utils.newline_node(node)])
simple_stmt_enable_static
_node = utils.code_repr(
'paddle.enable_static()').children[0].children[0]
_node.parent = None
simple_stmt_enable_static.insert_child(0, _node)
simple_stmt_enable_static.prefix = utils.get_indent(with_node)
suite_node = capture['suite']
# remove first newline
for node in suite_node.children:
if not isinstance(node, Leaf):
continue
if node.type == token.NEWLINE:
node.remove()
break
# remove first indent node, and add indent prefix to sibling node.
indent = None
for node in suite_node.children:
if not isinstance(node, Leaf):
continue
if node.type == token.INDENT:
indent = node.value
if node.next_sibling is not None:
node.next_sibling.prefix = node.prefix + indent
node.remove()
break
# transfer post leading dedent node prefix to sibling of with node
leaves = [leaf for leaf in suite_node.leaves()]
# visit all leaves in reversed order
last_dedent_leaf_idx = len(leaves)
for leaf in leaves[::-1]:
if leaf.type == token.DEDENT:
with_node.next_sibling.prefix = leaf.prefix + with_node.next_sibling.prefix
leaf.prefix = ""
else:
break
# remove dedenet node corresponding to with node
for node in suite_node.children[::-1]:
if not isinstance(node, Leaf):
continue
if node.type == token.DEDENT:
node.remove()
break
# unindent all code in suite
for node in suite_node.leaves():
if node.type == token.INDENT:
node.value = utils.dec_indent(node.value)
else:
node.prefix = utils.dec_indent(node.prefix)
with_node.remove()
parent.insert_child(idx, simple_stmt_disable_static)
idx += 1
for node in suite_node.children:
parent.insert_child(idx, node)
idx += 1
parent.insert_child(idx, simple_stmt_enable_static)
def process_import(node: LN, capture: Capture,
filename: Filename) -> Optional[LN]:
# Skip any imports at file scope
if node.parent.parent.type == python_symbols.file_input:
return
IMPORT_WHITELIST = {
"importlib",
"math",
"optparse",
"os",
"os.path",
"six.moves.cPickle as pickle",
"six.moves",
"sys",
"urllib2",
"uuid",
}
module_name = str(node.children[1]).strip()
always_float = module_name in IMPORT_WHITELIST
# if STDLIB_ONLY:
# if :
# return
# See if this is rejected
if ONLY_FLOAT:
isort_class = None
if any(x.isupper() for x in ONLY_FLOAT):
isort_class = isort.place_module(
module_name, config=get_isort_config_for(filename))
if (module_name not in ONLY_FLOAT and
not any(module_name.startswith(x + ".") for x in ONLY_FLOAT)
and isort_class not in ONLY_FLOAT):
return
# if not always_float:
# Bypass nodes with comments for now
if node.get_suffix().strip() or get_complete_prefix(node).strip():
print(
f"Not floating {filename}:{node.get_lineno()} ({module_name}) as has comments"
)
return
if "matplotlib" in str(node):
print(f"Not floating {filename}:{node.get_lineno()} as matplotlib")
return
# Find the root node. While doing so, check that we aren't inside a try
root = node
while root.parent:
# Handle always-float and try blocks - don't leave invalid
if root.type == python_symbols.try_stmt:
if always_float:
# Check that we aren't the only entry in this suite
assert node.parent.parent.type == python_symbols.suite
if len(node.parent.parent.children) == 4:
print(
f"Not floating always-float {filename}:{node.get_lineno()} ({module_name}) as only statement inside try"
)
return
else:
print(
f"Not floating {filename}:{node.get_lineno()} ({module_name}) as inside try"
)
return
if not always_float:
# Give a special message to the user for __main__ if non-floating
if (root.type == python_symbols.if_stmt and not IGNORE_IF
and "__main__" in str(root.children[1])):
print(
f"Not floating {filename}:{node.get_lineno()} ({module_name}) as inside __main__ test if"
)
return
if root.type == python_symbols.if_stmt and not IGNORE_IF:
print(
f"Not floating {filename}:{node.get_lineno()} ({module_name}) as inside if"
)
return
root = root.parent
# Find the insertion point for this root node
insert_point = find_import_insert_point(root)
# Get the actual statement node
statement = node.parent
prev_sibling = statement.prev_sibling
next_sibling = statement.next_sibling
assert statement.type == python_symbols.simple_stmt
# Are we are the start of a scope?
parent_index = statement.parent.children.index(statement)
# From suite definition; parent_index of first statement is either 0 or 2:
# suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT
# But for our purposes can be 3 if we have a docstring.
assert parent_index != 0, "Inline statement functions not supported ATM"
prev_sibiling_is_string = (prev_sibling.type == python_symbols.simple_stmt
and prev_sibling.children[0].type
== token.STRING)
if parent_index == 2 or (parent_index == 3 and prev_sibiling_is_string):
# We're the first statement, or the first non-docstring statement.
# If we have a trailing newline, remove it. Indentation handled later.
if next_sibling and next_sibling.prefix.startswith("\n"):
next_sibling.prefix = next_sibling.prefix[1:]
# Get the previous node. This might be the sibling, or some tree-child thereof
prev_node = list(prev_sibling.leaves())[-1]
# print_node(prev_node)
if prev_node.type in {token.INDENT, token.DEDENT}:
# If we just indented(dedented) then tree looks like:
# [INDENT] " "
# [simple_stmt] "" <- statement
# ...
# [NEWLINE] "" "\n"
# [LN] " "
# e.g. this next sibling node holds it's own indent but the
# statement node's indentation is handled by the indent. So we
# need to remove the indentation from the next sibling.
next_sibling.prefix = next_sibling.prefix.lstrip(" ")
if prev_node.type == token.INDENT and prev_node.prefix.isspace():
# We've got leading newlines we want to strip -
# If a function starts with a blank line(s), the "\n" and any
# stray indentation are attached to the [INDENT] as a prefix.
# Our reasoning for removing this blank line: It was showing the
# user that the import was special.
# This might be flimsy reasoning.
prev_node.prefix = ""
# We could be transplanting a node with a prefix. Move it to the sibling
next_sibling.prefix = node.prefix.rstrip(" ") + next_sibling.prefix
# Do the actual moving
statement.remove()
root.insert_child(insert_point, statement)
node.prefix = ""