def _patch_imports(self, resolver, output_path: Path) -> int:
# select modules to patch imports
query = Query()
query.paths = []
for package in resolver.graph.metainfo.package.packages:
for module_path in package:
query.paths.append(str(module_path))
# patch vendors if it's outside of main package
package_path = resolver.graph.metainfo.package.packages[0].path
if package_path.resolve() not in output_path.resolve().parents:
query.paths.append(str(output_path))
# set renamings
root = Path(self.config['project'])
for library in output_path.iterdir():
if library.name in self.config['vendor']['exclude']:
continue
library_module = '.'.join(library.resolve().relative_to(
str(root)).parts)
self.logger.debug('patch imports',
extra=dict(
old_name=library.name,
new_name=library_module,
))
query = transform_imports(
query=query,
old_name=library.name,
new_name=library_module,
)
# execute renaming
query.execute(interactive=False, write=True, silent=True)
return len(query.paths)
def run_bowler_modifier(
self,
input_text,
selector=None,
modifier=None,
selector_func=None,
modifier_func=None,
in_process=True,
):
"""Returns the modified text."""
if not (selector or selector_func):
raise ValueError("Pass selector")
if not (modifier or modifier_func):
raise ValueError("Pass modifier")
exception_queue = multiprocessing.Queue()
def local_modifier(node, capture, filename):
# When in_process=False, this runs in another process. See notes below.
try:
return modifier(node, capture, filename)
except Exception as e:
exception_queue.put(e)
with tempfile.NamedTemporaryFile(suffix=".py") as f:
# TODO: I'm almost certain this will not work on Windows, since
# NamedTemporaryFile has it already open for writing. Consider
# using mktemp directly?
with open(f.name, "w") as fw:
fw.write(input_text + "\n")
if selector_func:
query = selector_func([f.name])
else:
query = Query([f.name]).select(selector)
if modifier_func:
# N.b. exceptions may not work
query = modifier_func(query)
else:
query = query.modify(local_modifier)
# We require the in_process parameter in order to record coverage properly,
# but it also helps in bubbling exceptions and letting tests read state set
# by modifiers.
query.execute(interactive=False,
write=True,
silent=False,
in_process=in_process)
# In the case of in_process=False (mirroring normal use of the tool) we use
# the queue to ship back exceptions from local_process, which can actually
# fail the test. Normally exceptions in modifiers are not printed unless
# you pass --debug.
if not exception_queue.empty():
raise AssertionError from exception_queue.get()
with open(f.name, "r") as fr:
return fr.read().rstrip()
def replace_unicode(path):
"""
Run the bowler query on the input files for refactoring.
"""
(Query(path).select_function("__unicode__").rename('__str__').idiff()),
(Query(path).select_method("__unicode__").is_call().rename(
'__str__').idiff())
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--log-level", dest="log_level", type=str, choices=["DEBUG", "INFO", "WARNING", "ERROR"], help="set log level, default is INFO")
parser.add_argument("--no-log-file", dest="no_log_file", action='store_true', default=False, help="don't log to file")
parser.add_argument("--log-filepath", dest="log_filepath", type=str, help='set log file path, default is "report.log"')
parser.add_argument("--inpath", required=True, type=str, help='the file or directory path you want to upgrade.')
parser.add_argument("--backup", type=str, nargs='?', default=None, const=None, help='backup directory, default is the "~/.paddle1to2/".')
parser.add_argument("--write", action='store_true', default=False, help='modify files in-place.')
parser.add_argument("--no-confirm", dest="no_confirm", action='store_true', default=False, help='write files in-place without confirm, ignored without --write.')
parser.add_argument("--refactor", action='append', choices=refactor.__all__, help='this is a debug option. Specify refactor you want to run. If none, all refactors will be run.')
parser.add_argument("--print-match", action='store_true', default=False, help='this is a debug option. Print matched code and node for each file.')
args = parser.parse_args()
if args.refactor:
args.refactor = set(args.refactor)
if args.backup is None:
home = os.path.expanduser('~')
args.backup = os.path.join(home, '.paddle1to2')
else:
args.backup = os.path.expanduser(args.backup)
if args.log_level:
logger.setLevel(args.log_level)
if not args.no_log_file:
log_to_file(args.log_filepath)
if not should_convert(args.inpath):
logger.error("convert abort!")
sys.exit(1)
# refactor code via "Query" step by step.
q = Query(args.inpath)
for fn in refactor.__all__:
refactor_func = getattr(refactor, fn)
if args.refactor and fn not in args.refactor:
continue
assert callable(refactor_func), "{} is not callable.".format(fn)
logger.debug("run refactor: {}".format(fn))
if args.print_match:
refactor_func(q, change_spec).filter(filters.print_match)
else:
refactor_func(q, change_spec)
if args.write:
# backup args.inpath
backup = backup_inpath(args.inpath, args.backup)
# print diff to stdout, and modify file in place.
if utils.is_windows():
q.execute(write=True, silent=False, need_confirm=not args.no_confirm, backup=backup, in_process=True)
else:
q.execute(write=True, silent=False, need_confirm=not args.no_confirm, backup=backup)
else:
# print diff to stdout
if utils.is_windows():
q.execute(write=False, silent=False, in_process=True)
else:
q.execute(write=False, silent=False)
click.secho('Refactor finished without touching source files, add "--write" to modify source files in-place if everything is ok.', fg="red", bold=True)
def api_rename(q: Query, change_spec):
"""
1. rename old api to new api. e.g.
origin code snippet:
```
a = old_path.old_to.old_api(1, 2)
```
refactored code snippet:
```
a = new_path.new_to.new_api(1, 2)
```
2. print warning if specified api are used.
"""
# construct api rename mapping and api warning mapping
rename_map = {}
warning_map = {}
for main_alias, v in change_spec.items():
new_api_name = v.get('update_to', None)
if new_api_name is not None:
rename_map[main_alias] = new_api_name
warning = v.get('warning', None)
if warning is not None:
warning_map[main_alias] = warning
pattern = """ power< 'paddle' trailer< any* >* > """
def _api_rename(node: LN, capture: Capture, filename: Filename):
code = ''
for leaf in node.leaves():
code = code + leaf.value
found_rename = False
found_warning = False
api = None
for _api in rename_map.keys():
if utils.startswith(code, _api):
found_rename = True
api = _api
break
for _api in warning_map.keys():
if utils.startswith(code, _api):
found_warning = True
api = _api
break
if not found_rename and not found_warning:
return
# if found rename, replace old_api with new_api
if found_rename:
utils.replace_module_path(node, api, rename_map[api])
# if not found rename and found warning, print warning
elif found_warning:
log_warning(filename, node.get_lineno(), warning_map[api])
q.select(pattern).modify(_api_rename)
return q
def default_query_func(files):
if selector_func:
q = selector_func(files)
else:
q = Query(files).select(selector)
if modifier_func:
q = modifier_func(q)
else:
q = q.modify(modifier)
return q
def __init__(self, input, nobackups, show_diffs):
"""
Args:
input: path to file to be refactored.
nobackups: If true no backup '.bak' files will be created for those
files that are being refactored.
show_diffs: Should diffs of the refactoring be printed to stdout?
"""
self.nobackups = nobackups
self.show_diffs = show_diffs
self.fn = input
self.query = Query([self.fn])
def main():
"""Runs the query. Called by bowler if run as a script"""
do_write = "--do" in sys.argv
do_silent = "--silent" in sys.argv
(Query().select(PATTERN).filter(do_filter).modify(process_class).execute(
interactive=False, write=do_write, silent=do_silent))
def get_query(path):
return (
Query(path)
.select(PATTERN)
.modify(remove_debugger_statements)
.execute(interactive=False, write=True)
)
def rewrite(paths, interactive=False, silent=False):
"""
Rewrite the passed in paths
"""
(Query(paths).select_module("tornado").filter(filter_tornado_imports).
rename("salt.ext.tornado").select_root().select("classdef|funcdef").
filter(filter_not_decorated).modify(replace_decorators).execute(
write=True, interactive=interactive, silent=silent))
def run(models, interactive: bool = False) -> Query:
query = (
Query(models)
.select_method("ForeignKey")
.modify(on_delete_modifier)
.select_method("OneToOneField")
.modify(on_delete_modifier)
)
return query.diff(interactive=interactive)
def test_remove_marker(src, expected, tmpdir):
pyfile = tmpdir.join("t.py")
pyfile.write(src)
(Query([str(pyfile)]).select("funcdef").modify(remove_marker).execute(
interactive=False, write=True, silent=False, in_process=True))
modified = pyfile.read()
assert modified == expected
def norm_api_alias(q: Query, change_spec):
"""
rename all alias to main alias. e.g.
origin code snippet:
```
a = path1.to1.alias1()
```
refactored code snippet:
```
a = path2.to2.main_alias()
```
"""
# construct alias mapping
alias_map = {}
for main_alias, v in change_spec.items():
for alias in v.get('alias', []):
alias_map[alias] = main_alias
pattern = """ power< 'paddle' trailer< any* >* > """
def _norm(node: LN, capture: Capture, filename: Filename):
code = ''
for leaf in node.leaves():
code = code + leaf.value
found_alias = False
alias = None
for _alias in alias_map.keys():
if utils.startswith(code, _alias):
found_alias = True
alias = _alias
break
if not found_alias:
return
main_alias = alias_map[alias]
update_to = change_spec[main_alias].get('update_to', None)
# if main_alias contains "update_to" field, rename alias to "update_to" directly
utils.replace_module_path(node, alias, main_alias)
log_info(filename, node.get_lineno(),
'{} -> {}'.format(alias, main_alias))
q.select(pattern).modify(_norm)
return q
def filtered_testfuncs(src_file, filter):
results = []
def get_testfunc(node, capture, filename):
results.append(TestFunc.from_node(node, filename))
(Query([str(src_file)
]).select("funcdef").filter(filter).modify(get_testfunc).execute(
interactive=False, write=False, silent=False, in_process=True))
return results
def run_query(files, write=True, silent=False):
(
# Look for files in the current working directory
Query(*files).select_root().modify(callback=sort_imports)
# Actually run both of the above.
.execute(
# interactive diff implies write (for the bits the user says 'y' to)
interactive=False,
write=write,
silent=silent,
))
def rewrite(paths, interactive=False, silent=False):
"""
Rewrite the passed in paths
"""
# Don't waste time on non-test files
paths = utils.filter_test_files(paths)
if not paths:
return
(Query(paths).select("classdef|funcdef").filter(
filter_not_decorated).modify(replace_decorator).execute(
write=True, interactive=interactive, silent=silent))
def main():
(Query("fake.py").select("""
power<
obj=NAME
trailer<
'.' attr=NAME
>
>
""").modify(modify_attr).select(
"""atom< "{" dictsetmaker< body=any* > "}" >""").filter(
filter_dict_literal).modify(modify_dict_literal).diff())
def main(argv=None):
"""Runs the query. Called by bowler if run as a script"""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--do",
action="store_true",
help="Actually write the changes")
parser.add_argument("--silent",
action="store_true",
help="Do the processing quietly")
parser.add_argument("filenames",
metavar="FILE",
nargs="*",
help="Specific filenames to process")
parser.add_argument(
"--logger",
metavar="NAME",
default="logger",
help="The conventional name for loggers",
)
args = parser.parse_args(argv)
# Logger might be a lookup pattern. Split it.
logger_pattern = f"'{args.logger}'"
if "." in args.logger:
parts = args.logger.split(".")
logger_pattern = f"'{parts[0]}' " + " ".join(f"trailer < '.' '{x}' >"
for x in parts[1:])
PERCENT_PATTERN = f"""
power <
{logger_pattern}
trailer
call=trailer <
'(' term < formatstr=any '%' vars=any > ')'
>
>"""
FORMAT_PATTERN = f"""
power <
{logger_pattern}
trailer
call=trailer <
'(' formatblock=any <
formatstr=STRING trailer < "." "format" >
any < '(' arglist=any ')'>
> ')'
>
>
"""
(Query(
args.filenames).select(PERCENT_PATTERN).modify(process_percent_format).
select(FORMAT_PATTERN).modify(process_format_format).execute(
interactive=False, write=args.do, silent=args.silent))
def run_removal_query(path):
"""
Run the bowler query on the input files for refactoring.
"""
(
Query(path)
.select("decorator<'@' name='python_2_unicode_compatible' any>")
.modify(remove_node)
.select("import_from<'from' module_name=any 'import' 'python_2_unicode_compatible'>")
.modify(remove_node)
.write()
)
def transform_imports(query: Query, old_name: str, new_name: str) -> Query:
params = dict(
name=old_name,
dotted_name=' '.join(quoted_parts(old_name)),
power_name=' '.join(power_parts(old_name)),
)
for modifier_class in modifiers:
modifier = modifier_class(old_name=old_name, new_name=new_name)
selector = modifier.selector.format(**params)
query = query.select(selector).modify(modifier)
return query
def main():
parser = argparse.ArgumentParser(
description=
"Adds some py2&3 compatibility that modernize/futurize missed")
parser.add_argument(
'--no-input',
dest='interactive',
default=True,
action='store_false',
help="Non-interactive mode",
)
parser.add_argument(
'--no-write',
dest='write',
default=True,
action='store_false',
help=
"Don't write the changes to the source file, just output a diff to stdout",
)
parser.add_argument(
'--debug',
dest='debug',
default=False,
action='store_true',
help="Spit out debugging information",
)
parser.add_argument('files',
nargs='+',
help="The python source file(s) to operate on.")
args = parser.parse_args()
# No way to pass this to .modify() callables, so we just set it at module level
flags['debug'] = args.debug
(
# Look for files in the current working directory
Query(*args.files).select("""
classdef<
"class" classname=NAME any* ":"
suite=suite<
any*
func=funcdef< "def" funcname="__unicode__" parameters< "(" NAME ")" > any* >
any*
>
>
""").modify(callback=replace_unicode_methods)
# Actually run all of the above.
.execute(
# interactive diff implies write (for the bits the user says 'y' to)
interactive=(args.interactive and args.write),
write=args.write,
))
def test_transform_imports(code_in: str, code_out: str, old_name: str, new_name: str, temp_path: Path):
code_in += '\n'
code_out += '\n'
path = temp_path / 'tmp.py'
path.write_text(code_in)
query = transform_imports(query=Query(str(path)), old_name=old_name, new_name=new_name)
query.execute(silent=True, write=True, interactive=False)
result = path.read_text()
if code_in == code_out:
assert result == code_out, 'unexpected changes'
else:
assert result != code_in, 'nothing was changed'
assert result == code_out, 'invalid changes'
def main():
"""Runs the query. Called by bowler if run as a script"""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--do",
action="store_true",
help="Actually write the changes")
parser.add_argument("--silent",
action="store_true",
help="Do the processing quietly")
parser.add_argument("--ignoreif",
action="store_true",
help="Float from inside if statements")
parser.add_argument(
"--stdlib",
action="store_true",
help="Only float standard library imports. Same as --only=STDLIB",
)
parser.add_argument(
"--only",
action="store",
help=
"Comma-separated list of modules to float. All others will be ignored. Can pass in all caps e.g. STDLIB and the isort classification will be used.",
)
parser.add_argument("filenames",
metavar="FILE",
nargs="*",
help="Specific filenames to process")
args = parser.parse_args()
global IGNORE_IF, ONLY_FLOAT
IGNORE_IF = args.ignoreif
# Don't allow stdlib floating if isort is not available
if args.stdlib and isort is None:
sys.exit(
"Can not classify standard library modules; isort not installed")
# Handle specific subsets of floating
if args.only:
only_list_prep = [x.strip() for x in args.only.split(",")]
if any(x.isupper() for x in only_list_prep) and isort is None:
sys.exit(
"Can not use isort classification as isort is not available")
ONLY_FLOAT = set(only_list_prep)
if args.stdlib:
ONLY_FLOAT.add("STDLIB")
(Query(args.filenames).select(PATTERN)
# .select_root()
.modify(process_import).execute(interactive=False,
write=args.do,
silent=args.silent))
class RefactorTool:
def __init__(self, input, nobackups, show_diffs):
"""
Args:
input: path to file to be refactored.
nobackups: If true no backup '.bak' files will be created for those
files that are being refactored.
show_diffs: Should diffs of the refactoring be printed to stdout?
"""
self.nobackups = nobackups
self.show_diffs = show_diffs
self.fn = input
self.query = Query([self.fn])
def rename_methods(self):
for old_name, new_name in methods:
self.query.select_method(old_name).rename(new_name)
def fix_imports(self):
# Fix old style: from playwright import sync_playwright
self.query.select_module("sync_playwright").select_module(
"playwright").rename("playwright.sync_api")
pass
def output_diffs(self):
self.query.diff()
def write_file(self):
if not self.nobackups:
# Make a backup before refactor
backup = self.fn + ".bak"
if os.path.lexists(backup):
try:
os.remove(backup)
except OSError as err:
self.log_message("Cannot remove backup %s" % backup)
try:
shutil.copyfile(self.fn, backup)
except OSError as err:
self.log_message("Cannot copy %s to %s" % (self.fn, backup))
self.query.write()
def log_message(self, msg):
print("Info: " + msg)
def log_error(self, msg):
print("Error: " + msg)
def main():
parser = argparse.ArgumentParser(
description="Removes bytestring literals. Be careful with this!"
)
parser.add_argument(
'--no-input',
dest='interactive',
default=True,
action='store_false',
help="Non-interactive mode",
)
parser.add_argument(
'--no-write',
dest='write',
default=True,
action='store_false',
help="Don't write the changes to the source file, just output a diff to stdout",
)
parser.add_argument(
'--debug',
dest='debug',
default=False,
action='store_true',
help="Spit out debugging information",
)
parser.add_argument(
'files', nargs='+', help="The python source file(s) to operate on."
)
args = parser.parse_args()
# No way to pass this to .modify() callables, so we just set it at module level
flags['debug'] = args.debug
(
# Look for files in the current working directory
Query(*args.files)
.select(
"""
STRING
"""
)
.modify(callback=debytesify)
# Actually run all of the above.
.execute(
# interactive diff implies write (for the bits the user says 'y' to)
interactive=(args.interactive and args.write),
write=args.write,
)
)
def rewrite(paths, interactive=False, silent=False):
"""
Rewrite the passed in paths
"""
(Query(paths).select("""
(
file_input<
simple_stmt<
[STRING]
docstring=any*
>
any*
>
|
class_def=classdef<
any*
suite<
any*
simple_stmt<
[STRING]
docstring=any*
>
any*
>
any*
>
|
funcdef<
any*
suite<
any*
simple_stmt<
[STRING]
docstring=any*
>
any*
>
any*
>
)
""").filter(filter_no_module_docstrings).modify(
fix_module_docstrings).execute(write=True,
interactive=interactive,
silent=silent))
def _refactor_helper(refactor_func, input_src, change_spec):
try:
ntf = NamedTemporaryFile(suffix='.py', delete=False)
ntf.write(input_src.encode('utf-8'))
ntf.close()
q = Query(ntf.name)
if utils.is_windows():
refactor_func(q, change_spec).execute(write=True,
silent=True,
need_confirm=False,
print_hint=False,
in_process=True)
else:
refactor_func(q, change_spec).execute(write=True,
silent=True,
need_confirm=False,
print_hint=False)
with open(ntf.name, 'r') as f:
output_src = f.read()
return output_src
finally:
os.remove(ntf.name)
def main():
"""Runs the query. Called by bowler if run as a script"""
global LIBS
global LIB_USES
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--do", action="store_true", help="Actually write the changes")
parser.add_argument(
"--silent", action="store_true", help="Do the processing quietly"
)
# parser.add_argument(
# "--ignoreif", action="store_true", help="Float from inside if statements"
# )
parser.add_argument(
"library_path",
metavar="LIBDIR",
nargs="?",
help="Where to find built libraries",
)
args = parser.parse_args()
LIBS = [x.stem for x in Path(args.library_path).glob("*.so")]
print(f"Looking for {len(LIBS)} library imports")
# print(", ".join(LIBS))
# global IGNORE_IF
# IGNORE_IF = args.ignoreif
(
Query()
.select(PATTERN)
# .select_root()
.modify(process_import)
.execute(interactive=False, write=args.do, silent=args.silent)
)
with open("libs.list", "w") as f:
f.write(pprint.pformat(LIB_USES))
print("Wrote results to libs.list")
def main():
"""Runs the query. Called by bowler if run as a script"""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--do",
action="store_true",
help="Actually write the changes")
parser.add_argument("--silent",
action="store_true",
help="Do the processing quietly")
parser.add_argument("--onlyfloat",
action="store_true",
help="Only float, don't also fixup")
parser.add_argument("filenames",
metavar="FILE",
nargs="*",
help="Specific filenames to process")
args = parser.parse_args()
# Hack in a separate mode for updating everything for py3
global FLOAT_MODE
FLOAT_MODE = args.onlyfloat
(Query(args.filenames).select_root().modify(process_root).execute(
interactive=False, write=args.do, silent=args.silent))
def __init__(self):
self.qry = Query()
