def convert_with_2to3(filepath: str) -> str:
warnings.warn('convert_with_2to3() is deprecated',
RemovedInSphinx60Warning,
stacklevel=2)
try:
from lib2to3.pgen2.parse import ParseError
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
except ImportError as exc:
# python 3.9.0a6+ emits PendingDeprecationWarning for lib2to3.
# Additionally, removal of the module is still discussed at PEP-594.
# To support future python, this catches ImportError for lib2to3.
raise SyntaxError from exc
fixers = get_fixers_from_package('lib2to3.fixes')
refactoring_tool = RefactoringTool(fixers)
source = refactoring_tool._read_python_source(filepath)[0]
try:
tree = refactoring_tool.refactor_string(source, 'conf.py')
except ParseError as err:
# do not propagate lib2to3 exceptions
lineno, offset = err.context[1]
# try to match ParseError details with SyntaxError details
raise SyntaxError(err.msg,
(filepath, lineno, offset, err.value)) from err
return str(tree)
def convert2to3(cell):
original_code = cell['source']
try:
x = ast.parse(original_code)
y = tokenize_and_templatize(original_code)
return cell
except:
pass
# fixers = ['lib2to3.fixes.fix_print']
# refactor = RefactoringTool(fixer_names=fixers)
# tree = refactor.refactor_string(original_code, 'temp')
try:
# fixers = get_fixers_from_package('lib2to3.fixes')
# the only python2 issue is print statements so we only import this
# fixer since it makes RefactorTool way faster
fixers = ['lib2to3.fixes.fix_print']
refactor = RefactoringTool(fixer_names=fixers)
tree = refactor.refactor_string(original_code, 'temp')
converted_code = str(tree)
print(converted_code)
x = ast.parse(converted_code)
# y = tokenize_and_templatize(converted_code)
# print('===============\n', original_code, '\n', converted_code)
cell['source'] = converted_code
except Exception as e:
print(e)
# print('==========\n', print(original_code))
pass
return cell
def load_script(script_path: Path):
from . import sfm
from . import vs
from . import sfm_utils
hooks = {
'vs': vs,
'sfm': sfm.SFM(),
}
# if sys.meta_path[0]!=
importer = HookedImporter(hooks=hooks)
importer.install()
assert script_path.exists()
with script_path.open('r') as f:
script_data = f.read()
refactoring_tool = RefactoringTool(
fixer_names=get_fixers_from_package('lib2to3.fixes'))
node3 = refactoring_tool.refactor_string(script_data + '\n', 'script')
compiled = compile(str(node3), str(script_path), 'exec')
exec(
compiled, {
'sfm': sfm.SFM(),
'sfmUtils': sfm_utils.SfmUtils(),
'print': sfm.sfm.sfm_logger.print
})
def main(argv=None):
if argv is None:
argv = sys.argv
path = argv[1]
fixer_names = get_fixers_from_package('pyramid.fixers')
tool = RefactoringTool(fixer_names)
tool.refactor([path], write=True)
fix_zcml(path)
def main(argv=None):
if argv is None:
argv = sys.argv
path = argv[1]
fixer_names = get_fixers_from_package('pyramid.fixers')
tool = RefactoringTool(fixer_names)
tool.refactor([path], write=True)
fix_zcml(path)
def setup():
"""
Call this before using the refactoring tools to create them on demand
if needed.
"""
if None in [RTs._rt, RTs._rtp]:
RTs._rt = RefactoringTool(myfixes)
RTs._rtp = RefactoringTool(myfixes, {'print_function': True})
def __init__(self):
fixers = [
'lib2to3.fixes.fix_%s' % name for name in [
'apply',
'asserts',
'basestring',
'buffer',
'dict',
'except',
'exec',
'execfile',
'exitfunc',
'filter',
'funcattrs',
'future',
'getcwdu',
'has_key',
'idioms',
'import',
'imports',
'imports2',
'input',
'intern',
'isinstance',
'itertools',
'itertools_imports',
'long',
'map',
'metaclass',
'methodattrs',
'ne',
'nonzero',
'numliterals',
'operator',
'paren',
'print',
'raise',
'raw_input',
'reduce',
'reload',
'renames',
'repr',
'set_literal',
'standarderror',
'sys_exc',
'throw',
'tuple_params',
'types',
'unicode',
'urllib',
'ws_comma',
'xrange',
'xreadlines',
'zip',
]
]
RefactoringTool.__init__(self, fixers)
def setup_detect_python2():
"""
Call this before using the refactoring tools to create them on demand
if needed.
"""
if None in [RTs._rt_py2_detect, RTs._rtp_py2_detect]:
RTs._rt_py2_detect = RefactoringTool(py2_detect_fixers)
RTs._rtp_py2_detect = RefactoringTool(py2_detect_fixers,
{'print_function': True})
def pathlab_string(input):
sys.path.insert(0, Path(__file__).parent.absolute())
refactoring_tool = RefactoringTool(
fixer_names=get_fixers_from_package('pathlad.custom_fixers'))
result = input
result = str(refactoring_tool.refactor_string(result + '\n', 'paths'))[:-1]
result = str(
refactoring_tool.refactor_string(result + '\n', 'paths_nested'))[:-1]
return result
def convert_with_2to3(filepath):
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
from lib2to3.pgen2.parse import ParseError
fixers = get_fixers_from_package('lib2to3.fixes')
refactoring_tool = RefactoringTool(fixers)
source = refactoring_tool._read_python_source(filepath)[0]
try:
tree = refactoring_tool.refactor_string(source, 'conf.py')
except ParseError, err:
# do not propagate lib2to3 exceptions
lineno, offset = err.context[1]
# try to match ParseError details with SyntaxError details
raise SyntaxError(err.msg, (filepath, lineno, offset, err.value))
def convert_with_2to3(filepath):
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
from lib2to3.pgen2.parse import ParseError
fixers = get_fixers_from_package('lib2to3.fixes')
refactoring_tool = RefactoringTool(fixers)
source = refactoring_tool._read_python_source(filepath)[0]
try:
tree = refactoring_tool.refactor_string(source, 'conf.py')
except ParseError, err:
# do not propagate lib2to3 exceptions
lineno, offset = err.context[1]
# try to match ParseError details with SyntaxError details
raise SyntaxError(err.msg, (filepath, lineno, offset, err.value))
def run_2to3(cls, path):
from lib2to3.refactor import get_fixers_from_package, RefactoringTool
rt = RefactoringTool(get_fixers_from_package('lib2to3.fixes'))
with TRACER.timed('Translating %s' % path):
for root, dirs, files in os.walk(path):
for fn in files:
full_fn = os.path.join(root, fn)
if full_fn.endswith('.py'):
with TRACER.timed('%s' % fn, V=3):
try:
chmod_plus_w(full_fn)
rt.refactor_file(full_fn, write=True)
except IOError as e:
TRACER.log('Failed to translate %s: %s' % (fn, e))
def run_2to3(cls, path):
from lib2to3.refactor import get_fixers_from_package, RefactoringTool
rt = RefactoringTool(get_fixers_from_package('lib2to3.fixes'))
with TRACER.timed('Translating %s' % path):
for root, dirs, files in os.walk(path):
for fn in files:
full_fn = os.path.join(root, fn)
if full_fn.endswith('.py'):
with TRACER.timed('%s' % fn, V=3):
try:
chmod_plus_w(full_fn)
rt.refactor_file(full_fn, write=True)
except IOError as e:
TRACER.log('Failed to translate %s: %s' % (fn, e))
def test_convert_state_file_to_conf_74(self):
"""
Tests conversion of the pstate files (pre-7.4.0) files to .conf files. Tests for two different pstate files,
one corrupted with incorrect metainfo data, and the other one with correct working metadata.
"""
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
refactoring_tool = RefactoringTool(fixer_names=get_fixers_from_package('lib2to3.fixes'))
os.makedirs(os.path.join(self.state_dir, STATEDIR_CHECKPOINT_DIR))
# Copy a good working Ubuntu pstate file
src_path = os.path.join(self.CONFIG_PATH, "194257a7bf4eaea978f4b5b7fbd3b4efcdd99e43.state")
dest_path = os.path.join(self.state_dir, STATEDIR_CHECKPOINT_DIR, "ubuntu_ok.state")
shutil.copyfile(src_path, dest_path)
convert_state_file_to_conf_74(dest_path, refactoring_tool)
converted_file_path = os.path.join(self.state_dir, STATEDIR_CHECKPOINT_DIR, "ubuntu_ok.conf")
self.assertTrue(os.path.exists(converted_file_path))
self.assertFalse(os.path.exists(dest_path))
os.remove(converted_file_path)
# Copy Ubuntu pstate file with corrupted metainfo data
src_path = os.path.join(self.CONFIG_PATH, "194257a7bf4eaea978f4b5b7fbd3b4efcdd99e43_corrupted.state")
dest_path = os.path.join(self.state_dir, STATEDIR_CHECKPOINT_DIR, "ubuntu_corrupted.state")
shutil.copyfile(src_path, dest_path)
convert_state_file_to_conf_74(dest_path, refactoring_tool)
converted_file_path = os.path.join(self.state_dir, STATEDIR_CHECKPOINT_DIR, "ubuntu_corrupted.conf")
self.assertFalse(os.path.exists(converted_file_path))
self.assertFalse(os.path.exists(dest_path))
def convert_with_2to3(filepath: str) -> str:
warnings.warn('convert_with_2to3() is deprecated',
RemovedInSphinx60Warning, stacklevel=2)
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
from lib2to3.pgen2.parse import ParseError
fixers = get_fixers_from_package('lib2to3.fixes')
refactoring_tool = RefactoringTool(fixers)
source = refactoring_tool._read_python_source(filepath)[0]
try:
tree = refactoring_tool.refactor_string(source, 'conf.py')
except ParseError as err:
# do not propagate lib2to3 exceptions
lineno, offset = err.context[1]
# try to match ParseError details with SyntaxError details
raise SyntaxError(err.msg, (filepath, lineno, offset, err.value))
return str(tree)
def transform(source):
python_zip_path = str(Path(os.path.dirname(os.__file__)).parents[0])
python_zip_file = get_zip_file(python_zip_path)
python_zip_folder = zipfile.ZipFile(os.path.join(python_zip_path, python_zip_file))
fixers = get_all_fixers_from_zipfolder(python_zip_folder)
refactoring_tool = RefactoringTool(fixers)
return refactor_script(source, refactoring_tool)
def convert_config_to_tribler74(state_dir):
"""
Convert the download config files to Tribler 7.4 format. The extensions will also be renamed from .state to .conf
"""
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
refactoring_tool = RefactoringTool(fixer_names=get_fixers_from_package('lib2to3.fixes'))
for filename in (state_dir / STATEDIR_CHECKPOINT_DIR).glob('*.state'):
convert_state_file_to_conf_74(filename, refactoring_tool=refactoring_tool)
def run_2to3(files, doctests_only=False, fixer_names=None, options=None, explicit=None):
""" Wrapper function around the refactor() class which
performs the conversions on a list of python files.
Invoke 2to3 on a list of Python files. The files should all come
from the build area, as the modification is done in-place."""
# if not files:
# return
# Make this class local, to delay import of 2to3
from lib2to3.refactor import get_fixers_from_package, RefactoringTool
fixers = get_fixers_from_package("lib2to3.fixes")
if fixer_names:
for fixername in fixer_names:
fixers.extend(get_fixers_from_package(fixername))
r = RefactoringTool(fixers, options=options)
r.refactor(files, write=True, doctests_only=doctests_only)
def transform(source):
python_zip_path = str(Path(os.path.dirname(os.__file__)).parents[0])
python_zip_file = get_zip_file(python_zip_path)
python_zip_folder = zipfile.ZipFile(os.path.join(python_zip_path, python_zip_file))
fixers = get_all_fixers_from_zipfolder(python_zip_folder)
dir_path = os.getcwd()
sys.path.append(os.path.join(dir_path, '.\\python_migration_fixers'))
fixers.extend(['fix_none'])
refactoring_tool = RefactoringTool(fixers)
return refactor_script(source, refactoring_tool)
def main(argv=None):
ap = argparse.ArgumentParser()
ap.add_argument('path', type=pathlib.Path,
help="Notebook or directory containing notebooks")
options = ap.parse_args(argv)
avail_fixes = set(get_fixers_from_package('lib2to3.fixes'))
rt = RefactoringTool(avail_fixes)
if options.path.is_dir():
for nb_path in options.path.rglob('*.ipynb'):
refactor_notebook_inplace(rt, nb_path)
else:
refactor_notebook_inplace(rt, options.path)
def find_all_narrative_py2_code(ws_url: str, token: str, min_id: int,
max_id: int, outfile: str, report: str):
assert ws_url
assert token
assert min_id and min_id > 0
assert max_id and max_id >= min_id
assert outfile
ws = Workspace(url=ws_url, token=token)
all_results = {"fail": [], "no_narr": [], "no_change": [], "changes": []}
avail_fixes = set(get_fixers_from_package("lib2to3.fixes"))
rt = RefactoringTool(avail_fixes, options={"print_function": False})
for ws_id in range(min_id, max_id):
try:
result = _find_narrative_py2_code(ws_id, ws, rt, verbose=True)
if result is None:
all_results["no_narr"].append({"id": ws_id})
elif result.updated_cells == 0:
all_results["no_change"].append(result.to_dict())
else:
all_results["changes"].append(result.to_dict())
except baseclient.ServerError as e:
if "No workspace with id" in str(e):
print(f"WS:{ws_id} does not exist")
all_results["fail"].append({"id": ws_id, "error": str(e.message)})
except TypeError as e:
print(f"WS:{ws_id} metadata doesn't link to a Narrative type!")
all_results["fail"].append({"id": ws_id, "error": str(e)})
except json.JSONDecodeError as e:
print(f"WS:{ws_id} unable to unpack Narrative - not valid JSON!")
all_results["fail"].append({
"id":
ws_id,
"error":
f"Invalid JSON in Narrative object: {str(e)}"
})
with open(outfile, "w") as fjson:
fjson.write(json.dumps(all_results, indent=4))
print(f"Done. Results in {outfile}")
if report is not None:
with open(report, "w") as ftsv:
ftsv.write("user name\tnarrative id\tlast saved\n")
for n in all_results["changes"]:
ftsv.write(f"{n['owner']}\t{n['id']}\t{n['last_saved']}\n")
def _update_narrative(narr_obj: list, ws_info: list,
rt: RefactoringTool) -> NarrativeInfo:
"""
Core pieces of this taken from this gist by Thomas Takluyver and Fernando Perez:
https://gist.github.com/takluyver/c8839593c615bb2f6e80
"""
try:
nb = nbformat.reads(json.dumps(narr_obj['data']), 4.0)
except:
nb = nbformat.reads(json.dumps(narr_obj['data']), 3.0)
ninfo = NarrativeInfo(narr_obj['info'], ws_info[2])
cells = list()
if nb.nbformat == 4:
cells = nb.cells
else:
cells = nb.worksheets[0].cells
for idx, cell in enumerate(nb.cells):
if cell.cell_type != "code":
continue
head = ''
source = ''
if hasattr(cell, 'source'):
source = cell.source
elif hasattr(cell, 'input'):
source = cell.input
else:
continue
if source.startswith('%%'):
split_source = cell.source.split('\n', 1)
if len(split_source) == 2:
head, source = split_source
try:
tree = rt.refactor_string(source + "\n", f"{ws_info[0]}-cell{idx}")
result = str(tree)[:-1]
except (lib2to3.pgen2.parse.ParseError,
lib2to3.pgen2.tokenize.TokenError):
result = source
if head:
source = head + '\n' + source
result = head + '\n' + result
ninfo.add_updated_cell(idx, source, result)
return ninfo
class TestRefactoringTool:
def __init__(self, fixer):
self.refactoring_tool = RefactoringTool([fixer], {}, explicit=True)
def refactor(self, before):
print("INPUT: ", before)
print(
"PARSED: ",
repr(self.refactoring_tool.driver.parse_string(reformat(before))),
)
return str(
self.refactoring_tool.refactor_string(reformat(before),
"<string>"))
def refactor_and_check(self, before, expected):
__tracebackhide__ = True
after = self.refactor(before)
assert reformat(expected).rstrip("\n") == after.rstrip("\n")
def setUp(self):
self.refactor = RefactoringTool(['fix_name2']).refactor_string
def setUp(self):
self.refactor = RefactoringTool(['fix_indent']).refactor_string
def setUp(self):
self.refactor = RefactoringTool(['fix_constant']).refactor_string
'lib.%s-%s' % (get_platform(), sys.version[0:3]))
if os.path.exists(platlib):
sys.path.insert(0, platlib)
import psycopg2
if sys.version_info < (3,):
import tests
else:
from distutils.file_util import copy_file
import glob
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
if not os.path.isdir("py3tests"):
os.mkdir("py3tests")
new = []
for f in glob.glob("tests/*.py"):
f, copied = copy_file(f, "py3tests", preserve_times=0, update=1)
if copied:
new.append(f)
fixers = get_fixers_from_package('lib2to3.fixes')
r = RefactoringTool(fixers)
r.refactor(new, write=True)
import py3tests as tests
def test_suite():
return tests.test_suite()
if __name__ == '__main__':
unittest.main(defaultTest='test_suite')
# Converts an individual Python file or a directory of Python files to Python3
# Note - These functions will overwrite the files after conversion. Originals are not retained.
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
avail_fixes = set(get_fixers_from_package('lib2to3.fixes'))
rt = RefactoringTool(avail_fixes)
# Converts and overwrites a Python file to Python3
def convert_file(filename):
rt.refactor_file(filename, write=True)
# Converts and overwrites all files ending in .py in the directory to Python3
def convert_directory(directoryname):
rt.refactor_dir(directoryname,write=True)
# This is a Python 3.x script to build pywin32. It converts then executes
# the regular setup.py script.
import os
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
fixers = ['lib2to3.fixes.fix_print', 'lib2to3.fixes.fix_except']
options = dict(doctests_only=False, fix=[], list_fixes=[],
print_function=False, verbose=False,
write=True)
r = RefactoringTool(fixers, options)
script = os.path.join(os.path.dirname(__file__), "setup.py")
data = open(script).read()
print("Converting...")
got = r.refactor_string(data, "setup.py")
print("Executing...")
exec(str(got))
import traceback
from lib2to3.refactor import RefactoringTool
import packaging.version
from Cheetah.Compiler import Compiler
from Cheetah.NameMapper import NotFound
from Cheetah.Template import Template
from past.translation import myfixes
from . import unicodify
# Skip libpasteurize fixers, which make sure code is py2 and py3 compatible.
# This is not needed, we only translate code on py3.
if sys.version_info.major > 2:
myfixes = [f for f in myfixes if not f.startswith('libpasteurize')]
refactoring_tool = RefactoringTool(myfixes, {'print_function': True})
else:
myfixes = refactoring_tool = None
class FixedModuleCodeCompiler(Compiler):
module_code = None
def getModuleCode(self):
self._moduleDef = self.module_code
return self._moduleDef
def create_compiler_class(module_code):
class CustomCompilerClass(FixedModuleCodeCompiler):
def refactor(x):
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
fixer_names = get_fixers_from_package('lib2to3.fixes')
r = RefactoringTool(fixer_names, options=None)
r.refactor([x], write=True)
from libfuturize import fixes
__version__ = '0.1.0'
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
myfixes = (list(fixes.libfuturize_fix_names_stage1) +
list(fixes.lib2to3_fix_names_stage1) +
list(fixes.libfuturize_fix_names_stage2) +
list(fixes.lib2to3_fix_names_stage2))
# There are two possible grammars: with or without the print statement.
# Hence we have two possible refactoring tool implementations.
_rt = RefactoringTool(myfixes)
_rtp = RefactoringTool(myfixes, {'print_function': True})
#
# We detect whether the code is Py2 or Py3 by applying certain lib2to3 fixers
# to it. If the diff is empty, it's Python 3 code.
py2_detect_fixers = [
# From stage 1:
'lib2to3.fixes.fix_apply',
# 'lib2to3.fixes.fix_dict', # TODO: add support for utils.viewitems() etc. and move to stage2
'lib2to3.fixes.fix_except',
'lib2to3.fixes.fix_execfile',
'lib2to3.fixes.fix_exitfunc',
'lib2to3.fixes.fix_funcattrs',
'lib2to3.fixes.fix_filter',
long
except NameError:
long = int
try:
unicode
except NameError:
unicode = str
try:
unichr
except NameError:
unichr = chr
try:
literal_eval(r"u'\u4444'")
except SyntaxError:
# Remove 'u' prefixes in unicode literals in Python 3
rtp_fix_unicode = RefactoringTool(['lib2to3.fixes.fix_unicode'],
{'print_function': True})
else:
rtp_fix_unicode = None
def escape(text):
"""Escape unprintable or non-ASCII characters, double quotes and ampersands
in a string using XML character references.
"""
def fixup(m):
ch = m.group(0)
return '&#' + str(ord(ch)) + ';'
text = re.sub('[^ -~]|[&"]', fixup, text)
return text if isinstance(text, str) else str(text)
def refactor(self, source):
return RefactoringTool(self._fixers).refactor_string(
source, " ".join(chain([__name__], self._fixers)))
def convert_string(input):
tool = RefactoringTool(get_fixers_from_package("doc484.fixes"))
tree = tool.refactor_string(input, '<test.py>')
return str(tree)
def __init__(self, fixer):
self.refactoring_tool = RefactoringTool([fixer], {}, explicit=True)
import argparse
import os
import sys
import imp
import runpy
from io import StringIO
from pkgutil import ImpImporter, ImpLoader
import runpy
import sys
import tempfile
import lib2to3
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
fixes = get_fixers_from_package('lib2to3.fixes')
rt = RefactoringTool(fixes)
PACKAGES = []
DIRS = []
def maybe_2to3(filename, modname=None):
"""Returns a python3 version of filename."""
need_2to3 = False
filename = os.path.abspath(filename)
if any(filename.startswith(d) for d in DIRS):
need_2to3 = True
elif modname is not None and any(modname.startswith(p) for p in PACKAGES):
need_2to3 = True
if not need_2to3:
return filename
