def __build_fixer_names(self):
if self.fixer_names: return
self.fixer_names = []
for p in setuptools.lib2to3_fixer_packages:
self.fixer_names.extend(get_fixers_from_package(p))
if self.distribution.use_2to3_fixers is not None:
for p in self.distribution.use_2to3_fixers:
self.fixer_names.extend(get_fixers_from_package(p))
def get_py2to3_converter(self, options=None, proc_count=0):
from lib2to3 import refactor as ref_mod
from snakeoil.dist import caching_2to3
if ((sys.version_info >= (3, 0) and sys.version_info < (3, 1, 2)) or
(sys.version_info >= (2, 6) and sys.version_info < (2, 6, 5))):
if proc_count not in (0, 1):
log.warn(
"disabling parallelization: you're running a python version "
"with a broken multiprocessing.queue.JoinableQueue.put "
"(python bug 4660).")
proc_count = 1
elif proc_count == 0:
import multiprocessing
proc_count = multiprocessing.cpu_count()
assert proc_count >= 1
if proc_count > 1 and not caching_2to3.multiprocessing_available:
proc_count = 1
refactor_kls = caching_2to3.MultiprocessRefactoringTool
fixer_names = ref_mod.get_fixers_from_package('lib2to3.fixes')
f = refactor_kls(fixer_names, options=options).refactor
def f2(*args, **kwds):
if caching_2to3.multiprocessing_available:
kwds['num_processes'] = proc_count
return f(*args, **kwds)
return f2
def run_2to3(self, files):
# Add doctest parsing; this stuff copied from distutils.utils in
# python 3.2 source
if not files:
return
fixer_names, options, explicit = (self.fixer_names,
self.options,
self.explicit)
# Make this class local, to delay import of 2to3
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
class DistutilsRefactoringTool(RefactoringTool):
def log_error(self, msg, *args, **kw):
log.error(msg, *args)
def log_message(self, msg, *args):
log.info(msg, *args)
def log_debug(self, msg, *args):
log.debug(msg, *args)
if fixer_names is None:
fixer_names = get_fixers_from_package('lib2to3.fixes')
r = DistutilsRefactoringTool(fixer_names, options=options)
r.refactor(files, write=True)
# Then doctests
r.refactor(files, write=True, doctests_only=True)
def run(self):
py2k_rebuilds = []
if not self.inplace:
if not is_py3k:
py2k_rebuilds = list(self._compute_rebuilds(self.force))
dst_build_py.build_py.run(self)
if self.generate_verinfo:
self._run_generate_verinfo(py2k_rebuilds)
self._inner_run(py2k_rebuilds)
if is_py3k:
return
from lib3to2.build import run_3to2
from lib2to3 import refactor
# assume a few fixes are already handled in the code or aren't needed
# for py27
skip_list = (
'lib3to2.fixes.fix_str', 'lib3to2.fixes.fix_printfunction',
'lib3to2.fixes.fix_except', 'lib3to2.fixes.fix_with',
)
fixer_names = [x for x in refactor.get_fixers_from_package('lib3to2.fixes')
if x not in skip_list]
log.info("starting 3to2 conversion; this may take a while...")
run_3to2([x[0] for x in py2k_rebuilds], fixer_names=fixer_names)
for path, mtime in py2k_rebuilds:
os.utime(path, (-1, mtime))
log.info("completed py2k conversions")
def run_2to3(self, files):
excludes = build_py.opts_2to3.get('exclude')
nofix = build_py.opts_2to3.get('nofix')
groups = {'all': files}
for file in files[:]:
# Strip 'build/lib.*/kaa/module/' from name.
relfile = re.sub(r'build\/[^/]*\/kaa\/[^/]+\/', '', file)
if excludes:
for pattern in excludes:
if fnmatch.fnmatch(relfile, pattern):
files.remove(file)
if nofix:
for pattern, fixers in nofix.items():
if fnmatch.fnmatch(relfile, pattern) and file in files:
groups.setdefault(tuple(fixers), []).append(file)
files.remove(file)
from lib2to3.refactor import get_fixers_from_package
all_fixers = set(get_fixers_from_package('lib2to3.fixes'))
for fixers, files in groups.items():
if not files:
continue
if fixers == 'all':
self.fixer_names = all_fixers
else:
self.fixer_names = all_fixers.difference('lib2to3.fixes.fix_%s' % fixer for fixer in fixers)
print('Running 2to3 on %d files, this may take a while ...' % len(files))
distutils_build_py.run_2to3(self, files)
def run_2to3(files, fixer_names=None, options=None, explicit=None):
"""Invoke 2to3 on a list of Python files.
The files should all come from the build area, as the
modification is done in-place. To reduce the build time,
only files modified since the last invocation of this
function should be passed in the files argument."""
if not files:
return
# Make this class local, to delay import of 2to3
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
class DistutilsRefactoringTool(RefactoringTool):
def log_error(self, msg, *args, **kw):
log.error(msg, *args)
def log_message(self, msg, *args):
log.info(msg, *args)
def log_debug(self, msg, *args):
log.debug(msg, *args)
if fixer_names is None:
fixer_names = get_fixers_from_package('lib2to3.fixes')
r = DistutilsRefactoringTool(fixer_names, options=options)
r.refactor(files, write=True)
def test_fixers_importable():
fixers = refactor.get_fixers_from_package(LIBMODERNIZE_FIXES_PKG)
for module_name in fixers:
try:
__import__(module_name)
except ImportError:
raise AssertionError('{0!r} cannot be imported'.format(module_name))
def detect_version(filename):
"""Attempt to detect the Python major version of the source code in
`filename`.
`filename` may refer to a file or a directory.
Returns: `2` or `3`, depending on the detected version.
Raises:
ValueError: If a parsing error is detected.
"""
rt = VersionDetector(refactor.get_fixers_from_package("lib2to3.fixes"))
rt.filters.append(filter_tuple_printing)
try:
# File to parse is the first argument given.
rt.refactor([filename])
except ParseError as ex:
parseval = getattr(ex, "value", None)
if parseval in ("print", "from", "**"):
# Happens when parsing Python3 files
# though this still needs investigating.
return 3
else:
raise ValueError("Syntax/ParseError: {}".format(ex))
if rt.output:
return 2
else:
return 3
def test_fixer_loading_helpers(self):
contents = ["explicit", "first", "last", "parrot", "preorder"]
non_prefixed = refactor.get_all_fix_names("myfixes")
prefixed = refactor.get_all_fix_names("myfixes", False)
full_names = refactor.get_fixers_from_package("myfixes")
self.assertEqual(prefixed, ["fix_" + name for name in contents])
self.assertEqual(non_prefixed, contents)
self.assertEqual(full_names, ["myfixes.fix_" + name for name in contents])
def main(argv=None):
if argv is None:
argv = sys.argv
path = argv[1]
fixer_names = get_fixers_from_package('pyramid.fixers')
tool = RefactoringTool(fixer_names)
tool.refactor([path], write=True)
fix_zcml(path)
def test_fixer_loading_helpers(self):
contents = ["explicit", "first", "last", "parrot", "preorder"]
non_prefixed = refactor.get_all_fix_names("myfixes")
prefixed = refactor.get_all_fix_names("myfixes", False)
full_names = refactor.get_fixers_from_package("myfixes")
self.assertEqual(prefixed, ["fix_" + name for name in contents])
self.assertEqual(non_prefixed, contents)
self.assertEqual(full_names,
["myfixes.fix_" + name for name in contents])
def convert_config_to_tribler74(state_dir):
"""
Convert the download config files to Tribler 7.4 format. The extensions will also be renamed from .state to .conf
"""
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
refactoring_tool = RefactoringTool(fixer_names=get_fixers_from_package('lib2to3.fixes'))
for filename in (state_dir / STATEDIR_CHECKPOINT_DIR).glob('*.state'):
convert_state_file_to_conf_74(filename, refactoring_tool=refactoring_tool)
def refactor(x):
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
class DistutilsRefactoringTool(RefactoringTool):
def ignore(self, msg, *args, **kw):
pass
log_error = log_message = log_debug = ignore
fixer_names = get_fixers_from_package('lib2to3.fixes')
r = DistutilsRefactoringTool(fixer_names, options=None)
r.refactor([x], write=True)
def run_2to3(self, files, options=None):
from distutils import log
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
if not files:
return
# Make this class local, to delay import of 2to3
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
class DistutilsRefactoringTool(RefactoringTool):
def refactor(self, files, *args, **kwargs):
self._total_files = len(files)
self._refactored_files = 0
super(DistutilsRefactoringTool, self).refactor(files, *args, **kwargs)
del self._total_files
del self._refactored_files
def refactor_file(self, filename, *args, **kwargs):
if self._total_files // 10 != 0 and self._refactored_files % (self._total_files // 10) == 0:
print(
"Refactoring files: %i%% (%i on %i)."
% (
self._refactored_files / (self._total_files // 10) * 10,
self._refactored_files,
self._total_files,
)
)
self._refactored_files += 1
return super(DistutilsRefactoringTool, self).refactor_file(filename, *args, **kwargs)
def log_error(self, msg, *args, **kw):
log.error(msg, *args)
def log_message(self, msg, *args):
log.info(msg, *args)
def log_debug(self, msg, *args):
log.debug(msg, *args)
fixer_names = [
"fix_basestring",
"fix_dict",
"fix_imports",
"fix_long",
"fix_metaclass",
"fix_methodattrs",
"fix_numliterals",
"fix_types",
"fix_unicode",
"fix_urllib",
"fix_xrange",
]
fixers = list(map(lambda x: "lib2to3.fixes." + x, fixer_names))
fixers += get_fixers_from_package("2to3")
r = DistutilsRefactoringTool(fixers, options=options)
r.refactor(files, write=True)
def pathlab_string(input):
sys.path.insert(0, Path(__file__).parent.absolute())
refactoring_tool = RefactoringTool(
fixer_names=get_fixers_from_package('pathlad.custom_fixers'))
result = input
result = str(refactoring_tool.refactor_string(result + '\n', 'paths'))[:-1]
result = str(
refactoring_tool.refactor_string(result + '\n', 'paths_nested'))[:-1]
return result
def test_crlf_newlines(self):
old_sep = os.linesep
os.linesep = "\r\n"
try:
fn = os.path.join(TEST_DATA_DIR, "crlf.py")
fixes = refactor.get_fixers_from_package("lib2to3.fixes")
self.check_file_refactoring(fn, fixes)
finally:
os.linesep = old_sep
def test_crlf_newlines(self):
old_sep = os.linesep
os.linesep = "\r\n"
try:
fn = os.path.join(TEST_DATA_DIR, "crlf.py")
fixes = refactor.get_fixers_from_package("lib2to3.fixes")
self.check_file_refactoring(fn, fixes)
finally:
os.linesep = old_sep
def convert_2to3(file_content, file_name):
"""Quick helper function to convert python2 to python3 so that we can keep the ast buildin."""
# all default fixers
avail_fixes = set(refactor.get_fixers_from_package("lib2to3.fixes"))
# create default RefactoringTool, apply to passed file_content string and return fixed string
rt = refactor.RefactoringTool(avail_fixes)
tmp = rt.refactor_string(file_content, file_name)
return str(tmp)
def run_2to3(files, doctests_only=False, fixer_names=None, options=None, explicit=None):
""" Wrapper function around the refactor() class which
performs the conversions on a list of python files.
Invoke 2to3 on a list of Python files. The files should all come
from the build area, as the modification is done in-place."""
# if not files:
# return
# Make this class local, to delay import of 2to3
from lib2to3.refactor import get_fixers_from_package, RefactoringTool
fixers = get_fixers_from_package("lib2to3.fixes")
if fixer_names:
for fixername in fixer_names:
fixers.extend(get_fixers_from_package(fixername))
r = RefactoringTool(fixers, options=options)
r.refactor(files, write=True, doctests_only=doctests_only)
def run_2to3(self, files, options=None):
from distutils import log
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
if not files:
return
# Make this class local, to delay import of 2to3
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
class DistutilsRefactoringTool(RefactoringTool):
def refactor(self, files, *args, **kwargs):
self._total_files = len(files)
self._refactored_files = 0
super(DistutilsRefactoringTool,
self).refactor(files, *args, **kwargs)
del self._total_files
del self._refactored_files
def refactor_file(self, filename, *args, **kwargs):
if self._total_files//10 != 0 and \
self._refactored_files % (self._total_files//10) == 0:
print('Refactoring files: %i%% (%i on %i).' %
(self._refactored_files /
(self._total_files // 10) * 10,
self._refactored_files, self._total_files))
self._refactored_files += 1
return super(DistutilsRefactoringTool,
self).refactor_file(filename, *args, **kwargs)
def log_error(self, msg, *args, **kw):
log.error(msg, *args)
def log_message(self, msg, *args):
log.info(msg, *args)
def log_debug(self, msg, *args):
log.debug(msg, *args)
fixer_names = get_fixers_from_package('lib2to3.fixes')
fixer_names.remove('lib2to3.fixes.fix_import')
fixer_names += get_fixers_from_package('2to3')
r = DistutilsRefactoringTool(fixer_names, options=options)
r.refactor(files, write=True)
def __init__(self, *args, **kwargs):
_build_py.__init__(self, *args, **kwargs)
import logging
from lib2to3 import refactor
import lib3to2.main
rt_logger = logging.getLogger("RefactoringTool")
rt_logger.addHandler(logging.StreamHandler())
fixers = refactor.get_fixers_from_package("lib3to2.fixes")
fixers.remove("lib3to2.fixes.fix_print")
self.rtool = lib3to2.main.StdoutRefactoringTool(fixers, None, [], False, False)
def find_fixers(blacklist=None):
blacklist = blacklist or []
names = getattr(build_py, 'fixer_names', None) or []
from lib2to3.refactor import get_fixers_from_package
for p in setuptools.lib2to3_fixer_packages:
names.extend(get_fixers_from_package(p))
# explicitly remove all blacklisted fixers to trigger value errors on
# non-existing filters in the blacklist
for f in blacklist:
names.remove(f)
build_py.fixer_names = names
def refactor_python_files(root: Path, fix_package: str) -> None:
fixers = get_fixers_from_package(fix_package)
options = {"print_function": True}
refactoring_tool = StdoutRefactoringTool(
fixers=fixers,
options=options,
explicit=[],
nobackups=True,
show_diffs=False,
)
refactoring_tool.refactor([root], write=True)
def __init__(self, *args, **kwargs):
_build_py.__init__(self, *args, **kwargs)
import logging
from lib2to3 import refactor
import lib3to2.main
rt_logger = logging.getLogger("RefactoringTool")
rt_logger.addHandler(logging.StreamHandler())
fixers = refactor.get_fixers_from_package('lib3to2.fixes')
fixers.remove('lib3to2.fixes.fix_print')
self.rtool = lib3to2.main.StdoutRefactoringTool(
fixers, None, [], False, False)
def test_crlf_newlines(self):
if due_to_ironpython_bug("http://ironpython.codeplex.com/workitem/28171"):
return
old_sep = os.linesep
os.linesep = "\r\n"
try:
fn = os.path.join(TEST_DATA_DIR, "crlf.py")
fixes = refactor.get_fixers_from_package("lib2to3.fixes")
self.check_file_refactoring(fn, fixes)
finally:
os.linesep = old_sep
def run_2to3(self, files, doctests = False):
# See of the distribution option has been set, otherwise check the
# setuptools default.
if self.distribution.use_2to3 is not True:
return
if not files:
return
log.info("Fixing "+" ".join(files))
if not self.fixer_names:
self.fixer_names = []
for p in setuptools.lib2to3_fixer_packages:
self.fixer_names.extend(get_fixers_from_package(p))
if self.distribution.use_2to3_fixers is not None:
for p in self.distribution.use_2to3_fixers:
self.fixer_names.extend(get_fixers_from_package(p))
if doctests:
if setuptools.run_2to3_on_doctests:
r = DistutilsRefactoringTool(self.fixer_names)
r.refactor(files, write=True, doctests_only=True)
else:
_Mixin2to3.run_2to3(self, files)
def run_2to3(self, files, doctests=False):
# See of the distribution option has been set, otherwise check the
# setuptools default.
if self.distribution.use_2to3 is not True:
return
if not files:
return
log.info("Fixing " + " ".join(files))
if not self.fixer_names:
self.fixer_names = []
for p in setuptools.lib2to3_fixer_packages:
self.fixer_names.extend(get_fixers_from_package(p))
if self.distribution.use_2to3_fixers is not None:
for p in self.distribution.use_2to3_fixers:
self.fixer_names.extend(get_fixers_from_package(p))
if doctests:
if setuptools.run_2to3_on_doctests:
r = DistutilsRefactoringTool(self.fixer_names)
r.refactor(files, write=True, doctests_only=True)
else:
_Mixin2to3.run_2to3(self, files)
def refactor(x):
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
class DistutilsRefactoringTool(RefactoringTool):
def ignore(self, msg, *args, **kw):
pass
log_error = log_message = log_debug = ignore
fixer_names = get_fixers_from_package('lib2to3.fixes')
r = DistutilsRefactoringTool(fixer_names, options=None)
r.refactor([x], write=True)
def ConvertPy2to3(py_script_text):
"""
Converts contents of input string py_script_text to python 3 using lib2to3
and returns the result as a string.
"""
# As far as I can tell, lib2to3 isn't documented that well but once
# you find the right recipe it's very easy to use for this case.
# ref:
# https://stackoverflow.com/questions/30340151/using-2to3-on-in-memory-scripts
fixes = refactor.get_fixers_from_package('lib2to3.fixes')
converter = refactor.RefactoringTool(fixes)
ast = converter.refactor_string(py_script_text, '<script>')
return str(ast)
def run_3to2(files, fixer_names=None, options=None, explicit=None):
"""Invoke 3to2 on a list of Python files.
The files should all come from the build area, as the
modification is done in-place. To reduce the build time,
only files modified since the last invocation of this
function should be passed in the files argument."""
if not files:
return
if fixer_names is None:
fixer_names = refactor.get_fixers_from_package('lib3to2.fixes')
r = DistutilsRefactoringTool(fixer_names, options=options)
r.refactor(files, write=True)
def run_2to3(self, files, options=None):
from distutils import log
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
if not files:
return
# Make this class local, to delay import of 2to3
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
class DistutilsRefactoringTool(RefactoringTool):
def refactor(self, files, *args, **kwargs):
self._total_files = len(files)
self._refactored_files = 0
super(DistutilsRefactoringTool, self).refactor(files,
*args, **kwargs)
del self._total_files
del self._refactored_files
def refactor_file(self, filename, *args, **kwargs):
if self._total_files//10 != 0 and \
self._refactored_files % (self._total_files//10) == 0:
print('Refactoring files: %i%% (%i on %i).' %
(self._refactored_files/(self._total_files//10)*10,
self._refactored_files, self._total_files))
self._refactored_files += 1
return super(DistutilsRefactoringTool, self).refactor_file(
filename, *args, **kwargs)
def log_error(self, msg, *args, **kw):
log.error(msg, *args)
def log_message(self, msg, *args):
log.info(msg, *args)
def log_debug(self, msg, *args):
log.debug(msg, *args)
fixer_names = get_fixers_from_package('lib2to3.fixes')
fixer_names.remove('lib2to3.fixes.fix_import')
fixer_names += get_fixers_from_package('2to3')
r = DistutilsRefactoringTool(fixer_names, options=options)
r.refactor(files, write=True)
def run_2to3(self, files, options=None):
from distutils import log
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
if not files:
return
# Make this class local, to delay import of 2to3
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
class DistutilsRefactoringTool(RefactoringTool):
def log_error(self, msg, *args, **kw):
log.error(msg, *args)
def log_message(self, msg, *args):
log.info(msg, *args)
def log_debug(self, msg, *args):
log.debug(msg, *args)
fixer_names = get_fixers_from_package('lib2to3.fixes')
fixer_names += get_fixers_from_package('2to3')
r = DistutilsRefactoringTool(fixer_names, options=options)
r.refactor(files, write=True)
def convert_with_2to3(filepath):
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
from lib2to3.pgen2.parse import ParseError
fixers = get_fixers_from_package('lib2to3.fixes')
refactoring_tool = RefactoringTool(fixers)
source = refactoring_tool._read_python_source(filepath)[0]
try:
tree = refactoring_tool.refactor_string(source, 'conf.py')
except ParseError, err:
# do not propagate lib2to3 exceptions
lineno, offset = err.context[1]
# try to match ParseError details with SyntaxError details
raise SyntaxError(err.msg, (filepath, lineno, offset, err.value))
def convert_with_2to3(filepath):
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
from lib2to3.pgen2.parse import ParseError
fixers = get_fixers_from_package('lib2to3.fixes')
refactoring_tool = RefactoringTool(fixers)
source = refactoring_tool._read_python_source(filepath)[0]
try:
tree = refactoring_tool.refactor_string(source, 'conf.py')
except ParseError, err:
# do not propagate lib2to3 exceptions
lineno, offset = err.context[1]
# try to match ParseError details with SyntaxError details
raise SyntaxError(err.msg, (filepath, lineno, offset, err.value))
def run_2to3(cls, path):
from lib2to3.refactor import get_fixers_from_package, RefactoringTool
rt = RefactoringTool(get_fixers_from_package('lib2to3.fixes'))
with TRACER.timed('Translating %s' % path):
for root, dirs, files in os.walk(path):
for fn in files:
full_fn = os.path.join(root, fn)
if full_fn.endswith('.py'):
with TRACER.timed('%s' % fn, V=3):
try:
chmod_plus_w(full_fn)
rt.refactor_file(full_fn, write=True)
except IOError as e:
TRACER.log('Failed to translate %s: %s' % (fn, e))
def run_2to3(cls, path):
from lib2to3.refactor import get_fixers_from_package, RefactoringTool
rt = RefactoringTool(get_fixers_from_package('lib2to3.fixes'))
with TRACER.timed('Translating %s' % path):
for root, dirs, files in os.walk(path):
for fn in files:
full_fn = os.path.join(root, fn)
if full_fn.endswith('.py'):
with TRACER.timed('%s' % fn, V=3):
try:
chmod_plus_w(full_fn)
rt.refactor_file(full_fn, write=True)
except IOError as e:
TRACER.log('Failed to translate %s: %s' % (fn, e))
def build(log=simple_logger, force=False):
dreampielib_dir = dirname(abspath(__file__))
src_dir = dirname(dreampielib_dir)
build_dir = join(dreampielib_dir, 'data')
if py3_available:
avail_fixes = refactor.get_fixers_from_package('lib2to3.fixes')
rt = refactor.RefactoringTool(avail_fixes)
for ver in lib_vers:
lib_fn = join(build_dir, lib_fns[ver])
# Make dirs if they don't exist yet
if not os.path.exists(lib_fn):
os.mkdir(lib_fn)
for dir in dirs:
dir_fn = join(lib_fn, dir)
if not os.path.exists(dir_fn):
os.mkdir(dir_fn)
# Write files if not up to date
for fn in files:
src_fn = join(src_dir, fn)
dst_fn = join(lib_fn, fn)
if not force and not newer(src_fn, dst_fn):
continue
if ver == 3:
log.info("Converting %s to Python 3..." % fn)
else:
log.info("Copying %s..." % fn)
f = open(join(src_dir, fn), 'rb')
src = f.read()
f.close()
if ver == 3:
dst = str(rt.refactor_string(src+'\n', fn))[:-1]
else:
dst = src
dst = """\
# This file was automatically generated from a file in the source DreamPie dir.
# DO NOT EDIT IT, as your changes will be gone when the file is created again.
""" + dst
f = open(dst_fn, 'wb')
f.write(dst)
f.close()
def get_refactorer(fixer_pkg="lib2to3", fixers=None, options=None):
"""
A convenience function for creating a RefactoringTool for tests.
fixers is a list of fixers for the RefactoringTool to use. By default
"lib2to3.fixes.*" is used. options is an optional dictionary of options to
be passed to the RefactoringTool.
"""
if fixers is not None:
fixers = [fixer_pkg + ".fixes.fix_" + fix for fix in fixers]
else:
fixers = refactor.get_fixers_from_package(fixer_pkg + ".fixes")
options = options or {}
return refactor.RefactoringTool(fixers, options, explicit=True)
def build(log=simple_logger, force=False):
dreampielib_dir = dirname(abspath(__file__))
src_dir = dirname(dreampielib_dir)
build_dir = join(dreampielib_dir, 'data')
if py3_available:
avail_fixes = refactor.get_fixers_from_package('lib2to3.fixes')
rt = refactor.RefactoringTool(avail_fixes)
for ver in lib_vers:
lib_fn = join(build_dir, lib_fns[ver])
# Make dirs if they don't exist yet
if not os.path.exists(lib_fn):
os.mkdir(lib_fn)
for dir in dirs:
dir_fn = join(lib_fn, dir)
if not os.path.exists(dir_fn):
os.mkdir(dir_fn)
# Write files if not up to date
for fn in files:
src_fn = join(src_dir, fn)
dst_fn = join(lib_fn, fn)
if not force and not newer(src_fn, dst_fn):
continue
if ver == 3:
log.info("Converting %s to Python 3..." % fn)
else:
log.info("Copying %s..." % fn)
f = open(join(src_dir, fn), 'rb')
src = f.read()
f.close()
if ver == 3:
dst = str(rt.refactor_string(src + '\n', fn))[:-1]
else:
dst = src
dst = """\
# This file was automatically generated from a file in the source DreamPie dir.
# DO NOT EDIT IT, as your changes will be gone when the file is created again.
""" + dst
f = open(dst_fn, 'wb')
f.write(dst)
f.close()
def get_refactorer(fixer_pkg="lib2to3", fixers=None, options=None):
"""
A convenience function for creating a RefactoringTool for tests.
fixers is a list of fixers for the RefactoringTool to use. By default
"lib2to3.fixes.*" is used. options is an optional dictionary of options to
be passed to the RefactoringTool.
"""
if fixers is not None:
fixers = [fixer_pkg + ".fixes.fix_" + fix for fix in fixers]
else:
fixers = refactor.get_fixers_from_package(fixer_pkg + ".fixes")
options = options or {}
return refactor.RefactoringTool(fixers, options, explicit=True)
def main(argv=None):
ap = argparse.ArgumentParser()
ap.add_argument('path', type=pathlib.Path,
help="Notebook or directory containing notebooks")
options = ap.parse_args(argv)
avail_fixes = set(get_fixers_from_package('lib2to3.fixes'))
rt = RefactoringTool(avail_fixes)
if options.path.is_dir():
for nb_path in options.path.rglob('*.ipynb'):
refactor_notebook_inplace(rt, nb_path)
else:
refactor_notebook_inplace(rt, options.path)
def main(argv=None):
ap = argparse.ArgumentParser()
ap.add_argument('path', type=pathlib.Path,
help="Notebook or directory containing notebooks")
options = ap.parse_args(argv)
avail_fixes = set(get_fixers_from_package('lib2to3.fixes'))
rt = RefactoringTool(avail_fixes)
if options.path.is_dir():
for nb_path in options.path.rglob('*.ipynb'):
refactor_notebook_inplace(rt, nb_path)
else:
refactor_notebook_inplace(rt, options.path)
def run_2to3(self, files, options=None):
from distutils import log
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
if not files:
return
# Make this class local, to delay import of 2to3
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
class DistutilsRefactoringTool(RefactoringTool):
def refactor(self, files, *args, **kwargs):
self._total_files = len(files)
self._refactored_files = 0
super(DistutilsRefactoringTool,
self).refactor(files, *args, **kwargs)
del self._total_files
del self._refactored_files
def refactor_file(self, filename, *args, **kwargs):
if self._total_files//10 != 0 and \
self._refactored_files % (self._total_files//10) == 0:
print('Refactoring files: %i%% (%i on %i).' %
(self._refactored_files /
(self._total_files // 10) * 10,
self._refactored_files, self._total_files))
self._refactored_files += 1
return super(DistutilsRefactoringTool,
self).refactor_file(filename, *args, **kwargs)
def log_error(self, msg, *args, **kw):
log.error(msg, *args)
def log_message(self, msg, *args):
log.info(msg, *args)
def log_debug(self, msg, *args):
log.debug(msg, *args)
fixer_names = [
'fix_basestring', 'fix_dict', 'fix_imports', 'fix_long',
'fix_metaclass', 'fix_methodattrs', 'fix_numliterals',
'fix_types', 'fix_unicode', 'fix_urllib', 'fix_xrange'
]
fixers = list(map(lambda x: 'lib2to3.fixes.' + x, fixer_names))
fixers += get_fixers_from_package('2to3')
r = DistutilsRefactoringTool(fixers, options=options)
r.refactor(files, write=True)
def convert_with_2to3(filepath: str) -> str:
warnings.warn('convert_with_2to3() is deprecated',
RemovedInSphinx60Warning, stacklevel=2)
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
from lib2to3.pgen2.parse import ParseError
fixers = get_fixers_from_package('lib2to3.fixes')
refactoring_tool = RefactoringTool(fixers)
source = refactoring_tool._read_python_source(filepath)[0]
try:
tree = refactoring_tool.refactor_string(source, 'conf.py')
except ParseError as err:
# do not propagate lib2to3 exceptions
lineno, offset = err.context[1]
# try to match ParseError details with SyntaxError details
raise SyntaxError(err.msg, (filepath, lineno, offset, err.value))
return str(tree)
def find_all_narrative_py2_code(ws_url: str, token: str, min_id: int,
max_id: int, outfile: str, report: str):
assert ws_url
assert token
assert min_id and min_id > 0
assert max_id and max_id >= min_id
assert outfile
ws = Workspace(url=ws_url, token=token)
all_results = {"fail": [], "no_narr": [], "no_change": [], "changes": []}
avail_fixes = set(get_fixers_from_package("lib2to3.fixes"))
rt = RefactoringTool(avail_fixes, options={"print_function": False})
for ws_id in range(min_id, max_id):
try:
result = _find_narrative_py2_code(ws_id, ws, rt, verbose=True)
if result is None:
all_results["no_narr"].append({"id": ws_id})
elif result.updated_cells == 0:
all_results["no_change"].append(result.to_dict())
else:
all_results["changes"].append(result.to_dict())
except baseclient.ServerError as e:
if "No workspace with id" in str(e):
print(f"WS:{ws_id} does not exist")
all_results["fail"].append({"id": ws_id, "error": str(e.message)})
except TypeError as e:
print(f"WS:{ws_id} metadata doesn't link to a Narrative type!")
all_results["fail"].append({"id": ws_id, "error": str(e)})
except json.JSONDecodeError as e:
print(f"WS:{ws_id} unable to unpack Narrative - not valid JSON!")
all_results["fail"].append({
"id":
ws_id,
"error":
f"Invalid JSON in Narrative object: {str(e)}"
})
with open(outfile, "w") as fjson:
fjson.write(json.dumps(all_results, indent=4))
print(f"Done. Results in {outfile}")
if report is not None:
with open(report, "w") as ftsv:
ftsv.write("user name\tnarrative id\tlast saved\n")
for n in all_results["changes"]:
ftsv.write(f"{n['owner']}\t{n['id']}\t{n['last_saved']}\n")
def __init__(self, *args, **kwargs):
_build_py.__init__(self, *args, **kwargs)
import logging
import pip
pip.main(['install', '3to2'])
from lib2to3 import refactor
import lib3to2.main
rt_logger = logging.getLogger("RefactoringTool")
rt_logger.addHandler(logging.StreamHandler())
fixers = refactor.get_fixers_from_package('lib3to2.fixes')
self.rtool = lib3to2.main.StdoutRefactoringTool(
fixers,
None,
[],
False,
False
)
def create_fixer_names(fixes, nofixes):
"""Build a set of fixer names."""
# Taken from lib2to3.main:
fixer_pkg = 'lib2to3.fixes'
avail_fixes = set(refactor.get_fixers_from_package(fixer_pkg))
unwanted_fixes = set(fixer_pkg + ".fix_" + fix for fix in nofixes)
explicit = set()
if fixes:
all_present = False
for fix in fixes:
if fix == "all":
all_present = True
else:
explicit.add(fixer_pkg + ".fix_" + fix)
requested = avail_fixes.union(explicit) if all_present else explicit
else:
requested = avail_fixes.union(explicit)
fixer_names = requested.difference(unwanted_fixes)
return fixer_names
def __init__(self, *args, **kwargs):
_build_py.__init__(self, *args, **kwargs)
checkpoint_fn = os.path.join(os.path.dirname(__file__),
'3to2.conversion.ok')
if os.path.exists(checkpoint_fn):
return
import logging
from lib2to3 import refactor
import lib3to2.main
import lib3to2.fixes
rt_logger = logging.getLogger("RefactoringTool")
rt_logger.addHandler(logging.StreamHandler())
try:
fixers = refactor.get_fixers_from_package('lib3to2.fixes')
except OSError:
# fallback for .egg installs
fixers = [
'lib3to2.fixes.fix_{}'.format(s)
for s in ('absimport', 'annotations', 'bitlength', 'bool',
'bytes', 'classdecorator', 'collections',
'dctsetcomp', 'division', 'except', 'features',
'fullargspec', 'funcattrs', 'getcwd', 'imports',
'imports2', 'input', 'int', 'intern', 'itertools',
'kwargs', 'memoryview', 'metaclass', 'methodattrs',
'newstyle', 'next', 'numliterals', 'open', 'print',
'printfunction', 'raise', 'range', 'reduce',
'setliteral', 'str', 'super', 'throw', 'unittest',
'unpacking', 'with')
]
for fix in ('fix_except', 'fix_int', 'fix_print', 'fix_str',
'fix_throw', 'fix_unittest', 'fix_absimport',
'fix_dctsetcomp', 'fix_setliteral', 'fix_with',
'fix_open'):
fixers.remove('lib3to2.fixes.{}'.format(fix))
fixers.append('fix_pymor_futures')
print(fixers)
self.rtool = lib3to2.main.StdoutRefactoringTool(
fixers, None, [], True, False)
self.rtool.refactor_dir('src', write=True)
self.rtool.refactor_dir('docs', write=True)
open(checkpoint_fn, 'wt').write('converted')
def convert_with_2to3(filepath: str) -> str:
try:
from lib2to3.refactor import RefactoringTool, get_fixers_from_package
from lib2to3.pgen2.parse import ParseError
except ImportError:
# python 3.9.0a6+ emits PendingDeprecationWarning for lib2to3.
# Additionally, removal of the module is still discussed at PEP-594.
# To support future python, this catches ImportError for lib2to3.
raise SyntaxError
fixers = get_fixers_from_package('lib2to3.fixes')
refactoring_tool = RefactoringTool(fixers)
source = refactoring_tool._read_python_source(filepath)[0]
try:
tree = refactoring_tool.refactor_string(source, 'conf.py')
except ParseError as err:
# do not propagate lib2to3 exceptions
lineno, offset = err.context[1]
# try to match ParseError details with SyntaxError details
raise SyntaxError(err.msg, (filepath, lineno, offset, err.value))
return str(tree)
def run_2to3(self, files):
excludes = build_py.opts_2to3.get('exclude')
nofix = build_py.opts_2to3.get('nofix')
fixermap = {} # fname -> [fixers_to_disable, ...]
for file in files[:]:
if self.distribution.get_name().startswith('kaa-'):
# This is a kaa module, so strip 'build/lib.*/kaa/module/' from
# name.
relfile = re.sub(r'build\/[^/]*\/kaa\/[^/]+\/', '', file)
else:
# A non-kaa module, strip build/lib.*/module/
relfile = re.sub(r'build\/[^/]*\/[^/]+\/', '', file)
if excludes:
for pattern in excludes:
if fnmatch.fnmatch(relfile, pattern):
files.remove(file)
if nofix:
for pattern, fixers in nofix.items():
if fnmatch.fnmatch(relfile, pattern) and file in files:
fixermap.setdefault(file, []).extend(list(fixers))
# Now that we've got a list of all files and which fixers to disable
# for them, flip fixermap upside down and group by disabled fixer list.
groups = {} # (disabled_fixers, ...) -> [filenames]
for file, disabled_fixers in fixermap.items():
groups.setdefault(tuple(disabled_fixers) or 'none',
[]).append(file)
from lib2to3.refactor import get_fixers_from_package
all_fixers = set(get_fixers_from_package('lib2to3.fixes'))
for disabled_fixers, files in groups.items():
if disabled_fixers == 'none':
self.fixer_names = all_fixers
else:
self.fixer_names = all_fixers.difference(
'lib2to3.fixes.fix_%s' % fixer
for fixer in disabled_fixers)
print('Running 2to3 on %d files, this may take a while ...' %
len(files))
distutils_build_py.run_2to3(self, files)
def __init__(self, *args, **kwargs):
_build_py.__init__(self, *args, **kwargs)
checkpoint_fn = os.path.join(os.path.dirname(__file__), '3to2.conversion.ok')
if os.path.exists(checkpoint_fn):
return
import logging
from lib2to3 import refactor
import lib3to2.main
import lib3to2.fixes
rt_logger = logging.getLogger("RefactoringTool")
rt_logger.addHandler(logging.StreamHandler())
try:
fixers = refactor.get_fixers_from_package('lib3to2.fixes')
except OSError:
# fallback for .egg installs
fixers = ['lib3to2.fixes.fix_{}'.format(s) for s in ('absimport', 'annotations', 'bitlength', 'bool',
'bytes', 'classdecorator', 'collections', 'dctsetcomp', 'division', 'except', 'features',
'fullargspec', 'funcattrs', 'getcwd', 'imports', 'imports2', 'input', 'int', 'intern', 'itertools',
'kwargs', 'memoryview', 'metaclass', 'methodattrs', 'newstyle', 'next', 'numliterals', 'open', 'print',
'printfunction', 'raise', 'range', 'reduce', 'setliteral', 'str', 'super', 'throw', 'unittest',
'unpacking', 'with')]
for fix in ('fix_except', 'fix_int', 'fix_print', 'fix_range', 'fix_str', 'fix_throw',
'fix_unittest', 'fix_absimport', 'fix_dctsetcomp', 'fix_setliteral', 'fix_with', 'fix_open'):
fixers.remove('lib3to2.fixes.{}'.format(fix))
fixers.append('fix_pymor_futures')
print(fixers)
self.rtool = lib3to2.main.StdoutRefactoringTool(
fixers,
None,
[],
True,
False
)
self.rtool.refactor_dir('src', write=True)
self.rtool.refactor_dir('docs', write=True)
open(checkpoint_fn, 'wta').write('converted')
def run_2to3(self, files):
excludes = build_py.opts_2to3.get('exclude')
nofix = build_py.opts_2to3.get('nofix')
fixermap = {} # fname -> [fixers_to_disable, ...]
for file in files[:]:
if self.distribution.get_name().startswith('kaa-'):
# This is a kaa module, so strip 'build/lib.*/kaa/module/' from
# name.
relfile = re.sub(r'build\/[^/]*\/kaa\/[^/]+\/', '', file)
else:
# A non-kaa module, strip build/lib.*/module/
relfile = re.sub(r'build\/[^/]*\/[^/]+\/', '', file)
if excludes:
for pattern in excludes:
if fnmatch.fnmatch(relfile, pattern):
files.remove(file)
if nofix:
for pattern, fixers in nofix.items():
if fnmatch.fnmatch(relfile, pattern) and file in files:
fixermap.setdefault(file, []).extend(list(fixers))
# Now that we've got a list of all files and which fixers to disable
# for them, flip fixermap upside down and group by disabled fixer list.
groups = {} # (disabled_fixers, ...) -> [filenames]
for file, disabled_fixers in fixermap.items():
groups.setdefault(tuple(disabled_fixers) or 'none', []).append(file)
from lib2to3.refactor import get_fixers_from_package
all_fixers = set(get_fixers_from_package('lib2to3.fixes'))
for disabled_fixers, files in groups.items():
if disabled_fixers == 'none':
self.fixer_names = all_fixers
else:
self.fixer_names = all_fixers.difference('lib2to3.fixes.fix_%s' % fixer for fixer in disabled_fixers)
print('Running 2to3 on %d files, this may take a while ...' % len(files))
distutils_build_py.run_2to3(self, files)
def get_py2to3_converter(self, options=None, proc_count=0):
from lib2to3 import refactor as ref_mod
from snakeoil.dist import caching_2to3
if proc_count == 0:
import multiprocessing
proc_count = multiprocessing.cpu_count()
assert proc_count >= 1
if proc_count > 1 and not caching_2to3.multiprocessing_available:
proc_count = 1
refactor_kls = caching_2to3.MultiprocessRefactoringTool
fixer_names = ref_mod.get_fixers_from_package('lib2to3.fixes')
f = refactor_kls(fixer_names, options=options).refactor
def f2(*args, **kwds):
if caching_2to3.multiprocessing_available:
kwds['num_processes'] = proc_count
return f(*args, **kwds)
return f2
def run(self):
py2k_rebuilds = []
if not self.inplace:
if not is_py3k:
py2k_rebuilds = list(self._compute_rebuilds(self.force))
dst_build_py.build_py.run(self)
if self.generate_verinfo:
self._run_generate_verinfo(py2k_rebuilds)
self._inner_run(py2k_rebuilds)
if is_py3k:
return
from lib3to2.build import run_3to2
from lib2to3 import refactor
# assume a few fixes are already handled in the code or aren't needed
# for py27
skip_list = (
'lib3to2.fixes.fix_str',
'lib3to2.fixes.fix_printfunction',
'lib3to2.fixes.fix_except',
'lib3to2.fixes.fix_with',
)
fixer_names = [
x for x in refactor.get_fixers_from_package('lib3to2.fixes')
if x not in skip_list
]
log.info("starting 3to2 conversion; this may take a while...")
run_3to2([x[0] for x in py2k_rebuilds], fixer_names=fixer_names)
for path, mtime in py2k_rebuilds:
os.utime(path, (-1, mtime))
log.info("completed py2k conversions")
def get_py2to3_converter(self, options=None, proc_count=0):
from lib2to3 import refactor as ref_mod
from snakeoil.dist import caching_2to3
if proc_count == 0:
import multiprocessing
proc_count = multiprocessing.cpu_count()
assert proc_count >= 1
if proc_count > 1 and not caching_2to3.multiprocessing_available:
proc_count = 1
refactor_kls = caching_2to3.MultiprocessRefactoringTool
fixer_names = ref_mod.get_fixers_from_package('lib2to3.fixes')
f = refactor_kls(fixer_names, options=options).refactor
def f2(*args, **kwds):
if caching_2to3.multiprocessing_available:
kwds['num_processes'] = proc_count
return f(*args, **kwds)
return f2
from distutils.core import setup, Extension
try:
from distutils.command.build_py import build_py_2to3 as build_py
from distutils.command.build_scripts import build_scripts_2to3 as build_scripts
except ImportError:
from distutils.command.build_py import build_py
from distutils.command.build_scripts import build_scripts
else:
build_py.fixer_names = fixer_names
build_scripts.fixer_names = fixer_names
else:
if sys.version_info >= (3, ):
from lib2to3.refactor import get_fixers_from_package
all_fixers = set(get_fixers_from_package('lib2to3.fixes'))
exclude_fixers = sorted(all_fixers.difference(fixer_names))
extra['use_2to3'] = True
extra['use_2to3_fixers'] = []
extra['use_2to3_exclude_fixers'] = exclude_fixers
class gdal_config_error(Exception):
pass
from distutils.command.build_ext import build_ext
from distutils.ccompiler import get_default_compiler
from distutils.sysconfig import get_python_inc
# This will apply to all the doctests too:
from __future__ import print_function
from Bio._py3k import _universal_read_mode
import unittest
import doctest
import os
import sys
import warnings
from Bio import BiopythonExperimentalWarning
warnings.simplefilter('ignore', BiopythonExperimentalWarning)
if sys.version_info[0] >= 3:
from lib2to3 import refactor
fixers = refactor.get_fixers_from_package("lib2to3.fixes")
fixers.remove("lib2to3.fixes.fix_print") # Already using print function
rt = refactor.RefactoringTool(fixers)
assert rt.refactor_docstring(">>> print(2+2)\n4\n", "example1") == \
">>> print(2+2)\n4\n"
assert rt.refactor_docstring('>>> print("Two plus two is", 2+2)\n'
'Two plus two is 4\n', "example2") == \
'>>> print("Two plus two is", 2+2)\nTwo plus two is 4\n'
# Cache this to restore the cwd at the end of the tests
original_path = os.path.abspath(".")
if os.path.basename(sys.argv[0]) == "test_Tutorial.py":
# sys.argv[0] will be (relative) path to test_Turorial.py - use this to allow, e.g.
# [base]$ python Tests/test_Tutorial.py
# [Tests/]$ python test_Tutorial.py
import tempfile
import shutil
import unittest
import warnings
from lib2to3 import refactor, pygram, fixer_base
from lib2to3.pgen2 import token
from . import support
TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "data")
FIXER_DIR = os.path.join(TEST_DATA_DIR, "fixers")
sys.path.append(FIXER_DIR)
try:
_DEFAULT_FIXERS = refactor.get_fixers_from_package("myfixes")
finally:
sys.path.pop()
_2TO3_FIXERS = refactor.get_fixers_from_package("lib2to3.fixes")
class TestRefactoringTool(unittest.TestCase):
def setUp(self):
sys.path.append(FIXER_DIR)
def tearDown(self):
sys.path.pop()
def check_instances(self, instances, classes):
for inst, cls in zip(instances, classes):
from distutils.core import setup, Extension
try:
from distutils.command.build_py import build_py_2to3 as build_py
from distutils.command.build_scripts import build_scripts_2to3 as build_scripts
except ImportError:
from distutils.command.build_py import build_py
from distutils.command.build_scripts import build_scripts
else:
build_py.fixer_names = fixer_names
build_scripts.fixer_names = fixer_names
else:
if sys.version_info >= (3,):
from lib2to3.refactor import get_fixers_from_package
all_fixers = set(get_fixers_from_package("lib2to3.fixes"))
exclude_fixers = sorted(all_fixers.difference(fixer_names))
extra["use_2to3"] = True
extra["use_2to3_fixers"] = []
extra["use_2to3_exclude_fixers"] = exclude_fixers
class gdal_config_error(Exception):
pass
from distutils.command.build_ext import build_ext
from distutils.ccompiler import get_default_compiler
from distutils.sysconfig import get_python_inc
