def convert_with_2to3(filepath: str) -> str: warnings.warn('convert_with_2to3() is deprecated', RemovedInSphinx60Warning, stacklevel=2) try: from lib2to3.pgen2.parse import ParseError from lib2to3.refactor import RefactoringTool, get_fixers_from_package except ImportError as exc: # python 3.9.0a6+ emits PendingDeprecationWarning for lib2to3. # Additionally, removal of the module is still discussed at PEP-594. # To support future python, this catches ImportError for lib2to3. raise SyntaxError from exc fixers = get_fixers_from_package('lib2to3.fixes') refactoring_tool = RefactoringTool(fixers) source = refactoring_tool._read_python_source(filepath)[0] try: tree = refactoring_tool.refactor_string(source, 'conf.py') except ParseError as err: # do not propagate lib2to3 exceptions lineno, offset = err.context[1] # try to match ParseError details with SyntaxError details raise SyntaxError(err.msg, (filepath, lineno, offset, err.value)) from err return str(tree)
def convert2to3(cell): original_code = cell['source'] try: x = ast.parse(original_code) y = tokenize_and_templatize(original_code) return cell except: pass # fixers = ['lib2to3.fixes.fix_print'] # refactor = RefactoringTool(fixer_names=fixers) # tree = refactor.refactor_string(original_code, 'temp') try: # fixers = get_fixers_from_package('lib2to3.fixes') # the only python2 issue is print statements so we only import this # fixer since it makes RefactorTool way faster fixers = ['lib2to3.fixes.fix_print'] refactor = RefactoringTool(fixer_names=fixers) tree = refactor.refactor_string(original_code, 'temp') converted_code = str(tree) print(converted_code) x = ast.parse(converted_code) # y = tokenize_and_templatize(converted_code) # print('===============\n', original_code, '\n', converted_code) cell['source'] = converted_code except Exception as e: print(e) # print('==========\n', print(original_code)) pass return cell
def load_script(script_path: Path): from . import sfm from . import vs from . import sfm_utils hooks = { 'vs': vs, 'sfm': sfm.SFM(), } # if sys.meta_path[0]!= importer = HookedImporter(hooks=hooks) importer.install() assert script_path.exists() with script_path.open('r') as f: script_data = f.read() refactoring_tool = RefactoringTool( fixer_names=get_fixers_from_package('lib2to3.fixes')) node3 = refactoring_tool.refactor_string(script_data + '\n', 'script') compiled = compile(str(node3), str(script_path), 'exec') exec( compiled, { 'sfm': sfm.SFM(), 'sfmUtils': sfm_utils.SfmUtils(), 'print': sfm.sfm.sfm_logger.print })
def main(argv=None): if argv is None: argv = sys.argv path = argv[1] fixer_names = get_fixers_from_package('pyramid.fixers') tool = RefactoringTool(fixer_names) tool.refactor([path], write=True) fix_zcml(path)
def setup(): """ Call this before using the refactoring tools to create them on demand if needed. """ if None in [RTs._rt, RTs._rtp]: RTs._rt = RefactoringTool(myfixes) RTs._rtp = RefactoringTool(myfixes, {'print_function': True})
def __init__(self): fixers = [ 'lib2to3.fixes.fix_%s' % name for name in [ 'apply', 'asserts', 'basestring', 'buffer', 'dict', 'except', 'exec', 'execfile', 'exitfunc', 'filter', 'funcattrs', 'future', 'getcwdu', 'has_key', 'idioms', 'import', 'imports', 'imports2', 'input', 'intern', 'isinstance', 'itertools', 'itertools_imports', 'long', 'map', 'metaclass', 'methodattrs', 'ne', 'nonzero', 'numliterals', 'operator', 'paren', 'print', 'raise', 'raw_input', 'reduce', 'reload', 'renames', 'repr', 'set_literal', 'standarderror', 'sys_exc', 'throw', 'tuple_params', 'types', 'unicode', 'urllib', 'ws_comma', 'xrange', 'xreadlines', 'zip', ] ] RefactoringTool.__init__(self, fixers)
def setup_detect_python2(): """ Call this before using the refactoring tools to create them on demand if needed. """ if None in [RTs._rt_py2_detect, RTs._rtp_py2_detect]: RTs._rt_py2_detect = RefactoringTool(py2_detect_fixers) RTs._rtp_py2_detect = RefactoringTool(py2_detect_fixers, {'print_function': True})
def pathlab_string(input): sys.path.insert(0, Path(__file__).parent.absolute()) refactoring_tool = RefactoringTool( fixer_names=get_fixers_from_package('pathlad.custom_fixers')) result = input result = str(refactoring_tool.refactor_string(result + '\n', 'paths'))[:-1] result = str( refactoring_tool.refactor_string(result + '\n', 'paths_nested'))[:-1] return result
def convert_with_2to3(filepath): from lib2to3.refactor import RefactoringTool, get_fixers_from_package from lib2to3.pgen2.parse import ParseError fixers = get_fixers_from_package('lib2to3.fixes') refactoring_tool = RefactoringTool(fixers) source = refactoring_tool._read_python_source(filepath)[0] try: tree = refactoring_tool.refactor_string(source, 'conf.py') except ParseError, err: # do not propagate lib2to3 exceptions lineno, offset = err.context[1] # try to match ParseError details with SyntaxError details raise SyntaxError(err.msg, (filepath, lineno, offset, err.value))
def run_2to3(cls, path): from lib2to3.refactor import get_fixers_from_package, RefactoringTool rt = RefactoringTool(get_fixers_from_package('lib2to3.fixes')) with TRACER.timed('Translating %s' % path): for root, dirs, files in os.walk(path): for fn in files: full_fn = os.path.join(root, fn) if full_fn.endswith('.py'): with TRACER.timed('%s' % fn, V=3): try: chmod_plus_w(full_fn) rt.refactor_file(full_fn, write=True) except IOError as e: TRACER.log('Failed to translate %s: %s' % (fn, e))
def test_convert_state_file_to_conf_74(self): """ Tests conversion of the pstate files (pre-7.4.0) files to .conf files. Tests for two different pstate files, one corrupted with incorrect metainfo data, and the other one with correct working metadata. """ from lib2to3.refactor import RefactoringTool, get_fixers_from_package refactoring_tool = RefactoringTool(fixer_names=get_fixers_from_package('lib2to3.fixes')) os.makedirs(os.path.join(self.state_dir, STATEDIR_CHECKPOINT_DIR)) # Copy a good working Ubuntu pstate file src_path = os.path.join(self.CONFIG_PATH, "194257a7bf4eaea978f4b5b7fbd3b4efcdd99e43.state") dest_path = os.path.join(self.state_dir, STATEDIR_CHECKPOINT_DIR, "ubuntu_ok.state") shutil.copyfile(src_path, dest_path) convert_state_file_to_conf_74(dest_path, refactoring_tool) converted_file_path = os.path.join(self.state_dir, STATEDIR_CHECKPOINT_DIR, "ubuntu_ok.conf") self.assertTrue(os.path.exists(converted_file_path)) self.assertFalse(os.path.exists(dest_path)) os.remove(converted_file_path) # Copy Ubuntu pstate file with corrupted metainfo data src_path = os.path.join(self.CONFIG_PATH, "194257a7bf4eaea978f4b5b7fbd3b4efcdd99e43_corrupted.state") dest_path = os.path.join(self.state_dir, STATEDIR_CHECKPOINT_DIR, "ubuntu_corrupted.state") shutil.copyfile(src_path, dest_path) convert_state_file_to_conf_74(dest_path, refactoring_tool) converted_file_path = os.path.join(self.state_dir, STATEDIR_CHECKPOINT_DIR, "ubuntu_corrupted.conf") self.assertFalse(os.path.exists(converted_file_path)) self.assertFalse(os.path.exists(dest_path))
def convert_with_2to3(filepath: str) -> str: warnings.warn('convert_with_2to3() is deprecated', RemovedInSphinx60Warning, stacklevel=2) from lib2to3.refactor import RefactoringTool, get_fixers_from_package from lib2to3.pgen2.parse import ParseError fixers = get_fixers_from_package('lib2to3.fixes') refactoring_tool = RefactoringTool(fixers) source = refactoring_tool._read_python_source(filepath)[0] try: tree = refactoring_tool.refactor_string(source, 'conf.py') except ParseError as err: # do not propagate lib2to3 exceptions lineno, offset = err.context[1] # try to match ParseError details with SyntaxError details raise SyntaxError(err.msg, (filepath, lineno, offset, err.value)) return str(tree)
def transform(source): python_zip_path = str(Path(os.path.dirname(os.__file__)).parents[0]) python_zip_file = get_zip_file(python_zip_path) python_zip_folder = zipfile.ZipFile(os.path.join(python_zip_path, python_zip_file)) fixers = get_all_fixers_from_zipfolder(python_zip_folder) refactoring_tool = RefactoringTool(fixers) return refactor_script(source, refactoring_tool)
def convert_config_to_tribler74(state_dir): """ Convert the download config files to Tribler 7.4 format. The extensions will also be renamed from .state to .conf """ from lib2to3.refactor import RefactoringTool, get_fixers_from_package refactoring_tool = RefactoringTool(fixer_names=get_fixers_from_package('lib2to3.fixes')) for filename in (state_dir / STATEDIR_CHECKPOINT_DIR).glob('*.state'): convert_state_file_to_conf_74(filename, refactoring_tool=refactoring_tool)
def run_2to3(files, doctests_only=False, fixer_names=None, options=None, explicit=None): """ Wrapper function around the refactor() class which performs the conversions on a list of python files. Invoke 2to3 on a list of Python files. The files should all come from the build area, as the modification is done in-place.""" # if not files: # return # Make this class local, to delay import of 2to3 from lib2to3.refactor import get_fixers_from_package, RefactoringTool fixers = get_fixers_from_package("lib2to3.fixes") if fixer_names: for fixername in fixer_names: fixers.extend(get_fixers_from_package(fixername)) r = RefactoringTool(fixers, options=options) r.refactor(files, write=True, doctests_only=doctests_only)
def transform(source): python_zip_path = str(Path(os.path.dirname(os.__file__)).parents[0]) python_zip_file = get_zip_file(python_zip_path) python_zip_folder = zipfile.ZipFile(os.path.join(python_zip_path, python_zip_file)) fixers = get_all_fixers_from_zipfolder(python_zip_folder) dir_path = os.getcwd() sys.path.append(os.path.join(dir_path, '.\\python_migration_fixers')) fixers.extend(['fix_none']) refactoring_tool = RefactoringTool(fixers) return refactor_script(source, refactoring_tool)
def main(argv=None): ap = argparse.ArgumentParser() ap.add_argument('path', type=pathlib.Path, help="Notebook or directory containing notebooks") options = ap.parse_args(argv) avail_fixes = set(get_fixers_from_package('lib2to3.fixes')) rt = RefactoringTool(avail_fixes) if options.path.is_dir(): for nb_path in options.path.rglob('*.ipynb'): refactor_notebook_inplace(rt, nb_path) else: refactor_notebook_inplace(rt, options.path)
def find_all_narrative_py2_code(ws_url: str, token: str, min_id: int, max_id: int, outfile: str, report: str): assert ws_url assert token assert min_id and min_id > 0 assert max_id and max_id >= min_id assert outfile ws = Workspace(url=ws_url, token=token) all_results = {"fail": [], "no_narr": [], "no_change": [], "changes": []} avail_fixes = set(get_fixers_from_package("lib2to3.fixes")) rt = RefactoringTool(avail_fixes, options={"print_function": False}) for ws_id in range(min_id, max_id): try: result = _find_narrative_py2_code(ws_id, ws, rt, verbose=True) if result is None: all_results["no_narr"].append({"id": ws_id}) elif result.updated_cells == 0: all_results["no_change"].append(result.to_dict()) else: all_results["changes"].append(result.to_dict()) except baseclient.ServerError as e: if "No workspace with id" in str(e): print(f"WS:{ws_id} does not exist") all_results["fail"].append({"id": ws_id, "error": str(e.message)}) except TypeError as e: print(f"WS:{ws_id} metadata doesn't link to a Narrative type!") all_results["fail"].append({"id": ws_id, "error": str(e)}) except json.JSONDecodeError as e: print(f"WS:{ws_id} unable to unpack Narrative - not valid JSON!") all_results["fail"].append({ "id": ws_id, "error": f"Invalid JSON in Narrative object: {str(e)}" }) with open(outfile, "w") as fjson: fjson.write(json.dumps(all_results, indent=4)) print(f"Done. Results in {outfile}") if report is not None: with open(report, "w") as ftsv: ftsv.write("user name\tnarrative id\tlast saved\n") for n in all_results["changes"]: ftsv.write(f"{n['owner']}\t{n['id']}\t{n['last_saved']}\n")
def _update_narrative(narr_obj: list, ws_info: list, rt: RefactoringTool) -> NarrativeInfo: """ Core pieces of this taken from this gist by Thomas Takluyver and Fernando Perez: https://gist.github.com/takluyver/c8839593c615bb2f6e80 """ try: nb = nbformat.reads(json.dumps(narr_obj['data']), 4.0) except: nb = nbformat.reads(json.dumps(narr_obj['data']), 3.0) ninfo = NarrativeInfo(narr_obj['info'], ws_info[2]) cells = list() if nb.nbformat == 4: cells = nb.cells else: cells = nb.worksheets[0].cells for idx, cell in enumerate(nb.cells): if cell.cell_type != "code": continue head = '' source = '' if hasattr(cell, 'source'): source = cell.source elif hasattr(cell, 'input'): source = cell.input else: continue if source.startswith('%%'): split_source = cell.source.split('\n', 1) if len(split_source) == 2: head, source = split_source try: tree = rt.refactor_string(source + "\n", f"{ws_info[0]}-cell{idx}") result = str(tree)[:-1] except (lib2to3.pgen2.parse.ParseError, lib2to3.pgen2.tokenize.TokenError): result = source if head: source = head + '\n' + source result = head + '\n' + result ninfo.add_updated_cell(idx, source, result) return ninfo
class TestRefactoringTool: def __init__(self, fixer): self.refactoring_tool = RefactoringTool([fixer], {}, explicit=True) def refactor(self, before): print("INPUT: ", before) print( "PARSED: ", repr(self.refactoring_tool.driver.parse_string(reformat(before))), ) return str( self.refactoring_tool.refactor_string(reformat(before), "<string>")) def refactor_and_check(self, before, expected): __tracebackhide__ = True after = self.refactor(before) assert reformat(expected).rstrip("\n") == after.rstrip("\n")
def setUp(self): self.refactor = RefactoringTool(['fix_name2']).refactor_string
def setUp(self): self.refactor = RefactoringTool(['fix_indent']).refactor_string
def setUp(self): self.refactor = RefactoringTool(['fix_constant']).refactor_string
'lib.%s-%s' % (get_platform(), sys.version[0:3])) if os.path.exists(platlib): sys.path.insert(0, platlib) import psycopg2 if sys.version_info < (3,): import tests else: from distutils.file_util import copy_file import glob from lib2to3.refactor import RefactoringTool, get_fixers_from_package if not os.path.isdir("py3tests"): os.mkdir("py3tests") new = [] for f in glob.glob("tests/*.py"): f, copied = copy_file(f, "py3tests", preserve_times=0, update=1) if copied: new.append(f) fixers = get_fixers_from_package('lib2to3.fixes') r = RefactoringTool(fixers) r.refactor(new, write=True) import py3tests as tests def test_suite(): return tests.test_suite() if __name__ == '__main__': unittest.main(defaultTest='test_suite')
# Converts an individual Python file or a directory of Python files to Python3 # Note - These functions will overwrite the files after conversion. Originals are not retained. from lib2to3.refactor import RefactoringTool, get_fixers_from_package avail_fixes = set(get_fixers_from_package('lib2to3.fixes')) rt = RefactoringTool(avail_fixes) # Converts and overwrites a Python file to Python3 def convert_file(filename): rt.refactor_file(filename, write=True) # Converts and overwrites all files ending in .py in the directory to Python3 def convert_directory(directoryname): rt.refactor_dir(directoryname,write=True)
# This is a Python 3.x script to build pywin32. It converts then executes # the regular setup.py script. import os from lib2to3.refactor import RefactoringTool, get_fixers_from_package fixers = ['lib2to3.fixes.fix_print', 'lib2to3.fixes.fix_except'] options = dict(doctests_only=False, fix=[], list_fixes=[], print_function=False, verbose=False, write=True) r = RefactoringTool(fixers, options) script = os.path.join(os.path.dirname(__file__), "setup.py") data = open(script).read() print("Converting...") got = r.refactor_string(data, "setup.py") print("Executing...") exec(str(got))
import traceback from lib2to3.refactor import RefactoringTool import packaging.version from Cheetah.Compiler import Compiler from Cheetah.NameMapper import NotFound from Cheetah.Template import Template from past.translation import myfixes from . import unicodify # Skip libpasteurize fixers, which make sure code is py2 and py3 compatible. # This is not needed, we only translate code on py3. if sys.version_info.major > 2: myfixes = [f for f in myfixes if not f.startswith('libpasteurize')] refactoring_tool = RefactoringTool(myfixes, {'print_function': True}) else: myfixes = refactoring_tool = None class FixedModuleCodeCompiler(Compiler): module_code = None def getModuleCode(self): self._moduleDef = self.module_code return self._moduleDef def create_compiler_class(module_code): class CustomCompilerClass(FixedModuleCodeCompiler):
def refactor(x): from lib2to3.refactor import RefactoringTool, get_fixers_from_package fixer_names = get_fixers_from_package('lib2to3.fixes') r = RefactoringTool(fixer_names, options=None) r.refactor([x], write=True)
from libfuturize import fixes __version__ = '0.1.0' logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) myfixes = (list(fixes.libfuturize_fix_names_stage1) + list(fixes.lib2to3_fix_names_stage1) + list(fixes.libfuturize_fix_names_stage2) + list(fixes.lib2to3_fix_names_stage2)) # There are two possible grammars: with or without the print statement. # Hence we have two possible refactoring tool implementations. _rt = RefactoringTool(myfixes) _rtp = RefactoringTool(myfixes, {'print_function': True}) # # We detect whether the code is Py2 or Py3 by applying certain lib2to3 fixers # to it. If the diff is empty, it's Python 3 code. py2_detect_fixers = [ # From stage 1: 'lib2to3.fixes.fix_apply', # 'lib2to3.fixes.fix_dict', # TODO: add support for utils.viewitems() etc. and move to stage2 'lib2to3.fixes.fix_except', 'lib2to3.fixes.fix_execfile', 'lib2to3.fixes.fix_exitfunc', 'lib2to3.fixes.fix_funcattrs', 'lib2to3.fixes.fix_filter',
long except NameError: long = int try: unicode except NameError: unicode = str try: unichr except NameError: unichr = chr try: literal_eval(r"u'\u4444'") except SyntaxError: # Remove 'u' prefixes in unicode literals in Python 3 rtp_fix_unicode = RefactoringTool(['lib2to3.fixes.fix_unicode'], {'print_function': True}) else: rtp_fix_unicode = None def escape(text): """Escape unprintable or non-ASCII characters, double quotes and ampersands in a string using XML character references. """ def fixup(m): ch = m.group(0) return '&#' + str(ord(ch)) + ';' text = re.sub('[^ -~]|[&"]', fixup, text) return text if isinstance(text, str) else str(text)
def refactor(self, source): return RefactoringTool(self._fixers).refactor_string( source, " ".join(chain([__name__], self._fixers)))
def convert_string(input): tool = RefactoringTool(get_fixers_from_package("doc484.fixes")) tree = tool.refactor_string(input, '<test.py>') return str(tree)
def __init__(self, fixer): self.refactoring_tool = RefactoringTool([fixer], {}, explicit=True)
import argparse import os import sys import imp import runpy from io import StringIO from pkgutil import ImpImporter, ImpLoader import runpy import sys import tempfile import lib2to3 from lib2to3.refactor import RefactoringTool, get_fixers_from_package fixes = get_fixers_from_package('lib2to3.fixes') rt = RefactoringTool(fixes) PACKAGES = [] DIRS = [] def maybe_2to3(filename, modname=None): """Returns a python3 version of filename.""" need_2to3 = False filename = os.path.abspath(filename) if any(filename.startswith(d) for d in DIRS): need_2to3 = True elif modname is not None and any(modname.startswith(p) for p in PACKAGES): need_2to3 = True if not need_2to3: return filename