def transform_module(transformer: Codemod, code: str) -> TransformResult:
"""
Given a module in a string and a Codemod to transform the module with,
execute the codemod on the code and return a TransformResult. This will
never raise an exception, instead will return a TransformFailure.
"""
try:
input_tree = parse_module(code)
output_tree = transformer.transform_module(input_tree)
return TransformSuccess(code=output_tree.code,
warning_messages=transformer.context.warnings)
except KeyboardInterrupt:
return TransformExit()
except SkipFile as ex:
return TransformSkip(
skip_description=str(ex),
skip_reason=SkipReason.OTHER,
warning_messages=transformer.context.warnings,
)
except Exception as ex:
return TransformFailure(
error=ex,
traceback_str=traceback.format_exc(),
warning_messages=transformer.context.warnings,
)
def transform_module(
transformer: Codemod, code: str, *, python_version: Optional[str] = None
) -> TransformResult:
"""
Given a module as represented by a string and a :class:`~libcst.codemod.Codemod`
that we wish to run, execute the codemod on the code and return a
:class:`~libcst.codemod.TransformResult`. This should never raise an exception.
On success, this returns a :class:`~libcst.codemod.TransformSuccess` containing
any generated warnings as well as the transformed code. If the codemod is
interrupted with a Ctrl+C, this returns a :class:`~libcst.codemod.TransformExit`.
If the codemod elected to skip by throwing a :class:`~libcst.codemod.SkipFile`
exception, this will return a :class:`~libcst.codemod.TransformSkip` containing
the reason for skipping as well as any warnings that were generated before
the codemod decided to skip. If the codemod throws an unexpected exception,
this will return a :class:`~libcst.codemod.TransformFailure` containing the
exception that occured as well as any warnings that were generated before the
codemod crashed.
"""
try:
input_tree = parse_module(
code,
config=(
PartialParserConfig(python_version=python_version)
if python_version is not None
else PartialParserConfig()
),
)
output_tree = transformer.transform_module(input_tree)
return TransformSuccess(
code=output_tree.code, warning_messages=transformer.context.warnings
)
except KeyboardInterrupt:
return TransformExit()
except SkipFile as ex:
return TransformSkip(
skip_description=str(ex),
skip_reason=SkipReason.OTHER,
warning_messages=transformer.context.warnings,
)
except Exception as ex:
return TransformFailure(
error=ex,
traceback_str=traceback.format_exc(),
warning_messages=transformer.context.warnings,
)
def parallel_exec_transform_with_prettyprint( # noqa: C901
transform: Codemod,
files: Sequence[str],
*,
jobs: Optional[int] = None,
unified_diff: Optional[int] = None,
include_generated: bool = False,
generated_code_marker: str = _DEFAULT_GENERATED_CODE_MARKER,
format_code: bool = False,
formatter_args: Sequence[str] = (),
show_successes: bool = False,
hide_generated: bool = False,
hide_blacklisted: bool = False,
hide_progress: bool = False,
blacklist_patterns: Sequence[str] = (),
python_version: Optional[str] = None,
repo_root: Optional[str] = None,
) -> ParallelTransformResult:
"""
Given a list of files and an instantiated codemod we should apply to them,
fork and apply the codemod in parallel to all of the files, including any
configured formatter. The ``jobs`` parameter controls the maximum number of
in-flight transforms, and needs to be at least 1. If not included, the number
of jobs will automatically be set to the number of CPU cores. If ``unified_diff``
is set to a number, changes to files will be printed to stdout with
``unified_diff`` lines of context. If it is set to ``None`` or left out, files
themselves will be updated with changes and formatting. If a
``python_version`` is provided, then we will parse each source file using
this version. Otherwise, we will use the version of the currently executing python
binary.
A progress indicator as well as any generated warnings will be printed to stderr.
To supress the interactive progress indicator, set ``hide_progress`` to ``True``.
Files that include the generated code marker will be skipped unless the
``include_generated`` parameter is set to ``True``. Similarly, files that match
a supplied blacklist of regex patterns will be skipped. Warnings for skipping
both blacklisted and generated files will be printed to stderr along with
warnings generated by the codemod unless ``hide_blacklisted`` and
``hide_generated`` are set to ``True``. Files that were successfully codemodded
will not be printed to stderr unless ``show_successes`` is set to ``True``.
To make this API possible, we take an instantiated transform. This is due to
the fact that lambdas are not pickleable and pickling functions is undefined.
This means we're implicitly relying on fork behavior on UNIX-like systems, and
this function will not work on Windows systems. To create a command-line utility
that runs on Windows, please instead see
:func:`~libcst.codemod.exec_transform_with_prettyprint`.
"""
# Ensure that we have no duplicates, otherwise we might get race conditions
# on write.
files = sorted(list({os.path.abspath(f) for f in files}))
total = len(files)
progress = Progress(enabled=not hide_progress, total=total)
# Grab number of cores if we need to
jobs: int = jobs if jobs is not None else cpu_count()
if jobs < 1:
raise Exception("Must have at least one job to process!")
if total == 0:
return ParallelTransformResult(successes=0,
failures=0,
skips=0,
warnings=0)
if repo_root is not None:
# Make sure if there is a root that we have the absolute path to it.
repo_root = os.path.abspath(repo_root)
# Spin up a full repo metadata manager so that we can provide metadata
# like type inference to individual forked processes.
print("Calculating full-repo metadata...", file=sys.stderr)
metadata_manager = FullRepoManager(
repo_root,
files,
transform.get_inherited_dependencies(),
)
metadata_manager.resolve_cache()
transform.context = replace(
transform.context,
metadata_manager=metadata_manager,
)
print("Executing codemod...", file=sys.stderr)
config = ExecutionConfig(
repo_root=repo_root,
unified_diff=unified_diff,
include_generated=include_generated,
generated_code_marker=generated_code_marker,
format_code=format_code,
formatter_args=formatter_args,
blacklist_patterns=blacklist_patterns,
python_version=python_version,
)
if total == 1:
# Simple case, we should not pay for process overhead.
# Let's just use a dummy synchronous pool.
jobs = 1
pool_impl = DummyPool
else:
pool_impl = Pool
# Warm the parser, pre-fork.
parse_module(
"",
config=(PartialParserConfig(python_version=python_version)
if python_version is not None else PartialParserConfig()),
)
successes: int = 0
failures: int = 0
warnings: int = 0
skips: int = 0
with pool_impl(processes=jobs) as p: # type: ignore
args = [{
"transformer": transform,
"filename": filename,
"config": config,
} for filename in files]
try:
for result in p.imap_unordered(_execute_transform_wrap,
args,
chunksize=4):
# Print an execution result, keep track of failures
_print_parallel_result(
result,
progress,
unified_diff=bool(unified_diff),
show_successes=show_successes,
hide_generated=hide_generated,
hide_blacklisted=hide_blacklisted,
)
progress.print(successes + failures + skips)
if isinstance(result.transform_result, TransformFailure):
failures += 1
elif isinstance(result.transform_result, TransformSuccess):
successes += 1
elif isinstance(result.transform_result,
(TransformExit, TransformSkip)):
skips += 1
warnings += len(result.transform_result.warning_messages)
finally:
progress.clear()
# Return whether there was one or more failure.
return ParallelTransformResult(successes=successes,
failures=failures,
skips=skips,
warnings=warnings)
def _execute_transform( # noqa: C901
transformer: Codemod,
filename: str,
config: ExecutionConfig,
) -> ExecutionResult:
for pattern in config.blacklist_patterns:
if re.fullmatch(pattern, filename):
return ExecutionResult(
filename=filename,
changed=False,
transform_result=TransformSkip(
skip_reason=SkipReason.BLACKLISTED,
skip_description=f"Blacklisted by pattern {pattern}.",
),
)
try:
with open(filename, "rb") as fp:
oldcode = fp.read()
# Skip generated files
if (not config.include_generated
and config.generated_code_marker.encode("utf-8") in oldcode):
return ExecutionResult(
filename=filename,
changed=False,
transform_result=TransformSkip(
skip_reason=SkipReason.GENERATED,
skip_description="Generated file.",
),
)
# Somewhat gross hack to provide the filename in the transform's context.
# We do this after the fork so that a context that was initialized with
# some defaults before calling parallel_exec_transform_with_prettyprint
# will be updated per-file.
transformer.context = replace(
transformer.context,
filename=filename,
full_module_name=_calculate_module(config.repo_root, filename),
)
# Run the transform, bail if we failed or if we aren't formatting code
try:
input_tree = parse_module(
oldcode,
config=(PartialParserConfig(
python_version=str(config.python_version))
if config.python_version is not None else
PartialParserConfig()),
)
output_tree = transformer.transform_module(input_tree)
newcode = output_tree.bytes
encoding = output_tree.encoding
except KeyboardInterrupt:
return ExecutionResult(filename=filename,
changed=False,
transform_result=TransformExit())
except SkipFile as ex:
return ExecutionResult(
filename=filename,
changed=False,
transform_result=TransformSkip(
skip_reason=SkipReason.OTHER,
skip_description=str(ex),
warning_messages=transformer.context.warnings,
),
)
except Exception as ex:
return ExecutionResult(
filename=filename,
changed=False,
transform_result=TransformFailure(
error=ex,
traceback_str=traceback.format_exc(),
warning_messages=transformer.context.warnings,
),
)
# Call formatter if needed, but only if we actually changed something in this
# file
if config.format_code and newcode != oldcode:
try:
newcode = invoke_formatter(config.formatter_args, newcode)
except KeyboardInterrupt:
return ExecutionResult(
filename=filename,
changed=False,
transform_result=TransformExit(),
)
except Exception as ex:
return ExecutionResult(
filename=filename,
changed=False,
transform_result=TransformFailure(
error=ex,
traceback_str=traceback.format_exc(),
warning_messages=transformer.context.warnings,
),
)
# Format as unified diff if needed, otherwise save it back
changed = oldcode != newcode
if config.unified_diff:
newcode = diff_code(
oldcode.decode(encoding),
newcode.decode(encoding),
config.unified_diff,
filename=filename,
)
else:
# Write back if we changed
if changed:
with open(filename, "wb") as fp:
fp.write(newcode)
# Not strictly necessary, but saves space in pickle since we won't use it
newcode = ""
# Inform success
return ExecutionResult(
filename=filename,
changed=changed,
transform_result=TransformSuccess(
warning_messages=transformer.context.warnings, code=newcode),
)
except KeyboardInterrupt:
return ExecutionResult(filename=filename,
changed=False,
transform_result=TransformExit())
except Exception as ex:
return ExecutionResult(
filename=filename,
changed=False,
transform_result=TransformFailure(
error=ex,
traceback_str=traceback.format_exc(),
warning_messages=transformer.context.warnings,
),
)
def parallel_exec_transform_with_prettyprint( # noqa: C901
transform: Codemod,
files: Sequence[str],
*,
jobs: Optional[int] = None,
unified_diff: Optional[int] = None,
include_generated: bool = False,
generated_code_marker: str = _DEFAULT_GENERATED_CODE_MARKER,
format_code: bool = False,
formatter_args: Sequence[str] = (),
show_successes: bool = False,
hide_generated: bool = False,
hide_blacklisted: bool = False,
hide_progress: bool = False,
blacklist_patterns: Sequence[str] = (),
python_version: Optional[str] = None,
repo_root: Optional[str] = None,
) -> ParallelTransformResult:
"""
Given a list of files and an instantiated codemod we should apply to them,
fork and apply the codemod in parallel to all of the files, including any
configured formatter. The ``jobs`` parameter controls the maximum number of
in-flight transforms, and needs to be at least 1. If not included, the number
of jobs will automatically be set to the number of CPU cores. If ``unified_diff``
is set to a number, changes to files will be printed to stdout with
``unified_diff`` lines of context. If it is set to ``None`` or left out, files
themselves will be updated with changes and formatting. If a
``python_version`` is provided, then we will parse each source file using
this version. Otherwise, we will use the version of the currently executing python
binary.
A progress indicator as well as any generated warnings will be printed to stderr.
To supress the interactive progress indicator, set ``hide_progress`` to ``True``.
Files that include the generated code marker will be skipped unless the
``include_generated`` parameter is set to ``True``. Similarly, files that match
a supplied blacklist of regex patterns will be skipped. Warnings for skipping
both blacklisted and generated files will be printed to stderr along with
warnings generated by the codemod unless ``hide_blacklisted`` and
``hide_generated`` are set to ``True``. Files that were successfully codemodded
will not be printed to stderr unless ``show_successes`` is set to ``True``.
To make this API possible, we take an instantiated transform. This is due to
the fact that lambdas are not pickleable and pickling functions is undefined.
This means we're implicitly relying on fork behavior on UNIX-like systems, and
this function will not work on Windows systems. To create a command-line utility
that runs on Windows, please instead see
:func:`~libcst.codemod.exec_transform_with_prettyprint`.
"""
# Ensure that we have no duplicates, otherwise we might get race conditions
# on write.
files = sorted(list(set(os.path.abspath(f) for f in files)))
total = len(files)
progress = Progress(enabled=not hide_progress, total=total)
# Grab number of cores if we need to
jobs: int = jobs or cpu_count()
if jobs < 1:
raise Exception("Must have at least one job to process!")
if total == 0:
return ParallelTransformResult(successes=0,
failures=0,
skips=0,
warnings=0)
if repo_root:
# Make sure if there is a root that we have the absolute path to it.
repo_root = os.path.abspath(repo_root)
# Spin up a full repo metadata manager so that we can provide metadata
# like type inference to individual forked processes.
print("Calculating full-repo metadata...", file=sys.stderr)
metadata_manager = FullRepoManager(
repo_root,
files,
transform.get_inherited_dependencies(),
)
metadata_manager.resolve_cache()
transform.context = replace(
transform.context,
metadata_manager=metadata_manager,
)
print("Executing codemod...", file=sys.stderr)
# We place results in this queue inside _parallel_exec_process_stub
# so that we can control when things get printed to the console.
queue = Queue()
if total == 1:
# Simple case, we should not pay for process overhead. Lets still
# use the exec stub however, so we can share code.
progress.print(0)
_parallel_exec_process_stub(
queue,
transform,
files[0],
repo_root,
unified_diff=unified_diff,
include_generated=include_generated,
generated_code_marker=generated_code_marker,
format_code=format_code,
formatter_args=formatter_args,
blacklist_patterns=blacklist_patterns,
python_version=python_version,
)
result = queue.get()
_print_parallel_result(
result,
progress,
unified_diff=bool(unified_diff),
show_successes=show_successes,
hide_generated=hide_generated,
hide_blacklisted=hide_blacklisted,
)
if isinstance(result.transform_result, TransformFailure):
return ParallelTransformResult(
successes=0,
failures=1,
skips=0,
warnings=len(result.transform_result.warning_messages),
)
elif isinstance(result.transform_result,
(TransformSkip, TransformExit)):
return ParallelTransformResult(
successes=0,
failures=0,
skips=1,
warnings=len(result.transform_result.warning_messages),
)
elif isinstance(result.transform_result, TransformSuccess):
return ParallelTransformResult(
successes=1,
failures=0,
skips=0,
warnings=len(result.transform_result.warning_messages),
)
else:
raise Exception("Logic error, unaccounted for result!")
# Warm the parser, pre-fork.
parse_module(
"",
config=(PartialParserConfig(python_version=python_version)
if python_version is not None else PartialParserConfig()),
)
# Complex case, more than one file
successes: int = 0
failures: int = 0
warnings: int = 0
skips: int = 0
pending_processes: List[Process] = []
# Start processes
filename_to_process: Dict[str, Process] = {}
for f in files:
process = Process(
target=_parallel_exec_process_stub,
args=(
queue,
transform,
f,
repo_root,
unified_diff,
include_generated,
generated_code_marker,
format_code,
formatter_args,
blacklist_patterns,
python_version,
),
)
pending_processes.append(process)
filename_to_process[f] = process
# Start the processes, allowing no more than num_processes to be running
# at once.
results_left = len(pending_processes)
joinable_processes: Set[Process] = set()
processes_started = 0
interrupted = False
while results_left > 0 and not interrupted:
while processes_started < jobs and pending_processes:
try:
# Move this process to the joinables
process = pending_processes.pop(0)
joinable_processes.add(process)
# Start it, bookkeep that we did
process.start()
processes_started += 1
except KeyboardInterrupt:
interrupted = True
continue
try:
result = queue.get(block=True, timeout=0.005)
except KeyboardInterrupt:
interrupted = True
continue
except Empty:
progress.print(successes + failures + skips)
continue
# Bookkeep the result, since we know the process that returned this is done.
results_left -= 1
processes_started -= 1
# Print an execution result, keep track of failures
_print_parallel_result(
result,
progress,
unified_diff=bool(unified_diff),
show_successes=show_successes,
hide_generated=hide_generated,
hide_blacklisted=hide_blacklisted,
)
progress.print(successes + failures + skips)
if isinstance(result.transform_result, TransformFailure):
failures += 1
elif isinstance(result.transform_result, TransformSuccess):
successes += 1
elif isinstance(result.transform_result,
(TransformExit, TransformSkip)):
skips += 1
warnings += len(result.transform_result.warning_messages)
# Join the process to free any related resources.
# Remove all references to the process to allow the GC to
# clean up any file handles.
process = filename_to_process.pop(result.filename, None)
if process:
process.join()
joinable_processes.discard(process)
# Now, join on all of them so we don't leave zombies or hang
for p in joinable_processes:
p.join()
# Return whether there was one or more failure.
progress.clear()
# If we caught an interrupt, raise that
if interrupted:
raise KeyboardInterrupt()
return ParallelTransformResult(successes=successes,
failures=failures,
skips=skips,
warnings=warnings)
def __init__(self, context: CodemodContext) -> None:
Codemod.__init__(self, context)
MatcherDecoratableTransformer.__init__(self)
def _parallel_exec_process_stub( # noqa: C901
result_queue: "Queue[ParallelExecResult]",
transformer: Codemod,
filename: str,
unified_diff: Optional[int],
include_generated: bool,
generated_code_marker: str,
format_code: bool,
formatter_args: Sequence[str],
blacklist_patterns: Sequence[str],
) -> None:
for pattern in blacklist_patterns:
if re.fullmatch(pattern, filename):
result_queue.put(
ParallelExecResult(
filename=filename,
changed=False,
transform_result=TransformSkip(
skip_reason=SkipReason.BLACKLISTED,
skip_description=f"Blacklisted by pattern {pattern}.",
),
)
)
return
try:
with open(filename, "rb") as fp:
oldcode = fp.read()
# Skip generated files
if not include_generated and generated_code_marker.encode("utf-8") in oldcode:
result_queue.put(
ParallelExecResult(
filename=filename,
changed=False,
transform_result=TransformSkip(
skip_reason=SkipReason.GENERATED,
skip_description="Generated file.",
),
)
)
return
# Somewhat gross hack to provide the filename in the transform's context.
# We do this after the fork so that a context that was initialized with
# some defaults before calling parallel_exec_transform_with_prettyprint
# will be updated per-file.
transformer.context = replace(transformer.context, filename=filename)
# Run the transform, bail if we failed or if we aren't formatting code
try:
input_tree = parse_module(oldcode)
output_tree = transformer.transform_module(input_tree)
newcode = output_tree.bytes
encoding = output_tree.encoding
except KeyboardInterrupt:
result_queue.put(
ParallelExecResult(
filename=filename, changed=False, transform_result=TransformExit()
)
)
return
except SkipFile as ex:
result_queue.put(
ParallelExecResult(
filename=filename,
changed=False,
transform_result=TransformSkip(
skip_reason=SkipReason.OTHER,
skip_description=str(ex),
warning_messages=transformer.context.warnings,
),
)
)
return
except Exception as ex:
result_queue.put(
ParallelExecResult(
filename=filename,
changed=False,
transform_result=TransformFailure(
error=ex,
traceback_str=traceback.format_exc(),
warning_messages=transformer.context.warnings,
),
)
)
return
# Call formatter if needed, but only if we actually changed something in this
# file
if format_code and newcode != oldcode:
try:
newcode = invoke_formatter(formatter_args, newcode)
except KeyboardInterrupt:
result_queue.put(
ParallelExecResult(
filename=filename,
changed=False,
transform_result=TransformExit(),
)
)
return
except Exception as ex:
result_queue.put(
ParallelExecResult(
filename=filename,
changed=False,
transform_result=TransformFailure(
error=ex,
traceback_str=traceback.format_exc(),
warning_messages=transformer.context.warnings,
),
)
)
return
# Format as unified diff if needed, otherwise save it back
changed = oldcode != newcode
if unified_diff:
newcode = diff_code(
oldcode.decode(encoding),
newcode.decode(encoding),
unified_diff,
filename=filename,
)
else:
# Write back if we changed
if changed:
with open(filename, "wb") as fp:
fp.write(newcode)
# Not strictly necessary, but saves space in pickle since we won't use it
newcode = ""
# Inform success
result_queue.put(
ParallelExecResult(
filename=filename,
changed=changed,
transform_result=TransformSuccess(
warning_messages=transformer.context.warnings, code=newcode
),
)
)
except KeyboardInterrupt:
result_queue.put(
ParallelExecResult(
filename=filename, changed=False, transform_result=TransformExit()
)
)
except Exception as ex:
result_queue.put(
ParallelExecResult(
filename=filename,
changed=False,
transform_result=TransformFailure(
error=ex,
traceback_str=traceback.format_exc(),
warning_messages=transformer.context.warnings,
),
)
)
