def typecheck_results(self) -> TypecheckResults:
addresses = [config.address for config in self.field_sets]
return TypecheckResults(
[TypecheckResult(
self.exit_code(addresses),
"",
"",
)],
typechecker_name=self.typechecker_name,
)
def test_streaming_output_partitions() -> None:
results = TypecheckResults(
[
TypecheckResult(21, "", "", partition_description="ghc8.1"),
TypecheckResult(
0, "stdout", "stderr", partition_description="ghc9.2"),
],
typechecker_name="typchecker",
)
assert results.level() == LogLevel.WARN
assert results.message() == dedent("""\
failed (exit code 21).
Partition #1 - ghc8.1:
Partition #2 - ghc9.2:
stdout
stderr
""")
def test_streaming_output_failure() -> None:
results = TypecheckResults([TypecheckResult(18, "stdout", "stderr")],
typechecker_name="typchecker")
assert results.level() == LogLevel.WARN
assert results.message() == dedent("""\
failed (exit code 18).
stdout
stderr
""")
def test_streaming_output_success() -> None:
results = TypecheckResults([TypecheckResult(0, "stdout", "stderr")],
typechecker_name="typchecker")
assert results.level() == LogLevel.INFO
assert results.message() == dedent("""\
succeeded.
stdout
stderr
""")
async def mypy_typecheck(request: MyPyRequest, mypy: MyPy) -> TypecheckResults:
if mypy.skip:
return TypecheckResults([], typechecker_name="MyPy")
transitive_targets = await Get(
TransitiveTargets, Addresses(fs.address for fs in request.field_sets))
prepared_sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure),
)
pex_request = Get(
Pex,
PexRequest(
output_filename="mypy.pex",
internal_only=True,
requirements=PexRequirements(mypy.all_requirements),
# NB: This only determines what MyPy is run with. The user can specify what version
# their code is with `--python-version`. See
# https://mypy.readthedocs.io/en/stable/config_file.html#platform-configuration. We do
# not auto-configure this for simplicity and to avoid Pants magically setting values for
# users.
interpreter_constraints=PexInterpreterConstraints(
mypy.interpreter_constraints),
entry_point=mypy.entry_point,
),
)
config_digest_request = Get(
Digest,
PathGlobs(
globs=[mypy.config] if mypy.config else [],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--mypy-config`",
),
)
prepared_sources, pex, config_digest = await MultiGet(
prepared_sources_request, pex_request, config_digest_request)
srcs_snapshot = prepared_sources.source_files.snapshot
file_list_path = "__files.txt"
python_files = "\n".join(f for f in srcs_snapshot.files
if f.endswith(".py"))
file_list_digest = await Get(
Digest,
CreateDigest([FileContent(file_list_path, python_files.encode())]),
)
merged_input_files = await Get(
Digest,
MergeDigests([
file_list_digest, srcs_snapshot.digest, pex.digest, config_digest
]),
)
result = await Get(
FallibleProcessResult,
PexProcess(
pex,
argv=generate_args(mypy, file_list_path=file_list_path),
input_digest=merged_input_files,
extra_env={
"PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots)
},
description=
f"Run MyPy on {pluralize(len(srcs_snapshot.files), 'file')}.",
level=LogLevel.DEBUG,
),
)
return TypecheckResults(
[TypecheckResult.from_fallible_process_result(result)],
typechecker_name="MyPy")
async def mypy_typecheck_partition(partition: MyPyPartition, mypy: MyPy) -> TypecheckResult:
plugin_target_addresses = await Get(Addresses, UnparsedAddressInputs, mypy.source_plugins)
plugin_transitive_targets = await Get(
TransitiveTargets, TransitiveTargetsRequest(plugin_target_addresses)
)
plugin_requirements = PexRequirements.create_from_requirement_fields(
plugin_tgt[PythonRequirementsField]
for plugin_tgt in plugin_transitive_targets.closure
if plugin_tgt.has_field(PythonRequirementsField)
)
# If the user did not set `--python-version` already, we set it ourselves based on their code's
# interpreter constraints. This determines what AST is used by MyPy.
python_version = (
None
if partition.python_version_already_configured
else partition.interpreter_constraints.minimum_python_version()
)
# MyPy requires 3.5+ to run, but uses the typed-ast library to work with 2.7, 3.4, 3.5, 3.6,
# and 3.7. However, typed-ast does not understand 3.8+, so instead we must run MyPy with
# Python 3.8+ when relevant. We only do this if <3.8 can't be used, as we don't want a
# loose requirement like `>=3.6` to result in requiring Python 3.8+, which would error if
# 3.8+ is not installed on the machine.
tool_interpreter_constraints = (
partition.interpreter_constraints
if (
mypy.options.is_default("interpreter_constraints")
and partition.interpreter_constraints.requires_python38_or_newer()
)
else PexInterpreterConstraints(mypy.interpreter_constraints)
)
plugin_sources_get = Get(
PythonSourceFiles, PythonSourceFilesRequest(plugin_transitive_targets.closure)
)
closure_sources_get = Get(PythonSourceFiles, PythonSourceFilesRequest(partition.closure))
roots_sources_get = Get(
SourceFiles, SourceFilesRequest(tgt.get(PythonSources) for tgt in partition.root_targets)
)
requirements_pex_get = Get(
Pex,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(
(tgt.address for tgt in partition.root_targets),
hardcoded_interpreter_constraints=partition.interpreter_constraints,
internal_only=True,
),
)
# TODO(John Sirois): Scope the extra requirements to the partition.
# Right now we just use a global set of extra requirements and these might not be compatible
# with all partitions. See: https://github.com/pantsbuild/pants/issues/11556
mypy_extra_requirements_pex_get = Get(
Pex,
PexRequest(
output_filename="mypy_extra_requirements.pex",
internal_only=True,
requirements=PexRequirements(mypy.extra_requirements),
interpreter_constraints=partition.interpreter_constraints,
),
)
mypy_pex_get = Get(
VenvPex,
PexRequest(
output_filename="mypy.pex",
internal_only=True,
main=mypy.main,
requirements=PexRequirements((*mypy.all_requirements, *plugin_requirements)),
interpreter_constraints=tool_interpreter_constraints,
),
)
config_files_get = Get(ConfigFiles, ConfigFilesRequest, mypy.config_request)
(
plugin_sources,
closure_sources,
roots_sources,
mypy_pex,
requirements_pex,
mypy_extra_requirements_pex,
config_files,
) = await MultiGet(
plugin_sources_get,
closure_sources_get,
roots_sources_get,
mypy_pex_get,
requirements_pex_get,
mypy_extra_requirements_pex_get,
config_files_get,
)
python_files = determine_python_files(roots_sources.snapshot.files)
file_list_path = "__files.txt"
file_list_digest_request = Get(
Digest,
CreateDigest([FileContent(file_list_path, "\n".join(python_files).encode())]),
)
typechecked_venv_pex_request = Get(
VenvPex,
PexRequest(
output_filename="typechecked_venv.pex",
internal_only=True,
pex_path=[requirements_pex, mypy_extra_requirements_pex],
interpreter_constraints=partition.interpreter_constraints,
),
)
typechecked_venv_pex, file_list_digest = await MultiGet(
typechecked_venv_pex_request, file_list_digest_request
)
merged_input_files = await Get(
Digest,
MergeDigests(
[
file_list_digest,
plugin_sources.source_files.snapshot.digest,
closure_sources.source_files.snapshot.digest,
typechecked_venv_pex.digest,
config_files.snapshot.digest,
]
),
)
all_used_source_roots = sorted(
set(itertools.chain(plugin_sources.source_roots, closure_sources.source_roots))
)
env = {
"PEX_EXTRA_SYS_PATH": ":".join(all_used_source_roots),
"MYPYPATH": ":".join(all_used_source_roots),
}
result = await Get(
FallibleProcessResult,
VenvPexProcess(
mypy_pex,
argv=generate_argv(
mypy,
typechecked_venv_pex,
file_list_path=file_list_path,
python_version=python_version,
),
input_digest=merged_input_files,
extra_env=env,
description=f"Run MyPy on {pluralize(len(python_files), 'file')}.",
level=LogLevel.DEBUG,
),
)
return TypecheckResult.from_fallible_process_result(
result, partition_description=str(sorted(str(c) for c in partition.interpreter_constraints))
)
async def mypy_typecheck_partition(partition: MyPyPartition,
mypy: MyPy) -> TypecheckResult:
plugin_target_addresses = await Get(Addresses, UnparsedAddressInputs,
mypy.source_plugins)
plugin_transitive_targets_request = Get(
TransitiveTargets, TransitiveTargetsRequest(plugin_target_addresses))
plugin_transitive_targets, launcher_script = await MultiGet(
plugin_transitive_targets_request,
Get(Digest, CreateDigest([LAUNCHER_FILE])))
plugin_requirements = PexRequirements.create_from_requirement_fields(
plugin_tgt[PythonRequirementsField]
for plugin_tgt in plugin_transitive_targets.closure
if plugin_tgt.has_field(PythonRequirementsField))
# If the user did not set `--python-version` already, we set it ourselves based on their code's
# interpreter constraints. This determines what AST is used by MyPy.
python_version = (
None if partition.python_version_already_configured else
partition.interpreter_constraints.minimum_python_version())
# MyPy requires 3.5+ to run, but uses the typed-ast library to work with 2.7, 3.4, 3.5, 3.6,
# and 3.7. However, typed-ast does not understand 3.8, so instead we must run MyPy with
# Python 3.8 when relevant. We only do this if <3.8 can't be used, as we don't want a
# loose requirement like `>=3.6` to result in requiring Python 3.8, which would error if
# 3.8 is not installed on the machine.
tool_interpreter_constraints = PexInterpreterConstraints((
"CPython>=3.8", ) if (
mypy.options.is_default("interpreter_constraints")
and partition.interpreter_constraints.requires_python38_or_newer()
) else mypy.interpreter_constraints)
plugin_sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(plugin_transitive_targets.closure))
typechecked_sources_request = Get(
PythonSourceFiles, PythonSourceFilesRequest(partition.closure))
# Normally, this `requirements.pex` would be merged with mypy.pex via `--pex-path`. However,
# this will cause a runtime error if the interpreter constraints are different between the
# PEXes and they have incompatible wheels.
#
# Instead, we teach MyPy about the requirements by extracting the distributions from
# requirements.pex and setting EXTRACTED_WHEELS, which our custom launcher script then
# looks for.
#
# Conventionally, MyPy users might instead set `MYPYPATH` for this. However, doing this
# results in type checking the requirements themselves.
requirements_pex_request = Get(
Pex,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(
(addr for addr in partition.field_set_addresses),
hardcoded_interpreter_constraints=partition.
interpreter_constraints,
internal_only=True,
),
)
mypy_pex_request = Get(
Pex,
PexRequest(
output_filename="mypy.pex",
internal_only=True,
sources=launcher_script,
requirements=PexRequirements(
itertools.chain(mypy.all_requirements, plugin_requirements)),
interpreter_constraints=tool_interpreter_constraints,
entry_point=PurePath(LAUNCHER_FILE.path).stem,
),
)
config_digest_request = Get(Digest, PathGlobs, config_path_globs(mypy))
(
plugin_sources,
typechecked_sources,
mypy_pex,
requirements_pex,
config_digest,
) = await MultiGet(
plugin_sources_request,
typechecked_sources_request,
mypy_pex_request,
requirements_pex_request,
config_digest_request,
)
typechecked_srcs_snapshot = typechecked_sources.source_files.snapshot
file_list_path = "__files.txt"
python_files = "\n".join(
determine_python_files(
typechecked_sources.source_files.snapshot.files))
create_file_list_request = Get(
Digest,
CreateDigest([FileContent(file_list_path, python_files.encode())]),
)
file_list_digest, extracted_pex_distributions = await MultiGet(
create_file_list_request,
Get(ExtractedPexDistributions, Pex, requirements_pex))
merged_input_files = await Get(
Digest,
MergeDigests([
file_list_digest,
plugin_sources.source_files.snapshot.digest,
typechecked_srcs_snapshot.digest,
mypy_pex.digest,
extracted_pex_distributions.digest,
config_digest,
]),
)
all_used_source_roots = sorted(
set(
itertools.chain(plugin_sources.source_roots,
typechecked_sources.source_roots)))
env = {
"PEX_EXTRA_SYS_PATH":
":".join(all_used_source_roots),
"EXTRACTED_WHEELS":
":".join(extracted_pex_distributions.wheel_directory_paths),
}
result = await Get(
FallibleProcessResult,
PexProcess(
mypy_pex,
argv=generate_argv(mypy,
file_list_path=file_list_path,
python_version=python_version),
input_digest=merged_input_files,
extra_env=env,
description=
f"Run MyPy on {pluralize(len(typechecked_srcs_snapshot.files), 'file')}.",
level=LogLevel.DEBUG,
),
)
return TypecheckResult.from_fallible_process_result(
result,
partition_description=str(
sorted(str(c) for c in partition.interpreter_constraints)))
