async def mypy_typecheck_partition(partition: MyPyPartition, mypy: MyPy) -> TypecheckResult: plugin_target_addresses = await Get(Addresses, UnparsedAddressInputs, mypy.source_plugins) plugin_transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest(plugin_target_addresses) ) plugin_requirements = PexRequirements.create_from_requirement_fields( plugin_tgt[PythonRequirementsField] for plugin_tgt in plugin_transitive_targets.closure if plugin_tgt.has_field(PythonRequirementsField) ) # If the user did not set `--python-version` already, we set it ourselves based on their code's # interpreter constraints. This determines what AST is used by MyPy. python_version = ( None if partition.python_version_already_configured else partition.interpreter_constraints.minimum_python_version() ) # MyPy requires 3.5+ to run, but uses the typed-ast library to work with 2.7, 3.4, 3.5, 3.6, # and 3.7. However, typed-ast does not understand 3.8+, so instead we must run MyPy with # Python 3.8+ when relevant. We only do this if <3.8 can't be used, as we don't want a # loose requirement like `>=3.6` to result in requiring Python 3.8+, which would error if # 3.8+ is not installed on the machine. tool_interpreter_constraints = ( partition.interpreter_constraints if ( mypy.options.is_default("interpreter_constraints") and partition.interpreter_constraints.requires_python38_or_newer() ) else PexInterpreterConstraints(mypy.interpreter_constraints) ) plugin_sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(plugin_transitive_targets.closure) ) closure_sources_get = Get(PythonSourceFiles, PythonSourceFilesRequest(partition.closure)) roots_sources_get = Get( SourceFiles, SourceFilesRequest(tgt.get(PythonSources) for tgt in partition.root_targets) ) requirements_pex_get = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (tgt.address for tgt in partition.root_targets), hardcoded_interpreter_constraints=partition.interpreter_constraints, internal_only=True, ), ) # TODO(John Sirois): Scope the extra requirements to the partition. # Right now we just use a global set of extra requirements and these might not be compatible # with all partitions. See: https://github.com/pantsbuild/pants/issues/11556 mypy_extra_requirements_pex_get = Get( Pex, PexRequest( output_filename="mypy_extra_requirements.pex", internal_only=True, requirements=PexRequirements(mypy.extra_requirements), interpreter_constraints=partition.interpreter_constraints, ), ) mypy_pex_get = Get( VenvPex, PexRequest( output_filename="mypy.pex", internal_only=True, main=mypy.main, requirements=PexRequirements((*mypy.all_requirements, *plugin_requirements)), interpreter_constraints=tool_interpreter_constraints, ), ) config_files_get = Get(ConfigFiles, ConfigFilesRequest, mypy.config_request) ( plugin_sources, closure_sources, roots_sources, mypy_pex, requirements_pex, mypy_extra_requirements_pex, config_files, ) = await MultiGet( plugin_sources_get, closure_sources_get, roots_sources_get, mypy_pex_get, requirements_pex_get, mypy_extra_requirements_pex_get, config_files_get, ) python_files = determine_python_files(roots_sources.snapshot.files) file_list_path = "__files.txt" file_list_digest_request = Get( Digest, CreateDigest([FileContent(file_list_path, "\n".join(python_files).encode())]), ) typechecked_venv_pex_request = Get( VenvPex, PexRequest( output_filename="typechecked_venv.pex", internal_only=True, pex_path=[requirements_pex, mypy_extra_requirements_pex], interpreter_constraints=partition.interpreter_constraints, ), ) typechecked_venv_pex, file_list_digest = await MultiGet( typechecked_venv_pex_request, file_list_digest_request ) merged_input_files = await Get( Digest, MergeDigests( [ file_list_digest, plugin_sources.source_files.snapshot.digest, closure_sources.source_files.snapshot.digest, typechecked_venv_pex.digest, config_files.snapshot.digest, ] ), ) all_used_source_roots = sorted( set(itertools.chain(plugin_sources.source_roots, closure_sources.source_roots)) ) env = { "PEX_EXTRA_SYS_PATH": ":".join(all_used_source_roots), "MYPYPATH": ":".join(all_used_source_roots), } result = await Get( FallibleProcessResult, VenvPexProcess( mypy_pex, argv=generate_argv( mypy, typechecked_venv_pex, file_list_path=file_list_path, python_version=python_version, ), input_digest=merged_input_files, extra_env=env, description=f"Run MyPy on {pluralize(len(python_files), 'file')}.", level=LogLevel.DEBUG, ), ) return TypecheckResult.from_fallible_process_result( result, partition_description=str(sorted(str(c) for c in partition.interpreter_constraints)) )
async def mypy_typecheck(request: MyPyRequest, mypy: MyPy) -> TypecheckResults: if mypy.skip: return TypecheckResults([], typechecker_name="MyPy") transitive_targets = await Get( TransitiveTargets, Addresses(fs.address for fs in request.field_sets)) prepared_sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure), ) pex_request = Get( Pex, PexRequest( output_filename="mypy.pex", internal_only=True, requirements=PexRequirements(mypy.all_requirements), # NB: This only determines what MyPy is run with. The user can specify what version # their code is with `--python-version`. See # https://mypy.readthedocs.io/en/stable/config_file.html#platform-configuration. We do # not auto-configure this for simplicity and to avoid Pants magically setting values for # users. interpreter_constraints=PexInterpreterConstraints( mypy.interpreter_constraints), entry_point=mypy.entry_point, ), ) config_digest_request = Get( Digest, PathGlobs( globs=[mypy.config] if mypy.config else [], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--mypy-config`", ), ) prepared_sources, pex, config_digest = await MultiGet( prepared_sources_request, pex_request, config_digest_request) srcs_snapshot = prepared_sources.source_files.snapshot file_list_path = "__files.txt" python_files = "\n".join(f for f in srcs_snapshot.files if f.endswith(".py")) file_list_digest = await Get( Digest, CreateDigest([FileContent(file_list_path, python_files.encode())]), ) merged_input_files = await Get( Digest, MergeDigests([ file_list_digest, srcs_snapshot.digest, pex.digest, config_digest ]), ) result = await Get( FallibleProcessResult, PexProcess( pex, argv=generate_args(mypy, file_list_path=file_list_path), input_digest=merged_input_files, extra_env={ "PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots) }, description= f"Run MyPy on {pluralize(len(srcs_snapshot.files), 'file')}.", level=LogLevel.DEBUG, ), ) return TypecheckResults( [TypecheckResult.from_fallible_process_result(result)], typechecker_name="MyPy")
async def mypy_typecheck_partition(partition: MyPyPartition, mypy: MyPy) -> TypecheckResult: plugin_target_addresses = await Get(Addresses, UnparsedAddressInputs, mypy.source_plugins) plugin_transitive_targets_request = Get( TransitiveTargets, TransitiveTargetsRequest(plugin_target_addresses)) plugin_transitive_targets, launcher_script = await MultiGet( plugin_transitive_targets_request, Get(Digest, CreateDigest([LAUNCHER_FILE]))) plugin_requirements = PexRequirements.create_from_requirement_fields( plugin_tgt[PythonRequirementsField] for plugin_tgt in plugin_transitive_targets.closure if plugin_tgt.has_field(PythonRequirementsField)) # If the user did not set `--python-version` already, we set it ourselves based on their code's # interpreter constraints. This determines what AST is used by MyPy. python_version = ( None if partition.python_version_already_configured else partition.interpreter_constraints.minimum_python_version()) # MyPy requires 3.5+ to run, but uses the typed-ast library to work with 2.7, 3.4, 3.5, 3.6, # and 3.7. However, typed-ast does not understand 3.8, so instead we must run MyPy with # Python 3.8 when relevant. We only do this if <3.8 can't be used, as we don't want a # loose requirement like `>=3.6` to result in requiring Python 3.8, which would error if # 3.8 is not installed on the machine. tool_interpreter_constraints = PexInterpreterConstraints(( "CPython>=3.8", ) if ( mypy.options.is_default("interpreter_constraints") and partition.interpreter_constraints.requires_python38_or_newer() ) else mypy.interpreter_constraints) plugin_sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(plugin_transitive_targets.closure)) typechecked_sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(partition.closure)) # Normally, this `requirements.pex` would be merged with mypy.pex via `--pex-path`. However, # this will cause a runtime error if the interpreter constraints are different between the # PEXes and they have incompatible wheels. # # Instead, we teach MyPy about the requirements by extracting the distributions from # requirements.pex and setting EXTRACTED_WHEELS, which our custom launcher script then # looks for. # # Conventionally, MyPy users might instead set `MYPYPATH` for this. However, doing this # results in type checking the requirements themselves. requirements_pex_request = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (addr for addr in partition.field_set_addresses), hardcoded_interpreter_constraints=partition. interpreter_constraints, internal_only=True, ), ) mypy_pex_request = Get( Pex, PexRequest( output_filename="mypy.pex", internal_only=True, sources=launcher_script, requirements=PexRequirements( itertools.chain(mypy.all_requirements, plugin_requirements)), interpreter_constraints=tool_interpreter_constraints, entry_point=PurePath(LAUNCHER_FILE.path).stem, ), ) config_digest_request = Get(Digest, PathGlobs, config_path_globs(mypy)) ( plugin_sources, typechecked_sources, mypy_pex, requirements_pex, config_digest, ) = await MultiGet( plugin_sources_request, typechecked_sources_request, mypy_pex_request, requirements_pex_request, config_digest_request, ) typechecked_srcs_snapshot = typechecked_sources.source_files.snapshot file_list_path = "__files.txt" python_files = "\n".join( determine_python_files( typechecked_sources.source_files.snapshot.files)) create_file_list_request = Get( Digest, CreateDigest([FileContent(file_list_path, python_files.encode())]), ) file_list_digest, extracted_pex_distributions = await MultiGet( create_file_list_request, Get(ExtractedPexDistributions, Pex, requirements_pex)) merged_input_files = await Get( Digest, MergeDigests([ file_list_digest, plugin_sources.source_files.snapshot.digest, typechecked_srcs_snapshot.digest, mypy_pex.digest, extracted_pex_distributions.digest, config_digest, ]), ) all_used_source_roots = sorted( set( itertools.chain(plugin_sources.source_roots, typechecked_sources.source_roots))) env = { "PEX_EXTRA_SYS_PATH": ":".join(all_used_source_roots), "EXTRACTED_WHEELS": ":".join(extracted_pex_distributions.wheel_directory_paths), } result = await Get( FallibleProcessResult, PexProcess( mypy_pex, argv=generate_argv(mypy, file_list_path=file_list_path, python_version=python_version), input_digest=merged_input_files, extra_env=env, description= f"Run MyPy on {pluralize(len(typechecked_srcs_snapshot.files), 'file')}.", level=LogLevel.DEBUG, ), ) return TypecheckResult.from_fallible_process_result( result, partition_description=str( sorted(str(c) for c in partition.interpreter_constraints)))