async def merge_coverage_data( data_batch: PytestCoverageDataBatch, transitive_targets: TransitiveHydratedTargets, python_setup: PythonSetup, coverage_setup: CoverageSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> MergedCoverageData: """Takes all python test results and merges their coverage data into a single sql file.""" # We start with a bunch of test results, each of which has a coverage data file called `.coverage` # We prefix each of these with their address so that we can write them all into a single pex. coverage_directory_digests = await MultiGet( Get[Digest]( DirectoryWithPrefixToAdd( directory_digest=result.test_result.coverage_data. digest, # type: ignore[attr-defined] prefix=result.address.path_safe_spec, )) for result in data_batch.addresses_and_test_results if result.test_result is not None and result.test_result.coverage_data is not None) sources = await Get[SourceFiles](AllSourceFilesRequest( (ht.adaptor for ht in transitive_targets.closure), strip_source_roots=False)) sources_with_inits_snapshot = await Get[InitInjectedSnapshot]( InjectInitRequest(sources.snapshot)) coveragerc = await Get[Coveragerc](CoveragercRequest(HydratedTargets( transitive_targets.closure), test_time=True)) merged_input_files: Digest = await Get( Digest, DirectoriesToMerge(directories=( *coverage_directory_digests, sources_with_inits_snapshot.snapshot.directory_digest, coveragerc.digest, coverage_setup.requirements_pex.directory_digest, )), ) prefixes = [ f"{result.address.path_safe_spec}/.coverage" for result in data_batch.addresses_and_test_results ] coverage_args = ["combine", *prefixes] request = coverage_setup.requirements_pex.create_execute_request( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_path=f"./{coverage_setup.requirements_pex.output_filename}", pex_args=coverage_args, input_files=merged_input_files, output_files=(".coverage", ), description=f"Merge coverage reports.", ) result = await Get[ExecuteProcessResult](ExecuteProcessRequest, request) return MergedCoverageData(coverage_data=result.output_directory_digest)
async def generate_chroot(request: SetupPyChrootRequest) -> SetupPyChroot: if request.py2: # TODO: Implement Python 2 support. This will involve, among other things: merging ancestor # __init__.py files into the chroot, detecting packages based on the presence of __init__.py, # and inspecting all __init__.py files for the namespace package incantation. raise UnsupportedPythonVersion('Running setup.py commands not supported for Python 2.') owned_deps = await Get[OwnedDependencies](DependencyOwner(request.exported_target)) targets = HydratedTargets(od.hydrated_target for od in owned_deps) sources = await Get[SetupPySources](SetupPySourcesRequest(targets)) requirements = await Get[ExportedTargetRequirements](DependencyOwner(request.exported_target)) # Nest the sources under the src/ prefix. src_digest = await Get[Digest](DirectoryWithPrefixToAdd(sources.digest, CHROOT_SOURCE_ROOT)) # Generate the kwargs to the setup() call. setup_kwargs = request.exported_target.hydrated_target.adaptor.provides.setup_py_keywords.copy() setup_kwargs.update({ 'package_dir': {'': CHROOT_SOURCE_ROOT}, 'packages': sources.packages, 'namespace_packages': sources.namespace_packages, 'package_data': dict(sources.package_data), 'install_requires': requirements.requirement_strs }) ht = request.exported_target.hydrated_target key_to_binary_spec = getattr(ht.adaptor.provides, 'binaries', {}) keys = list(key_to_binary_spec.keys()) binaries = await MultiGet(Get[HydratedTarget]( Address, Address.parse(key_to_binary_spec[key], relative_to=ht.address.spec_path)) for key in keys) for key, binary in zip(keys, binaries): if (not isinstance(binary.adaptor, PythonBinaryAdaptor) or getattr(binary.adaptor, 'entry_point', None) is None): raise InvalidEntryPoint( f'The binary {key} exported by {ht.address.reference()} is not a valid entry point.') entry_points = setup_kwargs['entry_points'] = setup_kwargs.get('entry_points', {}) console_scripts = entry_points['console_scripts'] = entry_points.get('console_scripts', []) console_scripts.append(f'{key}={binary.adaptor.entry_point}') # Generate the setup script. setup_py_content = SETUP_BOILERPLATE.format( target_address_spec=ht.address.reference(), setup_kwargs_str=distutils_repr(setup_kwargs) ).encode() extra_files_digest = await Get[Digest]( InputFilesContent([ FileContent('setup.py', setup_py_content), FileContent('MANIFEST.in', 'include *.py'.encode()) # Make sure setup.py is included. ])) chroot_digest = await Get[Digest](DirectoriesToMerge((src_digest, extra_files_digest))) return SetupPyChroot(chroot_digest, json.dumps(setup_kwargs, sort_keys=True))
async def generate_coverage_report( transitive_targets: TransitiveHydratedTargets, python_setup: PythonSetup, coverage_setup: CoverageSetup, merged_coverage_data: MergedCoverageData, coverage_toolbase: PytestCoverage, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> CoverageReport: """Takes all python test results and generates a single coverage report.""" requirements_pex = coverage_setup.requirements_pex # TODO(#4535) We need a better way to do this kind of check that covers synthetic targets and rules extensibility. python_targets = [ target for target in transitive_targets.closure if target.adaptor.type_alias in ("python_library", "python_tests") ] coveragerc = await Get[Coveragerc](CoveragercRequest( HydratedTargets(python_targets))) sources = await Get[SourceFiles](AllSourceFilesRequest( (ht.adaptor for ht in transitive_targets.closure), strip_source_roots=False)) sources_with_inits_snapshot = await Get[InitInjectedSnapshot]( InjectInitRequest(sources.snapshot)) merged_input_files: Digest = await Get( Digest, DirectoriesToMerge(directories=( merged_coverage_data.coverage_data, coveragerc.digest, requirements_pex.directory_digest, sources_with_inits_snapshot.snapshot.directory_digest, )), ) report_type = coverage_toolbase.options.report coverage_args = [report_type.report_name] request = requirements_pex.create_execute_request( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_path=f"./{coverage_setup.requirements_pex.output_filename}", pex_args=coverage_args, input_files=merged_input_files, output_directories=("htmlcov", ), output_files=("coverage.xml", ), description=f"Generate coverage report.", ) result = await Get[ExecuteProcessResult](ExecuteProcessRequest, request) if report_type == ReportType.CONSOLE: return ConsoleCoverageReport(result.stdout.decode()) return FilesystemCoverageReport( result.output_directory_digest, coverage_toolbase.options.report_output_path)
def assert_ancestor_init_py(self, expected_init_pys: Iterable[str], addrs: Iterable[str]) -> None: ancestor_init_py_files = self.request_single_product( AncestorInitPyFiles, Params(HydratedTargets([self.tgt(addr) for addr in addrs]), SourceRootConfig.global_instance())) snapshots = [ self.request_single_product(Snapshot, Params(digest)) for digest in ancestor_init_py_files.digests ] init_py_files_found = set( [file for snapshot in snapshots for file in snapshot.files]) # NB: Doesn't include the root __init__.py or the missing src/python/foo/bar/__init__.py. assert sorted(expected_init_pys) == sorted(init_py_files_found)
def assert_sources(self, expected_files, expected_packages, expected_namespace_packages, expected_package_data, addrs): srcs = self.request_single_product( SetupPySources, Params( SetupPySourcesRequest( HydratedTargets([self.tgt(addr) for addr in addrs])), SourceRootConfig.global_instance())) chroot_snapshot = self.request_single_product(Snapshot, Params(srcs.digest)) assert sorted(expected_files) == sorted(chroot_snapshot.files) assert sorted(expected_packages) == sorted(srcs.packages) assert sorted(expected_namespace_packages) == sorted( srcs.namespace_packages) assert expected_package_data == dict(srcs.package_data)
def run_lint_rule( *, targets: List[HydratedTarget], mock_linter: Callable[[PythonTargetAdaptor], LintResult], ) -> Tuple[Lint, MockConsole]: console = MockConsole(use_colors=False) result: Lint = run_rule( lint, rule_args=[ console, HydratedTargets(targets), UnionMembership(union_rules={TargetWithSources: [PythonTargetAdaptor]}) ], mock_gets=[ MockGet(product_type=LintResult, subject_type=PythonTargetAdaptor, mock=mock_linter), ], ) return result, console
def run_fmt_rule( *, targets: List[HydratedTarget], mock_formatter: Callable[[PythonTargetAdaptor], FmtResult], ) -> Tuple[Fmt, MockConsole]: console = MockConsole(use_colors=False) result: Fmt = run_rule( fmt, rule_args=[ console, HydratedTargets(targets), UnionMembership(union_rules={TargetWithSources: [PythonTargetAdaptor]}) ], mock_gets=[ MockGet(product_type=FmtResult, subject_type=PythonTargetAdaptor, mock=mock_formatter), MockGet(product_type=FilesContent, subject_type=Digest, mock=lambda _: FilesContent([])) ], ) return result, console
async def legacy_pex_from_targets(request: LegacyPexFromTargetsRequest, python_setup: PythonSetup) -> PexRequest: transitive_hydrated_targets = await Get[TransitiveHydratedTargets]( Addresses, request.addresses) all_targets = transitive_hydrated_targets.closure python_targets = [ t for t in all_targets if isinstance(t.adaptor, PythonTargetAdaptor) ] resource_targets = [ t for t in all_targets if isinstance(t.adaptor, (FilesAdaptor, ResourcesAdaptor)) ] all_target_adaptors = [t.adaptor for t in all_targets] interpreter_constraints = PexInterpreterConstraints.create_from_adaptors( adaptors=all_target_adaptors, python_setup=python_setup) input_digests = [] if request.additional_input_files: input_digests.append(request.additional_input_files) if request.include_source_files: prepared_sources = await Get[ImportablePythonSources]( HydratedTargets(python_targets + resource_targets)) input_digests.append(prepared_sources.snapshot.directory_digest) merged_input_digest = await Get[Digest]( DirectoriesToMerge(directories=tuple(input_digests))) requirements = PexRequirements.create_from_adaptors( adaptors=all_target_adaptors, additional_requirements=request.additional_requirements) return PexRequest( output_filename=request.output_filename, requirements=requirements, interpreter_constraints=interpreter_constraints, entry_point=request.entry_point, input_files_digest=merged_input_digest, additional_args=request.additional_args, )
def run_lint_rule( *, targets: List[HydratedTarget], mock_linter: Optional[Callable[[PythonTargetAdaptor], LintResult]] = None, ) -> Tuple[Lint, str]: if mock_linter is None: mock_linter = lambda target_adaptor: LintResult( exit_code=1, stdout=f"Linted the target `{target_adaptor.name}`", stderr="" ) console = MockConsole(use_colors=False) result: Lint = run_rule( lint, rule_args=[ console, HydratedTargets(targets), UnionMembership(union_rules={TargetWithSources: [PythonTargetAdaptor]}) ], mock_gets=[ MockGet(product_type=LintResult, subject_type=PythonTargetAdaptor, mock=mock_linter), ], ) return result, console.stdout.getvalue()
async def setup_pytest_for_target( config: PythonTestConfiguration, pytest: PyTest, test_options: TestOptions, python_setup: PythonSetup, ) -> TestTargetSetup: # TODO: Rather than consuming the TestOptions subsystem, the TestRunner should pass on coverage # configuration via #7490. test_addresses = Addresses((config.address,)) # TODO(John Sirois): PexInterpreterConstraints are gathered in the same way by the # `create_pex_from_target_closure` rule, factor up. transitive_targets = await Get[TransitiveTargets](Addresses, test_addresses) all_targets = transitive_targets.closure # TODO: factor this up? It's mostly duplicated with pex_from_targets.py. python_targets = [] resource_targets = [] for tgt in all_targets: if tgt.has_field(PythonSources): python_targets.append(tgt) # NB: PythonRequirementsFileSources is a subclass of FilesSources. We filter it out so that # requirements.txt is not included in the PEX and so that irrelevant changes to it (e.g. # whitespace changes) do not invalidate the PEX. if tgt.has_field(ResourcesSources) or ( tgt.has_field(FilesSources) and not tgt.has_field(PythonRequirementsFileSources) ): resource_targets.append(tgt) interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields( (tgt.get(PythonInterpreterCompatibility) for tgt in python_targets), python_setup ) # Ensure all pexes we merge via PEX_PATH to form the test runner use the interpreter constraints # of the tests. This is handled by CreatePexFromTargetClosure, but we must pass this through for # CreatePex requests. pex_request = functools.partial(PexRequest, interpreter_constraints=interpreter_constraints) # NB: We set `--not-zip-safe` because Pytest plugin discovery, which uses # `importlib_metadata` and thus `zipp`, does not play nicely when doing import magic directly # from zip files. `zipp` has pathologically bad behavior with large zipfiles. # TODO: this does have a performance cost as the pex must now be expanded to disk. Long term, # it would be better to fix Zipp (whose fix would then need to be used by importlib_metadata # and then by Pytest). See https://github.com/jaraco/zipp/pull/26. additional_args_for_pytest = ("--not-zip-safe",) run_coverage = test_options.values.run_coverage plugin_file_digest: Optional[Digest] = ( await Get[Digest](InputFilesContent, COVERAGE_PLUGIN_INPUT) if run_coverage else None ) pytest_pex_request = pex_request( output_filename="pytest.pex", requirements=PexRequirements(pytest.get_requirement_strings()), additional_args=additional_args_for_pytest, sources=plugin_file_digest, ) requirements_pex_request = LegacyPexFromTargetsRequest( addresses=test_addresses, output_filename="requirements.pex", include_source_files=False, additional_args=additional_args_for_pytest, ) test_runner_pex_request = pex_request( output_filename="test_runner.pex", entry_point="pytest:main", interpreter_constraints=interpreter_constraints, additional_args=( "--pex-path", # TODO(John Sirois): Support shading python binaries: # https://github.com/pantsbuild/pants/issues/9206 # Right now any pytest transitive requirements will shadow corresponding user # requirements which will lead to problems when APIs that are used by either # `pytest:main` or the tests themselves break between the two versions. ":".join( (pytest_pex_request.output_filename, requirements_pex_request.output_filename) ), ), ) # Get the file names for the test_target so that we can specify to Pytest precisely which files # to test, rather than using auto-discovery. specified_source_files_request = SpecifiedSourceFilesRequest( [(config.sources, config.origin)], strip_source_roots=True ) # TODO(John Sirois): Support exploiting concurrency better: # https://github.com/pantsbuild/pants/issues/9294 # Some awkward code follows in order to execute 5-6 items concurrently given the current state # of MultiGet typing / API. Improve this since we should encourage full concurrency in general. requests: List[Get[Any]] = [ Get[Pex](PexRequest, pytest_pex_request), Get[Pex](LegacyPexFromTargetsRequest, requirements_pex_request), Get[Pex](PexRequest, test_runner_pex_request), Get[ImportablePythonSources](Targets(python_targets + resource_targets)), Get[SourceFiles](SpecifiedSourceFilesRequest, specified_source_files_request), ] if run_coverage: # TODO: update coverage to use the Target API. Also, add tests. hydrated_python_targets = await Get[HydratedTargets]( Addresses(tgt.address for tgt in python_targets) ) requests.append( Get[Coveragerc]( CoveragercRequest(HydratedTargets(hydrated_python_targets), test_time=True) ), ) ( pytest_pex, requirements_pex, test_runner_pex, prepared_sources, specified_source_files, *rest, ) = cast( Union[ Tuple[Pex, Pex, Pex, ImportablePythonSources, SourceFiles], Tuple[Pex, Pex, Pex, ImportablePythonSources, SourceFiles, Coveragerc], ], await MultiGet(requests), ) directories_to_merge = [ prepared_sources.snapshot.directory_digest, requirements_pex.directory_digest, pytest_pex.directory_digest, test_runner_pex.directory_digest, ] if run_coverage: coveragerc = rest[0] directories_to_merge.append(coveragerc.digest) merged_input_files = await Get[Digest]( DirectoriesToMerge(directories=tuple(directories_to_merge)) ) coverage_args = [] if run_coverage: coverage_args = [ "--cov-report=", # To not generate any output. https://pytest-cov.readthedocs.io/en/latest/config.html ] for package in config.coverage.determine_packages_to_cover( specified_source_files=specified_source_files ): coverage_args.extend(["--cov", package]) specified_source_file_names = sorted(specified_source_files.snapshot.files) return TestTargetSetup( test_runner_pex=test_runner_pex, args=(*pytest.options.args, *coverage_args, *specified_source_file_names), input_files_digest=merged_input_files, timeout_seconds=config.timeout.calculate_from_global_options(pytest), )
async def lint( linter: PylintLinter, pylint: Pylint, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> LintResult: if pylint.options.skip: return LintResult.noop() adaptors_with_origins = linter.adaptors_with_origins # Pylint needs direct dependencies in the chroot to ensure that imports are valid. However, it # doesn't lint those direct dependencies nor does it care about transitive dependencies. hydrated_targets = [ HydratedTarget(adaptor_with_origin.adaptor) for adaptor_with_origin in adaptors_with_origins ] dependencies = await MultiGet( Get[HydratedTarget](Address, dependency) for dependency in itertools.chain.from_iterable( ht.adaptor.dependencies for ht in hydrated_targets)) chrooted_python_sources = await Get[ImportablePythonSources]( HydratedTargets([*hydrated_targets, *dependencies])) # NB: Pylint output depends upon which Python interpreter version it's run with. We ensure that # each target runs with its own interpreter constraints. See # http://pylint.pycqa.org/en/latest/faq.html#what-versions-of-python-is-pylint-supporting. interpreter_constraints = PexInterpreterConstraints.create_from_adaptors( (adaptor_with_origin.adaptor for adaptor_with_origin in adaptors_with_origins), python_setup=python_setup, ) requirements_pex = await Get[Pex](PexRequest( output_filename="pylint.pex", requirements=PexRequirements(pylint.get_requirement_specs()), interpreter_constraints=interpreter_constraints, entry_point=pylint.get_entry_point(), )) config_path: Optional[str] = pylint.options.config config_snapshot = await Get[Snapshot](PathGlobs( globs=tuple([config_path] if config_path else []), glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--pylint-config`", )) merged_input_files = await Get[Digest](DirectoriesToMerge(directories=( requirements_pex.directory_digest, config_snapshot.directory_digest, chrooted_python_sources.snapshot.directory_digest, )), ) specified_source_files = await Get[SourceFiles]( LegacySpecifiedSourceFilesRequest(adaptors_with_origins, strip_source_roots=True)) address_references = ", ".join( sorted(adaptor_with_origin.adaptor.address.reference() for adaptor_with_origin in adaptors_with_origins)) request = requirements_pex.create_execute_request( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_path=f"./pylint.pex", pex_args=generate_args(specified_source_files=specified_source_files, pylint=pylint), input_files=merged_input_files, description=f"Run Pylint for {address_references}", ) result = await Get[FallibleExecuteProcessResult](ExecuteProcessRequest, request) return LintResult.from_fallible_execute_process_result(result)
async def setup_pytest_for_target( adaptor_with_origin: PythonTestsAdaptorWithOrigin, pytest: PyTest, test_options: TestOptions, python_setup: PythonSetup, ) -> TestTargetSetup: # TODO: Rather than consuming the TestOptions subsystem, the TestRunner should pass on coverage # configuration via #7490. adaptor = adaptor_with_origin.adaptor test_addresses = Addresses((adaptor.address,)) # TODO(John Sirois): PexInterpreterConstraints are gathered in the same way by the # `create_pex_from_target_closure` rule, factor up. transitive_hydrated_targets = await Get[TransitiveHydratedTargets](Addresses, test_addresses) all_targets = transitive_hydrated_targets.closure all_target_adaptors = [t.adaptor for t in all_targets] interpreter_constraints = PexInterpreterConstraints.create_from_adaptors( adaptors=all_target_adaptors, python_setup=python_setup ) # Ensure all pexes we merge via PEX_PATH to form the test runner use the interpreter constraints # of the tests. This is handled by CreatePexFromTargetClosure, but we must pass this through for # CreatePex requests. create_pex = functools.partial(CreatePex, interpreter_constraints=interpreter_constraints) # NB: We set `--not-zip-safe` because Pytest plugin discovery, which uses # `importlib_metadata` and thus `zipp`, does not play nicely when doing import magic directly # from zip files. `zipp` has pathologically bad behavior with large zipfiles. # TODO: this does have a performance cost as the pex must now be expanded to disk. Long term, # it would be better to fix Zipp (whose fix would then need to be used by importlib_metadata # and then by Pytest). See https://github.com/jaraco/zipp/pull/26. additional_args_for_pytest = ("--not-zip-safe",) run_coverage = test_options.values.run_coverage plugin_file_digest: Optional[Digest] = ( await Get[Digest](InputFilesContent, get_coverage_plugin_input()) if run_coverage else None ) pytest_pex = await Get[Pex]( CreatePex, create_pex( output_filename="pytest.pex", requirements=PexRequirements(pytest.get_requirement_strings()), additional_args=additional_args_for_pytest, input_files_digest=plugin_file_digest, ), ) requirements_pex = await Get[Pex]( CreatePexFromTargetClosure( addresses=test_addresses, output_filename="requirements.pex", include_source_files=False, additional_args=additional_args_for_pytest, ) ) test_runner_pex = await Get[Pex]( CreatePex, create_pex( output_filename="test_runner.pex", entry_point="pytest:main", interpreter_constraints=interpreter_constraints, additional_args=( "--pex-path", ":".join( pex_request.output_filename # TODO(John Sirois): Support shading python binaries: # https://github.com/pantsbuild/pants/issues/9206 # Right now any pytest transitive requirements will shadow corresponding user # requirements which will lead to problems when APIs that are used by either # `pytest:main` or the tests themselves break between the two versions. for pex_request in (pytest_pex, requirements_pex) ), ), ), ) chrooted_sources = await Get[ChrootedPythonSources](HydratedTargets(all_targets)) directories_to_merge = [ chrooted_sources.snapshot.directory_digest, requirements_pex.directory_digest, pytest_pex.directory_digest, test_runner_pex.directory_digest, ] # Get the file names for the test_target so that we can specify to Pytest precisely which files # to test, rather than using auto-discovery. specified_source_files = await Get[SourceFiles]( SpecifiedSourceFilesRequest([adaptor_with_origin], strip_source_roots=True) ) specified_source_file_names = specified_source_files.snapshot.files coverage_args = [] if run_coverage: coveragerc = await Get[Coveragerc]( CoveragercRequest(HydratedTargets(all_targets), test_time=True) ) directories_to_merge.append(coveragerc.digest) packages_to_cover = get_packages_to_cover( target=adaptor, specified_source_files=specified_source_files, ) coverage_args = [ "--cov-report=", # To not generate any output. https://pytest-cov.readthedocs.io/en/latest/config.html ] for package in packages_to_cover: coverage_args.extend(["--cov", package]) merged_input_files = await Get[Digest]( DirectoriesToMerge(directories=tuple(directories_to_merge)) ) timeout_seconds = calculate_timeout_seconds( timeouts_enabled=pytest.options.timeouts, target_timeout=getattr(adaptor, "timeout", None), timeout_default=pytest.options.timeout_default, timeout_maximum=pytest.options.timeout_maximum, ) return TestTargetSetup( test_runner_pex=test_runner_pex, args=(*pytest.options.args, *coverage_args, *sorted(specified_source_file_names)), input_files_digest=merged_input_files, timeout_seconds=timeout_seconds, )
async def generate_chroot(request: SetupPyChrootRequest) -> SetupPyChroot: owned_deps = await Get[OwnedDependencies](DependencyOwner( request.exported_target)) targets = HydratedTargets(od.hydrated_target for od in owned_deps) sources = await Get[SetupPySources](SetupPySourcesRequest(targets, py2=request.py2)) requirements = await Get[ExportedTargetRequirements](DependencyOwner( request.exported_target)) # Nest the sources under the src/ prefix. src_digest = await Get[Digest](DirectoryWithPrefixToAdd( sources.digest, CHROOT_SOURCE_ROOT)) # Generate the kwargs to the setup() call. setup_kwargs = request.exported_target.hydrated_target.adaptor.provides.setup_py_keywords.copy( ) setup_kwargs.update({ "package_dir": { "": CHROOT_SOURCE_ROOT }, "packages": sources.packages, "namespace_packages": sources.namespace_packages, "package_data": dict(sources.package_data), "install_requires": requirements.requirement_strs, }) adaptor = request.exported_target.hydrated_target.adaptor key_to_binary_spec = getattr(adaptor.provides, "binaries", {}) keys = list(key_to_binary_spec.keys()) binaries = await MultiGet(Get[HydratedTarget]( Address, Address.parse(key_to_binary_spec[key], relative_to=adaptor.address.spec_path)) for key in keys) for key, binary in zip(keys, binaries): if (not isinstance(binary.adaptor, PythonBinaryAdaptor) or getattr(binary.adaptor, "entry_point", None) is None): raise InvalidEntryPoint( f"The binary {key} exported by {adaptor.address.reference()} is not a valid entry point." ) entry_points = setup_kwargs["entry_points"] = setup_kwargs.get( "entry_points", {}) console_scripts = entry_points["console_scripts"] = entry_points.get( "console_scripts", []) console_scripts.append(f"{key}={binary.adaptor.entry_point}") # Generate the setup script. setup_py_content = SETUP_BOILERPLATE.format( target_address_spec=adaptor.address.reference(), setup_kwargs_str=distutils_repr(setup_kwargs), ).encode() extra_files_digest = await Get[Digest]( InputFilesContent([ FileContent("setup.py", setup_py_content), FileContent( "MANIFEST.in", "include *.py".encode()), # Make sure setup.py is included. ])) chroot_digest = await Get[Digest](DirectoriesToMerge( (src_digest, extra_files_digest))) return SetupPyChroot(chroot_digest, json.dumps(setup_kwargs, sort_keys=True))
async def setup_pytest_for_target( test_target: PythonTestsAdaptor, pytest: PyTest, test_options: TestOptions, ) -> TestTargetSetup: # TODO: Rather than consuming the TestOptions subsystem, the TestRunner should pass on coverage # configuration via #7490. transitive_hydrated_targets = await Get[TransitiveHydratedTargets]( BuildFileAddresses((test_target.address, ))) all_targets = transitive_hydrated_targets.closure resolved_requirements_pex = await Get[Pex]( CreatePexFromTargetClosure( build_file_addresses=BuildFileAddresses((test_target.address, )), output_filename='pytest-with-requirements.pex', entry_point="pytest:main", additional_requirements=pytest.get_requirement_strings(), # NB: We set `--not-zip-safe` because Pytest plugin discovery, which uses # `importlib_metadata` and thus `zipp`, does not play nicely when doing import magic directly # from zip files. `zipp` has pathologically bad behavior with large zipfiles. # TODO: this does have a performance cost as the pex must now be expanded to disk. Long term, # it would be better to fix Zipp (whose fix would then need to be used by importlib_metadata # and then by Pytest). See https://github.com/jaraco/zipp/pull/26. additional_args=("--not-zip-safe", ), include_source_files=False, )) chrooted_sources = await Get[ChrootedPythonSources]( HydratedTargets(all_targets)) directories_to_merge = [ chrooted_sources.digest, resolved_requirements_pex.directory_digest, ] # Get the file names for the test_target, adjusted for the source root. This allows us to # specify to Pytest which files to test and thus to avoid the test auto-discovery defined by # https://pytest.org/en/latest/goodpractices.html#test-discovery. In addition to a performance # optimization, this ensures that any transitive sources, such as a test project file named # test_fail.py, do not unintentionally end up being run as tests. source_root_stripped_test_target_sources = await Get[ SourceRootStrippedSources](Address, test_target.address.to_address()) coverage_args = [] test_target_sources_file_names = source_root_stripped_test_target_sources.snapshot.files if test_options.values.run_coverage: coveragerc_digest = await Get[Digest]( InputFilesContent, get_coveragerc_input(DEFAULT_COVERAGE_CONFIG)) directories_to_merge.append(coveragerc_digest) packages_to_cover = get_packages_to_cover( test_target, source_root_stripped_file_paths=test_target_sources_file_names, ) coverage_args = [ '--cov-report=', # To not generate any output. https://pytest-cov.readthedocs.io/en/latest/config.html ] for package in packages_to_cover: coverage_args.extend(['--cov', package]) merged_input_files = await Get[Digest]( DirectoriesToMerge(directories=tuple(directories_to_merge))) timeout_seconds = calculate_timeout_seconds( timeouts_enabled=pytest.options.timeouts, target_timeout=getattr(test_target, 'timeout', None), timeout_default=pytest.options.timeout_default, timeout_maximum=pytest.options.timeout_maximum, ) return TestTargetSetup( requirements_pex=resolved_requirements_pex, args=(*pytest.options.args, *coverage_args, *sorted(test_target_sources_file_names)), input_files_digest=merged_input_files, timeout_seconds=timeout_seconds, )