async def setup_ipython_lockfile( _: IPythonLockfileSentinel, ipython: IPython, python_setup: PythonSetup) -> GeneratePythonLockfile: if not ipython.uses_lockfile: return GeneratePythonLockfile.from_tool( ipython, use_pex=python_setup.generate_lockfiles_with_pex) # IPython is often run against the whole repo (`./pants repl ::`), but it is possible to run # on subsets of the codebase with disjoint interpreter constraints, such as # `./pants repl py2::` and then `./pants repl py3::`. Still, even with those subsets possible, # we need a single lockfile that works with all possible Python interpreters in use. # # This ORs all unique interpreter constraints. The net effect is that every possible Python # interpreter used will be covered. all_tgts = await Get(AllTargets, AllTargetsRequest()) unique_constraints = { InterpreterConstraints.create_from_compatibility_fields( [tgt[InterpreterConstraintsField]], python_setup) for tgt in all_tgts if tgt.has_field(InterpreterConstraintsField) } constraints = InterpreterConstraints( itertools.chain.from_iterable(unique_constraints)) return GeneratePythonLockfile.from_tool( ipython, constraints or InterpreterConstraints(python_setup.interpreter_constraints), use_pex=python_setup.generate_lockfiles_with_pex, )
async def setup_pytest_lockfile( _: PytestLockfileSentinel, pytest: PyTest, python_setup: PythonSetup) -> PythonLockfileRequest: if not pytest.uses_lockfile: return PythonLockfileRequest.from_tool(pytest) # Even though we run each python_tests target in isolation, we need a single lockfile that # works with them all (and their transitive deps). # # This first computes the constraints for each individual `python_test` target # (which will AND across each target in the closure). Then, it ORs all unique resulting # interpreter constraints. The net effect is that every possible Python interpreter used will # be covered. all_tgts = await Get(AllTargets, AllTargetsRequest()) transitive_targets_per_test = await MultiGet( Get(TransitiveTargets, TransitiveTargetsRequest([tgt.address])) for tgt in all_tgts if PythonTestFieldSet.is_applicable(tgt)) unique_constraints = { InterpreterConstraints.create_from_targets(transitive_targets.closure, python_setup) for transitive_targets in transitive_targets_per_test } constraints = InterpreterConstraints( itertools.chain.from_iterable(unique_constraints)) return PythonLockfileRequest.from_tool( pytest, constraints or InterpreterConstraints(python_setup.interpreter_constraints))
async def setup_setuptools_lockfile( _: SetuptoolsLockfileSentinel, setuptools: Setuptools, python_setup: PythonSetup ) -> GeneratePythonLockfile: if not setuptools.uses_custom_lockfile: return GeneratePythonLockfile.from_tool( setuptools, use_pex=python_setup.generate_lockfiles_with_pex ) all_tgts = await Get(AllTargets, AllTargetsRequest()) transitive_targets_per_python_dist = await MultiGet( Get(TransitiveTargets, TransitiveTargetsRequest([tgt.address])) for tgt in all_tgts if PythonDistributionFieldSet.is_applicable(tgt) ) unique_constraints = { InterpreterConstraints.create_from_targets(transitive_targets.closure, python_setup) or InterpreterConstraints(python_setup.interpreter_constraints) for transitive_targets in transitive_targets_per_python_dist } constraints = InterpreterConstraints(itertools.chain.from_iterable(unique_constraints)) return GeneratePythonLockfile.from_tool( setuptools, constraints or InterpreterConstraints(python_setup.interpreter_constraints), use_pex=python_setup.generate_lockfiles_with_pex, )
async def setup_mypy_lockfile( _: MyPyLockfileSentinel, first_party_plugins: MyPyFirstPartyPlugins, mypy: MyPy, python_setup: PythonSetup, ) -> GeneratePythonLockfile: if not mypy.uses_lockfile: return GeneratePythonLockfile.from_tool( mypy, use_pex=python_setup.generate_lockfiles_with_pex) constraints = mypy.interpreter_constraints if mypy.options.is_default("interpreter_constraints"): all_tgts = await Get(AllTargets, AllTargetsRequest()) all_transitive_targets = await MultiGet( Get(TransitiveTargets, TransitiveTargetsRequest([tgt.address])) for tgt in all_tgts if MyPyFieldSet.is_applicable(tgt)) unique_constraints = { InterpreterConstraints.create_from_targets( transitive_targets.closure, python_setup) for transitive_targets in all_transitive_targets } code_constraints = InterpreterConstraints( itertools.chain.from_iterable(unique_constraints)) if code_constraints.requires_python38_or_newer( python_setup.interpreter_universe): constraints = code_constraints return GeneratePythonLockfile.from_tool( mypy, constraints, extra_requirements=first_party_plugins.requirement_strings, use_pex=python_setup.generate_lockfiles_with_pex, )
async def setup_bandit_lockfile( _: BanditLockfileSentinel, bandit: Bandit, python_setup: PythonSetup ) -> GeneratePythonLockfile: if not bandit.uses_lockfile: return GeneratePythonLockfile.from_tool( bandit, use_pex=python_setup.generate_lockfiles_with_pex ) # While Bandit will run in partitions, we need a single lockfile that works with every # partition. # # This ORs all unique interpreter constraints. The net effect is that every possible Python # interpreter used will be covered. all_tgts = await Get(AllTargets, AllTargetsRequest()) unique_constraints = { InterpreterConstraints.create_from_targets([tgt], python_setup) for tgt in all_tgts if BanditFieldSet.is_applicable(tgt) } constraints = InterpreterConstraints(itertools.chain.from_iterable(unique_constraints)) return GeneratePythonLockfile.from_tool( bandit, constraints or InterpreterConstraints(python_setup.interpreter_constraints), use_pex=python_setup.generate_lockfiles_with_pex, )
async def _pylint_interpreter_constraints( first_party_plugins: PylintFirstPartyPlugins, python_setup: PythonSetup, ) -> InterpreterConstraints: # While Pylint will run in partitions, we need a set of constraints that works with every # partition. We must also consider any 3rd-party requirements used by 1st-party plugins. # # This first computes the constraints for each individual target. Then, it ORs all unique # resulting interpreter constraints. The net effect is that every possible Python interpreter # used will be covered. all_tgts = await Get(AllTargets, AllTargetsRequest()) unique_constraints = { InterpreterConstraints.create_from_compatibility_fields( ( tgt[InterpreterConstraintsField], *first_party_plugins.interpreter_constraints_fields, ), python_setup, ) for tgt in all_tgts if PylintFieldSet.is_applicable(tgt) } if not unique_constraints: unique_constraints.add( InterpreterConstraints.create_from_compatibility_fields( first_party_plugins.interpreter_constraints_fields, python_setup, ) ) constraints = InterpreterConstraints(itertools.chain.from_iterable(unique_constraints)) return constraints or InterpreterConstraints(python_setup.interpreter_constraints)
async def _flake8_interpreter_constraints( first_party_plugins: Flake8FirstPartyPlugins, python_setup: PythonSetup, ) -> InterpreterConstraints: # While Flake8 will run in partitions, we need a set of constraints that works with every # partition. # # This ORs all unique interpreter constraints. The net effect is that every possible Python # interpreter used will be covered. all_tgts = await Get(AllTargets, AllTargetsRequest()) unique_constraints = { InterpreterConstraints.create_from_compatibility_fields( ( tgt[InterpreterConstraintsField], *first_party_plugins.interpreter_constraints_fields, ), python_setup, ) for tgt in all_tgts if Flake8FieldSet.is_applicable(tgt) } if not unique_constraints: unique_constraints.add( InterpreterConstraints.create_from_compatibility_fields( first_party_plugins.interpreter_constraints_fields, python_setup, )) constraints = InterpreterConstraints( itertools.chain.from_iterable(unique_constraints)) return constraints or InterpreterConstraints( python_setup.interpreter_constraints)
async def setup_pylint_lockfile( _: PylintLockfileSentinel, first_party_plugins: PylintFirstPartyPlugins, pylint: Pylint, python_setup: PythonSetup, ) -> GeneratePythonLockfile: if not pylint.uses_lockfile: return GeneratePythonLockfile.from_tool( pylint, use_pex=python_setup.generate_lockfiles_with_pex) # While Pylint will run in partitions, we need a single lockfile that works with every # partition. We must also consider any 3rd-party requirements used by 1st-party plugins. # # This first computes the constraints for each individual target, including its direct # dependencies (which will AND across each target in the closure). Then, it ORs all unique # resulting interpreter constraints. The net effect is that every possible Python interpreter # used will be covered. all_tgts = await Get(AllTargets, AllTargetsRequest()) relevant_targets = tuple(tgt for tgt in all_tgts if PylintFieldSet.is_applicable(tgt)) direct_deps_per_target = await MultiGet( Get(Targets, DependenciesRequest(tgt.get(Dependencies))) for tgt in relevant_targets) unique_constraints = set() for tgt, direct_deps in zip(relevant_targets, direct_deps_per_target): constraints_fields = (t[InterpreterConstraintsField] for t in (tgt, *direct_deps) if t.has_field(InterpreterConstraintsField)) unique_constraints.add( InterpreterConstraints.create_from_compatibility_fields( (*constraints_fields, *first_party_plugins.interpreter_constraints_fields), python_setup, )) if not unique_constraints: unique_constraints.add( InterpreterConstraints.create_from_compatibility_fields( first_party_plugins.interpreter_constraints_fields, python_setup, )) constraints = InterpreterConstraints( itertools.chain.from_iterable(unique_constraints)) return GeneratePythonLockfile.from_tool( pylint, constraints or InterpreterConstraints(python_setup.interpreter_constraints), extra_requirements=first_party_plugins.requirement_strings, use_pex=python_setup.generate_lockfiles_with_pex, )
async def _black_interpreter_constraints( black: Black, python_setup: PythonSetup ) -> InterpreterConstraints: constraints = black.interpreter_constraints if black.options.is_default("interpreter_constraints"): all_tgts = await Get(AllTargets, AllTargetsRequest()) # TODO: fix to use `FieldSet.is_applicable()`. code_constraints = InterpreterConstraints.create_from_targets( (tgt for tgt in all_tgts if not tgt.get(SkipBlackField).value), python_setup ) if code_constraints is not None and code_constraints.requires_python38_or_newer( python_setup.interpreter_universe ): constraints = code_constraints return constraints
async def _bandit_interpreter_constraints( python_setup: PythonSetup) -> InterpreterConstraints: # While Bandit will run in partitions, we need a set of constraints that works with every # partition. # # This ORs all unique interpreter constraints. The net effect is that every possible Python # interpreter used will be covered. all_tgts = await Get(AllTargets, AllTargetsRequest()) unique_constraints = { InterpreterConstraints.create_from_targets([tgt], python_setup) for tgt in all_tgts if BanditFieldSet.is_applicable(tgt) } constraints = InterpreterConstraints( itertools.chain.from_iterable(ic for ic in unique_constraints if ic)) return constraints or InterpreterConstraints( python_setup.interpreter_constraints)
async def _mypy_interpreter_constraints( mypy: MyPy, python_setup: PythonSetup) -> InterpreterConstraints: constraints = mypy.interpreter_constraints if mypy.options.is_default("interpreter_constraints"): all_tgts = await Get(AllTargets, AllTargetsRequest()) unique_constraints = { InterpreterConstraints.create_from_targets([tgt], python_setup) for tgt in all_tgts if MyPyFieldSet.is_applicable(tgt) } code_constraints = InterpreterConstraints( itertools.chain.from_iterable(ic for ic in unique_constraints if ic)) if code_constraints.requires_python38_or_newer( python_setup.interpreter_universe): constraints = code_constraints return constraints
async def setup_black_lockfile( _: BlackLockfileSentinel, black: Black, python_setup: PythonSetup) -> PythonLockfileRequest: if not black.uses_lockfile: return PythonLockfileRequest.from_tool(black) constraints = black.interpreter_constraints if black.options.is_default("interpreter_constraints"): all_tgts = await Get(AllTargets, AllTargetsRequest()) # TODO: fix to use `FieldSet.is_applicable()`. code_constraints = InterpreterConstraints.create_from_targets( (tgt for tgt in all_tgts if not tgt.get(SkipBlackField).value), python_setup) if code_constraints.requires_python38_or_newer( python_setup.interpreter_universe): constraints = code_constraints return PythonLockfileRequest.from_tool(black, constraints)
async def setup_flake8_lockfile( _: Flake8LockfileSentinel, first_party_plugins: Flake8FirstPartyPlugins, flake8: Flake8, python_setup: PythonSetup, ) -> GeneratePythonLockfile: if not flake8.uses_lockfile: return GeneratePythonLockfile.from_tool( flake8, use_pex=python_setup.generate_lockfiles_with_pex ) # While Flake8 will run in partitions, we need a single lockfile that works with every # partition. # # This ORs all unique interpreter constraints. The net effect is that every possible Python # interpreter used will be covered. all_tgts = await Get(AllTargets, AllTargetsRequest()) relevant_targets = tuple(tgt for tgt in all_tgts if Flake8FieldSet.is_applicable(tgt)) unique_constraints = set() for tgt in relevant_targets: if tgt.has_field(InterpreterConstraintsField): constraints_field = tgt[InterpreterConstraintsField] unique_constraints.add( InterpreterConstraints.create_from_compatibility_fields( (constraints_field, *first_party_plugins.interpreter_constraints_fields), python_setup, ) ) if not unique_constraints: unique_constraints.add( InterpreterConstraints.create_from_compatibility_fields( first_party_plugins.interpreter_constraints_fields, python_setup, ) ) constraints = InterpreterConstraints(itertools.chain.from_iterable(unique_constraints)) return GeneratePythonLockfile.from_tool( flake8, constraints or InterpreterConstraints(python_setup.interpreter_constraints), extra_requirements=first_party_plugins.requirement_strings, use_pex=python_setup.generate_lockfiles_with_pex, )
async def setup_flake8_lockfile( _: Flake8LockfileSentinel, flake8: Flake8, python_setup: PythonSetup) -> PythonLockfileRequest: if not flake8.uses_lockfile: return PythonLockfileRequest.from_tool(flake8) # While Flake8 will run in partitions, we need a single lockfile that works with every # partition. # # This ORs all unique interpreter constraints. The net effect is that every possible Python # interpreter used will be covered. all_tgts = await Get(AllTargets, AllTargetsRequest()) unique_constraints = { InterpreterConstraints.create_from_targets([tgt], python_setup) for tgt in all_tgts if Flake8FieldSet.is_applicable(tgt) } constraints = InterpreterConstraints( itertools.chain.from_iterable(unique_constraints)) return PythonLockfileRequest.from_tool( flake8, constraints or InterpreterConstraints(python_setup.interpreter_constraints))
async def find_all_unexpanded_targets_singleton() -> AllUnexpandedTargets: return await Get(AllUnexpandedTargets, AllTargetsRequest())
async def py_constraints( addresses: Addresses, console: Console, py_constraints_subsystem: PyConstraintsSubsystem, python_setup: PythonSetup, registered_target_types: RegisteredTargetTypes, union_membership: UnionMembership, ) -> PyConstraintsGoal: if py_constraints_subsystem.summary: if addresses: console.print_stderr( "The `py-constraints --summary` goal does not take file/target arguments. Run " "`help py-constraints` for more details.") return PyConstraintsGoal(exit_code=1) all_targets = await Get(AllTargets, AllTargetsRequest()) all_python_targets = tuple(t for t in all_targets if t.has_field(InterpreterConstraintsField)) constraints_per_tgt = [ InterpreterConstraints.create_from_targets([tgt], python_setup) for tgt in all_python_targets ] transitive_targets_per_tgt = await MultiGet( Get(TransitiveTargets, TransitiveTargetsRequest([tgt.address])) for tgt in all_python_targets) transitive_constraints_per_tgt = [ InterpreterConstraints.create_from_targets( transitive_targets.closure, python_setup) for transitive_targets in transitive_targets_per_tgt ] dependees_per_root = await MultiGet( Get( Dependees, DependeesRequest( [tgt.address], transitive=True, include_roots=False)) for tgt in all_python_targets) data = [{ "Target": tgt.address.spec, "Constraints": str(constraints), "Transitive Constraints": str(transitive_constraints), "# Dependencies": len(transitive_targets.dependencies), "# Dependees": len(dependees), } for tgt, constraints, transitive_constraints, transitive_targets, dependees in zip( all_python_targets, constraints_per_tgt, transitive_constraints_per_tgt, transitive_targets_per_tgt, dependees_per_root, )] with py_constraints_subsystem.output_sink(console) as stdout: writer = csv.DictWriter( stdout, fieldnames=[ "Target", "Constraints", "Transitive Constraints", "# Dependencies", "# Dependees", ], ) writer.writeheader() for entry in data: writer.writerow(entry) return PyConstraintsGoal(exit_code=0) transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest(addresses)) final_constraints = InterpreterConstraints.create_from_targets( transitive_targets.closure, python_setup) if not final_constraints: target_types_with_constraints = sorted( tgt_type.alias for tgt_type in registered_target_types.types if tgt_type.class_has_field(InterpreterConstraintsField, union_membership)) logger.warning( "No Python files/targets matched for the `py-constraints` goal. All target types with " f"Python interpreter constraints: {', '.join(target_types_with_constraints)}" ) return PyConstraintsGoal(exit_code=0) constraints_to_addresses = defaultdict(set) for tgt in transitive_targets.closure: constraints = InterpreterConstraints.create_from_targets([tgt], python_setup) if not constraints: continue constraints_to_addresses[constraints].add(tgt.address) with py_constraints_subsystem.output(console) as output_stdout: output_stdout(f"Final merged constraints: {final_constraints}\n") if len(addresses) > 1: merged_constraints_warning = ( "(These are the constraints used if you were to depend on all of the input " "files/targets together, even though they may end up never being used together in " "the real world. Consider using a more precise query or running " f"`{bin_name()} py-constraints --summary`.)\n") output_stdout(indent(fill(merged_constraints_warning, 80), " ")) for constraint, addrs in sorted(constraints_to_addresses.items()): output_stdout(f"\n{constraint}\n") for addr in sorted(addrs): output_stdout(f" {addr}\n") return PyConstraintsGoal(exit_code=0)