def _validate_interpreter_constraints(self, constraint_tgts):
"""Validate that the transitive constraints of the given PythonBinary target are compatible.
If no (local) interpreter can satisfy all of the given targets, raises
PythonInterpreterCache.UnsatisfiableInterpreterConstraintsError.
TODO: This currently does so by finding a concrete local interpreter that matches all of the
constraints, but it is possible to do this in memory instead.
see https://github.com/pantsbuild/pants/issues/7775
"""
PythonInterpreterCache.global_instance(
).select_interpreter_for_targets(constraint_tgts)
def _gather_sources(self, target_roots):
with temporary_dir() as cache_dir:
interpreter = PythonInterpreter.get()
context = self.context(
target_roots=target_roots,
for_subsystems=[PythonInterpreterCache],
options={
PythonSetup.options_scope: {
'interpreter_cache_dir':
cache_dir,
'interpreter_search_paths':
[os.path.dirname(interpreter.binary)],
}
})
# We must get an interpreter via the cache, instead of using the value of
# PythonInterpreter.get() directly, to ensure that the interpreter has setuptools and
# wheel support.
interpreter_cache = PythonInterpreterCache.global_instance()
interpreters = interpreter_cache.setup(
filters=[str(interpreter.identity.requirement)])
context.products.get_data(PythonInterpreter,
lambda: interpreters[0])
task = self.create_task(context)
task.execute()
return context.products.get_data(GatherSources.PYTHON_SOURCES)
def execute(self):
if not self.will_be_invoked():
return
tool_subsystem = self.tool_subsystem_cls.scoped_instance(self)
interpreter_cache = PythonInterpreterCache.global_instance()
interpreters = interpreter_cache.setup(
filters=tool_subsystem.get_interpreter_constraints())
if not interpreters:
raise TaskError(
"Found no Python interpreter capable of running the {} tool with "
"constraints {}".format(
tool_subsystem.options_scope,
tool_subsystem.get_interpreter_constraints()))
interpreter = min(interpreters)
pex_path = self._generate_fingerprinted_pex_path(
tool_subsystem, interpreter)
if not os.path.exists(pex_path):
with self.context.new_workunit(
name=f"create-{tool_subsystem.options_scope}-pex",
labels=[WorkUnitLabel.PREP]):
self._build_tool_pex(tool_subsystem=tool_subsystem,
interpreter=interpreter,
pex_path=pex_path)
tool_instance = self.tool_instance_cls(pex_path, interpreter)
self.context.products.register_data(self.tool_instance_cls,
tool_instance)
def _create_interpreter_cache(self, setup_options=None):
Subsystem.reset(reset_options=True)
self.context(for_subsystems=[PythonInterpreterCache],
options={
'python-setup': setup_options,
})
return PythonInterpreterCache.global_instance()
def execute(self):
""""Run Checkstyle on all found non-synthetic source files."""
python_tgts = self.context.targets(
lambda tgt: isinstance(tgt, (PythonTarget))
)
if not python_tgts:
return 0
interpreter_cache = PythonInterpreterCache.global_instance()
with self.invalidated(self.get_targets(self._is_checked)) as invalidation_check:
failure_count = 0
tgts_by_compatibility, _ = interpreter_cache.partition_targets_by_compatibility(
[vt.target for vt in invalidation_check.invalid_vts]
)
for filters, targets in tgts_by_compatibility.items():
sources = self.calculate_sources([tgt for tgt in targets])
if sources:
allowed_interpreters = set(interpreter_cache.setup(filters=filters))
if not allowed_interpreters:
raise TaskError('No valid interpreters found for targets: {}\n(filters: {})'
.format(targets, filters))
interpreter = min(allowed_interpreters)
failure_count += self.checkstyle(interpreter, sources)
if failure_count > 0 and self.get_options().fail:
raise TaskError('{} Python Style issues found. You may try `./pants fmt <targets>`'
.format(failure_count))
return failure_count
def _resolve_requirements(self, target_roots, options=None):
with temporary_dir() as cache_dir:
options = options or {}
python_setup_opts = options.setdefault(PythonSetup.options_scope,
{})
python_setup_opts['interpreter_cache_dir'] = cache_dir
interpreter = PythonInterpreter.get()
python_setup_opts['interpreter_search_paths'] = [
os.path.dirname(interpreter.binary)
]
context = self.context(target_roots=target_roots,
options=options,
for_subsystems=[PythonInterpreterCache])
# We must get an interpreter via the cache, instead of using the value of
# PythonInterpreter.get() directly, to ensure that the interpreter has setuptools and
# wheel support.
interpreter_cache = PythonInterpreterCache.global_instance()
interpreters = interpreter_cache.setup(
filters=[str(interpreter.identity.requirement)])
context.products.get_data(PythonInterpreter,
lambda: interpreters[0])
task = self.create_task(context)
task.execute()
return context.products.get_data(
ResolveRequirements.REQUIREMENTS_PEX)
def _create_interpreter_path_file(self, interpreter_path_file, targets):
interpreter_cache = PythonInterpreterCache.global_instance()
interpreter = interpreter_cache.select_interpreter_for_targets(targets)
safe_mkdir_for(interpreter_path_file)
with open(interpreter_path_file, 'w') as outfile:
outfile.write('{}\n'.format(interpreter.binary))
for dist, location in interpreter.extras.items():
dist_name, dist_version = dist
outfile.write('{}\t{}\t{}\n'.format(dist_name, dist_version,
location))
def test_namespace_effective(self):
self.create_file('src/thrift/com/foo/one.thrift', contents=dedent("""
namespace py foo.bar
struct One {}
"""))
one = self.make_target(spec='src/thrift/com/foo:one',
target_type=PythonThriftLibrary,
sources=['one.thrift'])
apache_thrift_gen, synthetic_target_one = self.generate_single_thrift_target(one)
self.create_file('src/thrift2/com/foo/two.thrift', contents=dedent("""
namespace py foo.baz
struct Two {}
"""))
two = self.make_target(spec='src/thrift2/com/foo:two',
target_type=PythonThriftLibrary,
sources=['two.thrift'])
_, synthetic_target_two = self.generate_single_thrift_target(two)
# Confirm separate PYTHONPATH entries, which we need to test namespace packages.
self.assertNotEqual(synthetic_target_one.target_base, synthetic_target_two.target_base)
targets = (synthetic_target_one, synthetic_target_two)
self.context(for_subsystems=[PythonInterpreterCache, PythonRepos])
interpreter_cache = PythonInterpreterCache.global_instance()
python_repos = PythonRepos.global_instance()
interpreter = interpreter_cache.select_interpreter_for_targets(targets)
# We need setuptools to import namespace packages under python 2 (via pkg_resources), so we
# prime the PYTHONPATH with a known good version of setuptools.
# TODO(John Sirois): We really should be emitting setuptools in a
# `synthetic_target_extra_dependencies` override in `ApacheThriftPyGen`:
# https://github.com/pantsbuild/pants/issues/5975
pythonpath = [os.path.join(get_buildroot(), t.target_base) for t in targets]
for resolved_dist in resolve(['thrift=={}'.format(self.get_thrift_version(apache_thrift_gen)),
'setuptools==40.6.3'],
interpreter=interpreter,
context=python_repos.get_network_context(),
fetchers=python_repos.get_fetchers()):
pythonpath.append(resolved_dist.distribution.location)
process = subprocess.Popen([interpreter.binary,
'-c',
'from foo.bar.ttypes import One; from foo.baz.ttypes import Two'],
env={'PYTHONPATH': os.pathsep.join(pythonpath)},
stderr=subprocess.PIPE)
_, stderr = process.communicate()
self.assertEqual(0, process.returncode, stderr)
def test_namespace_effective(self):
self.create_file('src/thrift/com/foo/one.thrift', contents=dedent("""
namespace py foo.bar
struct One {}
"""))
one = self.make_target(spec='src/thrift/com/foo:one',
target_type=PythonThriftLibrary,
sources=['one.thrift'])
apache_thrift_gen, synthetic_target_one = self.generate_single_thrift_target(one)
self.create_file('src/thrift2/com/foo/two.thrift', contents=dedent("""
namespace py foo.baz
struct Two {}
"""))
two = self.make_target(spec='src/thrift2/com/foo:two',
target_type=PythonThriftLibrary,
sources=['two.thrift'])
_, synthetic_target_two = self.generate_single_thrift_target(two)
# Confirm separate PYTHONPATH entries, which we need to test namespace packages.
self.assertNotEqual(synthetic_target_one.target_base, synthetic_target_two.target_base)
targets = (synthetic_target_one, synthetic_target_two)
self.context(for_subsystems=[PythonInterpreterCache, PythonRepos])
interpreter_cache = PythonInterpreterCache.global_instance()
python_repos = PythonRepos.global_instance()
interpreter = interpreter_cache.select_interpreter_for_targets(targets)
# We need setuptools to import namespace packages (via pkg_resources), so we prime the
# PYTHONPATH with interpreter extras, which Pants always populates with setuptools and wheel.
# TODO(John Sirois): We really should be emitting setuptools in a
# `synthetic_target_extra_dependencies` override in `ApacheThriftPyGen`:
# https://github.com/pantsbuild/pants/issues/5975
pythonpath = list(interpreter.extras.values())
pythonpath.extend(os.path.join(get_buildroot(), t.target_base) for t in targets)
for resolved_dist in resolve(['thrift=={}'.format(self.get_thrift_version(apache_thrift_gen))],
interpreter=interpreter,
context=python_repos.get_network_context(),
fetchers=python_repos.get_fetchers()):
pythonpath.append(resolved_dist.distribution.location)
process = subprocess.Popen([interpreter.binary,
'-c',
'from foo.bar.ttypes import One; from foo.baz.ttypes import Two'],
env={'PYTHONPATH': os.pathsep.join(pythonpath)},
stderr=subprocess.PIPE)
_, stderr = process.communicate()
self.assertEqual(0, process.returncode, stderr)
def execute(self):
tool_subsystem = self.tool_subsystem_cls.scoped_instance(self)
pex_name = tool_subsystem.options_scope
pex_path = os.path.join(self.workdir, self.fingerprint, '{}.pex'.format(pex_name))
interpreter_cache = PythonInterpreterCache.global_instance()
interpreter = interpreter_cache.select_interpreter_for_targets([])
if not os.path.exists(pex_path):
with self.context.new_workunit(name='create-{}-pex'.format(pex_name),
labels=[WorkUnitLabel.PREP]):
self._build_tool_pex(tool_subsystem=tool_subsystem,
interpreter=interpreter,
pex_path=pex_path)
tool_instance = self.tool_instance_cls(pex_path, interpreter)
self.context.products.register_data(self.tool_instance_cls, tool_instance)
def execute(self):
""""Run Checkstyle on all found non-synthetic source files."""
python_tgts = self.context.targets(
lambda tgt: isinstance(tgt, (PythonTarget)))
if not python_tgts:
return 0
interpreter_cache = PythonInterpreterCache.global_instance()
with self.invalidated(self.get_targets(
self._is_checked)) as invalidation_check:
failure_count = 0
tgts_by_compatibility, _ = interpreter_cache.partition_targets_by_compatibility(
[vt.target for vt in invalidation_check.invalid_vts])
for filters, targets in tgts_by_compatibility.items():
if self.get_options(
).interpreter_constraints_whitelist is None and not self._constraints_are_whitelisted(
filters):
deprecated_conditional(
lambda: self.get_options(
).interpreter_constraints_whitelist is None,
'1.14.0.dev2',
"Python linting is currently restricted to targets that match the global "
"interpreter constraints: {}. Pants detected unacceptable filters: {}. "
"Use the `--interpreter-constraints-whitelist` lint option to whitelist "
"compatibiltiy constraints.".format(
PythonSetup.global_instance().
interpreter_constraints, filters))
else:
sources = self.calculate_sources([tgt for tgt in targets])
if sources:
allowed_interpreters = set(
interpreter_cache.setup(filters=filters))
if not allowed_interpreters:
raise TaskError(
'No valid interpreters found for targets: {}\n(filters: {})'
.format(targets, filters))
interpreter = min(allowed_interpreters)
failure_count += self.checkstyle(interpreter, sources)
if failure_count > 0 and self.get_options().fail:
raise TaskError(
'{} Python Style issues found. You may try `./pants fmt <targets>`'
.format(failure_count))
return failure_count
def execute(self):
tool_subsystem = self.tool_subsystem_cls.scoped_instance(self)
interpreter_cache = PythonInterpreterCache.global_instance()
interpreter = min(
interpreter_cache.setup(
filters=tool_subsystem.get_interpreter_constraints()))
pex_path = self._generate_fingerprinted_pex_path(
tool_subsystem, interpreter)
if not os.path.exists(pex_path):
with self.context.new_workunit(name='create-{}-pex'.format(
tool_subsystem.options_scope),
labels=[WorkUnitLabel.PREP]):
self._build_tool_pex(tool_subsystem=tool_subsystem,
interpreter=interpreter,
pex_path=pex_path)
tool_instance = self.tool_instance_cls(pex_path, interpreter)
self.context.products.register_data(self.tool_instance_cls,
tool_instance)
def _resolve_requirements(self, target_roots, options=None):
with temporary_dir() as cache_dir:
options = options or {}
python_setup_opts = options.setdefault(PythonSetup.options_scope, {})
python_setup_opts['interpreter_cache_dir'] = cache_dir
interpreter = PythonInterpreter.get()
python_setup_opts['interpreter_search_paths'] = [os.path.dirname(interpreter.binary)]
context = self.context(target_roots=target_roots, options=options,
for_subsystems=[PythonInterpreterCache])
# We must get an interpreter via the cache, instead of using the value of
# PythonInterpreter.get() directly, to ensure that the interpreter has setuptools and
# wheel support.
interpreter_cache = PythonInterpreterCache.global_instance()
interpreters = interpreter_cache.setup(filters=[str(interpreter.identity.requirement)])
context.products.get_data(PythonInterpreter, lambda: interpreters[0])
task = self.create_task(context)
task.execute()
return context.products.get_data(ResolveRequirements.REQUIREMENTS_PEX)
def _gather_sources(self, target_roots):
with temporary_dir() as cache_dir:
interpreter = PythonInterpreter.get()
context = self.context(target_roots=target_roots,
for_subsystems=[PythonInterpreterCache],
options={
PythonSetup.options_scope: {
'interpreter_cache_dir': cache_dir,
'interpreter_search_paths': [os.path.dirname(interpreter.binary)],
}})
# We must get an interpreter via the cache, instead of using the value of
# PythonInterpreter.get() directly, to ensure that the interpreter has setuptools and
# wheel support.
interpreter_cache = PythonInterpreterCache.global_instance()
interpreters = interpreter_cache.setup(filters=[str(interpreter.identity.requirement)])
context.products.get_data(PythonInterpreter, lambda: interpreters[0])
task = self.create_task(context)
task.execute()
return context.products.get_data(GatherSources.PYTHON_SOURCES)
def execute(self):
if not self.will_be_invoked():
return
tool_subsystem = self.tool_subsystem_cls.scoped_instance(self)
interpreter_cache = PythonInterpreterCache.global_instance()
interpreters = interpreter_cache.setup(filters=tool_subsystem.get_interpreter_constraints())
if not interpreters:
raise TaskError('Found no Python interpreter capable of running the {} tool with '
'constraints {}'.format(tool_subsystem.options_scope,
tool_subsystem.get_interpreter_constraints()))
interpreter = min(interpreters)
pex_path = self._generate_fingerprinted_pex_path(tool_subsystem, interpreter)
if not os.path.exists(pex_path):
with self.context.new_workunit(name='create-{}-pex'.format(tool_subsystem.options_scope),
labels=[WorkUnitLabel.PREP]):
self._build_tool_pex(tool_subsystem=tool_subsystem,
interpreter=interpreter,
pex_path=pex_path)
tool_instance = self.tool_instance_cls(pex_path, interpreter)
self.context.products.register_data(self.tool_instance_cls, tool_instance)
def _compatible_interpreter(self, unpacked_whls): constraints = PythonSetup.global_instance().compatibility_or_constraints(unpacked_whls.compatibility) allowable_interpreters = PythonInterpreterCache.global_instance().setup(filters=constraints) return min(allowable_interpreters)
def _create_interpreter_cache(self, setup_options=None):
Subsystem.reset(reset_options=True)
self.context(for_subsystems=[PythonInterpreterCache], options={
'python-setup': setup_options,
})
return PythonInterpreterCache.global_instance()
def test_namespace_effective(self):
self.create_file(
"src/thrift/com/foo/one.thrift",
contents=dedent(
"""
namespace py foo.bar
struct One {}
"""
),
)
one = self.make_target(
spec="src/thrift/com/foo:one", target_type=PythonThriftLibrary, sources=["one.thrift"]
)
apache_thrift_gen, synthetic_target_one = self.generate_single_thrift_target(one)
self.create_file(
"src/thrift2/com/foo/two.thrift",
contents=dedent(
"""
namespace py foo.baz
struct Two {}
"""
),
)
two = self.make_target(
spec="src/thrift2/com/foo:two", target_type=PythonThriftLibrary, sources=["two.thrift"]
)
_, synthetic_target_two = self.generate_single_thrift_target(two)
# Confirm separate PYTHONPATH entries, which we need to test namespace packages.
self.assertNotEqual(synthetic_target_one.target_base, synthetic_target_two.target_base)
targets = (synthetic_target_one, synthetic_target_two)
self.context(for_subsystems=[PythonInterpreterCache, PythonRepos])
interpreter_cache = PythonInterpreterCache.global_instance()
python_repos = PythonRepos.global_instance()
interpreter = interpreter_cache.select_interpreter_for_targets(targets)
# We need setuptools to import namespace packages under python 2 (via pkg_resources), so we
# prime the PYTHONPATH with a known good version of setuptools.
# TODO(John Sirois): We really should be emitting setuptools in a
# `synthetic_target_extra_dependencies` override in `ApacheThriftPyGen`:
# https://github.com/pantsbuild/pants/issues/5975
pythonpath = [os.path.join(get_buildroot(), t.target_base) for t in targets]
for resolved_dist in resolve(
[f"thrift=={self.get_thrift_version(apache_thrift_gen)}", "setuptools==40.6.3"],
interpreter=interpreter,
indexes=python_repos.indexes,
find_links=python_repos.repos,
):
pythonpath.append(resolved_dist.distribution.location)
process = subprocess.Popen(
[
interpreter.binary,
"-c",
"from foo.bar.ttypes import One; from foo.baz.ttypes import Two",
],
env={"PYTHONPATH": os.pathsep.join(pythonpath)},
stderr=subprocess.PIPE,
)
_, stderr = process.communicate()
self.assertEqual(0, process.returncode, stderr)
def _interpreter_cache(self): return PythonInterpreterCache.global_instance()
def _interpreter_cache(self):
return PythonInterpreterCache.global_instance()
def _compatible_interpreter(self, unpacked_whls):
constraints = PythonSetup.global_instance(
).compatibility_or_constraints(unpacked_whls.compatibility)
allowable_interpreters = PythonInterpreterCache.global_instance(
).setup(filters=constraints)
return min(allowable_interpreters)
