def base_requirements(self):
return [
PythonRequirement(
f"setuptools=={self.get_options().setuptools_version}"),
PythonRequirement(
f"wheel=={self.get_options().wheel_version}"),
]
def test_python_requirements_field(self):
req1 = PythonRequirement("foo==1.0")
req2 = PythonRequirement("bar==1.0")
self.assertNotEqual(
PythonRequirementsField([req1]).fingerprint(),
PythonRequirementsField([req2]).fingerprint(),
)
def _assert_unpacking(self, module_name):
# TODO: figure out how to generate a nice fake wheel that the pex resolve will accept instead of
# depending on a real wheel!
pex_requirement = PythonRequirement("pex==1.5.3")
unpacked_wheel_tgt = self._make_unpacked_wheel(
pex_requirement,
include_patterns=["pex/pex.py", "pex/__init__.py"],
module_name=module_name,
# TODO: `within_data_subdir` is only tested implicitly by the tensorflow_custom_op target
# in examples/! Make a fake wheel, resolve it, and test that `within_data_subdir`
# descends into the correct directory!
within_data_subdir=None,
)
context = self.context(target_roots=[unpacked_wheel_tgt])
unpack_task = self.create_task(context)
unpack_task.execute()
expected_files = {"pex/__init__.py", "pex/pex.py"}
with unpack_task.invalidated([unpacked_wheel_tgt
]) as invalidation_check:
vt = assert_single_element(invalidation_check.all_vts)
self.assertEqual(vt.target, unpacked_wheel_tgt)
archives = context.products.get_data(UnpackedArchives,
dict)[vt.target]
self.assertEqual(expected_files, set(archives.found_files))
def __call__(self, name=None, dist=None):
"""
:param string name: The name to use for the target, defaults to the dist name if specified and
otherwise the parent dir name.
:param string dist: The pants dist to create a requirement for. This must be a
'pantsbuild.pants*' distribution; eg:
'pantsbuild.pants.contrib.python.checks'.
"""
name = name or dist or os.path.basename(self._parse_context.rel_path)
dist = dist or "pantsbuild.pants"
if not (dist == "pantsbuild.pants"
or dist.startswith("pantsbuild.pants.")):
target = Address(spec_path=self._parse_context.rel_path,
target_name=name)
raise TargetDefinitionException(
target=target,
msg="The {} target only works for pantsbuild.pants "
"distributions, given {}".format(self.alias, dist),
)
requirement = PythonRequirement(requirement="{key}=={version}".format(
key=dist, version=pants_version()))
self._parse_context.create_object("python_requirement_library",
name=name,
requirements=[requirement])
def __call__(
self,
name: Optional[str] = None,
dist: Optional[str] = None,
*,
modules: Optional[Iterable[str]] = None,
):
"""
:param name: The name to use for the target, defaults to the dist name if specified and
otherwise the parent dir name.
:param dist: The Pants dist to create a requirement for. This must be a 'pantsbuild.pants*'
distribution; eg: 'pantsbuild.pants.testutil'.
:param modules: The modules exposed by the dist, e.g. `['pants.testutil']`. This defaults
to the name of the dist without the leading `pantsbuild`.
"""
name = name or dist or os.path.basename(self._parse_context.rel_path)
dist = dist or "pantsbuild.pants"
if not (dist == "pantsbuild.pants"
or dist.startswith("pantsbuild.pants.")):
target = Address(spec_path=self._parse_context.rel_path,
target_name=name)
raise TargetDefinitionException(
target=target,
msg=
(f"The {self.alias} target only works for pantsbuild.pants distributions, but "
f"given {dist}"),
)
if not modules:
modules = [dist.replace("pantsbuild.", "")]
requirement = PythonRequirement(
requirement=f"{dist}=={pants_version()}", modules=modules)
self._parse_context.create_object("python_requirement_library",
name=name,
requirements=[requirement])
def setUp(self):
super().setUp()
python_leaf = self.make_target(
"dependencies:python_leaf",
target_type=PythonLibrary,
sources=[],
)
python_inner = self.make_target(
"dependencies:python_inner",
target_type=PythonLibrary,
sources=[],
dependencies=[python_leaf],
)
python_inner_with_external = self.make_target(
"dependencies:python_inner_with_external",
target_type=PythonRequirementLibrary,
requirements=[PythonRequirement("ansicolors==1.1.8")],
)
self.make_target(
"dependencies:python_root",
target_type=PythonLibrary,
sources=[],
dependencies=[python_inner, python_inner_with_external],
)
def test_requirements_field() -> None:
raw_value = ("argparse==1.2.1", "configparser ; python_version<'3'")
parsed_value = tuple(Requirement.parse(v) for v in raw_value)
assert PythonRequirementsField(
raw_value, address=Address("demo")).value == parsed_value
# Macros can pass pre-parsed Requirement objects.
assert PythonRequirementsField(
parsed_value, address=Address("demo")).value == parsed_value
# Reject invalid types.
with pytest.raises(InvalidFieldTypeException):
PythonRequirementsField("sneaky_str", address=Address("demo"))
with pytest.raises(InvalidFieldTypeException):
PythonRequirementsField([1, 2], address=Address("demo"))
# Give a nice error message if the requirement can't be parsed.
with pytest.raises(InvalidFieldException) as exc:
PythonRequirementsField(["not valid! === 3.1"],
address=Address("demo"))
assert (
"Invalid requirement string 'not valid! === 3.1' in the 'requirements' field for the "
"target demo:") in str(exc.value)
# Check that we still support the deprecated `pants_requirement` object.
assert (PythonRequirementsField(
[PythonRequirement(v) for v in raw_value],
address=Address("demo")).value == parsed_value)
def resolve_requirement_strings(self, interpreter, requirement_strings):
"""Resolve a list of pip-style requirement strings."""
requirement_strings = sorted(requirement_strings)
if len(requirement_strings) == 0:
req_strings_id = "no_requirements"
elif len(requirement_strings) == 1:
req_strings_id = requirement_strings[0]
else:
req_strings_id = hash_all(requirement_strings)
path = os.path.realpath(
os.path.join(self.workdir, str(interpreter.identity),
req_strings_id))
if not os.path.isdir(path):
reqs = [
PythonRequirement(req_str) for req_str in requirement_strings
]
with safe_concurrent_creation(path) as safe_path:
pex_builder = PexBuilderWrapper.Factory.create(
builder=PEXBuilder(path=safe_path,
interpreter=interpreter,
copy=True),
log=self.context.log,
)
pex_builder.add_resolved_requirements(reqs)
pex_builder.freeze()
return PEX(path, interpreter=interpreter)
def setUp(self):
super().setUp()
python_leaf = self.make_target(
"dependencies:python_leaf",
target_type=PythonLibrary,
sources=[],
)
python_inner = self.make_target(
"dependencies:python_inner",
target_type=PythonLibrary,
sources=[],
dependencies=[
python_leaf,
],
)
python_inner_with_external = self.make_target(
"dependencies:python_inner_with_external",
target_type=PythonRequirementLibrary,
requirements=[PythonRequirement("antlr_python_runtime==3.1.3")],
)
self.make_target(
"dependencies:python_root",
target_type=PythonLibrary,
sources=[],
dependencies=[
python_inner,
python_inner_with_external,
],
)
def create(cls, builder, log=None, generate_ipex=False):
options = cls.global_instance().get_options()
setuptools_requirement = f"setuptools=={options.setuptools_version}"
pex_requirement = f"pex=={options.pex_version}"
log = log or logging.getLogger(__name__)
return PexBuilderWrapper(
builder=builder,
python_repos_subsystem=PythonRepos.global_instance(),
python_setup_subsystem=PythonSetup.global_instance(),
setuptools_requirement=PythonRequirement(
setuptools_requirement),
pex_requirement=PythonRequirement(pex_requirement),
log=log,
generate_ipex=generate_ipex,
)
def test_good_list(self) -> None:
build_file = dedent(
"""
python_requirement_library(
name='pyunit',
requirements=[
python_requirement('argparse==1.2.1'),
python_requirement('ansicolors>=1.18'),
]
)
"""
)
requirements = self.get_python_requirements(build_file, target_name="pyunit")
assert len(requirements) == 2
assert [req.requirement for req in requirements] == [
PythonRequirement("argparse==1.2.1").requirement,
PythonRequirement("ansicolors>=1.18").requirement,
]
def test_unpack_wheels_fingerprint_strategy(self):
fingerprint_strategy = UnpackWheelsFingerprintStrategy()
make_unpacked_wheel = functools.partial(self._make_unpacked_wheel,
include_patterns=["bar"])
req1 = PythonRequirement("com.example.bar==0.0.1")
target = make_unpacked_wheel(req1)
fingerprint1 = fingerprint_strategy.compute_fingerprint(target)
# Now, replace the build file with a different version.
self.reset_build_graph()
target = make_unpacked_wheel(
PythonRequirement("com.example.bar==0.0.2"))
fingerprint2 = fingerprint_strategy.compute_fingerprint(target)
self.assertNotEqual(fingerprint1, fingerprint2)
# Go back to the original library.
self.reset_build_graph()
target = make_unpacked_wheel(req1)
fingerprint3 = fingerprint_strategy.compute_fingerprint(target)
self.assertEqual(fingerprint1, fingerprint3)
def _build_tool_pex(self, tool_subsystem, interpreter, pex_path):
with safe_concurrent_creation(pex_path) as chroot:
pex_builder = PexBuilderWrapper.Factory.create(
builder=PEXBuilder(path=chroot, interpreter=interpreter),
log=self.context.log)
reqs = [
PythonRequirement(r)
for r in tool_subsystem.get_requirement_specs()
]
pex_builder.add_resolved_requirements(reqs=reqs,
platforms=["current"])
pex_builder.set_entry_point(tool_subsystem.get_entry_point())
pex_builder.freeze()
def _inject_synthetic_dist_requirements(self, dist, req_lib_addr):
"""Inject a synthetic requirements library that references a local wheel.
:param dist: Path of the locally built wheel to reference.
:param req_lib_addr: :class: `Address` to give to the synthetic target.
:return: a :class: `PythonRequirementLibrary` referencing the locally-built wheel.
"""
whl_dir, base = split_basename_and_dirname(dist)
whl_metadata = base.split("-")
req_name = "==".join([whl_metadata[0], whl_metadata[1]])
req = PythonRequirement(req_name, repository=whl_dir)
self.context.build_graph.inject_synthetic_target(
req_lib_addr, PythonRequirementLibrary, requirements=[req])
def test_bad_libraries_ref(self):
self.make_target(":right-type",
PythonRequirementLibrary,
requirements=[PythonRequirement("foo==123")])
# Making a target which is not a requirement library, which causes an error.
self.make_target(":wrong-type",
UnpackedWheels,
libraries=[":right-type"],
module_name="foo")
target = self.make_target(":foo",
UnpackedWheels,
libraries=[":wrong-type"],
module_name="foo")
with self.assertRaises(ImportWheelsMixin.WrongTargetTypeError):
target.imported_targets
def assert_pants_requirement(
self, python_requirement_library, expected_dist="pantsbuild.pants"
):
self.assertIsInstance(python_requirement_library, PythonRequirementLibrary)
expected = PythonRequirement(f"{expected_dist}=={pants_version()}")
def key(python_requirement):
return (
python_requirement.requirement.key,
python_requirement.requirement.specs,
python_requirement.requirement.extras,
)
self.assertEqual(
[key(expected)], [key(pr) for pr in python_requirement_library.payload.requirements]
)
def test_simple(self):
self.make_target(":import_whls",
PythonRequirementLibrary,
requirements=[PythonRequirement("foo==123")])
target = self.make_target(":foo",
UnpackedWheels,
libraries=[":import_whls"],
module_name="foo")
self.assertIsInstance(target, UnpackedWheels)
dependency_specs = [
spec for spec in target.compute_dependency_address_specs(
payload=target.payload)
]
self.assertSequenceEqual([":import_whls"], dependency_specs)
import_whl_dep = assert_single_element(
target.all_imported_requirements)
self.assertIsInstance(import_whl_dep, PythonRequirement)
def test_requirements_field() -> None:
raw_value = (
"argparse==1.2.1",
"configparser ; python_version<'3'",
"pip@ git+https://github.com/pypa/pip.git",
)
parsed_value = tuple(Requirement.parse(v) for v in raw_value)
assert PythonRequirementsField(
raw_value, address=Address("demo")).value == parsed_value
# Macros can pass pre-parsed Requirement objects.
assert PythonRequirementsField(
parsed_value, address=Address("demo")).value == parsed_value
# Reject invalid types.
with pytest.raises(InvalidFieldTypeException):
PythonRequirementsField("sneaky_str", address=Address("demo"))
with pytest.raises(InvalidFieldTypeException):
PythonRequirementsField([1, 2], address=Address("demo"))
# Give a nice error message if the requirement can't be parsed.
with pytest.raises(InvalidFieldException) as exc:
PythonRequirementsField(["not valid! === 3.1"],
address=Address("demo"))
assert (
"Invalid requirement 'not valid! === 3.1' in the 'requirements' field for the "
"target demo:") in str(exc.value)
# Give a nice error message if it looks like they're trying to use pip VCS-style requirements.
with pytest.raises(InvalidFieldException) as exc:
PythonRequirementsField(
["git+https://github.com/pypa/pip.git#egg=pip"],
address=Address("demo"))
assert "It looks like you're trying to use a pip VCS-style requirement?" in str(
exc.value)
# Check that we still support the deprecated `pants_requirement` object.
assert (PythonRequirementsField(
[PythonRequirement(v) for v in raw_value],
address=Address("demo")).value == parsed_value)
def assert_pants_requirement(
self,
build_file_entry: str,
*,
expected_target_name: str,
expected_dist: str = "pantsbuild.pants",
expected_module: str = "pants",
) -> None:
self.add_to_build_file("3rdparty/python", f"{build_file_entry}\n")
target = self.request_single_product(
WrappedTarget,
Address("3rdparty/python",
target_name=expected_target_name)).target
assert isinstance(target, PythonRequirementLibrary)
expected = PythonRequirement(f"{expected_dist}=={pants_version()}",
modules=[expected_module])
requirements = target[PythonRequirementsField].value
assert len(requirements) == 1
req = requirements[0]
assert req.requirement == expected.requirement
assert req.modules == expected.modules
def test_has_all_imported_req_libs(self):
def assert_dep(reqA, reqB):
self.assertEqual(reqA.requirement, reqB.requirement)
def sort_requirements(reqs):
return list(sorted(reqs, key=lambda r: str(r.requirement)))
self.add_to_build_file(
"BUILD",
dedent("""
python_requirement_library(name='lib1',
requirements=[
python_requirement('testName1==123'),
],
)
python_requirement_library(name='lib2',
requirements=[
python_requirement('testName2==456'),
python_requirement('testName3==789'),
],
)
unpacked_whls(name='unpacked-lib',
libraries=[':lib1', ':lib2'],
module_name='foo',
)
"""),
)
lib1 = self.target("//:lib1")
self.assertIsInstance(lib1, PythonRequirementLibrary)
assert_dep(assert_single_element(lib1.requirements),
PythonRequirement("testName1==123"))
lib2 = self.target("//:lib2")
self.assertIsInstance(lib2, PythonRequirementLibrary)
lib2_reqs = sort_requirements(lib2.requirements)
self.assertEqual(2, len(lib2_reqs))
assert_dep(lib2_reqs[0], PythonRequirement("testName2==456"))
assert_dep(lib2_reqs[1], PythonRequirement("testName3==789"))
unpacked_lib = self.target("//:unpacked-lib")
unpacked_req_libs = sort_requirements(
unpacked_lib.all_imported_requirements)
self.assertEqual(3, len(unpacked_req_libs))
assert_dep(unpacked_req_libs[0], PythonRequirement("testName1==123"))
assert_dep(unpacked_req_libs[1], PythonRequirement("testName2==456"))
assert_dep(unpacked_req_libs[2], PythonRequirement("testName3==789"))
def checker_pex(self, interpreter):
# TODO(John Sirois): Formalize in pants.base?
pants_dev_mode = os.environ.get("PANTS_DEV", "0") != "0"
if pants_dev_mode:
checker_id = self.checker_target.transitive_invalidation_hash()
else:
checker_id = hash_all([self._CHECKER_REQ])
pex_path = os.path.join(self.workdir, "checker", checker_id,
str(interpreter.identity))
if not os.path.exists(pex_path):
with self.context.new_workunit(name="build-checker"):
with safe_concurrent_creation(pex_path) as chroot:
pex_builder = PexBuilderWrapper.Factory.create(
builder=PEXBuilder(path=chroot,
interpreter=interpreter),
log=self.context.log,
)
# Constraining is required to guard against the case where the user
# has a pexrc file set.
pex_builder.add_interpreter_constraint(
str(interpreter.identity.requirement))
if pants_dev_mode:
pex_builder.add_sources_from(self.checker_target)
req_libs = [
tgt for tgt in self.checker_target.closure()
if isinstance(tgt, PythonRequirementLibrary)
]
pex_builder.add_requirement_libs_from(
req_libs=req_libs)
else:
try:
# The checker is already on sys.path, eg: embedded in pants.pex.
platform = Platform.current()
platform_name = platform.platform
env = Environment(
search_path=sys.path,
platform=platform_name,
python=interpreter.version_string,
)
working_set = WorkingSet(entries=sys.path)
for dist in working_set.resolve(
[Requirement.parse(self._CHECKER_REQ)],
env=env):
pex_builder.add_direct_requirements(
dist.requires())
# NB: We add the dist location instead of the dist itself to make sure its a
# distribution style pex knows how to package.
pex_builder.add_dist_location(dist.location)
pex_builder.add_direct_requirements(
[self._CHECKER_REQ])
except (DistributionNotFound,
PEXBuilder.InvalidDistribution):
# We need to resolve the checker from a local or remote distribution repo.
pex_builder.add_resolved_requirements(
[PythonRequirement(self._CHECKER_REQ)])
pex_builder.set_entry_point(self._CHECKER_ENTRYPOINT)
pex_builder.freeze()
return PEX(pex_path, interpreter=interpreter)
class TestBuildLocalPythonDistributions(BuildLocalPythonDistributionsTestBase):
dist_specs = OrderedDict([
(
"src/python/dist:universal_dist",
{
"key": "universal",
"target_type": PythonDistribution,
"sources": ["__init__.py", "setup.py"],
"filemap": {
"__init__.py":
"",
"setup.py":
"""\
from setuptools import find_packages, setup
setup(
name='universal_dist',
version='0.0.0',
packages=find_packages()
)
""",
},
},
),
(
"3rdparty/python:pycountry",
{
"key": "pycountry",
"target_type": PythonRequirementLibrary,
"requirements": [PythonRequirement("pycountry==18.5.20")],
},
),
(
"src/python/setup_requires:setup_requires",
{
"key": "setup_requires",
"target_type": PythonDistribution,
"setup_requires": ["3rdparty/python:pycountry"],
"sources": ["__init__.py", "setup.py"],
"filemap": {
"__init__.py":
"",
"setup.py":
"""\
from setuptools import find_packages, setup
import pycountry
us_country_string = pycountry.countries.get(alpha_2='US').name.replace(' ', '_').lower()
setup(
name='setup_requires_dist_{}'.format(us_country_string),
version='0.0.0',
packages=find_packages(),
)
""",
},
},
),
(
"src/python/install_requires:install_requires",
{
"key": "install_requires",
"target_type": PythonDistribution,
"sources": ["__init__.py", "setup.py"],
"filemap": {
"__init__.py":
"",
"setup.py":
"""\
from setuptools import setup
setup(
name='install_requires_dist',
version='0.0.0',
install_requires=['pycountry==17.1.2'],
)
""",
},
},
),
])
def test_create_distribution(self):
universal_dist = self.target_dict["universal"]
self._assert_dist_and_wheel_identity(
expected_name="universal_dist",
expected_version="0.0.0",
expected_platform=self.ExpectedPlatformType.any,
dist_target=universal_dist,
)
def test_python_dist_setup_requires(self):
setup_requires_dist = self.target_dict["setup_requires"]
self._assert_dist_and_wheel_identity(
expected_name="setup_requires_dist_united_states",
expected_version="0.0.0",
expected_platform=self.ExpectedPlatformType.any,
dist_target=setup_requires_dist,
extra_targets=[self.target_dict["pycountry"]],
)
def test_install_requires(self):
install_requires_dist = self.target_dict["install_requires"]
self._assert_dist_and_wheel_identity(
expected_name="install_requires_dist",
expected_version="0.0.0",
expected_platform=self.ExpectedPlatformType.any,
dist_target=install_requires_dist,
)
def _fake_target(self, spec, requirement_strs):
requirements = [PythonRequirement(r) for r in requirement_strs]
return self.make_target(spec=spec,
target_type=PythonRequirementLibrary,
requirements=requirements)