def get_request(self, plat: Platform) -> ExternalToolRequest:
"""Generate a request for this tool."""
version = self.get_options().version
known_versions = self.get_options().known_versions
for known_version in known_versions:
try:
ver, plat_val, sha256, length = (
x.strip() for x in known_version.split("|"))
except ValueError:
raise ExternalToolError(
f"Bad value for --known-versions (see ./pants "
f"help-advanced {self.options_scope}): {known_version}")
if plat_val == plat.value and ver == version:
digest = Digest(fingerprint=sha256,
serialized_bytes_length=int(length))
try:
url = self.generate_url(plat)
exe = self.generate_exe(plat) or url.rsplit("/", 1)[-1]
except ExternalToolError as e:
raise ExternalToolError(
f"Couldn't find {self.name} version {version} on {plat.value}"
) from e
return ExternalToolRequest(UrlToFetch(url=url, digest=digest),
exe)
raise UnknownVersion(
f"No known version of {self.name} {version} for {plat.value} found in {known_versions}"
)
def test_download_missing_file(self):
with self.isolated_local_store():
with http_server(StubHandler) as port:
url = UrlToFetch(f"http://localhost:{port}/notfound", self.pantsbuild_digest)
with self.assertRaises(ExecutionError) as cm:
self.scheduler.product_request(Snapshot, subjects=[url])
self.assertIn('404', str(cm.exception))
def download_cloc_script() -> DownloadedClocScript:
url = "https://binaries.pantsbuild.org/bin/cloc/1.80/cloc"
sha_256 = "2b23012b1c3c53bd6b9dd43cd6aa75715eed4feb2cb6db56ac3fbbd2dffeac9d"
digest = Digest(sha_256, 546279)
snapshot = yield Get(Snapshot, UrlToFetch(url, digest))
yield DownloadedClocScript(script_path=snapshot.files[0],
digest=snapshot.directory_digest)
def test_download_missing_file(self) -> None:
with self.isolated_local_store():
with http_server(StubHandler) as port:
url = UrlToFetch(f"http://localhost:{port}/notfound",
self.pantsbuild_digest)
with self.assertRaises(ExecutionError) as cm:
self.request_single_product(Snapshot, url)
assert "404" in str(cm.exception)
def test_download(self):
with self.isolated_local_store():
with http_server(StubHandler) as port:
url = UrlToFetch(f"http://localhost:{port}/CNAME", self.pantsbuild_digest)
snapshot, = self.scheduler.product_request(Snapshot, subjects=[url])
self.assert_snapshot_equals(snapshot, ["CNAME"], Digest(
"16ba2118adbe5b53270008790e245bbf7088033389461b08640a4092f7f647cf",
81
))
def download_pex_bin():
# TODO: Inject versions and digests here through some option, rather than hard-coding it.
url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.10/pex'
digest = Digest(
'f4ac0f61f0c469537f04418e9347658340b0bce4ba61d302c5b805d194dda482',
1868106)
snapshot = yield Get(Snapshot, UrlToFetch(url, digest))
yield DownloadedPexBin(executable=snapshot.files[0],
directory_digest=snapshot.directory_digest)
def download_pex_bin() -> DownloadedPexBin:
# TODO: Inject versions and digests here through some option, rather than hard-coding it.
url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.12/pex'
digest = Digest(
'ce64cb72cd23d2123dd48126af54ccf2b718d9ecb98c2ed3045ed1802e89e7e1',
1842359)
snapshot = yield Get(Snapshot, UrlToFetch(url, digest))
yield DownloadedPexBin(executable=snapshot.files[0],
directory_digest=snapshot.directory_digest)
def download_pex_bin() -> DownloadedPexBin:
# TODO: Inject versions and digests here through some option, rather than hard-coding it.
url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.11/pex'
digest = Digest(
'7a8fdfce2de22d25ba38afaa9df0282c33dd436959b3a5c3f788ded2ccc2cae9',
1867604)
snapshot = yield Get(Snapshot, UrlToFetch(url, digest))
yield DownloadedPexBin(executable=snapshot.files[0],
directory_digest=snapshot.directory_digest)
def do_test(expected_url: str, expected_length: int, expected_sha256: str,
plat: Platform, version: str) -> None:
foobar = create_subsystem(FooBar,
version=version,
known_versions=FooBar.default_known_versions)
assert ExternalToolRequest(
UrlToFetch(url=expected_url,
digest=Digest(expected_sha256, expected_length)),
f"foobar-{version}/bin/foobar",
) == foobar.get_request(plat)
def test_download_https(self):
with self.isolated_local_store():
url = UrlToFetch("https://www.pantsbuild.org/CNAME", Digest(
"63652768bd65af8a4938c415bdc25e446e97c473308d26b3da65890aebacf63f",
18,
))
snapshot, = self.scheduler.product_request(Snapshot, subjects=[url])
self.assert_snapshot_equals(snapshot, ["CNAME"], Digest(
"16ba2118adbe5b53270008790e245bbf7088033389461b08640a4092f7f647cf",
81
))
def test_download_wrong_digest(self):
with self.isolated_local_store():
with http_server(StubHandler) as port:
url = UrlToFetch(
f"http://localhost:{port}/CNAME",
Digest(
self.pantsbuild_digest.fingerprint,
self.pantsbuild_digest.serialized_bytes_length + 1,
),
)
with self.assertRaises(ExecutionError) as cm:
self.scheduler.product_request(Snapshot, subjects=[url])
self.assertIn('wrong digest', str(cm.exception).lower())
def test_download_wrong_digest(self) -> None:
with self.isolated_local_store():
with http_server(StubHandler) as port:
url = UrlToFetch(
f"http://localhost:{port}/CNAME",
Digest(
self.pantsbuild_digest.fingerprint,
self.pantsbuild_digest.serialized_bytes_length + 1,
),
)
with self.assertRaises(ExecutionError) as cm:
self.request_single_product(Snapshot, url)
assert "wrong digest" in str(cm.exception).lower()
async def fetch_binary_tool(req: BinaryToolFetchRequest,
url_set: BinaryToolUrlSet) -> Snapshot:
digest = url_set.tool_for_platform.digest
urls = url_set.get_urls()
if not urls:
raise ValueError(
f"binary tool url generator {url_set.url_generator} produced an empty list of "
f"urls for the request {req}")
# TODO: allow fetching a UrlToFetch with failure! Consider FallibleUrlToFetch analog to
# FallibleExecuteProcessResult!
url_to_fetch = urls[0]
return await Get[Snapshot](UrlToFetch(url_to_fetch, digest))
def resolve_requirements(request, python_setup, pex_build_environment):
"""Returns a PEX with the given requirements, optional entry point, and optional
interpreter constraints."""
# TODO: Inject versions and digests here through some option, rather than hard-coding it.
url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.8/pex'
digest = Digest(
'2ca320aede7e7bbcb907af54c9de832707a1df965fb5a0d560f2df29ba8a2f3d',
1866441)
pex_snapshot = yield Get(Snapshot, UrlToFetch(url, digest))
interpreter_search_paths = create_path_env_var(
python_setup.interpreter_search_paths)
env = {
"PATH": interpreter_search_paths,
**pex_build_environment.invocation_environment_dict
}
interpreter_constraint_args = []
for constraint in request.interpreter_constraints:
interpreter_constraint_args.extend(
["--interpreter-constraint", constraint])
# NB: we use the hardcoded and generic bin name `python`, rather than something dynamic like
# `sys.executable`, to ensure that the interpreter may be discovered both locally and in remote
# execution (so long as `env` is populated with a `PATH` env var and `python` is discoverable
# somewhere on that PATH). This is only used to run the downloaded PEX tool; it is not
# necessarily the interpreter that PEX will use to execute the generated .pex file.
# TODO(#7735): Set --python-setup-interpreter-search-paths differently for the host and target
# platforms, when we introduce platforms in https://github.com/pantsbuild/pants/issues/7735.
argv = [
"python", "./{}".format(pex_snapshot.files[0]), "-o",
request.output_filename
]
if request.entry_point is not None:
argv.extend(["-e", request.entry_point])
argv.extend(interpreter_constraint_args + list(request.requirements))
request = ExecuteProcessRequest(
argv=tuple(argv),
env=env,
input_files=pex_snapshot.directory_digest,
description='Resolve requirements: {}'.format(", ".join(
request.requirements)),
output_files=(request.output_filename, ),
)
result = yield Get(ExecuteProcessResult, ExecuteProcessRequest, request)
yield ResolvedRequirementsPex(
directory_digest=result.output_directory_digest, )
def test_caches_downloads(self):
with self.isolated_local_store():
with http_server(StubHandler) as port:
self.prime_store_with_roland_digest()
# This would error if we hit the HTTP server, because 404,
# but we're not going to hit the HTTP server because it's cached,
# so we shouldn't see an error...
url = UrlToFetch(
f"http://localhost:{port}/roland",
Digest('693d8db7b05e99c6b7a7c0616456039d89c555029026936248085193559a0b5d', 16),
)
snapshot, = self.scheduler.product_request(Snapshot, subjects=[url])
self.assert_snapshot_equals(snapshot, ["roland"], Digest(
"9341f76bef74170bedffe51e4f2e233f61786b7752d21c2339f8ee6070eba819",
82
))
def run_python_test(target):
# TODO: Inject versions and digests here through some option, rather than hard-coding it.
pex_snapshot = yield Get(Snapshot, UrlToFetch("https://github.com/pantsbuild/pex/releases/download/v1.5.2/pex27",
Digest('8053a79a5e9c2e6e9ace3999666c9df910d6289555853210c1bbbfa799c3ecda', 1757011)))
# TODO: This should be configurable, both with interpreter constraints, and for remote execution.
python_binary = sys.executable
argv = [
'./{}'.format(pex_snapshot.files[0].path),
'-e', 'pytest:main',
'--python', python_binary,
# TODO: This is non-hermetic because pytest will be resolved on the fly by pex27, where it should be hermetically provided in some way.
# We should probably also specify a specific version.
'pytest',
]
merged_input_files = yield Get(
Digest,
MergedDirectories,
MergedDirectories(directories=(target.adaptor.sources.snapshot.directory_digest, pex_snapshot.directory_digest)),
)
request = ExecuteProcessRequest(
argv=tuple(argv),
input_files=merged_input_files,
description='Run pytest for {}'.format(target.address.reference()),
# TODO: This should not be necessary
env={'PATH': os.path.dirname(python_binary)}
)
result = yield Get(FallibleExecuteProcessResult, ExecuteProcessRequest, request)
# TODO: Do something with stderr?
status = Status.SUCCESS if result.exit_code == 0 else Status.FAILURE
yield PyTestResult(status=status, stdout=str(result.stdout))
def run_python_test(transitive_hydrated_target):
target_root = transitive_hydrated_target.root
# TODO: Inject versions and digests here through some option, rather than hard-coding it.
pex_snapshot = yield Get(
Snapshot,
UrlToFetch(
"https://github.com/pantsbuild/pex/releases/download/v1.5.2/pex27",
Digest(
'8053a79a5e9c2e6e9ace3999666c9df910d6289555853210c1bbbfa799c3ecda',
1757011)))
all_targets = [target_root] + [
dep.root for dep in transitive_hydrated_target.dependencies
]
# Produce a pex containing pytest and all transitive 3rdparty requirements.
all_requirements = []
for maybe_python_req_lib in all_targets:
# This is a python_requirement()-like target.
if hasattr(maybe_python_req_lib.adaptor, 'requirement'):
all_requirements.append(str(maybe_python_req_lib.requirement))
# This is a python_requirement_library()-like target.
if hasattr(maybe_python_req_lib.adaptor, 'requirements'):
for py_req in maybe_python_req_lib.adaptor.requirements:
all_requirements.append(str(py_req.requirement))
# TODO: This should be configurable, both with interpreter constraints, and for remote execution.
python_binary = sys.executable
output_pytest_requirements_pex_filename = 'pytest-with-requirements.pex'
requirements_pex_argv = [
'./{}'.format(pex_snapshot.files[0].path),
'--python',
python_binary,
'-e',
'pytest:main',
'-o',
output_pytest_requirements_pex_filename,
# TODO: This is non-hermetic because pytest will be resolved on the fly by pex27, where it should be hermetically provided in some way.
# We should probably also specify a specific version.
'pytest',
# Sort all the requirement strings to increase the chance of cache hits across invocations.
] + sorted(all_requirements)
requirements_pex_request = ExecuteProcessRequest(
argv=tuple(requirements_pex_argv),
input_files=pex_snapshot.directory_digest,
description='Resolve requirements for {}'.format(
target_root.address.reference()),
# TODO: This should not be necessary
env={'PATH': os.path.dirname(python_binary)},
output_files=(output_pytest_requirements_pex_filename, ),
)
requirements_pex_response = yield Get(FallibleExecuteProcessResult,
ExecuteProcessRequest,
requirements_pex_request)
# Gather sources.
# TODO: make TargetAdaptor return a 'sources' field with an empty snapshot instead of raising to
# simplify the hasattr() checks here!
all_sources_digests = []
for maybe_source_target in all_targets:
if hasattr(maybe_source_target.adaptor, 'sources'):
sources_snapshot = maybe_source_target.adaptor.sources.snapshot
all_sources_digests.append(sources_snapshot.directory_digest)
all_input_digests = all_sources_digests + [
requirements_pex_response.output_directory_digest,
]
merged_input_files = yield Get(
Digest,
MergedDirectories,
MergedDirectories(directories=tuple(all_input_digests)),
)
request = ExecuteProcessRequest(
argv=('./pytest-with-requirements.pex', ),
input_files=merged_input_files,
description='Run pytest for {}'.format(
target_root.address.reference()),
# TODO: This should not be necessary
env={'PATH': os.path.dirname(python_binary)})
result = yield Get(FallibleExecuteProcessResult, ExecuteProcessRequest,
request)
# TODO: Do something with stderr?
status = Status.SUCCESS if result.exit_code == 0 else Status.FAILURE
yield PyTestResult(status=status, stdout=str(result.stdout))
def download_pex_bin():
# TODO: Inject versions and digests here through some option, rather than hard-coding it.
url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.8/pex'
digest = Digest('2ca320aede7e7bbcb907af54c9de832707a1df965fb5a0d560f2df29ba8a2f3d', 1866441)
snapshot = yield Get(Snapshot, UrlToFetch(url, digest))
yield DownloadedPexBin(executable=snapshot.files[0], directory_digest=snapshot.directory_digest)
def run_python_test(transitive_hydrated_target, pytest):
target_root = transitive_hydrated_target.root
# TODO: Inject versions and digests here through some option, rather than hard-coding it.
interpreter_major, interpreter_minor = sys.version_info[0:2]
pex_name, digest = {
(2, 7):
("pex27",
Digest(
'0ecbf48e3e240a413189194a9f829aec10446705c84db310affe36e23e741dbc',
1812737)),
(3, 6):
("pex36",
Digest(
'ba865e7ce7a840070d58b7ba24e7a67aff058435cfa34202abdd878e7b5d351d',
1812158)),
(3, 7):
("pex37",
Digest(
'51bf8e84d5290fe5ff43d45be78d58eaf88cf2a5e995101c8ff9e6a73a73343d',
1813189))
}.get((interpreter_major, interpreter_minor), (None, None))
if pex_name is None:
raise ValueError(
"Current interpreter {}.{} is not supported, as there is no corresponding PEX to download."
.format(interpreter_major, interpreter_minor))
pex_snapshot = yield Get(
Snapshot,
UrlToFetch(
"https://github.com/pantsbuild/pex/releases/download/v1.6.1/{}".
format(pex_name), digest))
all_targets = [target_root] + [
dep.root for dep in transitive_hydrated_target.dependencies
]
# Produce a pex containing pytest and all transitive 3rdparty requirements.
all_requirements = []
for maybe_python_req_lib in all_targets:
# This is a python_requirement()-like target.
if hasattr(maybe_python_req_lib.adaptor, 'requirement'):
all_requirements.append(str(maybe_python_req_lib.requirement))
# This is a python_requirement_library()-like target.
if hasattr(maybe_python_req_lib.adaptor, 'requirements'):
for py_req in maybe_python_req_lib.adaptor.requirements:
all_requirements.append(str(py_req.requirement))
# TODO: This should be configurable, both with interpreter constraints, and for remote execution.
python_binary = sys.executable
# TODO: This is non-hermetic because the requirements will be resolved on the fly by
# pex27, where it should be hermetically provided in some way.
output_pytest_requirements_pex_filename = 'pytest-with-requirements.pex'
requirements_pex_argv = [
'./{}'.format(pex_snapshot.files[0]),
'--python',
python_binary,
'-e',
'pytest:main',
'-o',
output_pytest_requirements_pex_filename,
# Sort all user requirement strings to increase the chance of cache hits across invocations.
] + list(pytest.get_requirement_strings()) + sorted(all_requirements)
requirements_pex_request = ExecuteProcessRequest(
argv=tuple(requirements_pex_argv),
input_files=pex_snapshot.directory_digest,
description='Resolve requirements for {}'.format(
target_root.address.reference()),
# TODO: This should not be necessary
env={'PATH': os.path.dirname(python_binary)},
output_files=(output_pytest_requirements_pex_filename, ),
)
requirements_pex_response = yield Get(ExecuteProcessResult,
ExecuteProcessRequest,
requirements_pex_request)
# Gather sources.
# TODO: make TargetAdaptor return a 'sources' field with an empty snapshot instead of raising to
# simplify the hasattr() checks here!
all_sources_digests = []
for maybe_source_target in all_targets:
if hasattr(maybe_source_target.adaptor, 'sources'):
sources_snapshot = maybe_source_target.adaptor.sources.snapshot
all_sources_digests.append(sources_snapshot.directory_digest)
all_input_digests = all_sources_digests + [
requirements_pex_response.output_directory_digest,
]
merged_input_files = yield Get(
Digest,
MergedDirectories,
MergedDirectories(directories=tuple(all_input_digests)),
)
request = ExecuteProcessRequest(
argv=('./pytest-with-requirements.pex', ),
input_files=merged_input_files,
description='Run pytest for {}'.format(
target_root.address.reference()),
# TODO: This should not be necessary
env={'PATH': os.path.dirname(python_binary)})
result = yield Get(FallibleExecuteProcessResult, ExecuteProcessRequest,
request)
# TODO: Do something with stderr?
status = Status.SUCCESS if result.exit_code == 0 else Status.FAILURE
yield PyTestResult(status=status, stdout=result.stdout.decode('utf-8'))
def run_python_test(transitive_hydrated_target, pytest):
target_root = transitive_hydrated_target.root
# TODO: Inject versions and digests here through some option, rather than hard-coding it.
url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.6/pex'
digest = Digest('61bb79384db0da8c844678440bd368bcbfac17bbdb865721ad3f9cb0ab29b629', 1826945)
pex_snapshot = yield Get(Snapshot, UrlToFetch(url, digest))
all_targets = [target_root] + [dep.root for dep in transitive_hydrated_target.dependencies]
# Produce a pex containing pytest and all transitive 3rdparty requirements.
all_requirements = []
for maybe_python_req_lib in all_targets:
# This is a python_requirement()-like target.
if hasattr(maybe_python_req_lib.adaptor, 'requirement'):
all_requirements.append(str(maybe_python_req_lib.requirement))
# This is a python_requirement_library()-like target.
if hasattr(maybe_python_req_lib.adaptor, 'requirements'):
for py_req in maybe_python_req_lib.adaptor.requirements:
all_requirements.append(str(py_req.requirement))
# TODO: This should be configurable, both with interpreter constraints, and for remote execution.
# TODO(#7061): This str() can be removed after we drop py2!
python_binary = text_type(sys.executable)
# TODO: This is non-hermetic because the requirements will be resolved on the fly by
# pex27, where it should be hermetically provided in some way.
output_pytest_requirements_pex_filename = 'pytest-with-requirements.pex'
requirements_pex_argv = [
python_binary,
'./{}'.format(pex_snapshot.files[0]),
# TODO(#7061): This text_type() can be removed after we drop py2!
'--python', text_type(python_binary),
'-e', 'pytest:main',
'-o', output_pytest_requirements_pex_filename,
# Sort all user requirement strings to increase the chance of cache hits across invocations.
] + [
# TODO(#7061): This text_type() wrapping can be removed after we drop py2!
text_type(req)
for req in sorted(
list(pytest.get_requirement_strings())
+ list(all_requirements))
]
requirements_pex_request = ExecuteProcessRequest(
argv=tuple(requirements_pex_argv),
input_files=pex_snapshot.directory_digest,
description='Resolve requirements for {}'.format(target_root.address.reference()),
output_files=(output_pytest_requirements_pex_filename,),
)
requirements_pex_response = yield Get(
ExecuteProcessResult, ExecuteProcessRequest, requirements_pex_request)
# Gather sources.
# TODO: make TargetAdaptor return a 'sources' field with an empty snapshot instead of raising to
# simplify the hasattr() checks here!
all_sources_digests = []
for maybe_source_target in all_targets:
if hasattr(maybe_source_target.adaptor, 'sources'):
sources_snapshot = maybe_source_target.adaptor.sources.snapshot
all_sources_digests.append(sources_snapshot.directory_digest)
all_input_digests = all_sources_digests + [
requirements_pex_response.output_directory_digest,
]
merged_input_files = yield Get(
Digest,
MergedDirectories,
MergedDirectories(directories=tuple(all_input_digests)),
)
request = ExecuteProcessRequest(
argv=(python_binary, './{}'.format(output_pytest_requirements_pex_filename)),
input_files=merged_input_files,
description='Run pytest for {}'.format(target_root.address.reference()),
)
result = yield Get(FallibleExecuteProcessResult, ExecuteProcessRequest, request)
# TODO: Do something with stderr?
status = Status.SUCCESS if result.exit_code == 0 else Status.FAILURE
yield PyTestResult(status=status, stdout=result.stdout.decode('utf-8'))
async def download_pex_bin() -> DownloadedPexBin:
# TODO: Inject versions and digests here through some option, rather than hard-coding it.
url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.12/pex'
digest = Digest('ce64cb72cd23d2123dd48126af54ccf2b718d9ecb98c2ed3045ed1802e89e7e1', 1842359)
snapshot = await Get[Snapshot](UrlToFetch(url, digest))
return DownloadedPexBin(SingleFileExecutable(snapshot))
async def download_cloc_script() -> DownloadedClocScript: url = "https://binaries.pantsbuild.org/bin/cloc/1.80/cloc" sha_256 = "2b23012b1c3c53bd6b9dd43cd6aa75715eed4feb2cb6db56ac3fbbd2dffeac9d" digest = Digest(sha_256, 546279) snapshot = await Get[Snapshot](UrlToFetch(url, digest)) return DownloadedClocScript(SingleFileExecutable(snapshot))
