def test_download_https() -> None:
# This also tests that the custom certs functionality works.
with temporary_dir() as temp_dir:
def write_resource(name: str) -> Path:
path = Path(temp_dir) / name
data = pkgutil.get_data("pants.engine.internals", f"fs_test_data/tls/rsa/{name}")
assert data is not None
path.write_bytes(data)
return path
server_cert = write_resource("server.crt")
server_key = write_resource("server.key")
cert_chain = write_resource("server.chain")
rule_runner = RuleRunner(
rules=[QueryRule(Snapshot, [DownloadFile])],
isolated_local_store=True,
ca_certs_path=str(cert_chain),
)
ssl_context = ssl.SSLContext()
ssl_context.load_cert_chain(certfile=str(server_cert), keyfile=str(server_key))
with http_server(StubHandler, ssl_context=ssl_context) as port:
snapshot = rule_runner.request(
Snapshot,
[DownloadFile(f"https://localhost:{port}/file.txt", DOWNLOADS_FILE_DIGEST)],
)
assert snapshot.files == ("file.txt",)
assert snapshot.digest == DOWNLOADS_EXPECTED_DIRECTORY_DIGEST
def test_download_missing_file(downloads_rule_runner: RuleRunner) -> None:
with pytest.raises(ExecutionError) as exc:
with http_server(StubHandler) as port:
downloads_rule_runner.request(
Snapshot, [DownloadFile(f"http://localhost:{port}/notfound", DOWNLOADS_FILE_DIGEST)]
)
assert "404" in str(exc.value)
def get_request(self, plat: Platform) -> ExternalToolRequest:
"""Generate a request for this tool."""
for known_version in self.known_versions:
try:
ver, plat_val, sha256, length = (
x.strip() for x in known_version.split("|"))
except ValueError:
raise ExternalToolError(
f"Bad value for --known-versions (see {self.options.pants_bin_name} "
f"help-advanced {self.options_scope}): {known_version}")
if plat_val == plat.value and ver == self.version:
digest = FileDigest(fingerprint=sha256,
serialized_bytes_length=int(length))
try:
url = self.generate_url(plat)
exe = self.generate_exe(plat)
except ExternalToolError as e:
raise ExternalToolError(
f"Couldn't find {self.name} version {self.version} on {plat.value}"
) from e
return ExternalToolRequest(
DownloadFile(url=url, expected_digest=digest), exe)
raise UnknownVersion(
f"No known version of {self.name} {self.version} for {plat.value} found in "
f"{self.known_versions}")
def test_download_https(self) -> None:
# Note that this also tests that the custom certs functionality works.
with temporary_dir() as temp_dir:
def write_resource(name: str) -> Path:
path = Path(temp_dir) / name
data = pkgutil.get_data("pants.engine.internals",
f"tls_testing/rsa/{name}")
assert data is not None
path.write_bytes(data)
return path
server_cert = write_resource("server.crt")
server_key = write_resource("server.key")
cert_chain = write_resource("server.chain")
scheduler = self.mk_scheduler(
rules=[*fs_rules(),
QueryRule(Snapshot, (DownloadFile, ))],
ca_certs_path=str(cert_chain),
)
with self.isolated_local_store():
ssl_context = ssl.SSLContext()
ssl_context.load_cert_chain(certfile=str(server_cert),
keyfile=str(server_key))
with http_server(StubHandler, ssl_context=ssl_context) as port:
snapshot = self.execute(
scheduler,
Snapshot,
DownloadFile(f"https://localhost:{port}/file.txt",
self.file_digest),
)[0]
self.assert_snapshot_equals(snapshot, ["file.txt"],
self.expected_snapshot_digest)
def test_download_valid(downloads_rule_runner: RuleRunner) -> None:
with http_server(StubHandler) as port:
snapshot = downloads_rule_runner.request(
Snapshot, [DownloadFile(f"http://localhost:{port}/file.txt", DOWNLOADS_FILE_DIGEST)]
)
assert snapshot.files == ("file.txt",)
assert snapshot.digest == DOWNLOADS_EXPECTED_DIRECTORY_DIGEST
def test_download_wrong_digest(downloads_rule_runner: RuleRunner) -> None:
file_digest = FileDigest(DOWNLOADS_FILE_DIGEST.fingerprint,
DOWNLOADS_FILE_DIGEST.serialized_bytes_length + 1)
with pytest.raises(ExecutionError) as exc:
with http_server(StubHandler) as port:
downloads_rule_runner.request(Snapshot, [
DownloadFile(f"http://localhost:{port}/file.txt", file_digest)
])
assert "wrong digest" in str(exc.value).lower()
def test_download_missing_file(self) -> None:
with self.isolated_local_store():
with http_server(StubHandler) as port:
with self.assertRaises(ExecutionError) as cm:
self.request_single_product(
Snapshot,
DownloadFile(f"http://localhost:{port}/notfound", self.pantsbuild_digest),
)
assert "404" in str(cm.exception)
def test_download_file(downloads_rule_runner: RuleRunner) -> None:
with temporary_dir() as temp_dir:
roland = Path(temp_dir, "roland")
roland.write_text("European Burmese")
snapshot = downloads_rule_runner.request(
Snapshot,
[DownloadFile(f"file:{roland}", ROLAND_FILE_DIGEST)],
)
assert snapshot.files == ("roland",)
assert snapshot.digest == ROLAND_DOWNLOAD_DIGEST
def do_test(expected_url: str, expected_length: int, expected_sha256: str,
plat: Platform, version: str) -> None:
foobar = create_subsystem(FooBar,
version=version,
known_versions=FooBar.default_known_versions)
assert (ExternalToolRequest(
DownloadFile(url=expected_url,
expected_digest=FileDigest(expected_sha256,
expected_length)),
f"foobar-{version}/bin/foobar",
) == foobar.get_request(plat))
def get_request_for(self, plat_val: str, sha256: str,
length: int) -> ExternalToolRequest:
"""Generate a request for this tool from the given info."""
plat = Platform(plat_val)
digest = FileDigest(fingerprint=sha256, serialized_bytes_length=length)
try:
url = self.generate_url(plat)
exe = self.generate_exe(plat)
except ExternalToolError as e:
raise ExternalToolError(
f"Couldn't find {self.name} version {self.version} on {plat.value}"
) from e
return ExternalToolRequest(
DownloadFile(url=url, expected_digest=digest), exe)
def test_download_https(self) -> None:
with self.isolated_local_store():
snapshot = self.request_single_product(
Snapshot,
DownloadFile(
"https://binaries.pantsbuild.org/do_not_remove_or_edit.txt",
Digest("f461fc99bcbe18e667687cf672c2dc68dc5c5db77c5bd426c9690e5c9cec4e3b", 184),
),
)
self.assert_snapshot_equals(
snapshot,
["do_not_remove_or_edit.txt"],
Digest("03bb499daabafc60212d2f4b2fab49b47b35b83a90c056224c768d52bce02691", 102),
)
def test_download(self) -> None:
with self.isolated_local_store():
with http_server(StubHandler) as port:
snapshot = self.request_single_product(
Snapshot,
DownloadFile(
f"http://localhost:{port}/do_not_remove_or_edit.txt", self.pantsbuild_digest
),
)
self.assert_snapshot_equals(
snapshot,
["do_not_remove_or_edit.txt"],
Digest("03bb499daabafc60212d2f4b2fab49b47b35b83a90c056224c768d52bce02691", 102),
)
def test_download_caches(downloads_rule_runner: RuleRunner) -> None:
# We put the expected content in the store, but because we have never fetched it from this
# URL, we confirm the URL and attempt to refetch. Once it is cached, it does not need to be
# refetched.
prime_store_with_roland_digest(downloads_rule_runner)
with temporary_dir() as temp_dir:
roland = Path(temp_dir, "roland")
roland.write_text("European Burmese")
snapshot = downloads_rule_runner.request(
Snapshot,
[DownloadFile(f"file:{roland}", ROLAND_FILE_DIGEST)],
)
assert snapshot.files == ("roland",)
assert snapshot.digest == ROLAND_DOWNLOAD_DIGEST
def test_download(self) -> None:
with self.isolated_local_store():
with http_server(StubHandler) as port:
snapshot = self.request(
Snapshot,
[
DownloadFile(f"http://localhost:{port}/file.txt",
self.file_digest)
],
)
self.assert_snapshot_equals(
snapshot,
["file.txt"],
self.expected_snapshot_digest,
)
def test_download_wrong_digest(self) -> None:
with self.isolated_local_store():
with http_server(StubHandler) as port:
with self.assertRaises(ExecutionError) as cm:
self.request_single_product(
Snapshot,
DownloadFile(
f"http://localhost:{port}/do_not_remove_or_edit.txt",
Digest(
self.pantsbuild_digest.fingerprint,
self.pantsbuild_digest.serialized_bytes_length + 1,
),
),
)
assert "wrong digest" in str(cm.exception).lower()
def test_download_caches(downloads_rule_runner: RuleRunner) -> None:
# We would error if we hit the HTTP server with 404, but we're not going to hit the HTTP
# server because it's cached, so we shouldn't see an error.
prime_store_with_roland_digest(downloads_rule_runner)
with http_server(StubHandler) as port:
download_file = DownloadFile(
f"http://localhost:{port}/roland",
FileDigest(
"693d8db7b05e99c6b7a7c0616456039d89c555029026936248085193559a0b5d",
16),
)
snapshot = downloads_rule_runner.request(Snapshot, [download_file])
assert snapshot.files == ("roland", )
assert snapshot.digest == Digest(
"9341f76bef74170bedffe51e4f2e233f61786b7752d21c2339f8ee6070eba819", 82)
def test_caches_downloads(self) -> None:
with self.isolated_local_store():
with http_server(StubHandler) as port:
self.prime_store_with_roland_digest()
# This would error if we hit the HTTP server, because 404,
# but we're not going to hit the HTTP server because it's cached,
# so we shouldn't see an error...
url = DownloadFile(
f"http://localhost:{port}/roland",
Digest("693d8db7b05e99c6b7a7c0616456039d89c555029026936248085193559a0b5d", 16),
)
snapshot = self.request_single_product(Snapshot, url)
self.assert_snapshot_equals(
snapshot,
["roland"],
Digest("9341f76bef74170bedffe51e4f2e233f61786b7752d21c2339f8ee6070eba819", 82),
)
def test_download_wrong_digest(self) -> None:
with self.isolated_local_store():
with http_server(StubHandler) as port:
with self.assertRaises(ExecutionError) as cm:
self.request(
Snapshot,
[
DownloadFile(
f"http://localhost:{port}/file.txt",
FileDigest(
self.file_digest.fingerprint,
self.file_digest.serialized_bytes_length +
1,
),
)
],
)
assert "wrong digest" in str(cm.exception).lower()
def do_test(expected_url: str, expected_length: int, expected_sha256: str,
plat: Platform, version: str) -> None:
foobar = create_subsystem(
FooBar,
version=version,
known_versions=FooBar.default_known_versions,
)
templated_foobar = create_subsystem(
TemplatedFooBar,
version=version,
known_versions=TemplatedFooBar.default_known_versions,
url_template=TemplatedFooBar.default_url_template,
url_platform_mapping=TemplatedFooBar.default_url_platform_mapping,
)
expected = ExternalToolRequest(
DownloadFile(url=expected_url,
expected_digest=FileDigest(expected_sha256,
expected_length)),
f"foobar-{version}/bin/foobar",
)
assert expected == foobar.get_request(plat)
assert expected == templated_foobar.get_request(plat)
async def _hydrate_asset_source(
request: GenerateSourcesRequest) -> GeneratedSources:
target = request.protocol_target
source_field = target[AssetSourceField]
if isinstance(source_field.value, str):
return GeneratedSources(request.protocol_sources)
http_source = source_field.value
file_digest = FileDigest(http_source.sha256, http_source.len)
# NB: This just has to run, we don't actually need the result because we know the Digest's
# FileEntry metadata.
await Get(Digest, DownloadFile(http_source.url, file_digest))
snapshot = await Get(
Snapshot,
CreateDigest([
FileEntry(
path=source_field.file_path,
file_digest=file_digest,
)
]),
)
return GeneratedSources(snapshot)
async def setup_shunit2_for_target(
request: TestSetupRequest,
shell_setup: ShellSetup,
test_subsystem: TestSubsystem,
test_extra_env: TestExtraEnv,
global_options: GlobalOptions,
) -> TestSetup:
shunit2_download_file = DownloadFile(
"https://raw.githubusercontent.com/kward/shunit2/b9102bb763cc603b3115ed30a5648bf950548097/shunit2",
FileDigest(
"1f11477b7948150d1ca50cdd41d89be4ed2acd137e26d2e0fe23966d0e272cc5",
40987),
)
shunit2_script, transitive_targets, built_package_dependencies, env = await MultiGet(
Get(Digest, DownloadFile, shunit2_download_file),
Get(TransitiveTargets,
TransitiveTargetsRequest([request.field_set.address])),
Get(
BuiltPackageDependencies,
BuildPackageDependenciesRequest(
request.field_set.runtime_package_dependencies),
),
Get(Environment, EnvironmentRequest(["PATH"])),
)
dependencies_source_files_request = Get(
SourceFiles,
SourceFilesRequest(
(tgt.get(SourcesField) for tgt in transitive_targets.dependencies),
for_sources_types=(ShellSourceField, FileSourceField,
ResourceSourceField),
enable_codegen=True,
),
)
dependencies_source_files, field_set_sources = await MultiGet(
dependencies_source_files_request,
Get(SourceFiles, SourceFilesRequest([request.field_set.sources])),
)
field_set_digest_content = await Get(DigestContents, Digest,
field_set_sources.snapshot.digest)
# `ShellTestSourceField` validates that there's exactly one file.
test_file_content = field_set_digest_content[0]
updated_test_file_content = add_source_shunit2(test_file_content)
updated_test_digest, runner = await MultiGet(
Get(Digest, CreateDigest([updated_test_file_content])),
Get(
Shunit2Runner,
Shunit2RunnerRequest(request.field_set.address, test_file_content,
request.field_set.shell),
),
)
input_digest = await Get(
Digest,
MergeDigests((
shunit2_script,
updated_test_digest,
dependencies_source_files.snapshot.digest,
*(pkg.digest for pkg in built_package_dependencies),
)),
)
env_dict = {
"PATH": create_path_env_var(shell_setup.executable_search_path(env)),
"SHUNIT_COLOR": "always" if global_options.colors else "none",
**test_extra_env.env,
}
argv = (
# Zsh requires extra args. See https://github.com/kward/shunit2/#-zsh.
[
runner.binary_path.path, "-o", "shwordsplit", "--",
*field_set_sources.snapshot.files
] if runner.shell == Shunit2Shell.zsh else
[runner.binary_path.path, *field_set_sources.snapshot.files])
cache_scope = (ProcessCacheScope.PER_SESSION
if test_subsystem.force else ProcessCacheScope.SUCCESSFUL)
process = Process(
argv=argv,
input_digest=input_digest,
description=f"Run shunit2 for {request.field_set.address}.",
level=LogLevel.DEBUG,
env=env_dict,
timeout_seconds=request.field_set.timeout.value,
cache_scope=cache_scope,
)
return TestSetup(process)
async def setup_shunit2_for_target(
request: TestSetupRequest,
bash_program: BashProgram,
bash_setup: BashSetup,
test_subsystem: TestSubsystem,
) -> TestSetup:
# Because shunit2 is a simple Bash file, we download it using `DownloadFile`. Normally, we
# would install the test runner through `ExternalTool`. See
# https://www.pantsbuild.org/v2.0/docs/rules-api-installing-tools and
# https://www.pantsbuild.org/v2.0/docs/rules-api-file-system.
shunit2_script_request = Get(
Digest,
DownloadFile(
url=
"https://raw.githubusercontent.com/kward/shunit2/b9102bb763cc603b3115ed30a5648bf950548097/shunit2",
expected_digest=Digest(
"1f11477b7948150d1ca50cdd41d89be4ed2acd137e26d2e0fe23966d0e272cc5",
40987,
),
),
)
transitive_targets_request = Get(
TransitiveTargets,
TransitiveTargetsRequest([request.field_set.address]))
shunit2_script, transitive_targets = await MultiGet(
shunit2_script_request, transitive_targets_request)
# We need to include all relevant transitive dependencies in the environment. We also get the
# test's sources so that we can check that it has `source ./shunit2` at the bottom of it.
#
# Because we might modify the test files, we leave the tests out of
# `dependencies_source_files_request` by using `transitive_targets.dependencies` instead of
# `transitive_targets.closure`. This makes sure that we don't accidentally include the
# unmodified test files and the modified test files in the same input. See
# https://www.pantsbuild.org/v2.0/docs/rules-api-and-target-api.
dependencies_source_files_request = Get(
SourceFiles,
SourceFilesRequest(
(tgt.get(Sources) for tgt in transitive_targets.dependencies),
for_sources_types=(BashSources, FilesSources, ResourcesSources),
),
)
test_source_files_request = Get(
SourceFiles, SourceFilesRequest([request.field_set.sources]))
dependencies_source_files, test_source_files = await MultiGet(
dependencies_source_files_request, test_source_files_request)
# To check if the test files already have `source ./shunit2` in them, we need to look at the
# actual file content. We use `DigestContents` for this, and then use `CreateDigest` to create
# a digest of the (possibly) updated test files. See
# https://www.pantsbuild.org/v2.0/docs/rules-api-file-system.
#
# Most test runners don't modify their test files like we do here, so most test runners can
# skip this step.
test_files_content = await Get(DigestContents, Digest,
test_source_files.snapshot.digest)
updated_test_files_content = []
for file_content in test_files_content:
if (b"source ./shunit2" in file_content.content
or b". ./shunit2" in file_content.content):
updated_test_files_content.append(file_content)
else:
updated_file_content = FileContent(
path=file_content.path,
content=file_content.content + b"\nsource ./shunit2\n",
)
updated_test_files_content.append(updated_file_content)
updated_test_source_files = await Get(
Digest, CreateDigest(updated_test_files_content))
# The Process needs one single `Digest`, so we merge everything together. See
# https://www.pantsbuild.org/v2.0/docs/rules-api-file-system.
input_digest = await Get(
Digest,
MergeDigests([
shunit2_script,
updated_test_source_files,
dependencies_source_files.snapshot.digest,
]),
)
# We must check if `test --force` was used, and if so, use a hack to invalidate the cache by
# mixing in a randomly generated UUID into the environment.
extra_env = {}
if test_subsystem.force and not request.is_debug:
uuid = await Get(UUID, UUIDRequest())
extra_env["__PANTS_FORCE_TEST_RUN__"] = str(uuid)
process = Process(
argv=[bash_program.exe, *test_source_files.snapshot.files],
input_digest=input_digest,
description=f"Run shunit2 on {request.field_set.address}.",
level=LogLevel.DEBUG,
env=bash_setup.env_dict,
timeout_seconds=request.field_set.timeout.value,
)
return TestSetup(process)
async def setup_shunit2_for_target(
request: TestSetupRequest,
shell_setup: ShellSetup,
test_subsystem: TestSubsystem,
test_extra_env: TestExtraEnv,
global_options: GlobalOptions,
) -> TestSetup:
shunit2_download_file = DownloadFile(
"https://raw.githubusercontent.com/kward/shunit2/b9102bb763cc603b3115ed30a5648bf950548097/shunit2",
FileDigest(
"1f11477b7948150d1ca50cdd41d89be4ed2acd137e26d2e0fe23966d0e272cc5",
40987),
)
shunit2_script, transitive_targets, built_package_dependencies, env = await MultiGet(
Get(Digest, DownloadFile, shunit2_download_file),
Get(TransitiveTargets,
TransitiveTargetsRequest([request.field_set.address])),
Get(
BuiltPackageDependencies,
BuildPackageDependenciesRequest(
request.field_set.runtime_package_dependencies),
),
Get(Environment, EnvironmentRequest(["PATH"])),
)
dependencies_source_files_request = Get(
SourceFiles,
SourceFilesRequest(
(tgt.get(Sources) for tgt in transitive_targets.dependencies),
for_sources_types=(ShellSources, FilesSources, ResourcesSources),
enable_codegen=True,
),
)
dependencies_source_files, field_set_sources = await MultiGet(
dependencies_source_files_request,
Get(SourceFiles, SourceFilesRequest([request.field_set.sources])),
)
field_set_digest_content = await Get(DigestContents, Digest,
field_set_sources.snapshot.digest)
# Because a FieldSet corresponds to a file address, there should be exactly 1 file in the
# sources. This assumption allows us to simplify determining which shell to use via inspecting
# the shebang.
if len(field_set_digest_content) != 1:
raise AssertionError(
f"The file address {request.field_set.address} had sources != 1, which is unexpected: "
f"{field_set_sources.snapshot.files}. Please file a bug at "
"https://github.com/pantsbuild/pants/issues/new with this error message copied."
)
original_test_file_content = field_set_digest_content[0]
updated_test_file_content = add_source_shunit2(original_test_file_content)
updated_test_digest, runner = await MultiGet(
Get(Digest, CreateDigest([updated_test_file_content])),
Get(
Shunit2Runner,
Shunit2RunnerRequest(request.field_set.address,
original_test_file_content,
request.field_set.shell),
),
)
input_digest = await Get(
Digest,
MergeDigests((
shunit2_script,
updated_test_digest,
dependencies_source_files.snapshot.digest,
*(pkg.digest for pkg in built_package_dependencies),
)),
)
env_dict = {
"PATH": create_path_env_var(shell_setup.executable_search_path(env)),
"SHUNIT_COLOR": "always" if global_options.options.colors else "none",
**test_extra_env.env,
}
argv = (
# Zsh requires extra args. See https://github.com/kward/shunit2/#-zsh.
[
runner.binary_path.path, "-o", "shwordsplit", "--",
*field_set_sources.snapshot.files
] if runner.shell == Shunit2Shell.zsh else
[runner.binary_path.path, *field_set_sources.snapshot.files])
cache_scope = ProcessCacheScope.NEVER if test_subsystem.force else ProcessCacheScope.SUCCESSFUL
process = Process(
argv=argv,
input_digest=input_digest,
description=f"Run shunit2 for {request.field_set.address}.",
level=LogLevel.DEBUG,
env=env_dict,
timeout_seconds=request.field_set.timeout.value,
cache_scope=cache_scope,
)
return TestSetup(process)