def test_download_https() -> None: # This also tests that the custom certs functionality works. with temporary_dir() as temp_dir: def write_resource(name: str) -> Path: path = Path(temp_dir) / name data = pkgutil.get_data("pants.engine.internals", f"fs_test_data/tls/rsa/{name}") assert data is not None path.write_bytes(data) return path server_cert = write_resource("server.crt") server_key = write_resource("server.key") cert_chain = write_resource("server.chain") rule_runner = RuleRunner( rules=[QueryRule(Snapshot, [DownloadFile])], isolated_local_store=True, ca_certs_path=str(cert_chain), ) ssl_context = ssl.SSLContext() ssl_context.load_cert_chain(certfile=str(server_cert), keyfile=str(server_key)) with http_server(StubHandler, ssl_context=ssl_context) as port: snapshot = rule_runner.request( Snapshot, [DownloadFile(f"https://localhost:{port}/file.txt", DOWNLOADS_FILE_DIGEST)], ) assert snapshot.files == ("file.txt",) assert snapshot.digest == DOWNLOADS_EXPECTED_DIRECTORY_DIGEST
def test_download_missing_file(downloads_rule_runner: RuleRunner) -> None: with pytest.raises(ExecutionError) as exc: with http_server(StubHandler) as port: downloads_rule_runner.request( Snapshot, [DownloadFile(f"http://localhost:{port}/notfound", DOWNLOADS_FILE_DIGEST)] ) assert "404" in str(exc.value)
def get_request(self, plat: Platform) -> ExternalToolRequest: """Generate a request for this tool.""" for known_version in self.known_versions: try: ver, plat_val, sha256, length = ( x.strip() for x in known_version.split("|")) except ValueError: raise ExternalToolError( f"Bad value for --known-versions (see {self.options.pants_bin_name} " f"help-advanced {self.options_scope}): {known_version}") if plat_val == plat.value and ver == self.version: digest = FileDigest(fingerprint=sha256, serialized_bytes_length=int(length)) try: url = self.generate_url(plat) exe = self.generate_exe(plat) except ExternalToolError as e: raise ExternalToolError( f"Couldn't find {self.name} version {self.version} on {plat.value}" ) from e return ExternalToolRequest( DownloadFile(url=url, expected_digest=digest), exe) raise UnknownVersion( f"No known version of {self.name} {self.version} for {plat.value} found in " f"{self.known_versions}")
def test_download_https(self) -> None: # Note that this also tests that the custom certs functionality works. with temporary_dir() as temp_dir: def write_resource(name: str) -> Path: path = Path(temp_dir) / name data = pkgutil.get_data("pants.engine.internals", f"tls_testing/rsa/{name}") assert data is not None path.write_bytes(data) return path server_cert = write_resource("server.crt") server_key = write_resource("server.key") cert_chain = write_resource("server.chain") scheduler = self.mk_scheduler( rules=[*fs_rules(), QueryRule(Snapshot, (DownloadFile, ))], ca_certs_path=str(cert_chain), ) with self.isolated_local_store(): ssl_context = ssl.SSLContext() ssl_context.load_cert_chain(certfile=str(server_cert), keyfile=str(server_key)) with http_server(StubHandler, ssl_context=ssl_context) as port: snapshot = self.execute( scheduler, Snapshot, DownloadFile(f"https://localhost:{port}/file.txt", self.file_digest), )[0] self.assert_snapshot_equals(snapshot, ["file.txt"], self.expected_snapshot_digest)
def test_download_valid(downloads_rule_runner: RuleRunner) -> None: with http_server(StubHandler) as port: snapshot = downloads_rule_runner.request( Snapshot, [DownloadFile(f"http://localhost:{port}/file.txt", DOWNLOADS_FILE_DIGEST)] ) assert snapshot.files == ("file.txt",) assert snapshot.digest == DOWNLOADS_EXPECTED_DIRECTORY_DIGEST
def test_download_wrong_digest(downloads_rule_runner: RuleRunner) -> None: file_digest = FileDigest(DOWNLOADS_FILE_DIGEST.fingerprint, DOWNLOADS_FILE_DIGEST.serialized_bytes_length + 1) with pytest.raises(ExecutionError) as exc: with http_server(StubHandler) as port: downloads_rule_runner.request(Snapshot, [ DownloadFile(f"http://localhost:{port}/file.txt", file_digest) ]) assert "wrong digest" in str(exc.value).lower()
def test_download_missing_file(self) -> None: with self.isolated_local_store(): with http_server(StubHandler) as port: with self.assertRaises(ExecutionError) as cm: self.request_single_product( Snapshot, DownloadFile(f"http://localhost:{port}/notfound", self.pantsbuild_digest), ) assert "404" in str(cm.exception)
def test_download_file(downloads_rule_runner: RuleRunner) -> None: with temporary_dir() as temp_dir: roland = Path(temp_dir, "roland") roland.write_text("European Burmese") snapshot = downloads_rule_runner.request( Snapshot, [DownloadFile(f"file:{roland}", ROLAND_FILE_DIGEST)], ) assert snapshot.files == ("roland",) assert snapshot.digest == ROLAND_DOWNLOAD_DIGEST
def do_test(expected_url: str, expected_length: int, expected_sha256: str, plat: Platform, version: str) -> None: foobar = create_subsystem(FooBar, version=version, known_versions=FooBar.default_known_versions) assert (ExternalToolRequest( DownloadFile(url=expected_url, expected_digest=FileDigest(expected_sha256, expected_length)), f"foobar-{version}/bin/foobar", ) == foobar.get_request(plat))
def get_request_for(self, plat_val: str, sha256: str, length: int) -> ExternalToolRequest: """Generate a request for this tool from the given info.""" plat = Platform(plat_val) digest = FileDigest(fingerprint=sha256, serialized_bytes_length=length) try: url = self.generate_url(plat) exe = self.generate_exe(plat) except ExternalToolError as e: raise ExternalToolError( f"Couldn't find {self.name} version {self.version} on {plat.value}" ) from e return ExternalToolRequest( DownloadFile(url=url, expected_digest=digest), exe)
def test_download_https(self) -> None: with self.isolated_local_store(): snapshot = self.request_single_product( Snapshot, DownloadFile( "https://binaries.pantsbuild.org/do_not_remove_or_edit.txt", Digest("f461fc99bcbe18e667687cf672c2dc68dc5c5db77c5bd426c9690e5c9cec4e3b", 184), ), ) self.assert_snapshot_equals( snapshot, ["do_not_remove_or_edit.txt"], Digest("03bb499daabafc60212d2f4b2fab49b47b35b83a90c056224c768d52bce02691", 102), )
def test_download(self) -> None: with self.isolated_local_store(): with http_server(StubHandler) as port: snapshot = self.request_single_product( Snapshot, DownloadFile( f"http://localhost:{port}/do_not_remove_or_edit.txt", self.pantsbuild_digest ), ) self.assert_snapshot_equals( snapshot, ["do_not_remove_or_edit.txt"], Digest("03bb499daabafc60212d2f4b2fab49b47b35b83a90c056224c768d52bce02691", 102), )
def test_download_caches(downloads_rule_runner: RuleRunner) -> None: # We put the expected content in the store, but because we have never fetched it from this # URL, we confirm the URL and attempt to refetch. Once it is cached, it does not need to be # refetched. prime_store_with_roland_digest(downloads_rule_runner) with temporary_dir() as temp_dir: roland = Path(temp_dir, "roland") roland.write_text("European Burmese") snapshot = downloads_rule_runner.request( Snapshot, [DownloadFile(f"file:{roland}", ROLAND_FILE_DIGEST)], ) assert snapshot.files == ("roland",) assert snapshot.digest == ROLAND_DOWNLOAD_DIGEST
def test_download(self) -> None: with self.isolated_local_store(): with http_server(StubHandler) as port: snapshot = self.request( Snapshot, [ DownloadFile(f"http://localhost:{port}/file.txt", self.file_digest) ], ) self.assert_snapshot_equals( snapshot, ["file.txt"], self.expected_snapshot_digest, )
def test_download_wrong_digest(self) -> None: with self.isolated_local_store(): with http_server(StubHandler) as port: with self.assertRaises(ExecutionError) as cm: self.request_single_product( Snapshot, DownloadFile( f"http://localhost:{port}/do_not_remove_or_edit.txt", Digest( self.pantsbuild_digest.fingerprint, self.pantsbuild_digest.serialized_bytes_length + 1, ), ), ) assert "wrong digest" in str(cm.exception).lower()
def test_download_caches(downloads_rule_runner: RuleRunner) -> None: # We would error if we hit the HTTP server with 404, but we're not going to hit the HTTP # server because it's cached, so we shouldn't see an error. prime_store_with_roland_digest(downloads_rule_runner) with http_server(StubHandler) as port: download_file = DownloadFile( f"http://localhost:{port}/roland", FileDigest( "693d8db7b05e99c6b7a7c0616456039d89c555029026936248085193559a0b5d", 16), ) snapshot = downloads_rule_runner.request(Snapshot, [download_file]) assert snapshot.files == ("roland", ) assert snapshot.digest == Digest( "9341f76bef74170bedffe51e4f2e233f61786b7752d21c2339f8ee6070eba819", 82)
def test_caches_downloads(self) -> None: with self.isolated_local_store(): with http_server(StubHandler) as port: self.prime_store_with_roland_digest() # This would error if we hit the HTTP server, because 404, # but we're not going to hit the HTTP server because it's cached, # so we shouldn't see an error... url = DownloadFile( f"http://localhost:{port}/roland", Digest("693d8db7b05e99c6b7a7c0616456039d89c555029026936248085193559a0b5d", 16), ) snapshot = self.request_single_product(Snapshot, url) self.assert_snapshot_equals( snapshot, ["roland"], Digest("9341f76bef74170bedffe51e4f2e233f61786b7752d21c2339f8ee6070eba819", 82), )
def test_download_wrong_digest(self) -> None: with self.isolated_local_store(): with http_server(StubHandler) as port: with self.assertRaises(ExecutionError) as cm: self.request( Snapshot, [ DownloadFile( f"http://localhost:{port}/file.txt", FileDigest( self.file_digest.fingerprint, self.file_digest.serialized_bytes_length + 1, ), ) ], ) assert "wrong digest" in str(cm.exception).lower()
def do_test(expected_url: str, expected_length: int, expected_sha256: str, plat: Platform, version: str) -> None: foobar = create_subsystem( FooBar, version=version, known_versions=FooBar.default_known_versions, ) templated_foobar = create_subsystem( TemplatedFooBar, version=version, known_versions=TemplatedFooBar.default_known_versions, url_template=TemplatedFooBar.default_url_template, url_platform_mapping=TemplatedFooBar.default_url_platform_mapping, ) expected = ExternalToolRequest( DownloadFile(url=expected_url, expected_digest=FileDigest(expected_sha256, expected_length)), f"foobar-{version}/bin/foobar", ) assert expected == foobar.get_request(plat) assert expected == templated_foobar.get_request(plat)
async def _hydrate_asset_source( request: GenerateSourcesRequest) -> GeneratedSources: target = request.protocol_target source_field = target[AssetSourceField] if isinstance(source_field.value, str): return GeneratedSources(request.protocol_sources) http_source = source_field.value file_digest = FileDigest(http_source.sha256, http_source.len) # NB: This just has to run, we don't actually need the result because we know the Digest's # FileEntry metadata. await Get(Digest, DownloadFile(http_source.url, file_digest)) snapshot = await Get( Snapshot, CreateDigest([ FileEntry( path=source_field.file_path, file_digest=file_digest, ) ]), ) return GeneratedSources(snapshot)
async def setup_shunit2_for_target( request: TestSetupRequest, shell_setup: ShellSetup, test_subsystem: TestSubsystem, test_extra_env: TestExtraEnv, global_options: GlobalOptions, ) -> TestSetup: shunit2_download_file = DownloadFile( "https://raw.githubusercontent.com/kward/shunit2/b9102bb763cc603b3115ed30a5648bf950548097/shunit2", FileDigest( "1f11477b7948150d1ca50cdd41d89be4ed2acd137e26d2e0fe23966d0e272cc5", 40987), ) shunit2_script, transitive_targets, built_package_dependencies, env = await MultiGet( Get(Digest, DownloadFile, shunit2_download_file), Get(TransitiveTargets, TransitiveTargetsRequest([request.field_set.address])), Get( BuiltPackageDependencies, BuildPackageDependenciesRequest( request.field_set.runtime_package_dependencies), ), Get(Environment, EnvironmentRequest(["PATH"])), ) dependencies_source_files_request = Get( SourceFiles, SourceFilesRequest( (tgt.get(SourcesField) for tgt in transitive_targets.dependencies), for_sources_types=(ShellSourceField, FileSourceField, ResourceSourceField), enable_codegen=True, ), ) dependencies_source_files, field_set_sources = await MultiGet( dependencies_source_files_request, Get(SourceFiles, SourceFilesRequest([request.field_set.sources])), ) field_set_digest_content = await Get(DigestContents, Digest, field_set_sources.snapshot.digest) # `ShellTestSourceField` validates that there's exactly one file. test_file_content = field_set_digest_content[0] updated_test_file_content = add_source_shunit2(test_file_content) updated_test_digest, runner = await MultiGet( Get(Digest, CreateDigest([updated_test_file_content])), Get( Shunit2Runner, Shunit2RunnerRequest(request.field_set.address, test_file_content, request.field_set.shell), ), ) input_digest = await Get( Digest, MergeDigests(( shunit2_script, updated_test_digest, dependencies_source_files.snapshot.digest, *(pkg.digest for pkg in built_package_dependencies), )), ) env_dict = { "PATH": create_path_env_var(shell_setup.executable_search_path(env)), "SHUNIT_COLOR": "always" if global_options.colors else "none", **test_extra_env.env, } argv = ( # Zsh requires extra args. See https://github.com/kward/shunit2/#-zsh. [ runner.binary_path.path, "-o", "shwordsplit", "--", *field_set_sources.snapshot.files ] if runner.shell == Shunit2Shell.zsh else [runner.binary_path.path, *field_set_sources.snapshot.files]) cache_scope = (ProcessCacheScope.PER_SESSION if test_subsystem.force else ProcessCacheScope.SUCCESSFUL) process = Process( argv=argv, input_digest=input_digest, description=f"Run shunit2 for {request.field_set.address}.", level=LogLevel.DEBUG, env=env_dict, timeout_seconds=request.field_set.timeout.value, cache_scope=cache_scope, ) return TestSetup(process)
async def setup_shunit2_for_target( request: TestSetupRequest, bash_program: BashProgram, bash_setup: BashSetup, test_subsystem: TestSubsystem, ) -> TestSetup: # Because shunit2 is a simple Bash file, we download it using `DownloadFile`. Normally, we # would install the test runner through `ExternalTool`. See # https://www.pantsbuild.org/v2.0/docs/rules-api-installing-tools and # https://www.pantsbuild.org/v2.0/docs/rules-api-file-system. shunit2_script_request = Get( Digest, DownloadFile( url= "https://raw.githubusercontent.com/kward/shunit2/b9102bb763cc603b3115ed30a5648bf950548097/shunit2", expected_digest=Digest( "1f11477b7948150d1ca50cdd41d89be4ed2acd137e26d2e0fe23966d0e272cc5", 40987, ), ), ) transitive_targets_request = Get( TransitiveTargets, TransitiveTargetsRequest([request.field_set.address])) shunit2_script, transitive_targets = await MultiGet( shunit2_script_request, transitive_targets_request) # We need to include all relevant transitive dependencies in the environment. We also get the # test's sources so that we can check that it has `source ./shunit2` at the bottom of it. # # Because we might modify the test files, we leave the tests out of # `dependencies_source_files_request` by using `transitive_targets.dependencies` instead of # `transitive_targets.closure`. This makes sure that we don't accidentally include the # unmodified test files and the modified test files in the same input. See # https://www.pantsbuild.org/v2.0/docs/rules-api-and-target-api. dependencies_source_files_request = Get( SourceFiles, SourceFilesRequest( (tgt.get(Sources) for tgt in transitive_targets.dependencies), for_sources_types=(BashSources, FilesSources, ResourcesSources), ), ) test_source_files_request = Get( SourceFiles, SourceFilesRequest([request.field_set.sources])) dependencies_source_files, test_source_files = await MultiGet( dependencies_source_files_request, test_source_files_request) # To check if the test files already have `source ./shunit2` in them, we need to look at the # actual file content. We use `DigestContents` for this, and then use `CreateDigest` to create # a digest of the (possibly) updated test files. See # https://www.pantsbuild.org/v2.0/docs/rules-api-file-system. # # Most test runners don't modify their test files like we do here, so most test runners can # skip this step. test_files_content = await Get(DigestContents, Digest, test_source_files.snapshot.digest) updated_test_files_content = [] for file_content in test_files_content: if (b"source ./shunit2" in file_content.content or b". ./shunit2" in file_content.content): updated_test_files_content.append(file_content) else: updated_file_content = FileContent( path=file_content.path, content=file_content.content + b"\nsource ./shunit2\n", ) updated_test_files_content.append(updated_file_content) updated_test_source_files = await Get( Digest, CreateDigest(updated_test_files_content)) # The Process needs one single `Digest`, so we merge everything together. See # https://www.pantsbuild.org/v2.0/docs/rules-api-file-system. input_digest = await Get( Digest, MergeDigests([ shunit2_script, updated_test_source_files, dependencies_source_files.snapshot.digest, ]), ) # We must check if `test --force` was used, and if so, use a hack to invalidate the cache by # mixing in a randomly generated UUID into the environment. extra_env = {} if test_subsystem.force and not request.is_debug: uuid = await Get(UUID, UUIDRequest()) extra_env["__PANTS_FORCE_TEST_RUN__"] = str(uuid) process = Process( argv=[bash_program.exe, *test_source_files.snapshot.files], input_digest=input_digest, description=f"Run shunit2 on {request.field_set.address}.", level=LogLevel.DEBUG, env=bash_setup.env_dict, timeout_seconds=request.field_set.timeout.value, ) return TestSetup(process)
async def setup_shunit2_for_target( request: TestSetupRequest, shell_setup: ShellSetup, test_subsystem: TestSubsystem, test_extra_env: TestExtraEnv, global_options: GlobalOptions, ) -> TestSetup: shunit2_download_file = DownloadFile( "https://raw.githubusercontent.com/kward/shunit2/b9102bb763cc603b3115ed30a5648bf950548097/shunit2", FileDigest( "1f11477b7948150d1ca50cdd41d89be4ed2acd137e26d2e0fe23966d0e272cc5", 40987), ) shunit2_script, transitive_targets, built_package_dependencies, env = await MultiGet( Get(Digest, DownloadFile, shunit2_download_file), Get(TransitiveTargets, TransitiveTargetsRequest([request.field_set.address])), Get( BuiltPackageDependencies, BuildPackageDependenciesRequest( request.field_set.runtime_package_dependencies), ), Get(Environment, EnvironmentRequest(["PATH"])), ) dependencies_source_files_request = Get( SourceFiles, SourceFilesRequest( (tgt.get(Sources) for tgt in transitive_targets.dependencies), for_sources_types=(ShellSources, FilesSources, ResourcesSources), enable_codegen=True, ), ) dependencies_source_files, field_set_sources = await MultiGet( dependencies_source_files_request, Get(SourceFiles, SourceFilesRequest([request.field_set.sources])), ) field_set_digest_content = await Get(DigestContents, Digest, field_set_sources.snapshot.digest) # Because a FieldSet corresponds to a file address, there should be exactly 1 file in the # sources. This assumption allows us to simplify determining which shell to use via inspecting # the shebang. if len(field_set_digest_content) != 1: raise AssertionError( f"The file address {request.field_set.address} had sources != 1, which is unexpected: " f"{field_set_sources.snapshot.files}. Please file a bug at " "https://github.com/pantsbuild/pants/issues/new with this error message copied." ) original_test_file_content = field_set_digest_content[0] updated_test_file_content = add_source_shunit2(original_test_file_content) updated_test_digest, runner = await MultiGet( Get(Digest, CreateDigest([updated_test_file_content])), Get( Shunit2Runner, Shunit2RunnerRequest(request.field_set.address, original_test_file_content, request.field_set.shell), ), ) input_digest = await Get( Digest, MergeDigests(( shunit2_script, updated_test_digest, dependencies_source_files.snapshot.digest, *(pkg.digest for pkg in built_package_dependencies), )), ) env_dict = { "PATH": create_path_env_var(shell_setup.executable_search_path(env)), "SHUNIT_COLOR": "always" if global_options.options.colors else "none", **test_extra_env.env, } argv = ( # Zsh requires extra args. See https://github.com/kward/shunit2/#-zsh. [ runner.binary_path.path, "-o", "shwordsplit", "--", *field_set_sources.snapshot.files ] if runner.shell == Shunit2Shell.zsh else [runner.binary_path.path, *field_set_sources.snapshot.files]) cache_scope = ProcessCacheScope.NEVER if test_subsystem.force else ProcessCacheScope.SUCCESSFUL process = Process( argv=argv, input_digest=input_digest, description=f"Run shunit2 for {request.field_set.address}.", level=LogLevel.DEBUG, env=env_dict, timeout_seconds=request.field_set.timeout.value, cache_scope=cache_scope, ) return TestSetup(process)