def get_request(self, plat: Platform) -> ExternalToolRequest: """Generate a request for this tool.""" version = self.get_options().version known_versions = self.get_options().known_versions for known_version in known_versions: try: ver, plat_val, sha256, length = ( x.strip() for x in known_version.split("|")) except ValueError: raise ExternalToolError( f"Bad value for --known-versions (see ./pants " f"help-advanced {self.options_scope}): {known_version}") if plat_val == plat.value and ver == version: digest = Digest(fingerprint=sha256, serialized_bytes_length=int(length)) try: url = self.generate_url(plat) exe = self.generate_exe(plat) or url.rsplit("/", 1)[-1] except ExternalToolError as e: raise ExternalToolError( f"Couldn't find {self.name} version {version} on {plat.value}" ) from e return ExternalToolRequest(UrlToFetch(url=url, digest=digest), exe) raise UnknownVersion( f"No known version of {self.name} {version} for {plat.value} found in {known_versions}" )
def test_download_missing_file(self): with self.isolated_local_store(): with http_server(StubHandler) as port: url = UrlToFetch(f"http://localhost:{port}/notfound", self.pantsbuild_digest) with self.assertRaises(ExecutionError) as cm: self.scheduler.product_request(Snapshot, subjects=[url]) self.assertIn('404', str(cm.exception))
def download_cloc_script() -> DownloadedClocScript: url = "https://binaries.pantsbuild.org/bin/cloc/1.80/cloc" sha_256 = "2b23012b1c3c53bd6b9dd43cd6aa75715eed4feb2cb6db56ac3fbbd2dffeac9d" digest = Digest(sha_256, 546279) snapshot = yield Get(Snapshot, UrlToFetch(url, digest)) yield DownloadedClocScript(script_path=snapshot.files[0], digest=snapshot.directory_digest)
def test_download_missing_file(self) -> None: with self.isolated_local_store(): with http_server(StubHandler) as port: url = UrlToFetch(f"http://localhost:{port}/notfound", self.pantsbuild_digest) with self.assertRaises(ExecutionError) as cm: self.request_single_product(Snapshot, url) assert "404" in str(cm.exception)
def test_download(self): with self.isolated_local_store(): with http_server(StubHandler) as port: url = UrlToFetch(f"http://localhost:{port}/CNAME", self.pantsbuild_digest) snapshot, = self.scheduler.product_request(Snapshot, subjects=[url]) self.assert_snapshot_equals(snapshot, ["CNAME"], Digest( "16ba2118adbe5b53270008790e245bbf7088033389461b08640a4092f7f647cf", 81 ))
def download_pex_bin(): # TODO: Inject versions and digests here through some option, rather than hard-coding it. url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.10/pex' digest = Digest( 'f4ac0f61f0c469537f04418e9347658340b0bce4ba61d302c5b805d194dda482', 1868106) snapshot = yield Get(Snapshot, UrlToFetch(url, digest)) yield DownloadedPexBin(executable=snapshot.files[0], directory_digest=snapshot.directory_digest)
def download_pex_bin() -> DownloadedPexBin: # TODO: Inject versions and digests here through some option, rather than hard-coding it. url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.12/pex' digest = Digest( 'ce64cb72cd23d2123dd48126af54ccf2b718d9ecb98c2ed3045ed1802e89e7e1', 1842359) snapshot = yield Get(Snapshot, UrlToFetch(url, digest)) yield DownloadedPexBin(executable=snapshot.files[0], directory_digest=snapshot.directory_digest)
def download_pex_bin() -> DownloadedPexBin: # TODO: Inject versions and digests here through some option, rather than hard-coding it. url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.11/pex' digest = Digest( '7a8fdfce2de22d25ba38afaa9df0282c33dd436959b3a5c3f788ded2ccc2cae9', 1867604) snapshot = yield Get(Snapshot, UrlToFetch(url, digest)) yield DownloadedPexBin(executable=snapshot.files[0], directory_digest=snapshot.directory_digest)
def do_test(expected_url: str, expected_length: int, expected_sha256: str, plat: Platform, version: str) -> None: foobar = create_subsystem(FooBar, version=version, known_versions=FooBar.default_known_versions) assert ExternalToolRequest( UrlToFetch(url=expected_url, digest=Digest(expected_sha256, expected_length)), f"foobar-{version}/bin/foobar", ) == foobar.get_request(plat)
def test_download_https(self): with self.isolated_local_store(): url = UrlToFetch("https://www.pantsbuild.org/CNAME", Digest( "63652768bd65af8a4938c415bdc25e446e97c473308d26b3da65890aebacf63f", 18, )) snapshot, = self.scheduler.product_request(Snapshot, subjects=[url]) self.assert_snapshot_equals(snapshot, ["CNAME"], Digest( "16ba2118adbe5b53270008790e245bbf7088033389461b08640a4092f7f647cf", 81 ))
def test_download_wrong_digest(self): with self.isolated_local_store(): with http_server(StubHandler) as port: url = UrlToFetch( f"http://localhost:{port}/CNAME", Digest( self.pantsbuild_digest.fingerprint, self.pantsbuild_digest.serialized_bytes_length + 1, ), ) with self.assertRaises(ExecutionError) as cm: self.scheduler.product_request(Snapshot, subjects=[url]) self.assertIn('wrong digest', str(cm.exception).lower())
def test_download_wrong_digest(self) -> None: with self.isolated_local_store(): with http_server(StubHandler) as port: url = UrlToFetch( f"http://localhost:{port}/CNAME", Digest( self.pantsbuild_digest.fingerprint, self.pantsbuild_digest.serialized_bytes_length + 1, ), ) with self.assertRaises(ExecutionError) as cm: self.request_single_product(Snapshot, url) assert "wrong digest" in str(cm.exception).lower()
async def fetch_binary_tool(req: BinaryToolFetchRequest, url_set: BinaryToolUrlSet) -> Snapshot: digest = url_set.tool_for_platform.digest urls = url_set.get_urls() if not urls: raise ValueError( f"binary tool url generator {url_set.url_generator} produced an empty list of " f"urls for the request {req}") # TODO: allow fetching a UrlToFetch with failure! Consider FallibleUrlToFetch analog to # FallibleExecuteProcessResult! url_to_fetch = urls[0] return await Get[Snapshot](UrlToFetch(url_to_fetch, digest))
def resolve_requirements(request, python_setup, pex_build_environment): """Returns a PEX with the given requirements, optional entry point, and optional interpreter constraints.""" # TODO: Inject versions and digests here through some option, rather than hard-coding it. url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.8/pex' digest = Digest( '2ca320aede7e7bbcb907af54c9de832707a1df965fb5a0d560f2df29ba8a2f3d', 1866441) pex_snapshot = yield Get(Snapshot, UrlToFetch(url, digest)) interpreter_search_paths = create_path_env_var( python_setup.interpreter_search_paths) env = { "PATH": interpreter_search_paths, **pex_build_environment.invocation_environment_dict } interpreter_constraint_args = [] for constraint in request.interpreter_constraints: interpreter_constraint_args.extend( ["--interpreter-constraint", constraint]) # NB: we use the hardcoded and generic bin name `python`, rather than something dynamic like # `sys.executable`, to ensure that the interpreter may be discovered both locally and in remote # execution (so long as `env` is populated with a `PATH` env var and `python` is discoverable # somewhere on that PATH). This is only used to run the downloaded PEX tool; it is not # necessarily the interpreter that PEX will use to execute the generated .pex file. # TODO(#7735): Set --python-setup-interpreter-search-paths differently for the host and target # platforms, when we introduce platforms in https://github.com/pantsbuild/pants/issues/7735. argv = [ "python", "./{}".format(pex_snapshot.files[0]), "-o", request.output_filename ] if request.entry_point is not None: argv.extend(["-e", request.entry_point]) argv.extend(interpreter_constraint_args + list(request.requirements)) request = ExecuteProcessRequest( argv=tuple(argv), env=env, input_files=pex_snapshot.directory_digest, description='Resolve requirements: {}'.format(", ".join( request.requirements)), output_files=(request.output_filename, ), ) result = yield Get(ExecuteProcessResult, ExecuteProcessRequest, request) yield ResolvedRequirementsPex( directory_digest=result.output_directory_digest, )
def test_caches_downloads(self): with self.isolated_local_store(): with http_server(StubHandler) as port: self.prime_store_with_roland_digest() # This would error if we hit the HTTP server, because 404, # but we're not going to hit the HTTP server because it's cached, # so we shouldn't see an error... url = UrlToFetch( f"http://localhost:{port}/roland", Digest('693d8db7b05e99c6b7a7c0616456039d89c555029026936248085193559a0b5d', 16), ) snapshot, = self.scheduler.product_request(Snapshot, subjects=[url]) self.assert_snapshot_equals(snapshot, ["roland"], Digest( "9341f76bef74170bedffe51e4f2e233f61786b7752d21c2339f8ee6070eba819", 82 ))
def run_python_test(target): # TODO: Inject versions and digests here through some option, rather than hard-coding it. pex_snapshot = yield Get(Snapshot, UrlToFetch("https://github.com/pantsbuild/pex/releases/download/v1.5.2/pex27", Digest('8053a79a5e9c2e6e9ace3999666c9df910d6289555853210c1bbbfa799c3ecda', 1757011))) # TODO: This should be configurable, both with interpreter constraints, and for remote execution. python_binary = sys.executable argv = [ './{}'.format(pex_snapshot.files[0].path), '-e', 'pytest:main', '--python', python_binary, # TODO: This is non-hermetic because pytest will be resolved on the fly by pex27, where it should be hermetically provided in some way. # We should probably also specify a specific version. 'pytest', ] merged_input_files = yield Get( Digest, MergedDirectories, MergedDirectories(directories=(target.adaptor.sources.snapshot.directory_digest, pex_snapshot.directory_digest)), ) request = ExecuteProcessRequest( argv=tuple(argv), input_files=merged_input_files, description='Run pytest for {}'.format(target.address.reference()), # TODO: This should not be necessary env={'PATH': os.path.dirname(python_binary)} ) result = yield Get(FallibleExecuteProcessResult, ExecuteProcessRequest, request) # TODO: Do something with stderr? status = Status.SUCCESS if result.exit_code == 0 else Status.FAILURE yield PyTestResult(status=status, stdout=str(result.stdout))
def run_python_test(transitive_hydrated_target): target_root = transitive_hydrated_target.root # TODO: Inject versions and digests here through some option, rather than hard-coding it. pex_snapshot = yield Get( Snapshot, UrlToFetch( "https://github.com/pantsbuild/pex/releases/download/v1.5.2/pex27", Digest( '8053a79a5e9c2e6e9ace3999666c9df910d6289555853210c1bbbfa799c3ecda', 1757011))) all_targets = [target_root] + [ dep.root for dep in transitive_hydrated_target.dependencies ] # Produce a pex containing pytest and all transitive 3rdparty requirements. all_requirements = [] for maybe_python_req_lib in all_targets: # This is a python_requirement()-like target. if hasattr(maybe_python_req_lib.adaptor, 'requirement'): all_requirements.append(str(maybe_python_req_lib.requirement)) # This is a python_requirement_library()-like target. if hasattr(maybe_python_req_lib.adaptor, 'requirements'): for py_req in maybe_python_req_lib.adaptor.requirements: all_requirements.append(str(py_req.requirement)) # TODO: This should be configurable, both with interpreter constraints, and for remote execution. python_binary = sys.executable output_pytest_requirements_pex_filename = 'pytest-with-requirements.pex' requirements_pex_argv = [ './{}'.format(pex_snapshot.files[0].path), '--python', python_binary, '-e', 'pytest:main', '-o', output_pytest_requirements_pex_filename, # TODO: This is non-hermetic because pytest will be resolved on the fly by pex27, where it should be hermetically provided in some way. # We should probably also specify a specific version. 'pytest', # Sort all the requirement strings to increase the chance of cache hits across invocations. ] + sorted(all_requirements) requirements_pex_request = ExecuteProcessRequest( argv=tuple(requirements_pex_argv), input_files=pex_snapshot.directory_digest, description='Resolve requirements for {}'.format( target_root.address.reference()), # TODO: This should not be necessary env={'PATH': os.path.dirname(python_binary)}, output_files=(output_pytest_requirements_pex_filename, ), ) requirements_pex_response = yield Get(FallibleExecuteProcessResult, ExecuteProcessRequest, requirements_pex_request) # Gather sources. # TODO: make TargetAdaptor return a 'sources' field with an empty snapshot instead of raising to # simplify the hasattr() checks here! all_sources_digests = [] for maybe_source_target in all_targets: if hasattr(maybe_source_target.adaptor, 'sources'): sources_snapshot = maybe_source_target.adaptor.sources.snapshot all_sources_digests.append(sources_snapshot.directory_digest) all_input_digests = all_sources_digests + [ requirements_pex_response.output_directory_digest, ] merged_input_files = yield Get( Digest, MergedDirectories, MergedDirectories(directories=tuple(all_input_digests)), ) request = ExecuteProcessRequest( argv=('./pytest-with-requirements.pex', ), input_files=merged_input_files, description='Run pytest for {}'.format( target_root.address.reference()), # TODO: This should not be necessary env={'PATH': os.path.dirname(python_binary)}) result = yield Get(FallibleExecuteProcessResult, ExecuteProcessRequest, request) # TODO: Do something with stderr? status = Status.SUCCESS if result.exit_code == 0 else Status.FAILURE yield PyTestResult(status=status, stdout=str(result.stdout))
def download_pex_bin(): # TODO: Inject versions and digests here through some option, rather than hard-coding it. url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.8/pex' digest = Digest('2ca320aede7e7bbcb907af54c9de832707a1df965fb5a0d560f2df29ba8a2f3d', 1866441) snapshot = yield Get(Snapshot, UrlToFetch(url, digest)) yield DownloadedPexBin(executable=snapshot.files[0], directory_digest=snapshot.directory_digest)
def run_python_test(transitive_hydrated_target, pytest): target_root = transitive_hydrated_target.root # TODO: Inject versions and digests here through some option, rather than hard-coding it. interpreter_major, interpreter_minor = sys.version_info[0:2] pex_name, digest = { (2, 7): ("pex27", Digest( '0ecbf48e3e240a413189194a9f829aec10446705c84db310affe36e23e741dbc', 1812737)), (3, 6): ("pex36", Digest( 'ba865e7ce7a840070d58b7ba24e7a67aff058435cfa34202abdd878e7b5d351d', 1812158)), (3, 7): ("pex37", Digest( '51bf8e84d5290fe5ff43d45be78d58eaf88cf2a5e995101c8ff9e6a73a73343d', 1813189)) }.get((interpreter_major, interpreter_minor), (None, None)) if pex_name is None: raise ValueError( "Current interpreter {}.{} is not supported, as there is no corresponding PEX to download." .format(interpreter_major, interpreter_minor)) pex_snapshot = yield Get( Snapshot, UrlToFetch( "https://github.com/pantsbuild/pex/releases/download/v1.6.1/{}". format(pex_name), digest)) all_targets = [target_root] + [ dep.root for dep in transitive_hydrated_target.dependencies ] # Produce a pex containing pytest and all transitive 3rdparty requirements. all_requirements = [] for maybe_python_req_lib in all_targets: # This is a python_requirement()-like target. if hasattr(maybe_python_req_lib.adaptor, 'requirement'): all_requirements.append(str(maybe_python_req_lib.requirement)) # This is a python_requirement_library()-like target. if hasattr(maybe_python_req_lib.adaptor, 'requirements'): for py_req in maybe_python_req_lib.adaptor.requirements: all_requirements.append(str(py_req.requirement)) # TODO: This should be configurable, both with interpreter constraints, and for remote execution. python_binary = sys.executable # TODO: This is non-hermetic because the requirements will be resolved on the fly by # pex27, where it should be hermetically provided in some way. output_pytest_requirements_pex_filename = 'pytest-with-requirements.pex' requirements_pex_argv = [ './{}'.format(pex_snapshot.files[0]), '--python', python_binary, '-e', 'pytest:main', '-o', output_pytest_requirements_pex_filename, # Sort all user requirement strings to increase the chance of cache hits across invocations. ] + list(pytest.get_requirement_strings()) + sorted(all_requirements) requirements_pex_request = ExecuteProcessRequest( argv=tuple(requirements_pex_argv), input_files=pex_snapshot.directory_digest, description='Resolve requirements for {}'.format( target_root.address.reference()), # TODO: This should not be necessary env={'PATH': os.path.dirname(python_binary)}, output_files=(output_pytest_requirements_pex_filename, ), ) requirements_pex_response = yield Get(ExecuteProcessResult, ExecuteProcessRequest, requirements_pex_request) # Gather sources. # TODO: make TargetAdaptor return a 'sources' field with an empty snapshot instead of raising to # simplify the hasattr() checks here! all_sources_digests = [] for maybe_source_target in all_targets: if hasattr(maybe_source_target.adaptor, 'sources'): sources_snapshot = maybe_source_target.adaptor.sources.snapshot all_sources_digests.append(sources_snapshot.directory_digest) all_input_digests = all_sources_digests + [ requirements_pex_response.output_directory_digest, ] merged_input_files = yield Get( Digest, MergedDirectories, MergedDirectories(directories=tuple(all_input_digests)), ) request = ExecuteProcessRequest( argv=('./pytest-with-requirements.pex', ), input_files=merged_input_files, description='Run pytest for {}'.format( target_root.address.reference()), # TODO: This should not be necessary env={'PATH': os.path.dirname(python_binary)}) result = yield Get(FallibleExecuteProcessResult, ExecuteProcessRequest, request) # TODO: Do something with stderr? status = Status.SUCCESS if result.exit_code == 0 else Status.FAILURE yield PyTestResult(status=status, stdout=result.stdout.decode('utf-8'))
def run_python_test(transitive_hydrated_target, pytest): target_root = transitive_hydrated_target.root # TODO: Inject versions and digests here through some option, rather than hard-coding it. url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.6/pex' digest = Digest('61bb79384db0da8c844678440bd368bcbfac17bbdb865721ad3f9cb0ab29b629', 1826945) pex_snapshot = yield Get(Snapshot, UrlToFetch(url, digest)) all_targets = [target_root] + [dep.root for dep in transitive_hydrated_target.dependencies] # Produce a pex containing pytest and all transitive 3rdparty requirements. all_requirements = [] for maybe_python_req_lib in all_targets: # This is a python_requirement()-like target. if hasattr(maybe_python_req_lib.adaptor, 'requirement'): all_requirements.append(str(maybe_python_req_lib.requirement)) # This is a python_requirement_library()-like target. if hasattr(maybe_python_req_lib.adaptor, 'requirements'): for py_req in maybe_python_req_lib.adaptor.requirements: all_requirements.append(str(py_req.requirement)) # TODO: This should be configurable, both with interpreter constraints, and for remote execution. # TODO(#7061): This str() can be removed after we drop py2! python_binary = text_type(sys.executable) # TODO: This is non-hermetic because the requirements will be resolved on the fly by # pex27, where it should be hermetically provided in some way. output_pytest_requirements_pex_filename = 'pytest-with-requirements.pex' requirements_pex_argv = [ python_binary, './{}'.format(pex_snapshot.files[0]), # TODO(#7061): This text_type() can be removed after we drop py2! '--python', text_type(python_binary), '-e', 'pytest:main', '-o', output_pytest_requirements_pex_filename, # Sort all user requirement strings to increase the chance of cache hits across invocations. ] + [ # TODO(#7061): This text_type() wrapping can be removed after we drop py2! text_type(req) for req in sorted( list(pytest.get_requirement_strings()) + list(all_requirements)) ] requirements_pex_request = ExecuteProcessRequest( argv=tuple(requirements_pex_argv), input_files=pex_snapshot.directory_digest, description='Resolve requirements for {}'.format(target_root.address.reference()), output_files=(output_pytest_requirements_pex_filename,), ) requirements_pex_response = yield Get( ExecuteProcessResult, ExecuteProcessRequest, requirements_pex_request) # Gather sources. # TODO: make TargetAdaptor return a 'sources' field with an empty snapshot instead of raising to # simplify the hasattr() checks here! all_sources_digests = [] for maybe_source_target in all_targets: if hasattr(maybe_source_target.adaptor, 'sources'): sources_snapshot = maybe_source_target.adaptor.sources.snapshot all_sources_digests.append(sources_snapshot.directory_digest) all_input_digests = all_sources_digests + [ requirements_pex_response.output_directory_digest, ] merged_input_files = yield Get( Digest, MergedDirectories, MergedDirectories(directories=tuple(all_input_digests)), ) request = ExecuteProcessRequest( argv=(python_binary, './{}'.format(output_pytest_requirements_pex_filename)), input_files=merged_input_files, description='Run pytest for {}'.format(target_root.address.reference()), ) result = yield Get(FallibleExecuteProcessResult, ExecuteProcessRequest, request) # TODO: Do something with stderr? status = Status.SUCCESS if result.exit_code == 0 else Status.FAILURE yield PyTestResult(status=status, stdout=result.stdout.decode('utf-8'))
async def download_pex_bin() -> DownloadedPexBin: # TODO: Inject versions and digests here through some option, rather than hard-coding it. url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.12/pex' digest = Digest('ce64cb72cd23d2123dd48126af54ccf2b718d9ecb98c2ed3045ed1802e89e7e1', 1842359) snapshot = await Get[Snapshot](UrlToFetch(url, digest)) return DownloadedPexBin(SingleFileExecutable(snapshot))
async def download_cloc_script() -> DownloadedClocScript: url = "https://binaries.pantsbuild.org/bin/cloc/1.80/cloc" sha_256 = "2b23012b1c3c53bd6b9dd43cd6aa75715eed4feb2cb6db56ac3fbbd2dffeac9d" digest = Digest(sha_256, 546279) snapshot = await Get[Snapshot](UrlToFetch(url, digest)) return DownloadedClocScript(SingleFileExecutable(snapshot))