def test_input_file_creation(self):
file_name = "some.filename"
file_contents = b"some file contents"
digest = self.request_single_product(
Digest,
CreateDigest([FileContent(path=file_name, content=file_contents)]))
req = Process(
argv=("/bin/cat", file_name),
input_digest=digest,
description="cat the contents of this file",
)
result = self.request_single_product(ProcessResult, req)
self.assertEqual(result.stdout, file_contents)
async def inject_missing_init_files(
request: InjectInitRequest) -> InitInjectedSnapshot:
"""Ensure that every package has an `__init__.py` file in it.
This will preserve any `__init__.py` files already in the input snapshot.
"""
snapshot = request.stripped_sources_snapshot
missing_init_files = sorted(identify_missing_init_files(snapshot.files))
if not missing_init_files:
return InitInjectedSnapshot(snapshot)
generated_inits_digest = await Get[Digest](InputFilesContent(
FileContent(path=fp, content=b"") for fp in missing_init_files))
result = await Get[Snapshot](MergeDigests(
(snapshot.digest, generated_inits_digest)))
return InitInjectedSnapshot(result)
def test_relative_import(self) -> None:
self.create_basic_library()
source = FileContent(
path="test_relative_import.py",
content=dedent("""\
from .library import add_two
def test():
assert add_two(2) == 4
""").encode(),
)
self.create_python_test_target([source], dependencies=[":library"])
result = self.run_pytest()
assert result.status == Status.SUCCESS
assert "test_relative_import.py ." in result.stdout
def test_executable(self):
file_name = "echo.sh"
file_contents = b'#!/bin/bash -eu\necho "Hello"\n'
input_file = InputFilesContent(
(FileContent(path=file_name, content=file_contents, is_executable=True),)
)
digest = self.request_single_product(Digest, input_file)
req = Process(
argv=("./echo.sh",), input_digest=digest, description="cat the contents of this file",
)
result = self.request_single_product(ProcessResult, req)
self.assertEqual(result.stdout, b"Hello\n")
def test_file_in_directory_creation(self):
path = "somedir/filename"
content = b"file contents"
input_file = InputFilesContent((FileContent(path=path, content=content),))
digest = self.request_single_product(Digest, input_file)
req = ExecuteProcessRequest(
argv=("/bin/cat", "somedir/filename"),
input_files=digest,
description="Cat a file in a directory to make sure that doesn't break",
)
result = self.request_single_product(ExecuteProcessResult, req)
self.assertEqual(result.stdout, content)
def test_input_file_creation(self):
file_name = "some.filename"
file_contents = b"some file contents"
input_file = InputFilesContent((FileContent(path=file_name, content=file_contents),))
digest = self.request_single_product(Digest, input_file)
req = ExecuteProcessRequest(
argv=("/bin/cat", file_name),
input_files=digest,
description="cat the contents of this file",
)
result = self.request_single_product(ExecuteProcessResult, req)
self.assertEqual(result.stdout, file_contents)
def test_absolute_import(self) -> None:
self.create_python_library([self.library_source])
source = FileContent(
path="test_absolute_import.py",
content=dedent("""\
from pants_test.library import add_two
def test():
assert add_two(2) == 4
""").encode(),
)
self.create_python_test_target([source], dependencies=[":library"])
result = self.run_pytest()
assert result.status == Status.SUCCESS
assert f"{self.package}/test_absolute_import.py ." in result.stdout
def test_write_digest(self):
# TODO(#8336): at some point, this test should require that Workspace only be invoked from
# an @goal_rule
workspace = Workspace(self.scheduler)
digest = self.request_single_product(
Digest,
CreateDigest([
FileContent("a.txt", b"hello"),
FileContent("subdir/b.txt", b"goodbye")
]),
)
path1 = Path(self.build_root, "a.txt")
path2 = Path(self.build_root, "subdir/b.txt")
assert not path1.is_file()
assert not path2.is_file()
workspace.write_digest(digest)
assert path1.is_file()
assert path2.is_file()
workspace.write_digest(digest, path_prefix="prefix")
assert Path(self.build_root, "prefix", path1).is_file()
assert Path(self.build_root, "prefix", path2).is_file()
def test_snapshot_subset_globs(self) -> None:
ss = SnapshotSubset(
self.generate_original_digest(),
PathGlobs(("a.txt", "c.txt", "subdir2/**")),
)
subset_snapshot = self.request_single_product(Snapshot, ss)
assert set(subset_snapshot.files) == {
"a.txt",
"c.txt",
"subdir2/a.txt",
"subdir2/nested_subdir/x.txt",
}
assert set(subset_snapshot.dirs) == {"subdir2/nested_subdir"}
content = b"dummy content"
subset_input = InputFilesContent((
FileContent(path="a.txt", content=content),
FileContent(path="c.txt", content=content),
FileContent(path="subdir2/a.txt", content=content),
FileContent(path="subdir2/nested_subdir/x.txt", content=content),
))
subset_digest = self.request_single_product(Digest, subset_input)
assert subset_snapshot.digest == subset_digest
def test_nonexistent_filename_globs(self) -> None:
# We expect to ignore, rather than error, on files that don't exist in the original snapshot.
ss = SnapshotSubset(directory_digest=self.generate_original_digest(),
globs=PathGlobs(
("some_file_not_in_snapshot.txt", "a.txt")))
subset_snapshot = self.request_single_product(Snapshot, ss)
assert set(subset_snapshot.files) == {"a.txt"}
content = b'dummy content'
subset_input = InputFilesContent((FileContent(path='a.txt',
content=content), ))
subset_digest = self.request_single_product(Digest, subset_input)
assert subset_snapshot.directory_digest == subset_digest
def _update_config(fc: FileContent) -> FileContent:
if PurePath(fc.path).suffix == ".toml":
all_config = _parse_toml_config(fc)
tool = all_config.setdefault("tool", {})
coverage = tool.setdefault("coverage", {})
run = coverage.setdefault("run", {})
run["relative_files"] = True
if "pytest.pex/*" not in run.get("omit", []):
run["omit"] = [*run.get("omit", []), "pytest.pex/*"]
return FileContent(fc.path, toml.dumps(all_config).encode())
cp = _parse_ini_config(fc)
run_section = "coverage:run" if fc.path in ("tox.ini",
"setup.cfg") else "run"
if not cp.has_section(run_section):
cp.add_section(run_section)
cp.set(run_section, "relative_files", "True")
omit_elements = cp[run_section].get("omit", "").split("\n") or ["\n"]
if "pytest.pex/*" not in omit_elements:
omit_elements.append("pytest.pex/*")
cp.set(run_section, "omit", "\n".join(omit_elements))
stream = StringIO()
cp.write(stream)
return FileContent(fc.path, stream.getvalue().encode())
async def create_or_update_coverage_config(coverage: CoverageSubsystem) -> CoverageConfig:
config_files = await Get(ConfigFiles, ConfigFilesRequest, coverage.config_request)
if config_files.snapshot.files:
digest_contents = await Get(DigestContents, Digest, config_files.snapshot.digest)
file_content = _update_config(digest_contents[0])
else:
cp = configparser.ConfigParser()
cp.add_section("run")
cp.set("run", "relative_files", "True")
cp.set("run", "omit", "\npytest.pex/*")
stream = StringIO()
cp.write(stream)
file_content = FileContent(".coveragerc", stream.getvalue().encode())
digest = await Get(Digest, CreateDigest([file_content]))
return CoverageConfig(digest, file_content.path)
def test_relative_import(rule_runner: RuleRunner) -> None:
create_python_library(rule_runner, [LIBRARY_SOURCE])
source = FileContent(
path=f"{PACKAGE}/test_relative_import.py",
content=dedent("""\
from .library import add_two
def test():
assert add_two(2) == 4
""").encode(),
)
tgt = create_test_target(rule_runner, [source], dependencies=[":library"])
result = run_pytest(rule_runner, tgt)
assert result.exit_code == 0
assert f"{PACKAGE}/test_relative_import.py ." in result.stdout
def test_add_prefix(self) -> None:
digest = self.request_product(
Digest,
[
CreateDigest((
FileContent(path="main.py", content=b'print("from main")'),
FileContent(path="subdir/sub.py",
content=b'print("from sub")'),
))
],
)
# Two components.
output_digest = self.request_product(
Digest, [AddPrefix(digest, "outer_dir/middle_dir")])
snapshot = self.request_product(Snapshot, [output_digest])
assert sorted(snapshot.files) == [
"outer_dir/middle_dir/main.py",
"outer_dir/middle_dir/subdir/sub.py",
]
assert sorted(snapshot.dirs) == [
"outer_dir",
"outer_dir/middle_dir",
"outer_dir/middle_dir/subdir",
]
# Empty.
output_digest = self.request_product(Digest, [AddPrefix(digest, "")])
assert digest == output_digest
# Illegal.
with self.assertRaisesRegex(
Exception,
r"Cannot add component .*ParentDir.* of path prefix `../something`."
):
self.request_product(Digest, [AddPrefix(digest, "../something")])
def test_3rdparty_plugin(rule_runner: RuleRunner) -> None:
target = make_target(
rule_runner,
[FileContent("bad.py", b"aws_key = 'JalrXUtnFEMI/K7MDENG/bPxRfiCYzEXAMPLEKEY'\n")],
# NB: `bandit-aws` does not currently work with Python 3.8. See
# https://github.com/pantsbuild/pants/issues/10545.
interpreter_constraints="CPython>=3.6,<3.8",
)
result = run_bandit(
rule_runner, [target], additional_args=["--bandit-extra-requirements=bandit-aws"]
)
assert len(result) == 1
assert result[0].exit_code == 1
assert "Issue: [C100:hardcoded_aws_key]" in result[0].stdout
assert result[0].report is None
def run_prelude_parsing_rule(prelude_content: str) -> BuildFilePreludeSymbols:
symbols = run_rule_with_mocks(
evaluate_preludes,
rule_args=[BuildFileOptions((), prelude_globs=("prelude",))],
mock_gets=[
MockGet(
output_type=DigestContents,
input_type=PathGlobs,
mock=lambda _: DigestContents(
[FileContent(path="/dev/null/prelude", content=prelude_content.encode())]
),
),
],
)
return cast(BuildFilePreludeSymbols, symbols)
def test_file_in_directory_creation(self):
path = "somedir/filename"
content = b"file contents"
digest = self.request(
Digest, [CreateDigest([FileContent(path=path, content=content)])])
req = Process(
argv=("/bin/cat", "somedir/filename"),
input_digest=digest,
description=
"Cat a file in a directory to make sure that doesn't break",
)
result = self.request(ProcessResult, [req])
self.assertEqual(result.stdout, content)
async def setup_analyzer() -> PackageAnalyzerSetup:
def get_file(filename: str) -> bytes:
content = pkgutil.get_data("pants.backend.go.util_rules", filename)
if not content:
raise AssertionError(f"Unable to find resource for `{filename}`.")
return content
analyer_sources_content = [
FileContent(filename, get_file(filename))
for filename in ("analyze_package.go", "read.go")
]
source_digest, import_config = await MultiGet(
Get(Digest, CreateDigest(analyer_sources_content)),
Get(ImportConfig, ImportConfigRequest,
ImportConfigRequest.stdlib_only()),
)
built_analyzer_pkg = await Get(
BuiltGoPackage,
BuildGoPackageRequest(
import_path="main",
subpath="",
digest=source_digest,
go_file_names=tuple(fc.path for fc in analyer_sources_content),
s_file_names=(),
direct_dependencies=(),
minimum_go_version=None,
),
)
main_pkg_a_file_path = built_analyzer_pkg.import_paths_to_pkg_a_files[
"main"]
input_digest = await Get(
Digest, MergeDigests([built_analyzer_pkg.digest,
import_config.digest]))
analyzer = await Get(
LinkedGoBinary,
LinkGoBinaryRequest(
input_digest=input_digest,
archives=(main_pkg_a_file_path, ),
import_config_path=import_config.CONFIG_PATH,
output_filename=PackageAnalyzerSetup.PATH,
description="Link Go package analyzer",
),
)
return PackageAnalyzerSetup(analyzer.digest)
def test_works_with_python27(rule_runner: RuleRunner) -> None:
"""A regression test that we can properly handle Python 2-only third-party dependencies.
There was a bug that this would cause the runner PEX to fail to execute because it did not have
Python 3 distributions of the requirements.
"""
rule_runner.add_to_build_file(
"",
dedent("""\
# Both requirements are a) typed and b) compatible with Py2 and Py3. However, `x690`
# has a distinct wheel for Py2 vs. Py3, whereas libumi has a universal wheel. We expect
# both to be usable, even though libumi is not compatible with Py3.
python_requirement_library(
name="libumi",
requirements=["libumi==0.0.2"],
)
python_requirement_library(
name="x690",
requirements=["x690==0.2.0"],
)
"""),
)
source_file = FileContent(
f"{PACKAGE}/py2.py",
dedent("""\
from libumi import hello_world
from x690 import types
print "Blast from the past!"
print hello_world() - 21 # MyPy should fail. You can't subtract an `int` from `bytes`.
""").encode(),
)
target = make_target(rule_runner, [source_file],
interpreter_constraints="==2.7.*")
result = run_mypy(rule_runner, [target], passthrough_args="--py2")
assert len(result) == 1
assert result[0].exit_code == 1
assert "Failed to execute PEX file" not in result[0].stderr
assert (
"Cannot find implementation or library stub for module named 'x690'"
not in result[0].stdout)
assert (
"Cannot find implementation or library stub for module named 'libumi'"
not in result[0].stdout)
assert f"{PACKAGE}/py2.py:5: error: Unsupported operand types" in result[
0].stdout
def test_pex_environment(rule_runner: RuleRunner, pex_type: type[Pex | VenvPex]) -> None:
sources = rule_runner.request(
Digest,
[
CreateDigest(
(
FileContent(
path="main.py",
content=textwrap.dedent(
"""
from os import environ
print(f"LANG={environ.get('LANG')}")
print(f"ftp_proxy={environ.get('ftp_proxy')}")
"""
).encode(),
),
)
),
],
)
pex_output = create_pex_and_get_all_data(
rule_runner,
pex_type=pex_type,
main=EntryPoint("main"),
sources=sources,
additional_pants_args=(
"--subprocess-environment-env-vars=LANG", # Value should come from environment.
"--subprocess-environment-env-vars=ftp_proxy=dummyproxy",
),
interpreter_constraints=PexInterpreterConstraints(["CPython>=3.6"]),
env={"LANG": "es_PY.UTF-8"},
)
pex = pex_output["pex"]
pex_process_type = PexProcess if isinstance(pex, Pex) else VenvPexProcess
process = rule_runner.request(
Process,
[
pex_process_type(
pex,
description="Run the pex and check its reported environment",
),
],
)
result = rule_runner.request(ProcessResult, [process])
assert b"LANG=es_PY.UTF-8" in result.stdout
assert b"ftp_proxy=dummyproxy" in result.stdout
def test_additional_inputs(rule_runner: RuleRunner) -> None:
# We use pex's --preamble-file option to set a custom preamble from a file.
# This verifies that the file was indeed provided as additional input to the pex call.
preamble_file = "custom_preamble.txt"
preamble = "#!CUSTOM PREAMBLE\n"
additional_inputs = rule_runner.request(
Digest, [CreateDigest([FileContent(path=preamble_file, content=preamble.encode())])]
)
additional_pex_args = (f"--preamble-file={preamble_file}",)
pex_output = create_pex_and_get_all_data(
rule_runner, additional_inputs=additional_inputs, additional_pex_args=additional_pex_args
)
with zipfile.ZipFile(pex_output["local_path"], "r") as zipfp:
with zipfp.open("__main__.py", "r") as main:
main_content = main.read().decode()
assert main_content[: len(preamble)] == preamble
def test_respects_passthrough_args(rule_runner: RuleRunner) -> None:
source = FileContent(
path=f"{PACKAGE}/test_config.py",
content=dedent("""\
def test_run_me():
pass
def test_ignore_me():
pass
""").encode(),
)
tgt = create_test_target(rule_runner, [source])
result = run_pytest(rule_runner, tgt, passthrough_args="-k test_run_me")
assert result.exit_code == 0
assert f"{PACKAGE}/test_config.py ." in result.stdout
assert "collected 2 items / 1 deselected / 1 selected" in result.stdout
def test_thirdparty_dep(rule_runner: RuleRunner) -> None:
setup_thirdparty_dep(rule_runner)
source = FileContent(
path=f"{PACKAGE}/test_3rdparty_dep.py",
content=dedent("""\
from ordered_set import OrderedSet
def test():
assert OrderedSet((1, 2)) == OrderedSet([1, 2])
""").encode(),
)
tgt = create_test_target(rule_runner, [source],
dependencies=["3rdparty/python:ordered-set"])
result = run_pytest(rule_runner, tgt)
assert result.exit_code == 0
assert f"{PACKAGE}/test_3rdparty_dep.py ." in result.stdout
def test_not_executable(self):
file_name = "echo.sh"
file_contents = b'#!/bin/bash -eu\necho "Hello"\n'
digest = self.request(Digest, [
CreateDigest([FileContent(path=file_name, content=file_contents)])
])
req = Process(
argv=("./echo.sh", ),
input_digest=digest,
description="cat the contents of this file",
)
with pytest.raises(ExecutionError) as exc:
self.request(ProcessResult, [req])
assert "Permission" in str(exc.value)
def test_digest_entries_handles_empty_directory(
rule_runner: RuleRunner) -> None:
digest = rule_runner.request(Digest, [
CreateDigest([Directory("a/b"),
FileContent("a/foo.txt", b"four\n")])
])
entries = rule_runner.request(DigestEntries, [digest])
assert entries == DigestEntries([
Directory("a/b"),
FileEntry(
"a/foo.txt",
FileDigest(
"ab929fcd5594037960792ea0b98caf5fdaf6b60645e4ef248c28db74260f393e",
5),
),
])
def generate_original_digest(rule_runner: RuleRunner) -> Digest:
files = [
FileContent(path, b"dummy content") for path in [
"a.txt",
"b.txt",
"c.txt",
"subdir/a.txt",
"subdir/b.txt",
"subdir2/a.txt",
"subdir2/nested_subdir/x.txt",
]
]
return rule_runner.request(
Digest,
[CreateDigest(files)],
)
def test_thirdparty_dep(self) -> None:
self.setup_thirdparty_dep()
source = FileContent(
path="test_3rdparty_dep.py",
content=dedent("""\
from ordered_set import OrderedSet
def test():
assert OrderedSet((1, 2)) == OrderedSet([1, 2])
""").encode(),
)
self.create_python_test_target(
[source], dependencies=["3rdparty/python:ordered-set"])
result = self.run_pytest()
assert result.status == Status.SUCCESS
assert "test_3rdparty_dep.py ." in result.stdout
def test_input_file_creation(self):
file_name = 'some.filename'
file_contents = b'some file contents'
input_file = InputFilesContent((FileContent(path=file_name,
content=file_contents), ))
digest, = self.scheduler.product_request(Digest, [input_file])
req = ExecuteProcessRequest(
argv=('/bin/cat', file_name),
input_files=digest,
description='cat the contents of this file',
)
result, = self.scheduler.product_request(ExecuteProcessResult, [req])
self.assertEqual(result.stdout, file_contents)
def test_respects_passthrough_args(self) -> None:
source = FileContent(
path="test_config.py",
content=dedent("""\
def test_run_me():
pass
def test_ignore_me():
pass
""").encode(),
)
self.create_python_test_target([source])
result = self.run_pytest(passthrough_args="-k test_run_me")
assert result.status == Status.SUCCESS
assert "test_config.py ." in result.stdout
assert "collected 2 items / 1 deselected / 1 selected" in result.stdout
def test_not_executable(self):
file_name = "echo.sh"
file_contents = b'#!/bin/bash -eu\necho "Hello"\n'
digest = self.request_single_product(
Digest,
CreateDigest([FileContent(path=file_name, content=file_contents)]))
req = Process(
argv=("./echo.sh", ),
input_digest=digest,
description="cat the contents of this file",
)
with self.assertRaisesWithMessageContaining(ExecutionError,
"Permission"):
self.request_single_product(ProcessResult, req)
