def _mock_fs_check(pathglobs: PathGlobs) -> Snapshot:
for glob in pathglobs.globs:
if glob in (existing_marker_files or []):
d, f = os.path.split(pathglobs.globs[0])
return Snapshot(digest=Digest("111", 111),
files=(f, ),
dirs=(d, ))
return Snapshot(digest=Digest("000", 000), files=tuple(), dirs=tuple())
def test_syntax_error(self) -> None:
address_mapper = unittest.mock.Mock()
address_mapper.prelude_glob_patterns = ("prelude",)
with self.assertRaisesRegex(
Exception, "Error parsing prelude file /dev/null/prelude: name 'blah' is not defined"
):
run_rule(
evaluate_preludes,
rule_args=[address_mapper,],
mock_gets=[
MockGet(
product_type=Snapshot,
subject_type=PathGlobs,
mock=lambda _: Snapshot(Digest("abc", 10), ("/dev/null/prelude",), ()),
),
MockGet(
product_type=FilesContent,
subject_type=Digest,
mock=lambda _: FilesContent(
[FileContent(path="/dev/null/prelude", content=b"blah")]
),
),
],
)
def resolve_addresses_with_origins_from_address_specs(
address_specs: AddressSpecs,
address_family: AddressFamily,
*,
tags: Optional[Iterable[str]] = None,
exclude_patterns: Optional[Iterable[str]] = None
) -> AddressesWithOrigins:
address_mapper = AddressMapper(
Parser(target_type_aliases=[], object_aliases=BuildFileAliases()),
tags=tags,
exclude_target_regexps=exclude_patterns,
)
snapshot = Snapshot(Digest("xx", 2), ("root/BUILD", ), ())
addresses_with_origins = run_rule(
addresses_with_origins_from_address_specs,
rule_args=[address_mapper, address_specs],
mock_gets=[
MockGet(product_type=Snapshot,
subject_type=PathGlobs,
mock=lambda _: snapshot),
MockGet(
product_type=AddressFamily,
subject_type=Dir,
mock=lambda _: address_family,
),
],
)
return cast(AddressesWithOrigins, addresses_with_origins)
def test_good_prelude(self) -> None:
address_mapper = unittest.mock.Mock()
address_mapper.prelude_glob_patterns = ("prelude", )
address_mapper.build_file_imports_behavior = BuildFileImportsBehavior.error
symbols = run_rule(
evalute_preludes,
rule_args=[
address_mapper,
],
mock_gets=[
MockGet(
product_type=Snapshot,
subject_type=PathGlobs,
mock=lambda _: Snapshot(Digest("abc", 10),
("/dev/null/prelude", ), ()),
),
MockGet(
product_type=FilesContent,
subject_type=Digest,
mock=lambda _: FilesContent([
FileContent(path="/dev/null/prelude",
content=b"def foo(): return 1")
]),
),
],
)
assert symbols.symbols["foo"]() == 1
def test_empty(self) -> None:
"""Test that parsing an empty BUILD file results in an empty AddressFamily."""
address_mapper = AddressMapper(
parser=JsonParser(TEST_TABLE),
prelude_glob_patterns=(),
build_file_imports_behavior=BuildFileImportsBehavior.error,
)
af = run_rule(
parse_address_family,
rule_args=[
address_mapper,
BuildFilePreludeSymbols(FrozenDict()),
Dir("/dev/null")
],
mock_gets=[
MockGet(
product_type=Snapshot,
subject_type=PathGlobs,
mock=lambda _: Snapshot(Digest("abc", 10),
("/dev/null/BUILD", ), ()),
),
MockGet(
product_type=FilesContent,
subject_type=Digest,
mock=lambda _: FilesContent(
[FileContent(path="/dev/null/BUILD", content=b"")]),
),
],
)
self.assertEqual(len(af.objects_by_name), 0)
def test_raises_empty_digest(self):
snapshot = Snapshot(EMPTY_DIRECTORY_DIGEST, files=('a.txt',), dirs=())
with self.assertRaisesWithMessage(
SingleFileExecutable.ValidationError,
f'snapshot {snapshot} used for {SingleFileExecutable} should have a non-empty digest!'):
SingleFileExecutable(snapshot)
def test_illegal_import(self) -> None:
prelude = dedent(
"""\
import os
def make_target():
python_library()
"""
).encode()
address_mapper = unittest.mock.Mock()
address_mapper.prelude_glob_patterns = ("prelude",)
with self.assertRaisesRegex(
Exception, "Import used in /dev/null/prelude at line 1\\. Import statements are banned"
):
run_rule(
evaluate_preludes,
rule_args=[address_mapper,],
mock_gets=[
MockGet(
product_type=Snapshot,
subject_type=PathGlobs,
mock=lambda _: Snapshot(Digest("abc", 10), ("/dev/null/prelude",), ()),
),
MockGet(
product_type=FilesContent,
subject_type=Digest,
mock=lambda _: FilesContent(
[FileContent(path="/dev/null/prelude", content=prelude)]
),
),
],
)
def test_empty(self):
"""Test that parsing an empty BUILD file results in an empty AddressFamily."""
address_mapper = AddressMapper(JsonParser(TestTable()))
af = run_rule(parse_address_family, address_mapper, Dir('/dev/null'), {
(Snapshot, PathGlobs): lambda _: Snapshot(DirectoryDigest('abc', 10), (File('/dev/null/BUILD'),)),
(FilesContent, DirectoryDigest): lambda _: FilesContent([FileContent('/dev/null/BUILD', b'')]),
})
self.assertEqual(len(af.objects_by_name), 0)
def test_snapshot_properties() -> None:
digest = Digest(
"691638f4d58abaa8cfdc9af2e00682f13f07f96ad1d177f146216a7341ca4982",
154)
snapshot = Snapshot._unsafe_create(digest, ["f.ext", "dir/f.ext"], ["dir"])
assert snapshot.digest == digest
assert snapshot.files == ("dir/f.ext", "f.ext")
assert snapshot.dirs == ("dir", )
def assert_hash(
expected: int,
*,
digest_char: str = "a",
files: Optional[List[str]] = None,
dirs: Optional[List[str]] = None,
) -> None:
digest = Digest(digest_char * 64, 1000)
snapshot = Snapshot._unsafe_create(digest, files or ["f.ext", "dir/f.ext"], dirs or ["dir"])
assert hash(snapshot) == expected
def test_duplicated(self):
"""Test that matching the same Spec twice succeeds."""
address = SingleAddress('a', 'a')
snapshot = Snapshot(Digest('xx', 2), ('a/BUILD',), ())
address_family = AddressFamily('a', {'a': ('a/BUILD', 'this is an object!')})
specs = Specs([address, address])
bfas = self._resolve_build_file_addresses(
specs, address_family, snapshot, self._address_mapper())
self.assertEqual(len(bfas.dependencies), 1)
self.assertEqual(bfas.dependencies[0].spec, 'a:a')
def test_duplicated(self) -> None:
"""Test that matching the same AddressSpec twice succeeds."""
address = SingleAddress("a", "a")
snapshot = Snapshot(Digest("xx", 2), ("a/BUILD",), ())
address_family = AddressFamily("a", {"a": ("a/BUILD", "this is an object!")})
address_specs = AddressSpecs([address, address])
addresses = self._resolve_addresses(
address_specs, address_family, snapshot, self._address_mapper()
)
assert len(addresses) == 1
assert addresses[0].spec == "a:a"
def test_duplicated(self):
"""Test that matching the same Spec twice succeeds."""
address = SingleAddress('a', 'a')
address_mapper = AddressMapper(JsonParser(TestTable()))
snapshot = Snapshot(DirectoryDigest(str('xx'), 2), (Path('a/BUILD', File('a/BUILD')),))
address_family = AddressFamily('a', {'a': ('a/BUILD', 'this is an object!')})
bfas = run_rule(addresses_from_address_families, address_mapper, Specs([address, address]), {
(Snapshot, PathGlobs): lambda _: snapshot,
(AddressFamily, Dir): lambda _: address_family,
})
self.assertEquals(len(bfas.dependencies), 1)
self.assertEquals(bfas.dependencies[0].spec, 'a:a')
def run_coordinator_of_tests(
self,
*,
address: Address,
origin: Optional[OriginSpec] = None,
test_target_type: bool = True,
include_sources: bool = True,
) -> AddressAndTestResult:
mocked_fileset = EagerFilesetWithSpec(
"src",
{"globs": []},
snapshot=Snapshot(
# TODO: this is not robust to set as an empty digest. Add a test util that provides
# some premade snapshots and possibly a generalized make_hydrated_target function.
directory_digest=EMPTY_DIRECTORY_DIGEST,
files=tuple(["test.py"] if include_sources else []),
dirs=(),
),
)
adaptor_cls = PythonTestsAdaptor if test_target_type else PythonBinaryAdaptor
type_alias = "python_tests" if test_target_type else "python_binary"
adaptor = adaptor_cls(address=address,
type_alias=type_alias,
sources=mocked_fileset)
union_membership = UnionMembership(union_rules=OrderedDict(
{TestTarget: OrderedSet([PythonTestsAdaptorWithOrigin])}))
with self.captured_logging(logging.INFO):
result: AddressAndTestResult = run_rule(
coordinator_of_tests,
rule_args=[
HydratedTargetWithOrigin(
target=HydratedTarget(adaptor),
origin=(origin
or SingleAddress(directory=address.spec_path,
name=address.target_name)),
),
union_membership,
],
mock_gets=[
MockGet(
product_type=TestResult,
subject_type=TestTarget,
mock=lambda _: TestResult(
status=Status.SUCCESS, stdout="foo", stderr=""),
),
],
union_membership=union_membership,
)
return result
def provider_rule(path_globs: PathGlobs) -> Snapshot:
dirs = (
"contrib/go/examples/3rdparty/go",
"contrib/go/examples/src/go/src",
"src/java",
"src/python",
"src/kotlin",
"my/project/src/java",
"src/example/java",
"src/example/python",
"fixed/root/jvm",
# subdirectories of source roots should not show up in final output
"src/kotlin/additional/directories/that/might/get/matched/src/foo",
)
return Snapshot(Digest("abcdef", 10), (), dirs)
def provider_rule(path_globs: PathGlobs) -> Snapshot:
dirs = (
'contrib/go/examples/3rdparty/go',
'contrib/go/examples/src/go/src',
'src/java',
'src/python',
'src/kotlin',
'my/project/src/java',
'src/example/java',
'src/example/python',
'fixed/root/jvm',
# subdirectories of source roots should not show up in final output
'src/kotlin/additional/directories/that/might/get/matched/src/foo',
)
return Snapshot(Digest('abcdef', 10), (), dirs)
def test_exclude_pattern_with_single_address(self):
"""Test that single address targets are filtered based on exclude patterns."""
spec = SingleAddress('root', 'not_me')
address_mapper = AddressMapper(JsonParser(TestTable()))
snapshot = Snapshot(DirectoryDigest('xx', 2),
(Path('root/BUILD', File('root/BUILD')),))
address_family = AddressFamily('root',
{
'not_me': ('root/BUILD', TargetAdaptor()),
}
)
targets = run_rule(
addresses_from_address_families, address_mapper, Specs([spec], exclude_patterns=tuple(['root.*'])),{
(Snapshot, PathGlobs): lambda _: snapshot,
(AddressFamily, Dir): lambda _: address_family,
})
self.assertEqual(len(targets.dependencies), 0)
def test_passes_eager_fileset_with_spec_through(self):
self.create_file('foo/foo/a.txt', 'a_contents')
fileset = self.sources_for(['foo/a.txt'], 'foo')
sf = SourcesField(sources=fileset)
self.assertIs(fileset, sf.sources)
self.assertEqual(['foo/a.txt'], list(sf.source_paths))
self.assertEqual(['foo/foo/a.txt'], list(sf.relative_to_buildroot()))
digest = '56001a7e48555f156420099a99da60a7a83acc90853046709341bf9f00a6f944'
want_snapshot = Snapshot(Digest(digest, 77), ('foo/foo/a.txt', ), ())
# We explicitly pass a None scheduler because we expect no scheduler lookups to be required
# in order to get a Snapshot.
self.assertEqual(sf.snapshot(scheduler=None), want_snapshot)
def hydrate_field(
*, raw_source_files: List[str], hydrated_source_files: Tuple[str, ...]
) -> FortranSourcesResult:
sources_field = FortranTarget(
{FortranSources.alias: raw_source_files}, address=Address.parse(":lib")
)[FortranSources]
result: FortranSourcesResult = run_rule(
hydrate_fortran_sources,
rule_args=[FortranSourcesRequest(sources_field)],
mock_gets=[
MockGet(
product_type=Snapshot,
subject_type=PathGlobs,
mock=lambda _: Snapshot(EMPTY_DIGEST, files=hydrated_source_files, dirs=()),
)
],
)
return result
def hydrate_field(
*, raw_source_files: List[str], hydrated_source_files: Tuple[str, ...]
) -> FortranSourcesResult:
sources_field = FortranTarget(
{FortranSources.alias: raw_source_files}, address=Address("", target_name="lib")
)[FortranSources]
result: FortranSourcesResult = run_rule_with_mocks(
hydrate_fortran_sources,
rule_args=[FortranSourcesRequest(sources_field)],
mock_gets=[
MockGet(
output_type=Snapshot,
input_type=PathGlobs,
mock=lambda _: Snapshot(EMPTY_DIGEST, files=hydrated_source_files, dirs=()),
)
],
)
return result
def run_coordinator_of_tests(
self,
*,
address: Address,
bfaddr_to_address_spec: Optional[Dict[BuildFileAddress,
AddressSpec]] = None,
test_target_type: bool = True,
include_sources: bool = True,
) -> AddressAndTestResult:
mocked_fileset = EagerFilesetWithSpec(
"src",
{"globs": []},
snapshot=Snapshot(
# TODO: this is not robust to set as an empty digest. Add a test util that provides
# some premade snapshots and possibly a generalized make_hydrated_target function.
directory_digest=EMPTY_DIRECTORY_DIGEST,
files=tuple(["test.py"] if include_sources else []),
dirs=()))
target_adaptor = (PythonTestsAdaptor(type_alias='python_tests',
sources=mocked_fileset)
if test_target_type else PythonBinaryAdaptor(
type_alias='python_binary',
sources=mocked_fileset))
with self.captured_logging(logging.INFO):
result: AddressAndTestResult = run_rule(
coordinator_of_tests,
rule_args=[
HydratedTarget(address, target_adaptor, ()),
UnionMembership(
union_rules={TestTarget: [PythonTestsAdaptor]}),
AddressProvenanceMap(
bfaddr_to_address_spec=bfaddr_to_address_spec or {}),
],
mock_gets=[
MockGet(
product_type=TestResult,
subject_type=PythonTestsAdaptor,
mock=lambda _: TestResult(
status=Status.SUCCESS, stdout='foo', stderr=''),
),
],
)
return result
def test_exclude_pattern(self):
"""Test that targets are filtered based on exclude patterns."""
spec = SiblingAddresses('root')
address_mapper = AddressMapper(JsonParser(TestTable()))
snapshot = Snapshot(DirectoryDigest(text_type('xx'), 2),
(Path('root/BUILD', File('root/BUILD')), ))
address_family = AddressFamily(
'root', {
'exclude_me': ('root/BUILD', TargetAdaptor()),
'not_me': ('root/BUILD', TargetAdaptor()),
})
targets = run_rule(
addresses_from_address_families, address_mapper,
Specs([spec], exclude_patterns=tuple(['.exclude*'])), {
(Snapshot, PathGlobs): lambda _: snapshot,
(AddressFamily, Dir): lambda _: address_family,
})
self.assertEquals(len(targets.dependencies), 1)
self.assertEquals(targets.dependencies[0].spec, 'root:not_me')
def test_tag_filter(self):
"""Test that targets are filtered based on `tags`."""
spec = SiblingAddresses('root')
address_mapper = AddressMapper(JsonParser(TestTable()))
snapshot = Snapshot(DirectoryDigest(str('xx'), 2), (Path('root/BUILD', File('root/BUILD')),))
address_family = AddressFamily('root',
{'a': ('root/BUILD', TargetAdaptor()),
'b': ('root/BUILD', TargetAdaptor(tags={'integration'})),
'c': ('root/BUILD', TargetAdaptor(tags={'not_integration'}))
}
)
targets = run_rule(
addresses_from_address_families, address_mapper, Specs([spec], tags=['+integration']), {
(Snapshot, PathGlobs): lambda _: snapshot,
(AddressFamily, Dir): lambda _: address_family,
})
self.assertEquals(len(targets.dependencies), 1)
self.assertEquals(targets.dependencies[0].spec, 'root:b')
def make_hydrated_target(
*,
name: str = "target",
adaptor_type: Type[TargetAdaptor] = PythonTargetAdaptor,
include_sources: bool = True,
) -> HydratedTarget:
mocked_snapshot = Snapshot(
# TODO: this is not robust to set as an empty digest. Add a test util that provides
# some premade snapshots and possibly a generalized make_hydrated_target function.
directory_digest=EMPTY_DIRECTORY_DIGEST,
files=("formatted.txt", "fake.txt") if include_sources else (),
dirs=())
return HydratedTarget(address=f"src/{name}",
adaptor=adaptor_type(
sources=EagerFilesetWithSpec(
"src", {"globs": []},
snapshot=mocked_snapshot),
name=name,
),
dependencies=())
def test_snapshot_equality() -> None:
# Only the digest is used for equality.
snapshot = Snapshot._unsafe_create(Digest("a" * 64, 1000),
["f.ext", "dir/f.ext"], ["dir"])
assert snapshot == Snapshot._unsafe_create(Digest("a" * 64, 1000),
["f.ext", "dir/f.ext"], ["dir"])
assert snapshot == Snapshot._unsafe_create(Digest("a" * 64, 1000),
["f.ext", "dir/f.ext"], ["foo"])
assert snapshot == Snapshot._unsafe_create(Digest("a" * 64, 1000),
["f.ext"], ["dir"])
assert snapshot != Snapshot._unsafe_create(Digest("a" * 64, 0),
["f.ext", "dir/f.ext"], ["dir"])
assert snapshot != Snapshot._unsafe_create(Digest("b" * 64, 1000),
["f.ext", "dir/f.ext"], ["dir"])
with pytest.raises(TypeError):
snapshot < snapshot # type: ignore[operator]
def test_empty(self) -> None:
"""Test that parsing an empty BUILD file results in an empty AddressFamily."""
address_mapper = AddressMapper(JsonParser(TEST_TABLE))
af = run_rule(
parse_address_family,
rule_args=[address_mapper, Dir("/dev/null")],
mock_gets=[
MockGet(
product_type=Snapshot,
subject_type=PathGlobs,
mock=lambda _: Snapshot(Digest("abc", 10),
("/dev/null/BUILD", ), ()),
),
MockGet(
product_type=FilesContent,
subject_type=Digest,
mock=lambda _: FilesContent(
[FileContent(path="/dev/null/BUILD", content=b"")]),
),
],
)
self.assertEqual(len(af.objects_by_name), 0)
def hydrate_field(
*, raw_source_files: List[str],
hydrated_source_files: Tuple[str, ...]) -> HaskellSourcesResult:
sources_field = HaskellTarget(
{
HaskellSources.alias: raw_source_files
},
address=Address.parse(":lib")).get(HaskellSources)
result: HaskellSourcesResult = run_rule(
hydrate_haskell_sources,
rule_args=[HaskellSourcesRequest(sources_field)],
mock_gets=[
MockGet(
product_type=Snapshot,
subject_type=PathGlobs,
mock=lambda _: Snapshot(
directory_digest=EMPTY_DIRECTORY_DIGEST,
files=hydrated_source_files,
dirs=(),
),
)
],
)
return result
def _snapshot(self) -> Snapshot:
return Snapshot(Digest("xx", 2), ("root/BUILD", ), ())
def _snapshot(self):
return Snapshot(
Digest('xx', 2),
(Path('root/BUILD', File('root/BUILD')),))
def _snapshot(self):
return Snapshot(Digest('xx', 2), ('root/BUILD', ), ())
