class StorageTest(unittest.TestCase):
TEST_KEY = b'hello'
TEST_VALUE = b'world'
TEST_PATH = File('/foo')
TEST_PATH2 = Dir('/bar')
class SomeException(Exception): pass
def setUp(self):
self.storage = Storage.create()
self.result = 'something'
self.request = Runnable(func=_runnable, args=('this is an arg',), cacheable=True)
def test_storage(self):
key = self.storage.put(self.TEST_PATH)
self.assertEquals(self.TEST_PATH, self.storage.get(key))
with self.assertRaises(InvalidKeyError):
self.assertFalse(self.storage.get(self.TEST_KEY))
def test_storage_key_mappings(self):
key1 = self.storage.put(self.TEST_PATH)
key2 = self.storage.put(self.TEST_PATH2)
self.storage.add_mapping(key1, key2)
self.assertEquals(key2, self.storage.get_mapping(key1))
# key2 isn't mapped to any other key.
self.assertIsNone(self.storage.get_mapping(key2))
def test_empty(self):
"""Test that parsing an empty BUILD file results in an empty AddressFamily."""
address_mapper = AddressMapper(JsonParser(TestTable()))
af = run_rule(parse_address_family, address_mapper, Dir('/dev/null'), {
(Snapshot, PathGlobs): lambda _: Snapshot(DirectoryDigest('abc', 10), (File('/dev/null/BUILD'),)),
(FilesContent, DirectoryDigest): lambda _: FilesContent([FileContent('/dev/null/BUILD', b'')]),
})
self.assertEqual(len(af.objects_by_name), 0)
def generate_subjects(self, filenames):
"""Given filenames, generate a set of subjects for invalidation predicate matching."""
for f in filenames:
# ReadLink, or FileContent for the literal path.
yield File(f)
yield Link(f)
# DirectoryListing for parent dirs.
yield Dir(dirname(f))
def generate_fs_subjects(filenames):
"""Given filenames, generate a set of subjects for invalidation predicate matching."""
for f in filenames:
# ReadLink, FileContent, or DirectoryListing for the literal path.
yield File(f)
yield Link(f)
yield Dir(f)
# Additionally, since the FS event service does not send invalidation events
# for the root directory, treat any changed file in the root as an invalidation
# of the root's listing.
if dirname(f) in ('.', ''):
yield Dir('')
def _lstat(self, relpath):
mode = type(self._reader.lstat(self._scm_relpath(relpath)))
if mode == NoneType:
return None
elif mode == self._reader.Symlink:
return Link(relpath)
elif mode == self._reader.Dir:
return Dir(relpath)
elif mode == self._reader.File:
return File(relpath)
else:
raise IOError('Unsupported file type in {}: {}'.format(self, relpath))
def test_duplicated(self):
"""Test that matching the same Spec twice succeeds."""
address = SingleAddress('a', 'a')
snapshot = Snapshot(Digest('xx', 2),
(Path('a/BUILD', File('a/BUILD')),))
address_family = AddressFamily('a', {'a': ('a/BUILD', 'this is an object!')})
specs = Specs([address, address])
bfas = self._resolve_build_file_addresses(
specs, address_family, snapshot, self._address_mapper())
self.assertEqual(len(bfas.dependencies), 1)
self.assertEqual(bfas.dependencies[0].spec, 'a:a')
def test_duplicated(self):
"""Test that matching the same Spec twice succeeds."""
address = SingleAddress('a', 'a')
address_mapper = AddressMapper(JsonParser(TestTable()))
snapshot = Snapshot(DirectoryDigest(str('xx'), 2), (Path('a/BUILD', File('a/BUILD')),))
address_family = AddressFamily('a', {'a': ('a/BUILD', 'this is an object!')})
bfas = run_rule(addresses_from_address_families, address_mapper, Specs([address, address]), {
(Snapshot, PathGlobs): lambda _: snapshot,
(AddressFamily, Dir): lambda _: address_family,
})
self.assertEquals(len(bfas.dependencies), 1)
self.assertEquals(bfas.dependencies[0].spec, 'a:a')
class StorageTest(unittest.TestCase):
TEST_KEY = b'hello'
TEST_VALUE = b'world'
TEST_PATH = File('/foo')
TEST_PATH2 = Dir('/bar')
class SomeException(Exception):
pass
def setUp(self):
self.storage = Storage.create()
self.result = 'something'
self.request = Runnable(func=_runnable,
args=('this is an arg', ),
cacheable=True)
def test_lmdb_key_value_store(self):
lmdb = Lmdb.create()[0]
with closing(lmdb) as kvs:
# Initially key does not exist.
self.assertFalse(kvs.get(self.TEST_KEY))
# Now write a key value pair and read back.
written = kvs.put(self.TEST_KEY, self.TEST_VALUE)
self.assertTrue(written)
self.assertEquals(self.TEST_VALUE,
kvs.get(self.TEST_KEY).getvalue())
# Write the same key again will not overwrite.
self.assertFalse(kvs.put(self.TEST_KEY, self.TEST_VALUE))
def test_storage(self):
with closing(self.storage) as storage:
key = storage.put(self.TEST_PATH)
self.assertEquals(self.TEST_PATH, storage.get(key))
with self.assertRaises(InvalidKeyError):
self.assertFalse(storage.get(self.TEST_KEY))
def test_storage_key_mappings(self):
with closing(self.storage) as storage:
key1 = storage.put(self.TEST_PATH)
key2 = storage.put(self.TEST_PATH2)
storage.add_mapping(key1, key2)
self.assertEquals(key2, storage.get_mapping(key1))
# key2 isn't mapped to any other key.
self.assertIsNone(storage.get_mapping(key2))
def test_exclude_pattern_with_single_address(self):
"""Test that single address targets are filtered based on exclude patterns."""
spec = SingleAddress('root', 'not_me')
address_mapper = AddressMapper(JsonParser(TestTable()))
snapshot = Snapshot(DirectoryDigest('xx', 2),
(Path('root/BUILD', File('root/BUILD')),))
address_family = AddressFamily('root',
{
'not_me': ('root/BUILD', TargetAdaptor()),
}
)
targets = run_rule(
addresses_from_address_families, address_mapper, Specs([spec], exclude_patterns=tuple(['root.*'])),{
(Snapshot, PathGlobs): lambda _: snapshot,
(AddressFamily, Dir): lambda _: address_family,
})
self.assertEqual(len(targets.dependencies), 0)
def test_exclude_pattern(self):
"""Test that targets are filtered based on exclude patterns."""
spec = SiblingAddresses('root')
address_mapper = AddressMapper(JsonParser(TestTable()))
snapshot = Snapshot(DirectoryDigest(text_type('xx'), 2),
(Path('root/BUILD', File('root/BUILD')), ))
address_family = AddressFamily(
'root', {
'exclude_me': ('root/BUILD', TargetAdaptor()),
'not_me': ('root/BUILD', TargetAdaptor()),
})
targets = run_rule(
addresses_from_address_families, address_mapper,
Specs([spec], exclude_patterns=tuple(['.exclude*'])), {
(Snapshot, PathGlobs): lambda _: snapshot,
(AddressFamily, Dir): lambda _: address_family,
})
self.assertEquals(len(targets.dependencies), 1)
self.assertEquals(targets.dependencies[0].spec, 'root:not_me')
def test_passes_eager_fileset_with_spec_through(self):
self.create_file('foo/foo/a.txt', 'a_contents')
fileset = self.sources_for(['foo/a.txt'], 'foo')
sf = SourcesField(sources=fileset)
self.assertIs(fileset, sf.sources)
self.assertEqual(['foo/a.txt'], list(sf.source_paths))
self.assertEqual(['foo/foo/a.txt'], list(sf.relative_to_buildroot()))
digest = str(
'56001a7e48555f156420099a99da60a7a83acc90853046709341bf9f00a6f944')
want_snapshot = Snapshot(DirectoryDigest(
digest, 77), (Path('foo/foo/a.txt', stat=File('foo/foo/a.txt')), ))
# We explicitly pass a None scheduler because we expect no scheduler lookups to be required
# in order to get a Snapshot.
self.assertEqual(sf.snapshot(scheduler=None), want_snapshot)
def test_tag_filter(self):
"""Test that targets are filtered based on `tags`."""
spec = SiblingAddresses('root')
address_mapper = AddressMapper(JsonParser(TestTable()))
snapshot = Snapshot(DirectoryDigest(str('xx'), 2), (Path('root/BUILD', File('root/BUILD')),))
address_family = AddressFamily('root',
{'a': ('root/BUILD', TargetAdaptor()),
'b': ('root/BUILD', TargetAdaptor(tags={'integration'})),
'c': ('root/BUILD', TargetAdaptor(tags={'not_integration'}))
}
)
targets = run_rule(
addresses_from_address_families, address_mapper, Specs([spec], tags=['+integration']), {
(Snapshot, PathGlobs): lambda _: snapshot,
(AddressFamily, Dir): lambda _: address_family,
})
self.assertEquals(len(targets.dependencies), 1)
self.assertEquals(targets.dependencies[0].spec, 'root:b')
def _scandir_raw(self, relpath):
# Sanity check. TODO: this should probably be added to the ProjectTree interface as
# an optional call, so that we can use it in fs.py rather than applying it by default.
abspath = os.path.normpath(self._join(relpath))
if os.path.realpath(abspath) != abspath:
raise ValueError(
'scandir for non-canonical path "{}" not supported in {}.'.
format(relpath, self))
for entry in os.scandir(abspath):
# NB: We don't use `DirEntry.stat`, as the scandir docs indicate that that always requires
# an additional syscall on Unixes.
entry_path = os.path.normpath(os.path.join(relpath, entry.name))
if entry.is_file(follow_symlinks=False):
yield File(entry_path)
elif entry.is_dir(follow_symlinks=False):
yield Dir(entry_path)
elif entry.is_symlink():
yield Link(entry_path)
else:
raise IOError('Unsupported file type in {}: {}'.format(
self, entry_path))
def _snapshot(self):
return Snapshot(
Digest('xx', 2),
(Path('root/BUILD', File('root/BUILD')),))
class StorageTest(unittest.TestCase):
TEST_KEY = b'hello'
TEST_VALUE = b'world'
TEST_PATH = File('/foo')
TEST_PATH2 = Dir('/bar')
class SomeException(Exception):
pass
def setUp(self):
self.storage = Storage.create()
self.result = 'something'
self.request = Runnable(func=_runnable, args=('this is an arg', ))
def test_lmdb_key_value_store(self):
lmdb = Lmdb.create()[0]
with closing(lmdb) as kvs:
# Initially key does not exist.
self.assertFalse(kvs.get(self.TEST_KEY))
# Now write a key value pair and read back.
written = kvs.put(self.TEST_KEY, self.TEST_VALUE)
self.assertTrue(written)
self.assertEquals(self.TEST_VALUE,
kvs.get(self.TEST_KEY).getvalue())
# Write the same key again will not overwrite.
self.assertFalse(kvs.put(self.TEST_KEY, self.TEST_VALUE))
def test_storage(self):
with closing(self.storage) as storage:
key = storage.put(self.TEST_PATH)
self.assertEquals(self.TEST_PATH, storage.get(key))
with self.assertRaises(InvalidKeyError):
self.assertFalse(storage.get(self.TEST_KEY))
def test_storage_key_mappings(self):
with closing(self.storage) as storage:
key1 = storage.put(self.TEST_PATH)
key2 = storage.put(self.TEST_PATH2)
storage.add_mapping(key1, key2)
self.assertEquals(key2, storage.get_mapping(key1))
# key2 isn't mapped to any other key.
self.assertIsNone(storage.get_mapping(key2))
def test_state_roundtrips(self):
states = [
Return('a'),
Throw(PickleableException()),
Waiting([TaskNode(None, None, None)]),
Runnable(_runnable, ('an arg', )),
Noop('nada {}', ('op', ))
]
with closing(self.storage) as storage:
for state in states:
key = storage.put_state(state)
actual = storage.get_state(key)
self.assertEquals(state, actual)
self.assertEquals(key, storage.put_state(actual))
class StorageTest(unittest.TestCase):
TEST_KEY = b'hello'
TEST_VALUE = b'world'
TEST_PATH = File('/foo')
TEST_PATH2 = Dir('/bar')
class SomeException(Exception): pass
def setUp(self):
self.storage = Storage.create()
self.result = StepResult(state='something')
self.request = StepRequest(step_id=123,
node='some node',
dependencies={'some dep': 'some state',
'another dep': 'another state'},
inline_nodes=False,
project_tree='some project tree')
def test_lmdb_key_value_store(self):
lmdb = Lmdb.create()[0]
with closing(lmdb) as kvs:
# Initially key does not exist.
self.assertFalse(kvs.get(self.TEST_KEY))
# Now write a key value pair and read back.
written = kvs.put(self.TEST_KEY, self.TEST_VALUE)
self.assertTrue(written)
self.assertEquals(self.TEST_VALUE, kvs.get(self.TEST_KEY).getvalue())
# Write the same key again will not overwrite.
self.assertFalse(kvs.put(self.TEST_KEY, self.TEST_VALUE))
def test_storage(self):
with closing(self.storage) as storage:
key = storage.put(self.TEST_PATH)
self.assertEquals(self.TEST_PATH, storage.get(key))
# The deserialized blob is equal by not the same as the input data.
self.assertFalse(storage.get(key) is self.TEST_PATH)
# Any other keys won't exist in the subjects.
self.assertNotEqual(self.TEST_KEY, key)
with self.assertRaises(InvalidKeyError):
self.assertFalse(storage.get(self.TEST_KEY))
# Verify key and value's types must match.
key._type = str
with self.assertRaises(ValueError):
storage.get(key)
def test_storage_key_mappings(self):
with closing(self.storage) as storage:
key1 = storage.put(self.TEST_PATH)
key2 = storage.put(self.TEST_PATH2)
storage.add_mapping(key1, key2)
self.assertEquals(key2, storage.get_mapping(key1))
# key2 isn't mapped to any other key.
self.assertIsNone(storage.get_mapping(key2))
def test_key_for_request(self):
with closing(self.storage) as storage:
keyed_request = storage.key_for_request(self.request)
for dep, dep_state in keyed_request.dependencies.items():
self.assertEquals(Key, type(dep))
self.assertEquals(Key, type(dep_state))
self.assertIs(self.request.node, keyed_request.node)
self.assertIs(self.request.project_tree, keyed_request.project_tree)
self.assertEquals(keyed_request, storage.key_for_request(keyed_request))
def test_resolve_request(self):
with closing(self.storage) as storage:
keyed_request = storage.key_for_request(self.request)
resolved_request = storage.resolve_request(keyed_request)
self.assertEquals(self.request, resolved_request)
self.assertIsNot(self.request, resolved_request)
self.assertEquals(resolved_request, self.storage.resolve_request(resolved_request))
def test_key_for_result(self):
with closing(self.storage) as storage:
keyed_result = storage.key_for_result(self.result)
self.assertEquals(Key, type(keyed_result.state))
self.assertEquals(keyed_result, storage.key_for_result(keyed_result))
def test_resolve_result(self):
with closing(self.storage) as storage:
keyed_result = storage.key_for_result(self.result)
resolved_result = storage.resolve_result(keyed_result)
self.assertEquals(self.result, resolved_result)
self.assertIsNot(self.result, resolved_result)
self.assertEquals(resolved_result, self.storage.resolve_result(resolved_result))
