class StorageTest(unittest.TestCase): TEST_KEY = b'hello' TEST_VALUE = b'world' TEST_PATH = File('/foo') TEST_PATH2 = Dir('/bar') class SomeException(Exception): pass def setUp(self): self.storage = Storage.create() self.result = 'something' self.request = Runnable(func=_runnable, args=('this is an arg',), cacheable=True) def test_storage(self): key = self.storage.put(self.TEST_PATH) self.assertEquals(self.TEST_PATH, self.storage.get(key)) with self.assertRaises(InvalidKeyError): self.assertFalse(self.storage.get(self.TEST_KEY)) def test_storage_key_mappings(self): key1 = self.storage.put(self.TEST_PATH) key2 = self.storage.put(self.TEST_PATH2) self.storage.add_mapping(key1, key2) self.assertEquals(key2, self.storage.get_mapping(key1)) # key2 isn't mapped to any other key. self.assertIsNone(self.storage.get_mapping(key2))
def test_empty(self): """Test that parsing an empty BUILD file results in an empty AddressFamily.""" address_mapper = AddressMapper(JsonParser(TestTable())) af = run_rule(parse_address_family, address_mapper, Dir('/dev/null'), { (Snapshot, PathGlobs): lambda _: Snapshot(DirectoryDigest('abc', 10), (File('/dev/null/BUILD'),)), (FilesContent, DirectoryDigest): lambda _: FilesContent([FileContent('/dev/null/BUILD', b'')]), }) self.assertEqual(len(af.objects_by_name), 0)
def generate_subjects(self, filenames): """Given filenames, generate a set of subjects for invalidation predicate matching.""" for f in filenames: # ReadLink, or FileContent for the literal path. yield File(f) yield Link(f) # DirectoryListing for parent dirs. yield Dir(dirname(f))
def generate_fs_subjects(filenames): """Given filenames, generate a set of subjects for invalidation predicate matching.""" for f in filenames: # ReadLink, FileContent, or DirectoryListing for the literal path. yield File(f) yield Link(f) yield Dir(f) # Additionally, since the FS event service does not send invalidation events # for the root directory, treat any changed file in the root as an invalidation # of the root's listing. if dirname(f) in ('.', ''): yield Dir('')
def _lstat(self, relpath): mode = type(self._reader.lstat(self._scm_relpath(relpath))) if mode == NoneType: return None elif mode == self._reader.Symlink: return Link(relpath) elif mode == self._reader.Dir: return Dir(relpath) elif mode == self._reader.File: return File(relpath) else: raise IOError('Unsupported file type in {}: {}'.format(self, relpath))
def test_duplicated(self): """Test that matching the same Spec twice succeeds.""" address = SingleAddress('a', 'a') snapshot = Snapshot(Digest('xx', 2), (Path('a/BUILD', File('a/BUILD')),)) address_family = AddressFamily('a', {'a': ('a/BUILD', 'this is an object!')}) specs = Specs([address, address]) bfas = self._resolve_build_file_addresses( specs, address_family, snapshot, self._address_mapper()) self.assertEqual(len(bfas.dependencies), 1) self.assertEqual(bfas.dependencies[0].spec, 'a:a')
def test_duplicated(self): """Test that matching the same Spec twice succeeds.""" address = SingleAddress('a', 'a') address_mapper = AddressMapper(JsonParser(TestTable())) snapshot = Snapshot(DirectoryDigest(str('xx'), 2), (Path('a/BUILD', File('a/BUILD')),)) address_family = AddressFamily('a', {'a': ('a/BUILD', 'this is an object!')}) bfas = run_rule(addresses_from_address_families, address_mapper, Specs([address, address]), { (Snapshot, PathGlobs): lambda _: snapshot, (AddressFamily, Dir): lambda _: address_family, }) self.assertEquals(len(bfas.dependencies), 1) self.assertEquals(bfas.dependencies[0].spec, 'a:a')
class StorageTest(unittest.TestCase): TEST_KEY = b'hello' TEST_VALUE = b'world' TEST_PATH = File('/foo') TEST_PATH2 = Dir('/bar') class SomeException(Exception): pass def setUp(self): self.storage = Storage.create() self.result = 'something' self.request = Runnable(func=_runnable, args=('this is an arg', ), cacheable=True) def test_lmdb_key_value_store(self): lmdb = Lmdb.create()[0] with closing(lmdb) as kvs: # Initially key does not exist. self.assertFalse(kvs.get(self.TEST_KEY)) # Now write a key value pair and read back. written = kvs.put(self.TEST_KEY, self.TEST_VALUE) self.assertTrue(written) self.assertEquals(self.TEST_VALUE, kvs.get(self.TEST_KEY).getvalue()) # Write the same key again will not overwrite. self.assertFalse(kvs.put(self.TEST_KEY, self.TEST_VALUE)) def test_storage(self): with closing(self.storage) as storage: key = storage.put(self.TEST_PATH) self.assertEquals(self.TEST_PATH, storage.get(key)) with self.assertRaises(InvalidKeyError): self.assertFalse(storage.get(self.TEST_KEY)) def test_storage_key_mappings(self): with closing(self.storage) as storage: key1 = storage.put(self.TEST_PATH) key2 = storage.put(self.TEST_PATH2) storage.add_mapping(key1, key2) self.assertEquals(key2, storage.get_mapping(key1)) # key2 isn't mapped to any other key. self.assertIsNone(storage.get_mapping(key2))
def test_exclude_pattern_with_single_address(self): """Test that single address targets are filtered based on exclude patterns.""" spec = SingleAddress('root', 'not_me') address_mapper = AddressMapper(JsonParser(TestTable())) snapshot = Snapshot(DirectoryDigest('xx', 2), (Path('root/BUILD', File('root/BUILD')),)) address_family = AddressFamily('root', { 'not_me': ('root/BUILD', TargetAdaptor()), } ) targets = run_rule( addresses_from_address_families, address_mapper, Specs([spec], exclude_patterns=tuple(['root.*'])),{ (Snapshot, PathGlobs): lambda _: snapshot, (AddressFamily, Dir): lambda _: address_family, }) self.assertEqual(len(targets.dependencies), 0)
def test_exclude_pattern(self): """Test that targets are filtered based on exclude patterns.""" spec = SiblingAddresses('root') address_mapper = AddressMapper(JsonParser(TestTable())) snapshot = Snapshot(DirectoryDigest(text_type('xx'), 2), (Path('root/BUILD', File('root/BUILD')), )) address_family = AddressFamily( 'root', { 'exclude_me': ('root/BUILD', TargetAdaptor()), 'not_me': ('root/BUILD', TargetAdaptor()), }) targets = run_rule( addresses_from_address_families, address_mapper, Specs([spec], exclude_patterns=tuple(['.exclude*'])), { (Snapshot, PathGlobs): lambda _: snapshot, (AddressFamily, Dir): lambda _: address_family, }) self.assertEquals(len(targets.dependencies), 1) self.assertEquals(targets.dependencies[0].spec, 'root:not_me')
def test_passes_eager_fileset_with_spec_through(self): self.create_file('foo/foo/a.txt', 'a_contents') fileset = self.sources_for(['foo/a.txt'], 'foo') sf = SourcesField(sources=fileset) self.assertIs(fileset, sf.sources) self.assertEqual(['foo/a.txt'], list(sf.source_paths)) self.assertEqual(['foo/foo/a.txt'], list(sf.relative_to_buildroot())) digest = str( '56001a7e48555f156420099a99da60a7a83acc90853046709341bf9f00a6f944') want_snapshot = Snapshot(DirectoryDigest( digest, 77), (Path('foo/foo/a.txt', stat=File('foo/foo/a.txt')), )) # We explicitly pass a None scheduler because we expect no scheduler lookups to be required # in order to get a Snapshot. self.assertEqual(sf.snapshot(scheduler=None), want_snapshot)
def test_tag_filter(self): """Test that targets are filtered based on `tags`.""" spec = SiblingAddresses('root') address_mapper = AddressMapper(JsonParser(TestTable())) snapshot = Snapshot(DirectoryDigest(str('xx'), 2), (Path('root/BUILD', File('root/BUILD')),)) address_family = AddressFamily('root', {'a': ('root/BUILD', TargetAdaptor()), 'b': ('root/BUILD', TargetAdaptor(tags={'integration'})), 'c': ('root/BUILD', TargetAdaptor(tags={'not_integration'})) } ) targets = run_rule( addresses_from_address_families, address_mapper, Specs([spec], tags=['+integration']), { (Snapshot, PathGlobs): lambda _: snapshot, (AddressFamily, Dir): lambda _: address_family, }) self.assertEquals(len(targets.dependencies), 1) self.assertEquals(targets.dependencies[0].spec, 'root:b')
def _scandir_raw(self, relpath): # Sanity check. TODO: this should probably be added to the ProjectTree interface as # an optional call, so that we can use it in fs.py rather than applying it by default. abspath = os.path.normpath(self._join(relpath)) if os.path.realpath(abspath) != abspath: raise ValueError( 'scandir for non-canonical path "{}" not supported in {}.'. format(relpath, self)) for entry in os.scandir(abspath): # NB: We don't use `DirEntry.stat`, as the scandir docs indicate that that always requires # an additional syscall on Unixes. entry_path = os.path.normpath(os.path.join(relpath, entry.name)) if entry.is_file(follow_symlinks=False): yield File(entry_path) elif entry.is_dir(follow_symlinks=False): yield Dir(entry_path) elif entry.is_symlink(): yield Link(entry_path) else: raise IOError('Unsupported file type in {}: {}'.format( self, entry_path))
def _snapshot(self): return Snapshot( Digest('xx', 2), (Path('root/BUILD', File('root/BUILD')),))
class StorageTest(unittest.TestCase): TEST_KEY = b'hello' TEST_VALUE = b'world' TEST_PATH = File('/foo') TEST_PATH2 = Dir('/bar') class SomeException(Exception): pass def setUp(self): self.storage = Storage.create() self.result = 'something' self.request = Runnable(func=_runnable, args=('this is an arg', )) def test_lmdb_key_value_store(self): lmdb = Lmdb.create()[0] with closing(lmdb) as kvs: # Initially key does not exist. self.assertFalse(kvs.get(self.TEST_KEY)) # Now write a key value pair and read back. written = kvs.put(self.TEST_KEY, self.TEST_VALUE) self.assertTrue(written) self.assertEquals(self.TEST_VALUE, kvs.get(self.TEST_KEY).getvalue()) # Write the same key again will not overwrite. self.assertFalse(kvs.put(self.TEST_KEY, self.TEST_VALUE)) def test_storage(self): with closing(self.storage) as storage: key = storage.put(self.TEST_PATH) self.assertEquals(self.TEST_PATH, storage.get(key)) with self.assertRaises(InvalidKeyError): self.assertFalse(storage.get(self.TEST_KEY)) def test_storage_key_mappings(self): with closing(self.storage) as storage: key1 = storage.put(self.TEST_PATH) key2 = storage.put(self.TEST_PATH2) storage.add_mapping(key1, key2) self.assertEquals(key2, storage.get_mapping(key1)) # key2 isn't mapped to any other key. self.assertIsNone(storage.get_mapping(key2)) def test_state_roundtrips(self): states = [ Return('a'), Throw(PickleableException()), Waiting([TaskNode(None, None, None)]), Runnable(_runnable, ('an arg', )), Noop('nada {}', ('op', )) ] with closing(self.storage) as storage: for state in states: key = storage.put_state(state) actual = storage.get_state(key) self.assertEquals(state, actual) self.assertEquals(key, storage.put_state(actual))
class StorageTest(unittest.TestCase): TEST_KEY = b'hello' TEST_VALUE = b'world' TEST_PATH = File('/foo') TEST_PATH2 = Dir('/bar') class SomeException(Exception): pass def setUp(self): self.storage = Storage.create() self.result = StepResult(state='something') self.request = StepRequest(step_id=123, node='some node', dependencies={'some dep': 'some state', 'another dep': 'another state'}, inline_nodes=False, project_tree='some project tree') def test_lmdb_key_value_store(self): lmdb = Lmdb.create()[0] with closing(lmdb) as kvs: # Initially key does not exist. self.assertFalse(kvs.get(self.TEST_KEY)) # Now write a key value pair and read back. written = kvs.put(self.TEST_KEY, self.TEST_VALUE) self.assertTrue(written) self.assertEquals(self.TEST_VALUE, kvs.get(self.TEST_KEY).getvalue()) # Write the same key again will not overwrite. self.assertFalse(kvs.put(self.TEST_KEY, self.TEST_VALUE)) def test_storage(self): with closing(self.storage) as storage: key = storage.put(self.TEST_PATH) self.assertEquals(self.TEST_PATH, storage.get(key)) # The deserialized blob is equal by not the same as the input data. self.assertFalse(storage.get(key) is self.TEST_PATH) # Any other keys won't exist in the subjects. self.assertNotEqual(self.TEST_KEY, key) with self.assertRaises(InvalidKeyError): self.assertFalse(storage.get(self.TEST_KEY)) # Verify key and value's types must match. key._type = str with self.assertRaises(ValueError): storage.get(key) def test_storage_key_mappings(self): with closing(self.storage) as storage: key1 = storage.put(self.TEST_PATH) key2 = storage.put(self.TEST_PATH2) storage.add_mapping(key1, key2) self.assertEquals(key2, storage.get_mapping(key1)) # key2 isn't mapped to any other key. self.assertIsNone(storage.get_mapping(key2)) def test_key_for_request(self): with closing(self.storage) as storage: keyed_request = storage.key_for_request(self.request) for dep, dep_state in keyed_request.dependencies.items(): self.assertEquals(Key, type(dep)) self.assertEquals(Key, type(dep_state)) self.assertIs(self.request.node, keyed_request.node) self.assertIs(self.request.project_tree, keyed_request.project_tree) self.assertEquals(keyed_request, storage.key_for_request(keyed_request)) def test_resolve_request(self): with closing(self.storage) as storage: keyed_request = storage.key_for_request(self.request) resolved_request = storage.resolve_request(keyed_request) self.assertEquals(self.request, resolved_request) self.assertIsNot(self.request, resolved_request) self.assertEquals(resolved_request, self.storage.resolve_request(resolved_request)) def test_key_for_result(self): with closing(self.storage) as storage: keyed_result = storage.key_for_result(self.result) self.assertEquals(Key, type(keyed_result.state)) self.assertEquals(keyed_result, storage.key_for_result(keyed_result)) def test_resolve_result(self): with closing(self.storage) as storage: keyed_result = storage.key_for_result(self.result) resolved_result = storage.resolve_result(keyed_result) self.assertEquals(self.result, resolved_result) self.assertIsNot(self.result, resolved_result) self.assertEquals(resolved_result, self.storage.resolve_result(resolved_result))