def test_sync_children_datawatch(self): """Test data sync.""" # accessing protexted members. # pylint: disable=W0212 zk_content = { 'a': { 'x': '1', 'y': '2', 'z': '3', }, } self.make_mock_zk(zk_content) zk2fs_sync = zksync.Zk2Fs(kazoo.client.KazooClient(), self.root) fs.mkdir_safe(os.path.join(self.root, 'a')) zk2fs_sync._children_watch('/a', ['x', 'y', 'z'], True, zk2fs_sync._default_on_add, zk2fs_sync._default_on_del) self._check_file('a/x', '1') self._check_file('a/y', '2') self._check_file('a/z', '3') self.assertIn('/a/x', zk2fs_sync.watches) self.assertIn('/a/y', zk2fs_sync.watches) self.assertIn('/a/z', zk2fs_sync.watches)
def test_sync_data(self): """Test data sync.""" # accessing protexted members. # pylint: disable=W0212 zk_content = { 'a': { 'x': '1', 'y': '2', 'z': '3', }, } self.make_mock_zk(zk_content) zk2fs_sync = zksync.Zk2Fs(kazoo.client.KazooClient(), self.root) fs.mkdir_safe(os.path.join(self.root, 'a')) event = kazoo.protocol.states.WatchedEvent( 'CREATED', 'CONNECTED', '/a/x') zk2fs_sync._data_watch('/a/x', 'aaa', None, event) self._check_file('a/x', 'aaa') event = kazoo.protocol.states.WatchedEvent( 'DELETED', 'CONNECTED', '/a/x') zk2fs_sync._data_watch('/a/x', 'aaa', None, event) self.assertFalse(os.path.exists(os.path.join(self.root, 'a/x'))) event = kazoo.protocol.states.WatchedEvent( 'CREATED', 'CONNECTED', '/a/x') zk2fs_sync._data_watch('/a/x', 'aaa', None, event) self._check_file('a/x', 'aaa') zk2fs_sync._data_watch('/a/x', None, None, None) self.assertFalse(os.path.exists(os.path.join(self.root, 'a/x')))
def test_sync_children_unordered(self): """Test zk2fs sync with unordered data.""" # Disable W0212: accessing protected members. # pylint: disable=W0212 zk_content = { 'a': { 'z': '1', 'x': '2', 'y': '3', }, } self.make_mock_zk(zk_content) glob.glob.return_value = ['a/b', 'a/a', 'a/y'] add = [] rm = [] zk2fs_sync = zksync.Zk2Fs(kazoo.client.KazooClient(), self.root) zk2fs_sync._children_watch('/a', ['z', 'x', 'y'], False, lambda x: add.append(os.path.basename(x)), lambda x: rm.append(os.path.basename(x))) # y first because its common self.assertSequenceEqual(['y', 'x', 'z'], add) self.assertSequenceEqual(['a', 'b'], rm)
def zk2fs_cmd(root, endpoints, identity_groups, appgroups, running, scheduled, servers, placement, trace, once): """Starts appcfgmgr process.""" fs.mkdir_safe(root) zk2fs_sync = zksync.Zk2Fs(context.GLOBAL.zk.conn, root) if servers: zk2fs_sync.sync_children(z.path.server(), watch_data=False) if running: # Running are ephemeral, and will be added/remove automatically. zk2fs_sync.sync_children(z.path.running()) if endpoints: zk2fs_sync.sync_children( z.ENDPOINTS, on_add=lambda p: _on_add_endpoint_proid(zk2fs_sync, p), on_del=lambda p: _on_del_endpoint_proid(zk2fs_sync, p)) if identity_groups: zk2fs_sync.sync_children( z.IDENTITY_GROUPS, on_add=lambda p: _on_add_identity(zk2fs_sync, p), on_del=lambda p: _on_del_identity(zk2fs_sync, p)) if scheduled: zk2fs_sync.sync_children(z.path.scheduled()) if appgroups: zk2fs_sync.sync_children(z.path.appgroup(), watch_data=True) if placement: zk2fs_sync.sync_children(z.path.placement(), watch_data=True) if trace: zk2fs_sync.sync_children( z.TRACE, on_add=lambda p: _on_add_trace_shard(zk2fs_sync, p), on_del=lambda p: _on_del_trace_shard(zk2fs_sync, p)) trace_sow = os.path.join(zk2fs_sync.fsroot, apptrace.TRACE_SOW_DIR) _LOGGER.info('Using trace sow db: %s', trace_sow) fs.mkdir_safe(trace_sow) zk2fs_sync.sync_children( z.TRACE_HISTORY, on_add=lambda p: _on_add_trace_db(zk2fs_sync, p, trace_sow), on_del=lambda p: _on_del_trace_db(zk2fs_sync, p, trace_sow), ) zk2fs_sync.mark_ready() if not once: while True: time.sleep(100000)
def test_sync_children(self): """Test zk2fs sync with no data.""" # Disable W0212: accessing protected members. # pylint: disable=W0212 zk_content = { 'a': { 'x': '1', 'y': '2', 'z': '3', }, } self.make_mock_zk(zk_content) zk2fs_sync = zksync.Zk2Fs(kazoo.client.KazooClient(), self.root) fs.mkdir_safe(os.path.join(self.root, 'a')) zk2fs_sync._children_watch('/a', ['x', 'y', 'z'], False, zk2fs_sync._default_on_add, zk2fs_sync._default_on_del) self._check_file('a/x', '1') self._check_file('a/y', '2') self._check_file('a/z', '3') self.assertNotIn('/a/x', zk2fs_sync.watches) # Common files are ignored in sync, 'x' content will not be updated. zk_content['a']['x'] = '123' zk_content['a']['q'] = 'qqq' zk2fs_sync._children_watch('/a', ['x', 'y', 'z', 'q'], False, zk2fs_sync._default_on_add, zk2fs_sync._default_on_del) self._check_file('a/x', '1') self._check_file('a/q', 'qqq') # Removing node from zk will delete it from file system. del zk_content['a']['x'] zk2fs_sync._children_watch('/a', ['y', 'z', 'q'], False, zk2fs_sync._default_on_add, zk2fs_sync._default_on_del) self.assertFalse(os.path.exists(os.path.join(self.root, 'a/x')))
def test_sync_children_immutable(self): """Test zk2fs sync with no watch needed.""" # Disable W0212: accessing protected members. # pylint: disable=W0212 zk_content = { 'a': { 'x': '1', 'y': '2', 'z': '3', }, } self.make_mock_zk(zk_content) zk2fs_sync = zksync.Zk2Fs(kazoo.client.KazooClient(), self.root) fs.mkdir_safe(os.path.join(self.root, 'a')) zk2fs_sync.sync_children('/a', watch_data=False, on_add=zk2fs_sync._default_on_add, on_del=zk2fs_sync._default_on_del, need_watch_predicate=lambda *args: False, cont_watch_predicate=lambda *args: False) self._check_file('a/x', '1') self._check_file('a/y', '2') self._check_file('a/z', '3') self.assertNotIn('/a/x', zk2fs_sync.watches) self.assertNotIn('/a', zk2fs_sync.watches) self.assertTrue(os.path.exists(os.path.join(self.root, 'a', '.done'))) kazoo.client.KazooClient.get_children.reset_mock() zk2fs_sync.sync_children('/a', watch_data=False, on_add=zk2fs_sync._default_on_add, on_del=zk2fs_sync._default_on_del, need_watch_predicate=lambda *args: False, cont_watch_predicate=lambda *args: False) self.assertFalse(kazoo.client.KazooClient.get_children.called)
def zk2fs_cmd(root, endpoints, identity_groups, appgroups, running, scheduled, servers, placement, tasks, once): """Starts appcfgmgr process.""" fs.mkdir_safe(root) zk2fs_sync = zksync.Zk2Fs(context.GLOBAL.zk.conn, root) if servers: zk2fs_sync.sync_children(z.path.server(), watch_data=False) if running: # Running are ephemeral, and will be added/remove automatically. zk2fs_sync.sync_children(z.path.running()) if endpoints: zk2fs_sync.sync_children( z.ENDPOINTS, on_add=lambda p: _on_add_proid(zk2fs_sync, p), on_del=lambda p: _on_del_proid(zk2fs_sync, p)) if identity_groups: zk2fs_sync.sync_children( z.IDENTITY_GROUPS, on_add=lambda p: _on_add_identity(zk2fs_sync, p), on_del=lambda p: _on_del_identity(zk2fs_sync, p)) if scheduled: zk2fs_sync.sync_children(z.path.scheduled()) if appgroups: zk2fs_sync.sync_children(z.path.appgroup(), watch_data=True) if placement: zk2fs_sync.sync_placement(z.path.placement(), watch_data=True) if tasks: regex = fnmatch.translate(tasks) _LOGGER.info('Using pattern: %s', regex) reobj = re.compile(regex) zk2fs_sync.sync_children( z.TASKS, on_add=lambda p: _on_add_app(zk2fs_sync, p, reobj), on_del=lambda p: _on_del_app(zk2fs_sync, p, reobj), ) tasks_sow_db = os.path.join(zk2fs_sync.fsroot, '.tasks-sow.db') _LOGGER.info('Using tasks sow db: %s', tasks_sow_db) if not os.path.exists(tasks_sow_db): _init_sow_db(tasks_sow_db) zk2fs_sync.sync_children( z.TASKS_HISTORY, on_add=lambda p: _on_add_task_db(zk2fs_sync, p, tasks_sow_db), on_del=lambda p: _on_del_task_db(zk2fs_sync, p, tasks_sow_db), ) zk2fs_sync.mark_ready() if not once: while True: time.sleep(100000)