def test_sync_children_unordered(self):
"""Test zk2fs sync with unordered data."""
# Disable W0212: accessing protected members.
# pylint: disable=W0212
zk_content = {
'a': {
'z': b'1',
'x': b'2',
'y': b'3',
},
}
self.make_mock_zk(zk_content)
glob.glob.return_value = ['a/b', 'a/a', 'a/y']
add = []
rm = []
zk2fs_sync = zk2fs.Zk2Fs(kazoo.client.KazooClient(), self.root)
zk2fs_sync._children_watch('/a', ['z', 'x', 'y'], False,
lambda x: add.append(os.path.basename(x)),
lambda x: rm.append(os.path.basename(x)))
# y first because its common
self.assertSequenceEqual(['y', 'x', 'z'], add)
self.assertSequenceEqual(['a', 'b'], rm)
def test_sync_data(self):
"""Test data sync."""
# accessing protexted members.
# pylint: disable=W0212
zk_content = {
'a': {
'x': b'1',
'y': b'2',
'z': b'3',
},
}
mock_stat = collections.namedtuple('ZkStat', ['last_modified'])(0)
self.make_mock_zk(zk_content)
zk2fs_sync = zk2fs.Zk2Fs(kazoo.client.KazooClient(), self.root)
fs.mkdir_safe(os.path.join(self.root, 'a'))
event = kazoo.protocol.states.WatchedEvent('CREATED', 'CONNECTED',
'/a/x')
zk2fs_sync._data_watch('/a/x', b'aaa', mock_stat, event)
self._check_file('a/x', 'aaa')
event = kazoo.protocol.states.WatchedEvent('DELETED', 'CONNECTED',
'/a/x')
zk2fs_sync._data_watch('/a/x', b'aaa', mock_stat, event)
self.assertFalse(os.path.exists(os.path.join(self.root, 'a/x')))
event = kazoo.protocol.states.WatchedEvent('CREATED', 'CONNECTED',
'/a/x')
zk2fs_sync._data_watch('/a/x', b'aaa', mock_stat, event)
self._check_file('a/x', 'aaa')
zk2fs_sync._data_watch('/a/x', None, None, None)
self.assertFalse(os.path.exists(os.path.join(self.root, 'a/x')))
def test_sync_children_datawatch(self):
"""Test data sync."""
# accessing protexted members.
# pylint: disable=W0212
zk_content = {
'a': {
'x': b'1',
'y': b'2',
'z': b'3',
},
}
self.make_mock_zk(zk_content)
zk2fs_sync = zk2fs.Zk2Fs(kazoo.client.KazooClient(), self.root)
fs.mkdir_safe(os.path.join(self.root, 'a'))
zk2fs_sync._children_watch('/a', ['x', 'y', 'z'], True,
zk2fs_sync._default_on_add,
zk2fs_sync._default_on_del)
self._check_file('a/x', '1')
self._check_file('a/y', '2')
self._check_file('a/z', '3')
self.assertIn('/a/x', zk2fs_sync.watches)
self.assertIn('/a/y', zk2fs_sync.watches)
self.assertIn('/a/z', zk2fs_sync.watches)
def zk2fs_cmd(root, endpoints, identity_groups, identity_groups_meta,
appgroups, running, scheduled, servers, servers_data,
placement, trace, server_trace, app_monitors, once):
"""Starts appcfgmgr process."""
fs.mkdir_safe(root)
tmp_dir = os.path.join(root, '.tmp')
fs.mkdir_safe(tmp_dir)
zk2fs_sync = zk2fs.Zk2Fs(context.GLOBAL.zk.conn, root, tmp_dir)
if servers or servers_data:
zk2fs_sync.sync_children(z.path.server(), watch_data=servers_data)
if running:
# Running are ephemeral, and will be added/remove automatically.
zk2fs_sync.sync_children(z.path.running())
if endpoints:
zk2fs_sync.sync_children(
z.ENDPOINTS,
on_add=lambda p: _on_add_endpoint_proid(zk2fs_sync, p),
on_del=lambda p: _on_del_endpoint_proid(zk2fs_sync, p))
if identity_groups:
zk2fs_sync.sync_children(
z.IDENTITY_GROUPS,
on_add=lambda p: _on_add_identity(zk2fs_sync, p,
identity_groups_meta),
on_del=lambda p: _on_del_identity(zk2fs_sync, p))
if scheduled:
zk2fs_sync.sync_children(z.path.scheduled())
if appgroups:
zk2fs_sync.sync_children(z.path.appgroup(), watch_data=True)
if app_monitors:
zk2fs_sync.sync_children(z.path.appmonitor(), watch_data=True)
if placement:
zk2fs_sync.sync_children(
z.path.placement(),
on_add=lambda p: _on_add_placement_server(zk2fs_sync, p),
on_del=lambda p: _on_del_placement_server(zk2fs_sync, p))
if trace:
_sync_trace(zk2fs_sync, z.TRACE, z.TRACE_HISTORY,
app_zk.TRACE_SOW_DIR)
if server_trace:
_sync_trace(zk2fs_sync, z.SERVER_TRACE, z.SERVER_TRACE_HISTORY,
server_zk.SERVER_TRACE_SOW_DIR)
zk2fs_sync.mark_ready()
if not once:
while True:
time.sleep(100000)
def test_sync_children(self):
"""Test zk2fs sync with no data."""
# Disable W0212: accessing protected members.
# pylint: disable=W0212
zk_content = {
'a': {
'x': b'1',
'y': b'2',
'z': b'3',
},
}
self.make_mock_zk(zk_content)
zk2fs_sync = zk2fs.Zk2Fs(kazoo.client.KazooClient(), self.root)
fs.mkdir_safe(os.path.join(self.root, 'a'))
zk2fs_sync._children_watch('/a', ['x', 'y', 'z'],
False,
zk2fs_sync._default_on_add,
zk2fs_sync._default_on_del)
self._check_file('a/x', '1')
self._check_file('a/y', '2')
self._check_file('a/z', '3')
self.assertNotIn('/a/x', zk2fs_sync.watches)
# Common files are ignored in sync, 'x' content will not be updated.
zk_content['a']['x'] = b'123'
zk_content['a']['q'] = b'qqq'
zk2fs_sync._children_watch('/a', ['x', 'y', 'z', 'q'],
False,
zk2fs_sync._default_on_add,
zk2fs_sync._default_on_del)
self._check_file('a/x', '1')
self._check_file('a/q', 'qqq')
# Removing node from zk will delete it from file system.
del zk_content['a']['x']
zk2fs_sync._children_watch('/a', ['y', 'z', 'q'],
False,
zk2fs_sync._default_on_add,
zk2fs_sync._default_on_del)
self.assertFalse(os.path.exists(os.path.join(self.root, 'a/x')))
def test_sync_children_immutable(self):
"""Test zk2fs sync with no watch needed."""
# Disable W0212: accessing protected members.
# pylint: disable=W0212
zk_content = {
'a': {
'x': b'1',
'y': b'2',
'z': b'3',
},
}
self.make_mock_zk(zk_content)
zk2fs_sync = zk2fs.Zk2Fs(kazoo.client.KazooClient(), self.root)
fs.mkdir_safe(os.path.join(self.root, 'a'))
zk2fs_sync.sync_children('/a',
watch_data=False,
on_add=zk2fs_sync._default_on_add,
on_del=zk2fs_sync._default_on_del,
need_watch_predicate=lambda *args: False,
cont_watch_predicate=lambda *args: False)
self._check_file('a/x', '1')
self._check_file('a/y', '2')
self._check_file('a/z', '3')
self.assertNotIn('/a/x', zk2fs_sync.watches)
self.assertNotIn('/a', zk2fs_sync.watches)
self.assertTrue(os.path.exists(os.path.join(self.root, 'a', '.done')))
kazoo.client.KazooClient.get_children.reset_mock()
zk2fs_sync.sync_children('/a',
watch_data=False,
on_add=zk2fs_sync._default_on_add,
on_del=zk2fs_sync._default_on_del,
need_watch_predicate=lambda *args: False,
cont_watch_predicate=lambda *args: False)
self.assertFalse(kazoo.client.KazooClient.get_children.called)
