def test_sync_children_datawatch(self):
"""Test data sync."""
# accessing protexted members.
# pylint: disable=W0212
zk_content = {
'a': {
'x': '1',
'y': '2',
'z': '3',
},
}
self.make_mock_zk(zk_content)
zk2fs_sync = zksync.Zk2Fs(kazoo.client.KazooClient(), self.root)
fs.mkdir_safe(os.path.join(self.root, 'a'))
zk2fs_sync._children_watch('/a', ['x', 'y', 'z'],
True,
zk2fs_sync._default_on_add,
zk2fs_sync._default_on_del)
self._check_file('a/x', '1')
self._check_file('a/y', '2')
self._check_file('a/z', '3')
self.assertIn('/a/x', zk2fs_sync.watches)
self.assertIn('/a/y', zk2fs_sync.watches)
self.assertIn('/a/z', zk2fs_sync.watches)
def test_sync_data(self):
"""Test data sync."""
# accessing protexted members.
# pylint: disable=W0212
zk_content = {
'a': {
'x': '1',
'y': '2',
'z': '3',
},
}
self.make_mock_zk(zk_content)
zk2fs_sync = zksync.Zk2Fs(kazoo.client.KazooClient(), self.root)
fs.mkdir_safe(os.path.join(self.root, 'a'))
event = kazoo.protocol.states.WatchedEvent(
'CREATED', 'CONNECTED', '/a/x')
zk2fs_sync._data_watch('/a/x', 'aaa', None, event)
self._check_file('a/x', 'aaa')
event = kazoo.protocol.states.WatchedEvent(
'DELETED', 'CONNECTED', '/a/x')
zk2fs_sync._data_watch('/a/x', 'aaa', None, event)
self.assertFalse(os.path.exists(os.path.join(self.root, 'a/x')))
event = kazoo.protocol.states.WatchedEvent(
'CREATED', 'CONNECTED', '/a/x')
zk2fs_sync._data_watch('/a/x', 'aaa', None, event)
self._check_file('a/x', 'aaa')
zk2fs_sync._data_watch('/a/x', None, None, None)
self.assertFalse(os.path.exists(os.path.join(self.root, 'a/x')))
def test_sync_children_unordered(self):
"""Test zk2fs sync with unordered data."""
# Disable W0212: accessing protected members.
# pylint: disable=W0212
zk_content = {
'a': {
'z': '1',
'x': '2',
'y': '3',
},
}
self.make_mock_zk(zk_content)
glob.glob.return_value = ['a/b', 'a/a', 'a/y']
add = []
rm = []
zk2fs_sync = zksync.Zk2Fs(kazoo.client.KazooClient(), self.root)
zk2fs_sync._children_watch('/a', ['z', 'x', 'y'],
False,
lambda x: add.append(os.path.basename(x)),
lambda x: rm.append(os.path.basename(x)))
# y first because its common
self.assertSequenceEqual(['y', 'x', 'z'], add)
self.assertSequenceEqual(['a', 'b'], rm)
def zk2fs_cmd(root, endpoints, identity_groups, appgroups, running,
scheduled, servers, placement, trace, once):
"""Starts appcfgmgr process."""
fs.mkdir_safe(root)
zk2fs_sync = zksync.Zk2Fs(context.GLOBAL.zk.conn, root)
if servers:
zk2fs_sync.sync_children(z.path.server(), watch_data=False)
if running:
# Running are ephemeral, and will be added/remove automatically.
zk2fs_sync.sync_children(z.path.running())
if endpoints:
zk2fs_sync.sync_children(
z.ENDPOINTS,
on_add=lambda p: _on_add_endpoint_proid(zk2fs_sync, p),
on_del=lambda p: _on_del_endpoint_proid(zk2fs_sync, p))
if identity_groups:
zk2fs_sync.sync_children(
z.IDENTITY_GROUPS,
on_add=lambda p: _on_add_identity(zk2fs_sync, p),
on_del=lambda p: _on_del_identity(zk2fs_sync, p))
if scheduled:
zk2fs_sync.sync_children(z.path.scheduled())
if appgroups:
zk2fs_sync.sync_children(z.path.appgroup(), watch_data=True)
if placement:
zk2fs_sync.sync_children(z.path.placement(), watch_data=True)
if trace:
zk2fs_sync.sync_children(
z.TRACE,
on_add=lambda p: _on_add_trace_shard(zk2fs_sync, p),
on_del=lambda p: _on_del_trace_shard(zk2fs_sync, p))
trace_sow = os.path.join(zk2fs_sync.fsroot, apptrace.TRACE_SOW_DIR)
_LOGGER.info('Using trace sow db: %s', trace_sow)
fs.mkdir_safe(trace_sow)
zk2fs_sync.sync_children(
z.TRACE_HISTORY,
on_add=lambda p: _on_add_trace_db(zk2fs_sync, p, trace_sow),
on_del=lambda p: _on_del_trace_db(zk2fs_sync, p, trace_sow),
)
zk2fs_sync.mark_ready()
if not once:
while True:
time.sleep(100000)
def test_sync_children(self):
"""Test zk2fs sync with no data."""
# Disable W0212: accessing protected members.
# pylint: disable=W0212
zk_content = {
'a': {
'x': '1',
'y': '2',
'z': '3',
},
}
self.make_mock_zk(zk_content)
zk2fs_sync = zksync.Zk2Fs(kazoo.client.KazooClient(), self.root)
fs.mkdir_safe(os.path.join(self.root, 'a'))
zk2fs_sync._children_watch('/a', ['x', 'y', 'z'],
False,
zk2fs_sync._default_on_add,
zk2fs_sync._default_on_del)
self._check_file('a/x', '1')
self._check_file('a/y', '2')
self._check_file('a/z', '3')
self.assertNotIn('/a/x', zk2fs_sync.watches)
# Common files are ignored in sync, 'x' content will not be updated.
zk_content['a']['x'] = '123'
zk_content['a']['q'] = 'qqq'
zk2fs_sync._children_watch('/a', ['x', 'y', 'z', 'q'],
False,
zk2fs_sync._default_on_add,
zk2fs_sync._default_on_del)
self._check_file('a/x', '1')
self._check_file('a/q', 'qqq')
# Removing node from zk will delete it from file system.
del zk_content['a']['x']
zk2fs_sync._children_watch('/a', ['y', 'z', 'q'],
False,
zk2fs_sync._default_on_add,
zk2fs_sync._default_on_del)
self.assertFalse(os.path.exists(os.path.join(self.root, 'a/x')))
def test_sync_children_immutable(self):
"""Test zk2fs sync with no watch needed."""
# Disable W0212: accessing protected members.
# pylint: disable=W0212
zk_content = {
'a': {
'x': '1',
'y': '2',
'z': '3',
},
}
self.make_mock_zk(zk_content)
zk2fs_sync = zksync.Zk2Fs(kazoo.client.KazooClient(), self.root)
fs.mkdir_safe(os.path.join(self.root, 'a'))
zk2fs_sync.sync_children('/a',
watch_data=False,
on_add=zk2fs_sync._default_on_add,
on_del=zk2fs_sync._default_on_del,
need_watch_predicate=lambda *args: False,
cont_watch_predicate=lambda *args: False)
self._check_file('a/x', '1')
self._check_file('a/y', '2')
self._check_file('a/z', '3')
self.assertNotIn('/a/x', zk2fs_sync.watches)
self.assertNotIn('/a', zk2fs_sync.watches)
self.assertTrue(os.path.exists(os.path.join(self.root, 'a', '.done')))
kazoo.client.KazooClient.get_children.reset_mock()
zk2fs_sync.sync_children('/a',
watch_data=False,
on_add=zk2fs_sync._default_on_add,
on_del=zk2fs_sync._default_on_del,
need_watch_predicate=lambda *args: False,
cont_watch_predicate=lambda *args: False)
self.assertFalse(kazoo.client.KazooClient.get_children.called)
def zk2fs_cmd(root, endpoints, identity_groups, appgroups, running,
scheduled, servers, placement, tasks, once):
"""Starts appcfgmgr process."""
fs.mkdir_safe(root)
zk2fs_sync = zksync.Zk2Fs(context.GLOBAL.zk.conn, root)
if servers:
zk2fs_sync.sync_children(z.path.server(), watch_data=False)
if running:
# Running are ephemeral, and will be added/remove automatically.
zk2fs_sync.sync_children(z.path.running())
if endpoints:
zk2fs_sync.sync_children(
z.ENDPOINTS,
on_add=lambda p: _on_add_proid(zk2fs_sync, p),
on_del=lambda p: _on_del_proid(zk2fs_sync, p))
if identity_groups:
zk2fs_sync.sync_children(
z.IDENTITY_GROUPS,
on_add=lambda p: _on_add_identity(zk2fs_sync, p),
on_del=lambda p: _on_del_identity(zk2fs_sync, p))
if scheduled:
zk2fs_sync.sync_children(z.path.scheduled())
if appgroups:
zk2fs_sync.sync_children(z.path.appgroup(), watch_data=True)
if placement:
zk2fs_sync.sync_placement(z.path.placement(), watch_data=True)
if tasks:
regex = fnmatch.translate(tasks)
_LOGGER.info('Using pattern: %s', regex)
reobj = re.compile(regex)
zk2fs_sync.sync_children(
z.TASKS,
on_add=lambda p: _on_add_app(zk2fs_sync, p, reobj),
on_del=lambda p: _on_del_app(zk2fs_sync, p, reobj),
)
tasks_sow_db = os.path.join(zk2fs_sync.fsroot, '.tasks-sow.db')
_LOGGER.info('Using tasks sow db: %s', tasks_sow_db)
if not os.path.exists(tasks_sow_db):
_init_sow_db(tasks_sow_db)
zk2fs_sync.sync_children(
z.TASKS_HISTORY,
on_add=lambda p: _on_add_task_db(zk2fs_sync, p, tasks_sow_db),
on_del=lambda p: _on_del_task_db(zk2fs_sync, p, tasks_sow_db),
)
zk2fs_sync.mark_ready()
if not once:
while True:
time.sleep(100000)
