def _place_watch(config, zk_path, command,
watch_type, max_wait_in_secs=0, alert_disabled=False):
"""
Place the watch for Config or Serverset giving the information extracted
from metaconfig.
"""
# If the ZK path is already added, skip the following steps
# in order to resolve conflicts.
if zk_path in _PATH_TO_COMMAND:
log.warn("Path %s has already been added to PATH_TO_COMMAND" % zk_path)
return False
if not _zk_path_exists(ZK_HOSTS, zk_path):
log.error("zk_path %s does not exist in zk_hosts %s, no watch is set."
% (zk_path, ZK_HOSTS))
# Save the configs that contain the nonexistent zk path for retry
_CONFIGS_WITH_NONEXISTENT_PATH.append(config)
return False
log.info("Creating Zookeeper data watcher for path %s of zk_hosts %s"
% (zk_path, ZK_HOSTS))
# Get the local file modification time if the file exists, else set to 0.
_get_and_set_local_file_modification_time(zk_path, command, watch_type)
if watch_type is not None and watch_type.lower() == 'serverset':
watcher = ServerSet(zk_path, ZK_HOSTS)
else:
watcher = DataWatcher(zk_path, ZK_HOSTS)
log.info(
"Associating command '%s' with watcher of path %s"
% (command, zk_path)
)
_PATH_TO_COMMAND[zk_path] = command
_PATH_TO_ALERT_DISABLED[zk_path] = alert_disabled
callback_func = functools.partial(
_run_command, zk_path, command, max_wait_in_secs, watch_type)
_PATH_TO_WATCHER[zk_path] = \
{"watcher": watcher, "func": callback_func, "watch_type": watch_type}
if watch_type == 'serverset':
watcher.monitor(callback_func)
else:
watcher.watch(callback_func)
return True
def __init__(self,
zk_hosts,
aws_keyfile,
s3_bucket,
file_path,
read_config_callback,
s3_endpoint="s3.amazonaws.com",
zk_path_suffix=''):
"""
Args:
file_path: config file path to watch.
read_config_callback: callback function when new config is detected. It should take
one argument which is the new config string. If it is None, no config file watcher
will be registered.
"""
self.zk_original_path = get_zk_path(file_path)
self.zk_path = self.zk_original_path + zk_path_suffix
super(ZKConfigManager, self).__init__(zk_hosts,
self.zk_path,
aws_keyfile,
s3_bucket,
s3_endpoint=s3_endpoint)
self.file_path = file_path
self.zk_lock_path = ZK_LOCK_PATH_FORMAT % self.zk_path
self.s3_file_path = S3_CONFIG_FILE_PATH_FORMAT % self.zk_original_path
if read_config_callback is not None:
assert hasattr(read_config_callback, '__call__')
self.read_config_callback = read_config_callback
self.version = -1
self.watcher = DataWatcher(None, zk_hosts, file_path=self.file_path)
if file_path:
try:
os.path.getmtime(file_path)
except OSError:
log.error("%s does not exist or is unaccessible" % file_path)
return
self.watcher.watch(self._read_config)
def update_dependency(dependents):
# For all children under dependency, set datawatch or children watches
# If this function is triggered not in the placing watchers time,
# restart ZUM to refresh to dependency list.
if _INITIALIZATION_COMPLETED:
_kill("Watched dependency changed, restart ZUM to catch the change")
for dependent in dependents:
if dependent.endswith(".dep"):
_load_metaconfigs_from_one_dependency(dependent)
else:
if dependent in _WATCHED_METACONFIGS:
continue
# Set watch on the MetaConfig znode and load the content from S3
metaconfig_zk_path = METACONFIG_ZK_PATH_FORMAT.format(dependent)
if not _kazoo_client(ZK_HOSTS).exists(metaconfig_zk_path):
log.error("The metaconfig %s does not exist" % dependent)
continue
log.info("Watching Metaconfig %s" % dependent)
watcher = DataWatcher(metaconfig_zk_path, ZK_HOSTS)
watcher.watch(functools.partial(update_metaconfig, dependent))
_WATCHED_METACONFIGS.add(dependent)
def test_data_watcher(self):
"""Test various scenarios for data watcher:
1. When data get changed, watcher callback should be invoked.
2. When the underlying zk client disconnects and then recovers,
the watcher callback should be invoked.
3. When the underlying zk client messes up beyond recovery,
the underlying client should be replaced, and once the new client
is in place, the watcher callback should be invoked again.
"""
data_stat = []
watcher_triggered = Event()
def data_watch(data, stat):
while data_stat:
data_stat.pop()
data_stat.append(data)
data_stat.append(stat)
watcher_triggered.set()
testutil.initialize_kazoo_client_manager(ZK_HOSTS)
client = KazooClientManager().get_client()
client.create(DataWatcherTestCase.TEST_PATH,
DataWatcherTestCase.DATA_0)
data_watcher = DataWatcher(DataWatcherTestCase.TEST_PATH,
ZK_HOSTS,
waiting_in_secs=0.01)
data_watcher.watch(data_watch).join()
watcher_triggered.wait(1)
# Now the data and version should be foo and 0.
self.assertEqual(data_stat[0], DataWatcherTestCase.DATA_0)
self.assertEqual(data_stat[1].version, 0)
watcher_triggered.clear()
client.set(DataWatcherTestCase.TEST_PATH, DataWatcherTestCase.DATA_1)
watcher_triggered.wait(1)
# Make sure that watch callback is triggered.
self.assertEqual(data_stat[0], DataWatcherTestCase.DATA_1)
self.assertEqual(data_stat[1].version, 1)
data_stat.pop()
data_stat.pop()
# Test recoverable failure
watcher_triggered.clear()
client.stop()
client.start()
# Here the client actually will call check the znode in the
# background.
watcher_triggered.wait(1)
# Since nothing changed, no notification from the client.
self.assertFalse(data_stat)
# Test client change
client.stop()
watcher_triggered.clear()
# give the monit greenlet a chance to detect failures.
gevent.sleep(1)
# Assert the client has been replaced with a new one.
self.assertFalse(KazooClientManager().get_client() is client)
watcher_triggered.wait(1)
# Make sure that watch callback is triggered when client is replaced.
self.assertEqual(data_stat[0], DataWatcherTestCase.DATA_1)
self.assertEqual(data_stat[1].version, 1)
def test_data_watcher(self):
"""Test data watcher with a local file:
1. When data get changed, watcher callback should be invoked.
2. When the underlying zk client disconnects and then recovers,
the watcher callback should be invoked.
3. When the underlying zk client messes up beyond recovery,
the underlying client should be replaced, and once the new client
is in place, the watcher callback should be invoked again.
Although when a local file is being watched, now all the code paths
about the above behaviors got affected, we still want to test all the
scenarios to make sure nothing breaks when a file is used.
"""
data_stat = []
watcher_triggered = Event()
fd, tmp_file = tempfile.mkstemp()
with open(tmp_file, 'w') as f:
f.write(self.DATA_0)
def data_watch(data, stat):
while data_stat:
data_stat.pop()
data_stat.append(data)
data_stat.append(stat)
watcher_triggered.set()
data_watcher = DataWatcher(DataWatcherWithFileTestCase.TEST_PATH,
ZK_HOSTS,
waiting_in_secs=0.01,
file_path=tmp_file)
data_watcher.watch(data_watch).join()
watcher_triggered.wait(1)
# Now the data and version should be foo and the mtime of file.
mtime = os.path.getmtime(tmp_file)
self.assertEqual(data_stat[0], DataWatcherWithFileTestCase.DATA_0)
self.assertEqual(data_stat[1].version, mtime)
self.assertEqual(data_watcher.get_data()[0],
DataWatcherWithFileTestCase.DATA_0)
self.assertEqual(data_watcher.get_data()[1].version, mtime)
watcher_triggered.clear()
gevent.sleep(1)
with open(tmp_file, 'w') as f:
f.write(self.DATA_1)
watcher_triggered.wait(1)
# Make sure that watch callback is triggered.
mtime = os.path.getmtime(tmp_file)
self.assertEqual(data_stat[0], DataWatcherWithFileTestCase.DATA_1)
self.assertEqual(data_stat[1].version, mtime)
self.assertEqual(data_watcher.get_data()[0],
DataWatcherWithFileTestCase.DATA_1)
self.assertEqual(data_watcher.get_data()[1].version, mtime)
data_stat.pop()
data_stat.pop()
# Test recoverable failure, even though the watcher with a file path
# is not changing any implementation or behavior in this part, we want
# to keep the tests here to ensure.
watcher_triggered.clear()
self.FILE_WATCH._clear_all_watches()
os.remove(tmp_file)
