def __init__(self, zk_hosts, aws_keyfile, s3_bucket, file_path, read_config_callback, s3_endpoint="s3.amazonaws.com", zk_path_suffix=''):
"""
Args:
file_path: config file path to watch.
read_config_callback: callback function when new config is detected. It should take
one argument which is the new config string. If it is None, no config file watcher
will be registered.
"""
self.zk_original_path = get_zk_path(file_path)
self.zk_path = self.zk_original_path + zk_path_suffix
super(ZKConfigManager, self).__init__(zk_hosts, self.zk_path, aws_keyfile, s3_bucket, s3_endpoint=s3_endpoint)
self.file_path = file_path
self.zk_lock_path = ZK_LOCK_PATH_FORMAT % self.zk_path
self.s3_file_path = S3_CONFIG_FILE_PATH_FORMAT % self.zk_original_path
if read_config_callback is not None:
assert hasattr(read_config_callback, '__call__')
self.read_config_callback = read_config_callback
self.version = -1
self.watcher = DataWatcher(None, zk_hosts, file_path=self.file_path)
if file_path:
try:
os.path.getmtime(file_path)
except OSError:
log.error("%s does not exist or is unaccessible" % file_path)
return
self.watcher.watch(self._read_config)
def __init__(self,
zk_hosts,
aws_keyfile,
s3_bucket,
file_path,
read_config_callback,
s3_endpoint="s3.amazonaws.com",
zk_path_suffix=''):
"""
Args:
file_path: config file path to watch.
read_config_callback: callback function when new config is detected. It should take
one argument which is the new config string. If it is None, no config file watcher
will be registered.
"""
self.zk_original_path = get_zk_path(file_path)
self.zk_path = self.zk_original_path + zk_path_suffix
super(ZKConfigManager, self).__init__(zk_hosts,
self.zk_path,
aws_keyfile,
s3_bucket,
s3_endpoint=s3_endpoint)
self.file_path = file_path
self.zk_lock_path = ZK_LOCK_PATH_FORMAT % self.zk_path
self.s3_file_path = S3_CONFIG_FILE_PATH_FORMAT % self.zk_original_path
if read_config_callback is not None:
assert hasattr(read_config_callback, '__call__')
self.read_config_callback = read_config_callback
self.version = -1
self.watcher = DataWatcher(None, zk_hosts, file_path=self.file_path)
if file_path:
try:
os.path.getmtime(file_path)
except OSError:
log.error("%s does not exist or is unaccessible" % file_path)
return
self.watcher.watch(self._read_config)
def update_dependency(dependents):
# For all children under dependency, set datawatch or children watches
# If this function is triggered not in the placing watchers time,
# restart ZUM to refresh to dependency list.
if _INITIALIZATION_COMPLETED:
_kill("Watched dependency changed, restart ZUM to catch the change")
for dependent in dependents:
if dependent.endswith(".dep"):
_load_metaconfigs_from_one_dependency(dependent)
else:
if dependent in _WATCHED_METACONFIGS:
continue
# Set watch on the MetaConfig znode and load the content from S3
metaconfig_zk_path = METACONFIG_ZK_PATH_FORMAT.format(dependent)
if not _kazoo_client(ZK_HOSTS).exists(metaconfig_zk_path):
log.error("The metaconfig %s does not exist" % dependent)
continue
log.info("Watching Metaconfig %s" % dependent)
watcher = DataWatcher(metaconfig_zk_path, ZK_HOSTS)
watcher.watch(functools.partial(update_metaconfig, dependent))
_WATCHED_METACONFIGS.add(dependent)
def update_dependency(dependents):
# For all children under dependency, set datawatch or children watches
# If this function is triggered not in the placing watchers time,
# restart ZUM to refresh to dependency list.
if _INITIALIZATION_COMPLETED:
_kill("Watched dependency changed, restart ZUM to catch the change")
for dependent in dependents:
if dependent.endswith(".dep"):
_load_metaconfigs_from_one_dependency(dependent)
else:
if dependent in _WATCHED_METACONFIGS:
continue
# Set watch on the MetaConfig znode and load the content from S3
metaconfig_zk_path = METACONFIG_ZK_PATH_FORMAT.format(dependent)
if not _kazoo_client(ZK_HOSTS).exists(metaconfig_zk_path):
log.error("The metaconfig %s does not exist" % dependent)
continue
log.info("Watching Metaconfig %s" % dependent)
watcher = DataWatcher(metaconfig_zk_path, ZK_HOSTS)
watcher.watch(functools.partial(update_metaconfig, dependent))
_WATCHED_METACONFIGS.add(dependent)
def _place_watch(config, zk_path, command,
watch_type, max_wait_in_secs=0, alert_disabled=False):
"""
Place the watch for Config or Serverset giving the information extracted
from metaconfig.
"""
# If the ZK path is already added, skip the following steps
# in order to resolve conflicts.
if zk_path in _PATH_TO_COMMAND:
log.warn("Path %s has already been added to PATH_TO_COMMAND" % zk_path)
return False
if not _zk_path_exists(ZK_HOSTS, zk_path):
log.error("zk_path %s does not exist in zk_hosts %s, no watch is set."
% (zk_path, ZK_HOSTS))
# Save the configs that contain the nonexistent zk path for retry
_CONFIGS_WITH_NONEXISTENT_PATH.append(config)
return False
log.info("Creating Zookeeper data watcher for path %s of zk_hosts %s"
% (zk_path, ZK_HOSTS))
# Get the local file modification time if the file exists, else set to 0.
_get_and_set_local_file_modification_time(zk_path, command, watch_type)
if watch_type is not None and watch_type.lower() == 'serverset':
watcher = ServerSet(zk_path, ZK_HOSTS)
else:
watcher = DataWatcher(zk_path, ZK_HOSTS)
log.info(
"Associating command '%s' with watcher of path %s"
% (command, zk_path)
)
_PATH_TO_COMMAND[zk_path] = command
_PATH_TO_ALERT_DISABLED[zk_path] = alert_disabled
callback_func = functools.partial(
_run_command, zk_path, command, max_wait_in_secs, watch_type)
_PATH_TO_WATCHER[zk_path] = \
{"watcher": watcher, "func": callback_func, "watch_type": watch_type}
if watch_type == 'serverset':
watcher.monitor(callback_func)
else:
watcher.watch(callback_func)
return True
class ZKConfigManager(ZKBaseConfigManager):
"""An extention of ZKBaseConfigManager that provides watcher for local config file.
Internally, the config file is stored in s3, which is downloaded to local config file by
zk_update_monitor process(when it gets notified by zk). To change the config file, we need to
push the data to s3 and update zk_node, which triggers zk_update_monitor to download the file
on each box.
Before using this class, puppet change needs to be made to let zk_update_monitor download your
file. To make zk_path and config file path consistent, we use ``get_zk_path()`` and
``get_config_file_path()`` to do the mutual conversion. The zk node created and the config file
to watch needs to have the consistent path.
If you only intend to use ``ZKConfigManager`` to get notified when config file is changed, you
can ignore ``update_zk()``. ``update_zk()`` provides a convenient way to change the original
data for the config file.
Usage:
def read_new_config(new_config):
print "New config is here", new_config
watcher = ZKConfigManager('/var/config/test_config', read_new_config)
# ``read_new_config()`` is called during watcher initialization, and from now on, if config
# file is changed, ``read_new_config()`` is called with the new config.
"""
def __init__(self, zk_hosts, aws_keyfile, s3_bucket, file_path, read_config_callback, s3_endpoint="s3.amazonaws.com", zk_path_suffix=''):
"""
Args:
file_path: config file path to watch.
read_config_callback: callback function when new config is detected. It should take
one argument which is the new config string. If it is None, no config file watcher
will be registered.
"""
self.zk_original_path = get_zk_path(file_path)
self.zk_path = self.zk_original_path + zk_path_suffix
super(ZKConfigManager, self).__init__(zk_hosts, self.zk_path, aws_keyfile, s3_bucket, s3_endpoint=s3_endpoint)
self.file_path = file_path
self.zk_lock_path = ZK_LOCK_PATH_FORMAT % self.zk_path
self.s3_file_path = S3_CONFIG_FILE_PATH_FORMAT % self.zk_original_path
if read_config_callback is not None:
assert hasattr(read_config_callback, '__call__')
self.read_config_callback = read_config_callback
self.version = -1
self.watcher = DataWatcher(None, zk_hosts, file_path=self.file_path)
if file_path:
try:
os.path.getmtime(file_path)
except OSError:
log.error("%s does not exist or is unaccessible" % file_path)
return
self.watcher.watch(self._read_config)
def _read_config(self, value, stat):
"""Callback function when config file is changed.
It checks the current and new version to decide whether to call the caller's callback
function.
"""
new_version = 0 if stat is None else stat.version
if self.version >= new_version:
return
if self.read_config_callback:
self.read_config_callback(value)
self.version = new_version
def reload_config_data(self):
"""Get the data from config file and invoke callback function.
Raises:
``IOError`` if the config file does not exist or accessible.
"""
value, stat = self.watcher.get_data()
self._read_config(value, stat)
def test_data_watcher(self):
"""Test various scenarios for data watcher:
1. When data get changed, watcher callback should be invoked.
2. When the underlying zk client disconnects and then recovers,
the watcher callback should be invoked.
3. When the underlying zk client messes up beyond recovery,
the underlying client should be replaced, and once the new client
is in place, the watcher callback should be invoked again.
"""
data_stat = []
watcher_triggered = Event()
def data_watch(data, stat):
while data_stat:
data_stat.pop()
data_stat.append(data)
data_stat.append(stat)
watcher_triggered.set()
testutil.initialize_kazoo_client_manager(ZK_HOSTS)
client = KazooClientManager().get_client()
client.create(DataWatcherTestCase.TEST_PATH,
DataWatcherTestCase.DATA_0)
data_watcher = DataWatcher(DataWatcherTestCase.TEST_PATH,
ZK_HOSTS,
waiting_in_secs=0.01)
data_watcher.watch(data_watch).join()
watcher_triggered.wait(1)
# Now the data and version should be foo and 0.
self.assertEqual(data_stat[0], DataWatcherTestCase.DATA_0)
self.assertEqual(data_stat[1].version, 0)
watcher_triggered.clear()
client.set(DataWatcherTestCase.TEST_PATH, DataWatcherTestCase.DATA_1)
watcher_triggered.wait(1)
# Make sure that watch callback is triggered.
self.assertEqual(data_stat[0], DataWatcherTestCase.DATA_1)
self.assertEqual(data_stat[1].version, 1)
data_stat.pop()
data_stat.pop()
# Test recoverable failure
watcher_triggered.clear()
client.stop()
client.start()
# Here the client actually will call check the znode in the
# background.
watcher_triggered.wait(1)
# Since nothing changed, no notification from the client.
self.assertFalse(data_stat)
# Test client change
client.stop()
watcher_triggered.clear()
# give the monit greenlet a chance to detect failures.
gevent.sleep(1)
# Assert the client has been replaced with a new one.
self.assertFalse(KazooClientManager().get_client() is client)
watcher_triggered.wait(1)
# Make sure that watch callback is triggered when client is replaced.
self.assertEqual(data_stat[0], DataWatcherTestCase.DATA_1)
self.assertEqual(data_stat[1].version, 1)
def test_data_watcher(self):
"""Test data watcher with a local file:
1. When data get changed, watcher callback should be invoked.
2. When the underlying zk client disconnects and then recovers,
the watcher callback should be invoked.
3. When the underlying zk client messes up beyond recovery,
the underlying client should be replaced, and once the new client
is in place, the watcher callback should be invoked again.
Although when a local file is being watched, now all the code paths
about the above behaviors got affected, we still want to test all the
scenarios to make sure nothing breaks when a file is used.
"""
data_stat = []
watcher_triggered = Event()
fd, tmp_file = tempfile.mkstemp()
with open(tmp_file, 'w') as f:
f.write(self.DATA_0)
def data_watch(data, stat):
while data_stat:
data_stat.pop()
data_stat.append(data)
data_stat.append(stat)
watcher_triggered.set()
data_watcher = DataWatcher(DataWatcherWithFileTestCase.TEST_PATH,
ZK_HOSTS,
waiting_in_secs=0.01,
file_path=tmp_file)
data_watcher.watch(data_watch).join()
watcher_triggered.wait(1)
# Now the data and version should be foo and the mtime of file.
mtime = os.path.getmtime(tmp_file)
self.assertEqual(data_stat[0], DataWatcherWithFileTestCase.DATA_0)
self.assertEqual(data_stat[1].version, mtime)
self.assertEqual(data_watcher.get_data()[0], DataWatcherWithFileTestCase.DATA_0)
self.assertEqual(data_watcher.get_data()[1].version, mtime)
watcher_triggered.clear()
gevent.sleep(1)
with open(tmp_file, 'w') as f:
f.write(self.DATA_1)
watcher_triggered.wait(1)
# Make sure that watch callback is triggered.
mtime = os.path.getmtime(tmp_file)
self.assertEqual(data_stat[0], DataWatcherWithFileTestCase.DATA_1)
self.assertEqual(data_stat[1].version, mtime)
self.assertEqual(data_watcher.get_data()[0], DataWatcherWithFileTestCase.DATA_1)
self.assertEqual(data_watcher.get_data()[1].version, mtime)
data_stat.pop()
data_stat.pop()
# Test recoverable failure, even though the watcher with a file path
# is not changing any implementation or behavior in this part, we want
# to keep the tests here to ensure.
watcher_triggered.clear()
self.FILE_WATCH._clear_all_watches()
os.remove(tmp_file)
class ZKConfigManager(ZKBaseConfigManager):
"""An extention of ZKBaseConfigManager that provides watcher for local config file.
Internally, the config file is stored in s3, which is downloaded to local config file by
zk_update_monitor process(when it gets notified by zk). To change the config file, we need to
push the data to s3 and update zk_node, which triggers zk_update_monitor to download the file
on each box.
Before using this class, puppet change needs to be made to let zk_update_monitor download your
file. To make zk_path and config file path consistent, we use ``get_zk_path()`` and
``get_config_file_path()`` to do the mutual conversion. The zk node created and the config file
to watch needs to have the consistent path.
If you only intend to use ``ZKConfigManager`` to get notified when config file is changed, you
can ignore ``update_zk()``. ``update_zk()`` provides a convenient way to change the original
data for the config file.
Usage:
def read_new_config(new_config):
print "New config is here", new_config
watcher = ZKConfigManager('/var/config/test_config', read_new_config)
# ``read_new_config()`` is called during watcher initialization, and from now on, if config
# file is changed, ``read_new_config()`` is called with the new config.
"""
def __init__(self,
zk_hosts,
aws_keyfile,
s3_bucket,
file_path,
read_config_callback,
s3_endpoint="s3.amazonaws.com",
zk_path_suffix=''):
"""
Args:
file_path: config file path to watch.
read_config_callback: callback function when new config is detected. It should take
one argument which is the new config string. If it is None, no config file watcher
will be registered.
"""
self.zk_original_path = get_zk_path(file_path)
self.zk_path = self.zk_original_path + zk_path_suffix
super(ZKConfigManager, self).__init__(zk_hosts,
self.zk_path,
aws_keyfile,
s3_bucket,
s3_endpoint=s3_endpoint)
self.file_path = file_path
self.zk_lock_path = ZK_LOCK_PATH_FORMAT % self.zk_path
self.s3_file_path = S3_CONFIG_FILE_PATH_FORMAT % self.zk_original_path
if read_config_callback is not None:
assert hasattr(read_config_callback, '__call__')
self.read_config_callback = read_config_callback
self.version = -1
self.watcher = DataWatcher(None, zk_hosts, file_path=self.file_path)
if file_path:
try:
os.path.getmtime(file_path)
except OSError:
log.error("%s does not exist or is unaccessible" % file_path)
return
self.watcher.watch(self._read_config)
def _read_config(self, value, stat):
"""Callback function when config file is changed.
It checks the current and new version to decide whether to call the caller's callback
function.
"""
new_version = 0 if stat is None else stat.version
if self.version >= new_version:
return
if self.read_config_callback:
self.read_config_callback(value)
self.version = new_version
def reload_config_data(self):
"""Get the data from config file and invoke callback function.
Raises:
``IOError`` if the config file does not exist or accessible.
"""
value, stat = self.watcher.get_data()
self._read_config(value, stat)
def test_data_watcher(self):
"""Test various scenarios for data watcher:
1. When data get changed, watcher callback should be invoked.
2. When the underlying zk client disconnects and then recovers,
the watcher callback should be invoked.
3. When the underlying zk client messes up beyond recovery,
the underlying client should be replaced, and once the new client
is in place, the watcher callback should be invoked again.
"""
data_stat = []
watcher_triggered = Event()
def data_watch(data, stat):
while data_stat:
data_stat.pop()
data_stat.append(data)
data_stat.append(stat)
watcher_triggered.set()
testutil.initialize_kazoo_client_manager(ZK_HOSTS)
client = KazooClientManager().get_client()
client.create(DataWatcherTestCase.TEST_PATH,
DataWatcherTestCase.DATA_0)
data_watcher = DataWatcher(DataWatcherTestCase.TEST_PATH,
ZK_HOSTS,
waiting_in_secs=0.01)
data_watcher.watch(data_watch).join()
watcher_triggered.wait(1)
# Now the data and version should be foo and 0.
self.assertEqual(data_stat[0], DataWatcherTestCase.DATA_0)
self.assertEqual(data_stat[1].version, 0)
watcher_triggered.clear()
client.set(DataWatcherTestCase.TEST_PATH, DataWatcherTestCase.DATA_1)
watcher_triggered.wait(1)
# Make sure that watch callback is triggered.
self.assertEqual(data_stat[0], DataWatcherTestCase.DATA_1)
self.assertEqual(data_stat[1].version, 1)
data_stat.pop()
data_stat.pop()
# Test recoverable failure
watcher_triggered.clear()
client.stop()
client.start()
# Here the client actually will call check the znode in the
# background.
watcher_triggered.wait(1)
# Since nothing changed, no notification from the client.
self.assertFalse(data_stat)
# Test client change
client.stop()
watcher_triggered.clear()
# give the monit greenlet a chance to detect failures.
gevent.sleep(1)
# Assert the client has been replaced with a new one.
self.assertFalse(KazooClientManager().get_client() is client)
watcher_triggered.wait(1)
# Make sure that watch callback is triggered when client is replaced.
self.assertEqual(data_stat[0], DataWatcherTestCase.DATA_1)
self.assertEqual(data_stat[1].version, 1)
def test_data_watcher(self):
"""Test data watcher with a local file:
1. When data get changed, watcher callback should be invoked.
2. When the underlying zk client disconnects and then recovers,
the watcher callback should be invoked.
3. When the underlying zk client messes up beyond recovery,
the underlying client should be replaced, and once the new client
is in place, the watcher callback should be invoked again.
Although when a local file is being watched, now all the code paths
about the above behaviors got affected, we still want to test all the
scenarios to make sure nothing breaks when a file is used.
"""
data_stat = []
watcher_triggered = Event()
fd, tmp_file = tempfile.mkstemp()
with open(tmp_file, 'w') as f:
f.write(self.DATA_0)
def data_watch(data, stat):
while data_stat:
data_stat.pop()
data_stat.append(data)
data_stat.append(stat)
watcher_triggered.set()
data_watcher = DataWatcher(DataWatcherWithFileTestCase.TEST_PATH,
ZK_HOSTS,
waiting_in_secs=0.01,
file_path=tmp_file)
data_watcher.watch(data_watch).join()
watcher_triggered.wait(1)
# Now the data and version should be foo and the mtime of file.
mtime = os.path.getmtime(tmp_file)
self.assertEqual(data_stat[0], DataWatcherWithFileTestCase.DATA_0)
self.assertEqual(data_stat[1].version, mtime)
self.assertEqual(data_watcher.get_data()[0],
DataWatcherWithFileTestCase.DATA_0)
self.assertEqual(data_watcher.get_data()[1].version, mtime)
watcher_triggered.clear()
gevent.sleep(1)
with open(tmp_file, 'w') as f:
f.write(self.DATA_1)
watcher_triggered.wait(1)
# Make sure that watch callback is triggered.
mtime = os.path.getmtime(tmp_file)
self.assertEqual(data_stat[0], DataWatcherWithFileTestCase.DATA_1)
self.assertEqual(data_stat[1].version, mtime)
self.assertEqual(data_watcher.get_data()[0],
DataWatcherWithFileTestCase.DATA_1)
self.assertEqual(data_watcher.get_data()[1].version, mtime)
data_stat.pop()
data_stat.pop()
# Test recoverable failure, even though the watcher with a file path
# is not changing any implementation or behavior in this part, we want
# to keep the tests here to ensure.
watcher_triggered.clear()
self.FILE_WATCH._clear_all_watches()
os.remove(tmp_file)
