def _get_watcher(self, path, callback=None):
"""Creates a Watcher object for the supplid path.
Creates a Watcher object for the supplied path and returns the
object. Triggers callbacks immediately.
This is broken into its own function so that it can leverage
the @_health_check decorator. This function should never be
called from outside of this class.
Args:
path: A string representing the path to the servers.
callback: (optional) reference to function to call if the path
changes.
Returns:
nd_service_registry.Watcher object
"""
# Ok, so the cache is missing the key. Lets look for it in Zookeeper
self.log.debug('[%s] Creating Watcher object...' % path)
watcher = Watcher(self._zk,
path,
watch_children=True,
callback=callback)
# Always register our dictionary saver callback, in addition to
# whatever our user has supplied
watcher.add_callback(self._save_watcher_to_dict)
return watcher
def _get_watcher(self, path, callback=None):
"""Creates a Watcher object for the supplid path.
Creates a Watcher object for the supplied path and returns the
object. Triggers callbacks immediately.
This is broken into its own function so that it can leverage
the @_health_check decorator. This function should never be
called from outside of this class.
Args:
path: A string representing the path to the servers.
callback: (optional) reference to function to call if the path
changes.
Returns:
nd_service_registry.Watcher object
"""
# Ok, so the cache is missing the key. Lets look for it in Zookeeper
log.debug('[%s] Creating Watcher object...' % path)
watcher = Watcher(self._zk,
path,
watch_children=True,
callback=callback)
# Always register our dictionary saver callback, in addition to
# whatever our user has supplied
watcher.add_callback(self._save_watcher_to_dict)
return watcher
class Registration(object):
"""An object that registers a znode with ZooKeeper."""
def __init__(self, zk, path, data=None, state=True):
# Create our logger
self.log = logging.getLogger('%s.Registration.%s' % (__name__, path))
# Set our local variables
self._zk = zk
self._path = path
self._state = state
# Store both encdoed-string and decoded-dict versions of our 'data'
# for comparison purposes later.
self._data = data
self._encoded_data = funcs.encode(data)
self._decoded_data = funcs.decode(self._encoded_data)
# Make sure that we have a watcher on the path we care about
self._watcher = Watcher(self._zk,
path=self._path,
watch_children=False,
callback=self._update)
def data(self):
"""Returns live node data from Watcher object."""
return self._watcher.data()
def get(self):
"""Returns live node information from Watcher object."""
return self._watcher.get()
def set_data(self, data):
"""Sets self._data.
Args:
data: String or Dict of data to register with this object."""
if not data == self._data:
self._data = data
self._encoded_data = funcs.encode(data)
self._decoded_data = funcs.decode(self._encoded_data)
self._update_data()
def _update_data(self):
try:
self._zk.retry(self._zk.set, self._path, value=self._encoded_data)
self.log.debug('Updated with data: %s' % self._encoded_data)
except kazoo.exceptions.NoAuthError, e:
self.log.error('No authorization to set node.')
pass
except Exception, e:
self.log.error('Received exception. Moving on, will re-attempt '
'when Watcher notifies us of a state change: %s '
% e)
pass
def __init__(self, zk, path, data=None, state=False, ephemeral=False):
# Set our local variables
self._ephemeral = ephemeral
self._zk = zk
self._path = path
self._state = state
# Store both encdoed-string and decoded-dict versions of our 'data'
# for comparison purposes later.
self._data = data
self._encoded_data = funcs.encode(data)
self._decoded_data = funcs.decode(self._encoded_data)
# Set a default watcher without a callback.
self._watcher = Watcher(self._zk,
path=self._path,
watch_children=False)
def test_deleting_node_watcher_is_watching(self):
# Create our test path
path = '%s/path-to-be-deleted-%s' % (self.sandbox, uuid.uuid4().hex)
self.zk.create(path, makepath=True)
# Watch it and verify that we got back the proper stat data for
# the existing node.
watch = Watcher(zk=self.zk, path=path)
self.assertTrue(watch.get()['stat'] is not None)
# Now, delete it
self.zk.delete(path)
# Now, wait until the Zookeeper callback has been executed, at which
# point, our Watcher object should cleanly update itself indicating
# that the path no longer exists.
expected_get_results = {
'path': path, 'stat': None, 'data': None, 'children': []}
waituntil(watch.get, expected_get_results, 5, mode=2)
self.assertEquals(expected_get_results, watch.get())
def test_watch_on_nonexistent_path_with_children_change(self):
path = '%s/nonexistent-path-%s' % (self.sandbox, uuid.uuid4().hex)
watch = Watcher(zk=self.zk, path=path)
# Initially, we expect the data, children count and stat data
# to be None because the node doesn't exist.
self.assertEquals(None, watch.get()['data'])
self.assertEquals(None, watch.get()['stat'])
self.assertEquals([], watch.get()['children'])
# Now register a child node under the path, and expect the
# data to change
child_path = '%s/my_child_test' % path
self.zk.create(child_path, makepath=True)
# Wait until the children list changes
def get_children():
return watch.get()['children']
waituntil(get_children, [], timeout=5)
# Now verify that the data has been updated in our Watcher
self.assertTrue('my_child_test' in watch.get()['children'])
def test_watch_on_nonexistent_path_with_children_change(self):
path = '%s/nonexistent-path-%s' % (self.sandbox, uuid.uuid4().hex)
watch = Watcher(zk=self.zk, path=path)
# Initially, we expect the data, children count and stat data
# to be None because the node doesn't exist.
self.assertEquals(None, watch.get()['data'])
self.assertEquals(None, watch.get()['stat'])
self.assertEquals([], watch.get()['children'])
# Now register a child node under the path, and expect the
# data to change
child_path = '%s/my_child_test' % path
self.zk.create(child_path, makepath=True)
# Wait until the children list changes
def get_children():
return watch.get()['children']
waituntil(get_children, [], timeout=5)
# Now verify that the data has been updated in our Watcher
self.assertTrue('my_child_test' in watch.get()['children'])
def test_watch_on_existing_path(self):
path = '%s/existing-path-%s' % (self.sandbox, uuid.uuid4().hex)
self.zk.create(path, makepath=True)
watch = Watcher(zk=self.zk, path=path)
# We expect there to be valid data...
self.assertEquals(None, watch.get()['data'])
self.assertNotEquals(None, watch.get()['stat'])
self.assertEquals([], watch.get()['children'])
# Now register a child node, and expect the data to change
child_path = '%s/my_child_test' % path
self.zk.create(child_path)
# Before progressing, wait for Zookeeper to have kicked off
# all of its notifications to the client.
def get_children():
print "got: %s" % watch.get()['children']
return watch.get()['children']
waituntil(get_children, [], timeout=5)
# Now expect the watch to be kicked off
self.assertTrue('my_child_test' in watch.get()['children'])
def test_deleting_node_watcher_is_watching(self):
# Create our test path
path = '%s/path-to-be-deleted-%s' % (self.sandbox, uuid.uuid4().hex)
self.zk.create(path, makepath=True)
# Watch it and verify that we got back the proper stat data for
# the existing node.
watch = Watcher(zk=self.zk, path=path)
self.assertTrue(watch.get()['stat'] is not None)
# Now, delete it
self.zk.delete(path)
# Now, wait until the Zookeeper callback has been executed, at which
# point, our Watcher object should cleanly update itself indicating
# that the path no longer exists.
expected_get_results = {
'path': path,
'stat': None,
'data': None,
'children': []
}
waituntil(watch.get, expected_get_results, 5, mode=2)
self.assertEquals(expected_get_results, watch.get())
def __init__(self, zk, path, data=None, state=False, ephemeral=False):
# Set our local variables
self._ephemeral = ephemeral
self._zk = zk
self._path = path
self._state = state
# Store both encdoed-string and decoded-dict versions of our 'data'
# for comparison purposes later.
self._data = data
self._encoded_data = funcs.encode(data)
self._decoded_data = funcs.decode(self._encoded_data)
# Set a default watcher without a callback.
self._watcher = Watcher(self._zk,
path=self._path,
watch_children=False)
def __init__(self, zk, path, data=None, state=True, ephemeral=False):
# Set our local variables
self._ephemeral = ephemeral
self._zk = zk
self._path = path
self._state = state
# Store both encdoed-string and decoded-dict versions of our 'data'
# for comparison purposes later.
self._data = data
self._encoded_data = funcs.encode(data)
self._decoded_data = funcs.decode(self._encoded_data)
# Make sure that we have a watcher on the path we care about
self._watcher = Watcher(self._zk,
path=self._path,
watch_children=False,
callback=self._update)
def test_watch_on_nonexistent_path_with_multiple_data_changes(self):
path = '%s/nonexistent-path-%s' % (self.sandbox, uuid.uuid4().hex)
watch = Watcher(zk=self.zk, path=path)
# In order to track the number of callbacks that are registered,
# we register a mocked method in the callbacks.
callback_checker = mock.Mock()
callback_checker.test.return_value = True
watch.add_callback(callback_checker.test)
# Initially, we expect the data, children count and stat data
# to be None because the node doesn't exist.
self.assertEquals(None, watch.get()['data'])
self.assertEquals(None, watch.get()['stat'])
self.assertEquals([], watch.get()['children'])
# Now register a child node under the path, and expect the
# data to change
child_path = '%s/my_child_test' % path
self.zk.create(child_path, makepath=True)
# Wait until the children list changes
def get_children():
return watch.get()['children']
waituntil(get_children, [], timeout=5)
# Now verify that the data has been updated in our Watcher
self.assertTrue('my_child_test' in watch.get()['children'])
# Now make a series of data changes and wait for each one to
# take effect before moving on to the next one.
def get_data():
return watch.get()['data']
for i in range(1, 5):
self.zk.set(path, value=('%s' % i).encode('UTF-8'))
waituntil(get_data, {'string_value': '%s' % i}, timeout=5, mode=2)
# The right number of calls is 6. There are a few upfront calls when
# the Watcher object is initialized, and after that there are a few
# calls for the updated node-data when new children are added. The
# call count would be MUCH higher (11++) if we were recursively
# creating Watches though.
self.assertEquals(6, len(callback_checker.test.mock_calls))
def __init__(self, zk, path, data=None, state=True):
# Create our logger
self.log = logging.getLogger('%s.Registration.%s' % (__name__, path))
# Set our local variables
self._zk = zk
self._path = path
self._state = state
# Store both encdoed-string and decoded-dict versions of our 'data'
# for comparison purposes later.
self._data = data
self._encoded_data = funcs.encode(data)
self._decoded_data = funcs.decode(self._encoded_data)
# Make sure that we have a watcher on the path we care about
self._watcher = Watcher(self._zk,
path=self._path,
watch_children=False,
callback=self._update)
def test_watch_on_nonexistent_path_with_multiple_data_changes(self):
path = '%s/nonexistent-path-%s' % (self.sandbox, uuid.uuid4().hex)
watch = Watcher(zk=self.zk, path=path)
# In order to track the number of callbacks that are registered,
# we register a mocked method in the callbacks.
callback_checker = mock.Mock()
callback_checker.test.return_value = True
watch.add_callback(callback_checker.test)
# Initially, we expect the data, children count and stat data
# to be None because the node doesn't exist.
self.assertEquals(None, watch.get()['data'])
self.assertEquals(None, watch.get()['stat'])
self.assertEquals([], watch.get()['children'])
# Now register a child node under the path, and expect the
# data to change
child_path = '%s/my_child_test' % path
self.zk.create(child_path, makepath=True)
# Wait until the children list changes
def get_children():
return watch.get()['children']
waituntil(get_children, [], timeout=5)
# Now verify that the data has been updated in our Watcher
self.assertTrue('my_child_test' in watch.get()['children'])
# Now make a series of data changes and wait for each one to
# take effect before moving on to the next one.
def get_data():
return watch.get()['data']
for i in xrange(1, 5):
self.zk.set(path, value='%s' % i)
waituntil(get_data, {'string_value': '%s' % i}, timeout=5, mode=2)
# The right number of calls is 6. There are a few upfront calls when
# the Watcher object is initialized, and after that there are a few
# calls for the updated node-data when new children are added. The
# call count would be MUCH higher (11++) if we were recursively
# creating Watches though.
self.assertEquals(6, len(callback_checker.test.mock_calls))
def test_watch_on_nonexistent_path_with_data_change(self):
path = '%s/nonexistent-path-%s' % (self.sandbox, uuid.uuid4().hex)
watch = Watcher(zk=self.zk, path=path)
# Initially, we expect the data, children count and stat data
# to be None because the node doesn't exist.
self.assertEquals(None, watch.get()['data'])
self.assertEquals(None, watch.get()['stat'])
self.assertEquals([], watch.get()['children'])
# Now, lets go and set the data path with something. We expect the
# watcher now to actually register the change. Use the waituntil()
# method to wait until the data has been updated.
def get_data():
return watch.get()['data']
self.zk.create(path, value=b"{'foo': 'bar'}", makepath=True)
waituntil(get_data, None, timeout=5, mode=1)
# Now verify that the data has been updated in our Watcher
self.assertEquals({'string_value': "{'foo': 'bar'}"},
watch.get()['data'])
self.assertNotEquals(None, watch.get()['stat'])
self.assertEquals([], watch.get()['children'])
def test_watch_on_existing_path(self):
path = '%s/existing-path-%s' % (self.sandbox, uuid.uuid4().hex)
self.zk.create(path, makepath=True)
watch = Watcher(zk=self.zk, path=path)
# We expect there to be valid data...
self.assertEquals(None, watch.get()['data'])
self.assertNotEquals(None, watch.get()['stat'])
self.assertEquals([], watch.get()['children'])
# Now register a child node, and expect the data to change
child_path = '%s/my_child_test' % path
self.zk.create(child_path)
# Before progressing, wait for Zookeeper to have kicked off
# all of its notifications to the client.
def get_children():
print("got: %s" % watch.get()['children'])
return watch.get()['children']
waituntil(get_children, [], timeout=5)
# Now expect the watch to be kicked off
self.assertTrue('my_child_test' in watch.get()['children'])
def test_watch_on_nonexistent_path_with_data_change(self):
path = '%s/nonexistent-path-%s' % (self.sandbox, uuid.uuid4().hex)
watch = Watcher(zk=self.zk, path=path)
# Initially, we expect the data, children count and stat data
# to be None because the node doesn't exist.
self.assertEquals(None, watch.get()['data'])
self.assertEquals(None, watch.get()['stat'])
self.assertEquals([], watch.get()['children'])
# Now, lets go and set the data path with something. We expect the
# watcher now to actually register the change. Use the waituntil()
# method to wait until the data has been updated.
def get_data():
return watch.get()['data']
self.zk.create(path, value="{'foo': 'bar'}", makepath=True)
waituntil(get_data, None, timeout=5, mode=1)
# Now verify that the data has been updated in our Watcher
self.assertEquals(
{'string_value': "{'foo': 'bar'}"}, watch.get()['data'])
self.assertNotEquals(None, watch.get()['stat'])
self.assertEquals([], watch.get()['children'])
class RegistrationBase(object):
"""Base object model for registering data in Zookeeper.
This object is not meant to be used directly -- its not complete, and
will not behave properly. It is simply a shell that other data types can
subclass."""
GENERAL_EXC_MSG = ('[%s] Received exception. Moving on.'
' Will not re-attempt this command: %s')
def __init__(self, zk, path, data=None, state=False, ephemeral=False):
# Set our local variables
self._ephemeral = ephemeral
self._zk = zk
self._path = path
self._state = state
# Store both encoded-string and decoded-dict versions of our 'data'
# for comparison purposes later.
self._data = data
self._encoded_data = funcs.encode(data)
self._decoded_data = funcs.decode(self._encoded_data)
# Set a default watcher without a callback.
self._watcher = Watcher(self._zk,
path=self._path,
watch_children=False)
def data(self):
"""Returns live node data from Watcher object."""
return self._watcher.get()['data']
def get(self):
"""Returns live node information from Watcher object."""
return self._watcher.get()
def set_data(self, data):
"""Sets self._data.
Args:
data: String or Dict of data to register with this object."""
if not data == self._data:
self._data = data
self._encoded_data = funcs.encode(data)
self._decoded_data = funcs.decode(self._encoded_data)
self._update_data()
def stop(self):
"""Disables our registration of the node."""
self.set_state(False)
def start(self):
"""Enables our registration of the node."""
self.set_state(True)
def state(self):
"""Returns self._state"""
return self._state
def set_state(self, state):
"""Sets the state of our registration.
Updates the internal 'wanted' state of this object. If True, we want
to be actively registered with Zookeeper. If False, we want to make
sure we're not registered with Zookeeper.
Args:
state: True or False"""
if self._state == state:
return
self._state = state
self._update_state(self._state)
def update(self, data=None, state=None):
"""If data or state are supplied, these are updated before triggering
the update.
Args:
data: (String/Dict) data to register with this object.
state: (Boolean) whether to register or unregister this object
"""
if type(state) is bool:
log.debug('[%s] Got updated state: %s' % (self._path, state))
self.set_state(state)
if data:
log.debug('[%s] Got updated data: %s' % (self._path, data))
self.set_data(data)
def _update_state(self, state):
"""Creates/Destroys a registered Zookeeper node.
If the underlying path does not exist, then the path is created. If
the path is multi-leveled (/foo/bar/baz/host:22), we only set the
ACL on the '/baz/' subdirectory. We do not set it on /foo/bar. This
leaves the ACL protection only at the final dangling endpoint, rather
than blocking out all of /foo for future use by other clients.
args:
state: Boolean True/False whether or not to register the path.
"""
if state is True:
try:
self._create_node()
except kazoo.exceptions.NoNodeError:
# The underlying path doesn't exist to put the node into.
# Create the path, and re-call ourselves. If the
# create_node_path method raises an exception, we break out and
# throw an alert rather than continuing this recursive call.
# If it returns cleanly, we call self._create_node()
# recursively.
# Try to create the root path for the node. If this raises
# an exception, we catch it, log it, and return.
self._create_node_path()
# If we got here, then the root path has been created and we
# recursively re-call ourselves to create the final path node.
self._create_node()
elif state is False:
self._delete_node()
def _create_node(self):
"""Creates the registered Zookeeper node endpoint.
If the path does not exist, raise the exception and allow the
_update_state() method to handle it.
"""
try:
log.debug('[%s] Registering...' % self._path)
self._zk.retry(self._zk.create, self._path,
value=self._encoded_data,
ephemeral=self._ephemeral, makepath=False)
log.info('[%s] Registered with data: %s' %
(self._path, self._encoded_data))
except kazoo.exceptions.NoNodeError:
# The underlying path does not exist. Raise this exception, and
# _update_state() handle it.
raise
except kazoo.exceptions.NodeExistsError as e:
# Node exists ... possible this callback got called multiple
# times
pass
except kazoo.exceptions.NoAuthError as e:
log.error('[%s] No authorization to create node.' % self._path)
except Exception as e:
log.error(RegistrationBase.GENERAL_EXC_MSG % (self._path, e))
def _create_node_path(self):
"""Recursively creates the underlying path for our node registration.
Note, if the path is multi-levels, does not apply an ACL to anything
but the deepest level of the path. For example, on
/foo/bar/baz/host:22, the ACL would only be applied to /foo/bar/baz,
not /foo or /foo/bar.
Note, no exceptions are caught here. This method is really meant to be
called only by _create_node(), which handles the exceptions behavior.
"""
# Strip the path down into a few components...
(path, node) = split(self._path)
# Count the number of levels in the path. If its >1, then we split
# the path into a 'root' path and a 'deep' path. We create the
# 'root' path with no ACL at all (the default OPEN acl). The
# final path that will hold our node registration will get created
# with whatever ACL settings were used when creating the Kazoo
# connection object.
if len([_f for _f in path.split('/') if _f]) > 1:
(root_path, deep_path) = split(path)
self._zk.retry(
self._zk.ensure_path, root_path,
acl=security.OPEN_ACL_UNSAFE)
# Create the final destination path folder that the node will be
# registered in -- and allow Kazoo to use the ACL if appropriate.
self._zk.retry(self._zk.ensure_path, path)
def _delete_node(self):
"""Deletes a registered Zookeeper node endpoint.
"""
# Try to delete the node
log.debug('[%s] Attempting de-registration...' % self._path)
try:
self._zk.retry(self._zk.delete, self._path)
except kazoo.exceptions.NoAuthError as e:
# The node exists, but we don't even have authorization to read
# it. We certainly will not have access then to change it below
# so return false. We'll retry again very soon.
log.error('[%s] No authorization to delete node.' % self._path)
except Exception as e:
log.error(RegistrationBase.GENERAL_EXC_MSG % (self._path, e))
def _update_data(self):
"""Updates an existing node in Zookeeper with fresh data."""
try:
self._zk.retry(self._zk.set, self._path, value=self._encoded_data)
log.debug('[%s] Updated with data: %s' %
(self._path, self._encoded_data))
except kazoo.exceptions.NoAuthError as e:
log.error('[%s] No authorization to set node.' % self._path)
except Exception as e:
log.error(RegistrationBase.GENERAL_EXC_MSG % (self._path, e))
class RegistrationBase(object):
"""Base object model for registering data in Zookeeper.
This object is not meant to be used directly -- its not complete, and
will not behave properly. It is simply a shell that other data types can
subclass."""
GENERAL_EXC_MSG = ('[%s] Received exception. Moving on.'
' Will not re-attempt this command: %s')
def __init__(self, zk, path, data=None, state=False, ephemeral=False):
# Set our local variables
self._ephemeral = ephemeral
self._zk = zk
self._path = path
self._state = state
# Store both encdoed-string and decoded-dict versions of our 'data'
# for comparison purposes later.
self._data = data
self._encoded_data = funcs.encode(data)
self._decoded_data = funcs.decode(self._encoded_data)
# Set a default watcher without a callback.
self._watcher = Watcher(self._zk,
path=self._path,
watch_children=False)
def data(self):
"""Returns live node data from Watcher object."""
return self._watcher.get()['data']
def get(self):
"""Returns live node information from Watcher object."""
return self._watcher.get()
def set_data(self, data):
"""Sets self._data.
Args:
data: String or Dict of data to register with this object."""
if not data == self._data:
self._data = data
self._encoded_data = funcs.encode(data)
self._decoded_data = funcs.decode(self._encoded_data)
self._update_data()
def stop(self):
"""Disables our registration of the node."""
self.set_state(False)
def start(self):
"""Enables our registration of the node."""
self.set_state(True)
def state(self):
"""Returns self._state"""
return self._state
def set_state(self, state):
"""Sets the state of our registration.
Updates the internal 'wanted' state of this object. If True, we want
to be actively registered with Zookeeper. If False, we want to make
sure we're not registered with Zookeeper.
Args:
state: True or False"""
if self._state == state:
return
self._state = state
self._update_state(self._state)
def update(self, data=None, state=None):
"""If data or state are supplied, these are updated before triggering
the update.
Args:
data: (String/Dict) data to register with this object.
state: (Boolean) whether to register or unregister this object
"""
if type(state) is bool:
log.debug('[%s] Got updated state: %s' % (self._path, state))
self.set_state(state)
if data:
log.debug('[%s] Got updated data: %s' % (self._path, data))
self.set_data(data)
def _update_state(self, state):
"""Creates/Destroys a registered Zookeeper node.
If the underlying path does not exist, then the path is created. If
the path is multi-leveled (/foo/bar/baz/host:22), we only set the
ACL on the '/baz/' subdirectory. We do not set it on /foo/bar. This
leaves the ACL protection only at the final dangling endpoint, rather
than blocking out all of /foo for future use by other clients.
args:
state: Boolean True/False whether or not to register the path.
"""
if state is True:
try:
self._create_node()
except kazoo.exceptions.NoNodeError:
# The underlying path doesn't exist to put the node into.
# Create the path, and re-call ourselves. If the
# create_node_path method raises an exception, we break out and
# throw an alert rather than continueing this recursive call.
# If it returns cleanly, we call self._create_node()
# recursively.
# Try to create the root path for the node. If this raises
# an exception, we catch it, log it, and return.
self._create_node_path()
# If we got here, then the root path has been created and we
# recursively re-call ourselves to create the final path node.
self._create_node()
elif state is False:
self._delete_node()
def _create_node(self):
"""Creates the registered Zookeeper node endpoint.
If the path does not exist, raise the exception and allow the
_update_state() method to handle it.
"""
try:
log.debug('[%s] Registering...' % self._path)
self._zk.retry(self._zk.create, self._path,
value=self._encoded_data,
ephemeral=self._ephemeral, makepath=False)
log.info('[%s] Registered with data: %s' %
(self._path, self._encoded_data))
except kazoo.exceptions.NoNodeError:
# The underlying path does not exist. Raise this exception, and
# _update_state() handle it.
raise
except kazoo.exceptions.NodeExistsError, e:
# Node exists ... possible this callback got called multiple
# times
pass
except kazoo.exceptions.NoAuthError, e:
log.error('[%s] No authorization to create node.' % self._path)
class RegistrationBase(object):
"""Base object model for registering data in Zookeeper.
This object is not meant to be used directly -- its not complete, and
will not behave properly. It is simply a shell that other data types can
subclass."""
GENERAL_EXC_MSG = ('[%s] Received exception. Moving on.'
' Will not re-attempt this command: %s')
def __init__(self, zk, path, data=None, state=False, ephemeral=False):
# Set our local variables
self._ephemeral = ephemeral
self._zk = zk
self._path = path
self._state = state
# Store both encdoed-string and decoded-dict versions of our 'data'
# for comparison purposes later.
self._data = data
self._encoded_data = funcs.encode(data)
self._decoded_data = funcs.decode(self._encoded_data)
# Set a default watcher without a callback.
self._watcher = Watcher(self._zk,
path=self._path,
watch_children=False)
def data(self):
"""Returns live node data from Watcher object."""
return self._watcher.get()['data']
def get(self):
"""Returns live node information from Watcher object."""
return self._watcher.get()
def set_data(self, data):
"""Sets self._data.
Args:
data: String or Dict of data to register with this object."""
if not data == self._data:
self._data = data
self._encoded_data = funcs.encode(data)
self._decoded_data = funcs.decode(self._encoded_data)
self._update_data()
def stop(self):
"""Disables our registration of the node."""
self.set_state(False)
def start(self):
"""Enables our registration of the node."""
self.set_state(True)
def state(self):
"""Returns self._state"""
return self._state
def set_state(self, state):
"""Sets the state of our registration.
Updates the internal 'wanted' state of this object. If True, we want
to be actively registered with Zookeeper. If False, we want to make
sure we're not registered with Zookeeper.
Args:
state: True or False"""
if self._state == state:
return
self._state = state
self._update_state(self._state)
def update(self, data=None, state=None):
"""If data or state are supplied, these are updated before triggering
the update.
Args:
data: (String/Dict) data to register with this object.
state: (Boolean) whether to register or unregister this object
"""
if type(state) is bool:
log.debug('[%s] Got updated state: %s' % (self._path, state))
self.set_state(state)
if data:
log.debug('[%s] Got updated data: %s' % (self._path, data))
self.set_data(data)
def _update_state(self, state):
"""Creates/Destroys a registered Zookeeper node.
If the underlying path does not exist, then the path is created. If
the path is multi-leveled (/foo/bar/baz/host:22), we only set the
ACL on the '/baz/' subdirectory. We do not set it on /foo/bar. This
leaves the ACL protection only at the final dangling endpoint, rather
than blocking out all of /foo for future use by other clients.
args:
state: Boolean True/False whether or not to register the path.
"""
if state is True:
try:
self._create_node()
except kazoo.exceptions.NoNodeError:
# The underlying path doesn't exist to put the node into.
# Create the path, and re-call ourselves. If the
# create_node_path method raises an exception, we break out and
# throw an alert rather than continueing this recursive call.
# If it returns cleanly, we call self._create_node()
# recursively.
# Try to create the root path for the node. If this raises
# an exception, we catch it, log it, and return.
self._create_node_path()
# If we got here, then the root path has been created and we
# recursively re-call ourselves to create the final path node.
self._create_node()
elif state is False:
self._delete_node()
def _create_node(self):
"""Creates the registered Zookeeper node endpoint.
If the path does not exist, raise the exception and allow the
_update_state() method to handle it.
"""
try:
log.debug('[%s] Registering...' % self._path)
self._zk.retry(self._zk.create,
self._path,
value=self._encoded_data,
ephemeral=self._ephemeral,
makepath=False)
log.info('[%s] Registered with data: %s' %
(self._path, self._encoded_data))
except kazoo.exceptions.NoNodeError:
# The underlying path does not exist. Raise this exception, and
# _update_state() handle it.
raise
except kazoo.exceptions.NodeExistsError, e:
# Node exists ... possible this callback got called multiple
# times
pass
except kazoo.exceptions.NoAuthError, e:
log.error('[%s] No authorization to create node.' % self._path)
class RegistrationBase(object):
"""Base object model for registering data in Zookeeper.
This object is not meant to be used directly -- its not complete, and
will not behave properly. It is simply a shell that other data types can
subclass."""
GENERAL_EXC_MSG = ('[%s] Received exception. Moving on.'
' Will not re-attempt this command: %s')
def __init__(self, zk, path, data=None, state=False, ephemeral=False):
# Set our local variables
self._ephemeral = ephemeral
self._zk = zk
self._path = path
self._state = state
# Store both encoded-string and decoded-dict versions of our 'data'
# for comparison purposes later.
self._data = data
self._encoded_data = funcs.encode(data)
self._decoded_data = funcs.decode(self._encoded_data)
# Set a default watcher without a callback.
self._watcher = Watcher(self._zk,
path=self._path,
watch_children=False)
def data(self):
"""Returns live node data from Watcher object."""
return self._watcher.get()['data']
def get(self):
"""Returns live node information from Watcher object."""
return self._watcher.get()
def set_data(self, data):
"""Sets self._data.
Args:
data: String or Dict of data to register with this object."""
if not data == self._data:
self._data = data
self._encoded_data = funcs.encode(data)
self._decoded_data = funcs.decode(self._encoded_data)
self._update_data()
def stop(self):
"""Disables our registration of the node."""
self.set_state(False)
def start(self):
"""Enables our registration of the node."""
self.set_state(True)
def state(self):
"""Returns self._state"""
return self._state
def set_state(self, state):
"""Sets the state of our registration.
Updates the internal 'wanted' state of this object. If True, we want
to be actively registered with Zookeeper. If False, we want to make
sure we're not registered with Zookeeper.
Args:
state: True or False"""
if self._state == state:
return
self._state = state
self._update_state(self._state)
def update(self, data=None, state=None):
"""If data or state are supplied, these are updated before triggering
the update.
Args:
data: (String/Dict) data to register with this object.
state: (Boolean) whether to register or unregister this object
"""
if type(state) is bool:
log.debug('[%s] Got updated state: %s' % (self._path, state))
self.set_state(state)
if data:
log.debug('[%s] Got updated data: %s' % (self._path, data))
self.set_data(data)
def _update_state(self, state):
"""Creates/Destroys a registered Zookeeper node.
If the underlying path does not exist, then the path is created. If
the path is multi-leveled (/foo/bar/baz/host:22), we only set the
ACL on the '/baz/' subdirectory. We do not set it on /foo/bar. This
leaves the ACL protection only at the final dangling endpoint, rather
than blocking out all of /foo for future use by other clients.
args:
state: Boolean True/False whether or not to register the path.
"""
if state is True:
try:
self._create_node()
except kazoo.exceptions.NoNodeError:
# The underlying path doesn't exist to put the node into.
# Create the path, and re-call ourselves. If the
# create_node_path method raises an exception, we break out and
# throw an alert rather than continuing this recursive call.
# If it returns cleanly, we call self._create_node()
# recursively.
# Try to create the root path for the node. If this raises
# an exception, we catch it, log it, and return.
self._create_node_path()
# If we got here, then the root path has been created and we
# recursively re-call ourselves to create the final path node.
self._create_node()
elif state is False:
self._delete_node()
def _create_node(self):
"""Creates the registered Zookeeper node endpoint.
If the path does not exist, raise the exception and allow the
_update_state() method to handle it.
"""
try:
log.debug('[%s] Registering...' % self._path)
self._zk.retry(self._zk.create,
self._path,
value=self._encoded_data,
ephemeral=self._ephemeral,
makepath=False)
log.info('[%s] Registered with data: %s' %
(self._path, self._encoded_data))
except kazoo.exceptions.NoNodeError:
# The underlying path does not exist. Raise this exception, and
# _update_state() handle it.
raise
except kazoo.exceptions.NodeExistsError as e:
# Node exists ... possible this callback got called multiple
# times
pass
except kazoo.exceptions.NoAuthError as e:
log.error('[%s] No authorization to create node.' % self._path)
except Exception as e:
log.error(RegistrationBase.GENERAL_EXC_MSG % (self._path, e))
def _create_node_path(self):
"""Recursively creates the underlying path for our node registration.
Note, if the path is multi-levels, does not apply an ACL to anything
but the deepest level of the path. For example, on
/foo/bar/baz/host:22, the ACL would only be applied to /foo/bar/baz,
not /foo or /foo/bar.
Note, no exceptions are caught here. This method is really meant to be
called only by _create_node(), which handles the exceptions behavior.
"""
# Strip the path down into a few components...
(path, node) = split(self._path)
# Count the number of levels in the path. If its >1, then we split
# the path into a 'root' path and a 'deep' path. We create the
# 'root' path with no ACL at all (the default OPEN acl). The
# final path that will hold our node registration will get created
# with whatever ACL settings were used when creating the Kazoo
# connection object.
if len([_f for _f in path.split('/') if _f]) > 1:
(root_path, deep_path) = split(path)
self._zk.retry(self._zk.ensure_path,
root_path,
acl=security.OPEN_ACL_UNSAFE)
# Create the final destination path folder that the node will be
# registered in -- and allow Kazoo to use the ACL if appropriate.
self._zk.retry(self._zk.ensure_path, path)
def _delete_node(self):
"""Deletes a registered Zookeeper node endpoint.
"""
# Try to delete the node
log.debug('[%s] Attempting de-registration...' % self._path)
try:
self._zk.retry(self._zk.delete, self._path)
except kazoo.exceptions.NoAuthError as e:
# The node exists, but we don't even have authorization to read
# it. We certainly will not have access then to change it below
# so return false. We'll retry again very soon.
log.error('[%s] No authorization to delete node.' % self._path)
except Exception as e:
log.error(RegistrationBase.GENERAL_EXC_MSG % (self._path, e))
def _update_data(self):
"""Updates an existing node in Zookeeper with fresh data."""
try:
self._zk.retry(self._zk.set, self._path, value=self._encoded_data)
log.debug('[%s] Updated with data: %s' %
(self._path, self._encoded_data))
except kazoo.exceptions.NoAuthError as e:
log.error('[%s] No authorization to set node.' % self._path)
except Exception as e:
log.error(RegistrationBase.GENERAL_EXC_MSG % (self._path, e))