def update_zk(self, old_value, value, force_update=True):
"""Update the s3 file and update the zk_node.
All the operations inside this function is guarded by a distributed zk lock. It prevents
race condition where multiple calls to try to update the config.
Inside the lock, the given ``old_value`` is checked against the value in s3(located by zk
node value) by default. Abort if they do not match unless force_update is True. Then the
new data is uploaded to s3 whose key is suffixed with the current timestamp. Finally the
zk node is updated with the current timestamp(which triggers zk_update_monitor to download).
The last two steps cannot be reversed because we can only trigger zk_update_monitor to
download when the new data is already in s3.
If enable_audit_history is True, and author and comment are both given,
we will log this change to audit history.
Args:
old_value: A string, which should be equal to the current value in zk.
old_value will be ignored if force_update is True
value: A string, value to update to.
force_update: Boolean, force update zk regardless if old_value matches s3_value or not. Default to be True.
Returns:
True if successfully updated, otherwise False.
"""
# Avoid potential kazoo client problem.
if not KazooClientManager(self.zk_hosts).get_client():
KazooClientManager(self.zk_hosts)._reconnect()
KazooClientManager(self.zk_hosts).get_client().ensure_path(self.zk_path)
# Try to get the lock.
lock = KazooClientManager(self.zk_hosts).get_client().Lock(self.zk_lock_path)
if not lock.acquire(blocking=False):
raise Exception('ZK lock is hold by someone else. Try later.')
try:
znode_data, znode = KazooClientManager(self.zk_hosts).get_client().get(self.zk_path)
# Only allow update if the given old value is the current s3 value, or the value of the
# zk_node is empty(which is the case where the zk node is updated for the first time).
if not force_update and znode_data:
s3_path_with_timestamp = self._get_s3_path_with_timestamp(znode_data)
try:
s3_value = s3config.S3Config(self.aws_keyfile, self.s3_bucket, self.s3_endpoint).get_config_string(
s3_path_with_timestamp)
except ValueError as e:
log.error("Failed to get s3 value from s3 path %s: %s" %
(s3_path_with_timestamp, str(e)))
raise Exception('Old s3 key %s located by the zk node value does not exist: '
'%s This is possibly due to s3 inconsistency. Try later.' %
(s3_path_with_timestamp, str(e)))
if old_value != s3_value:
raise Exception('Old value is not equal to s3 value for zk path %s, old_value: %s, s3_value: %s' %
(self.zk_path, old_value, s3_value))
update_time = time.time()
current_timestamp_str = str(update_time)
s3_path_with_timestamp = self._get_s3_path_with_timestamp(current_timestamp_str)
result = s3config.S3Config(self.aws_keyfile, self.s3_bucket, self.s3_endpoint).put_config_string(s3_path_with_timestamp, value)
if result is not None:
raise Exception('Error writing to s3 path %s for zk path %s: %s' % (
s3_path_with_timestamp, self.zk_path, result))
# Write the index also to S3(this will be the same data stored in zk). It is used to
# provide easy access to the S3 data in case zk is down.
s3config.S3Config(self.aws_keyfile, self.s3_bucket, self.s3_endpoint).put_config_string(
self.s3_file_path, current_timestamp_str)
# Try 10 times in case the write to zk failed. We want to make sure zk is changed because
# s3 is already overwritten. Otherwise, there will be inconsistency between s3 file and
# local config file.
for i in xrange(0, 10):
try:
KazooClientManager(self.zk_hosts).get_client().set(
self.zk_path, current_timestamp_str)
return True
except Exception as e:
print e
log.info('Zk write failed for zk path %s with %s for the %d time' % (
self.zk_path, e, i + 1))
KazooClientManager(self.zk_hosts)._reconnect()
raise Exception('Failed to write to zk path %s even though we already wrote to s3.' % (
self.zk_path))
finally:
lock.release()
def update_zk(self, old_value, value, force_update=True):
"""Update the s3 file and update the zk_node.
All the operations inside this function is guarded by a distributed zk lock. It prevents
race condition where multiple calls to try to update the config.
Inside the lock, the given ``old_value`` is checked against the value in s3(located by zk
node value) by default. Abort if they do not match unless force_update is True. Then the
new data is uploaded to s3 whose key is suffixed with the current timestamp. Finally the
zk node is updated with the current timestamp(which triggers zk_update_monitor to download).
The last two steps cannot be reversed because we can only trigger zk_update_monitor to
download when the new data is already in s3.
If enable_audit_history is True, and author and comment are both given,
we will log this change to audit history.
Args:
old_value: A string, which should be equal to the current value in zk.
old_value will be ignored if force_update is True
value: A string, value to update to.
force_update: Boolean, force update zk regardless if old_value matches s3_value or not. Default to be True.
Returns:
True if successfully updated, otherwise False.
"""
# Avoid potential kazoo client problem.
if not KazooClientManager(self.zk_hosts).get_client():
KazooClientManager(self.zk_hosts)._reconnect()
KazooClientManager(self.zk_hosts).get_client().ensure_path(
self.zk_path)
# Try to get the lock.
lock = KazooClientManager(self.zk_hosts).get_client().Lock(
self.zk_lock_path)
if not lock.acquire(blocking=False):
raise Exception('ZK lock is hold by someone else. Try later.')
try:
znode_data, znode = KazooClientManager(
self.zk_hosts).get_client().get(self.zk_path)
# Only allow update if the given old value is the current s3 value, or the value of the
# zk_node is empty(which is the case where the zk node is updated for the first time).
if not force_update and znode_data:
s3_path_with_timestamp = self._get_s3_path_with_timestamp(
znode_data)
try:
s3_value = s3config.S3Config(
self.aws_keyfile, self.s3_bucket,
self.s3_endpoint).get_config_string(
s3_path_with_timestamp)
except ValueError as e:
log.error("Failed to get s3 value from s3 path %s: %s" %
(s3_path_with_timestamp, str(e)))
raise Exception(
'Old s3 key %s located by the zk node value does not exist: '
'%s This is possibly due to s3 inconsistency. Try later.'
% (s3_path_with_timestamp, str(e)))
if old_value != s3_value:
raise Exception(
'Old value is not equal to s3 value for zk path %s, old_value: %s, s3_value: %s'
% (self.zk_path, old_value, s3_value))
update_time = time.time()
current_timestamp_str = str(update_time)
s3_path_with_timestamp = self._get_s3_path_with_timestamp(
current_timestamp_str)
result = s3config.S3Config(self.aws_keyfile, self.s3_bucket,
self.s3_endpoint).put_config_string(
s3_path_with_timestamp, value)
if result is not None:
raise Exception(
'Error writing to s3 path %s for zk path %s: %s' %
(s3_path_with_timestamp, self.zk_path, result))
# Write the index also to S3(this will be the same data stored in zk). It is used to
# provide easy access to the S3 data in case zk is down.
s3config.S3Config(self.aws_keyfile, self.s3_bucket,
self.s3_endpoint).put_config_string(
self.s3_file_path, current_timestamp_str)
# Try 10 times in case the write to zk failed. We want to make sure zk is changed because
# s3 is already overwritten. Otherwise, there will be inconsistency between s3 file and
# local config file.
for i in xrange(0, 10):
try:
KazooClientManager(self.zk_hosts).get_client().set(
self.zk_path, current_timestamp_str)
return True
except Exception as e:
print e
log.info(
'Zk write failed for zk path %s with %s for the %d time'
% (self.zk_path, e, i + 1))
KazooClientManager(self.zk_hosts)._reconnect()
raise Exception(
'Failed to write to zk path %s even though we already wrote to s3.'
% (self.zk_path))
finally:
lock.release()
