def _load_metaconfigs_from_one_dependency(dependency_name):
""" Load the metaconfigs from one dependency and finally from S3
It does 3 things:
1. Set children watch on dependencies. If the root dependency contains other
dependencies, set watchers.
2. Set znode watch on each MetaConfig Znode
3. Load the MetaConfig content from S3 using S3Utils.
If dependency/metaconfig does not exist, log and skip.
"""
if dependency_name in _WATCHED_DEPENDENCIES:
log.info("Dependency %s already watched" % dependency_name)
return
dependency_zk_path = DEPENDENCY_ZK_PATH_FORMAT.format(dependency_name)
if not _kazoo_client(ZK_HOSTS).exists(dependency_zk_path):
log.error("The dependency %s does not exist" % dependency_zk_path)
return
# Set ZK children watch on dependency
log.info("Watching dependency %s" % dependency_name)
_WATCHED_DEPENDENCIES.add(dependency_name)
_kazoo_client(ZK_HOSTS).ChildrenWatch(dependency_zk_path, update_dependency)
def _load_metaconfigs_from_one_dependency(dependency_name):
""" Load the metaconfigs from one dependency and finally from S3
It does 3 things:
1. Set children watch on dependencies. If the root dependency contains other
dependencies, set watchers.
2. Set znode watch on each MetaConfig Znode
3. Load the MetaConfig content from S3 using S3Utils.
If dependency/metaconfig does not exist, log and skip.
"""
if dependency_name in _WATCHED_DEPENDENCIES:
log.info("Dependency %s already watched" % dependency_name)
return
dependency_zk_path = DEPENDENCY_ZK_PATH_FORMAT.format(dependency_name)
if not _kazoo_client(ZK_HOSTS).exists(dependency_zk_path):
log.error("The dependency %s does not exist" % dependency_zk_path)
return
# Set ZK children watch on dependency
log.info("Watching dependency %s" % dependency_name)
_kazoo_client(ZK_HOSTS).ChildrenWatch(dependency_zk_path, update_dependency)
_WATCHED_DEPENDENCIES.add(dependency_name)
def _check_local_session_state():
global _ZK_SESSION_ID
while True:
client = _kazoo_client()
log.info("Current zk session id %s", client._session_id)
if _ZK_SESSION_ID is None:
_ZK_SESSION_ID = client._session_id
elif _ZK_SESSION_ID != client._session_id:
log.warning("Zookeeper session changes from %s to %s", _ZK_SESSION_ID,client._session_id)
since_start = datetime.datetime.utcnow() - _START_TIME
if since_start.total_seconds()>180:
_kill("Restart since ZK session changes")
gevent.sleep(60)
def update_metaconfig(metaconfig_name, *args, **kwargs):
zk_value, version = _kazoo_client(ZK_HOSTS).get(
METACONFIG_ZK_PATH_FORMAT.format(metaconfig_name))
s3_key = METACONFIG_S3_KEY_FORMAT.format(metaconfig_name)
s3_path = zk_util.construct_s3_path(s3_key, zk_value)
try:
metaconfig_data = s3config.S3Config(AWS_KEY_FILE, S3_BUCKET, s3_endpoint=S3_ENDPOINT).get_config_string(s3_path)
except ValueError as ve:
log.error("Abort downloading from s3 key %s due to ValueError: %s" % (s3_path, ve))
return
except Exception as e:
log.error("Abort downloading from s3 key %s due to unexpected s3 exception: %s" %
(s3_path, e))
return
metaconfig_list = json.loads(metaconfig_data)
for metaconfig in metaconfig_list:
_place_watch_from_metaconfig(metaconfig)
def update_metaconfig(metaconfig_name, *args, **kwargs):
zk_value, version = _kazoo_client(ZK_HOSTS).get(
METACONFIG_ZK_PATH_FORMAT.format(metaconfig_name))
s3_key = METACONFIG_S3_KEY_FORMAT.format(metaconfig_name)
s3_path = zk_util.construct_s3_path(s3_key, zk_value)
try:
metaconfig_data = s3config.S3Config(AWS_KEY_FILE, S3_BUCKET, s3_endpoint=S3_ENDPOINT).get_config_string(s3_path)
except ValueError as ve:
log.error("Abort downloading from s3 key %s due to ValueError: %s" % (s3_path, ve))
return
except Exception as e:
log.error("Abort downloading from s3 key %s due to unexpected s3 exception: %s" %
(s3_path, e))
return
metaconfig_list = json.loads(metaconfig_data)
for metaconfig in metaconfig_list:
_place_watch_from_metaconfig(metaconfig)
def update_dependency(dependents):
# For all children under dependency, set datawatch or children watches
# If this function is triggered not in the placing watchers time,
# restart ZUM to refresh to dependency list.
if _INITIALIZATION_COMPLETED:
_kill("Watched dependency changed, restart ZUM to catch the change")
for dependent in dependents:
if dependent.endswith(".dep"):
_load_metaconfigs_from_one_dependency(dependent)
else:
if dependent in _WATCHED_METACONFIGS:
continue
# Set watch on the MetaConfig znode and load the content from S3
metaconfig_zk_path = METACONFIG_ZK_PATH_FORMAT.format(dependent)
if not _kazoo_client(ZK_HOSTS).exists(metaconfig_zk_path):
log.error("The metaconfig %s does not exist" % dependent)
continue
log.info("Watching Metaconfig %s" % dependent)
watcher = DataWatcher(metaconfig_zk_path, ZK_HOSTS)
watcher.watch(functools.partial(update_metaconfig, dependent))
_WATCHED_METACONFIGS.add(dependent)
def update_dependency(dependents):
# For all children under dependency, set datawatch or children watches
# If this function is triggered not in the placing watchers time,
# restart ZUM to refresh to dependency list.
if _INITIALIZATION_COMPLETED:
_kill("Watched dependency changed, restart ZUM to catch the change")
for dependent in dependents:
if dependent.endswith(".dep"):
_load_metaconfigs_from_one_dependency(dependent)
else:
if dependent in _WATCHED_METACONFIGS:
continue
# Set watch on the MetaConfig znode and load the content from S3
metaconfig_zk_path = METACONFIG_ZK_PATH_FORMAT.format(dependent)
if not _kazoo_client(ZK_HOSTS).exists(metaconfig_zk_path):
log.error("The metaconfig %s does not exist" % dependent)
continue
log.info("Watching Metaconfig %s" % dependent)
watcher = DataWatcher(metaconfig_zk_path, ZK_HOSTS)
watcher.watch(functools.partial(update_metaconfig, dependent))
_WATCHED_METACONFIGS.add(dependent)