def gevent_load_metaconfigs(dependency):
try:
log.info("Sleep 3 seconds for FLASK to start before actually loading")
gevent.sleep(3)
# We prioritize the bootstraping of configs from ConfigV3
_load_metaconfigs_from_one_dependency(dependency)
global _INITIALIZATION_COMPLETED
_INITIALIZATION_COMPLETED = True
except BaseException:
_kill("Restart because initialization error")
def gevent_load_metaconfigs(dependency):
try:
log.info("Sleep 3 seconds for FLASK to start before actually loading")
gevent.sleep(3)
_load_metaconfigs_from_one_dependency(_GLOBAL_SHARED_DEPENDENCY)
if dependency != _GLOBAL_SHARED_DEPENDENCY:
_load_metaconfigs_from_one_dependency(dependency)
global _INITIALIZATION_COMPLETED
_INITIALIZATION_COMPLETED = True
except BaseException:
_kill("Restart because initialization error")
def _check_local_session_state():
global _ZK_SESSION_ID
while True:
client = _kazoo_client()
log.info("Current zk session id %s", client._session_id)
if _ZK_SESSION_ID is None:
_ZK_SESSION_ID = client._session_id
elif _ZK_SESSION_ID != client._session_id:
log.warning("Zookeeper session changes from %s to %s", _ZK_SESSION_ID,client._session_id)
since_start = datetime.datetime.utcnow() - _START_TIME
if since_start.total_seconds()>180:
_kill("Restart since ZK session changes")
gevent.sleep(60)
def _periodic_checking():
while True:
try:
log.info('sleeping... for %s seconds'
% (str(WAKEUP_FREQUENCY_IN_SECS)))
gevent.sleep(float(WAKEUP_FREQUENCY_IN_SECS))
# Wait up to MAX_JITTER_TIME_IN_SECS seconds before invoking to lower
# the likelihood that many machines invoke do this at the same time
gevent.sleep(random.randint(0, MAX_JITTER_TIME_IN_SECS))
log.info('Executing all associated commands...')
if random.uniform(0, 1) < RESTART_PROBABILITY:
_kill("Restart from periodic_checking")
_refresh_all_commands()
_retry_nonexistent_zk_paths(list(_CONFIGS_WITH_NONEXISTENT_PATH))
except Exception:
log.exception("Periodic checking error")
def _notification_processor():
while True:
(zk_path, command, value, version, max_wait_in_secs,
watch_type, notification_timestamp) = _NOTIFICATION_EVENT_QUEUE.get()
if zk_path == "kill":
_kill("Restart via kill api")
# ignore all notifications with an older version
if _is_older_version(zk_path, version, notification_timestamp):
continue
# TODO: we need to deal with it if the number of spawned greenlets
# becomes an issue.
gevent.spawn(_process_notification, command, value, version,
max_wait_in_secs, watch_type, zk_path,
notification_timestamp)
def _notification_processor():
while True:
(zk_path, command, value, version, max_wait_in_secs,
watch_type, notification_timestamp) = _NOTIFICATION_EVENT_QUEUE.get()
if zk_path == "kill":
_kill("Restart via kill api")
# ignore all notifications with an older version
if _is_older_version(zk_path, version, notification_timestamp):
continue
# TODO: we need to deal with it if the number of spawned greenlets
# becomes an issue.
gevent.spawn(_process_notification, command, value, version,
max_wait_in_secs, watch_type, zk_path,
notification_timestamp)
def _periodic_checking():
while True:
global _LAST_SUCCESS_PERIODIC_CHECK
try:
log.info('sleeping... for %s seconds'
% (str(WAKEUP_FREQUENCY_IN_SECS)))
gevent.sleep(float(WAKEUP_FREQUENCY_IN_SECS))
# Wait up to MAX_JITTER_TIME_IN_SECS seconds before invoking to lower
# the likelihood that many machines invoke do this at the same time
gevent.sleep(random.randint(0, MAX_JITTER_TIME_IN_SECS))
log.info('Executing all associated commands...')
if random.uniform(0, 1) < RESTART_PROBABILITY:
_kill("Restart from periodic_checking")
_refresh_all_commands()
_retry_nonexistent_zk_paths(list(_CONFIGS_WITH_NONEXISTENT_PATH))
_LAST_SUCCESS_PERIODIC_CHECK = datetime.datetime.now()
except Exception:
log.exception("Periodic checking error")
def update_dependency(dependents):
# For all children under dependency, set datawatch or children watches
# If this function is triggered not in the placing watchers time,
# restart ZUM to refresh to dependency list.
if _INITIALIZATION_COMPLETED:
_kill("Watched dependency changed, restart ZUM to catch the change")
for dependent in dependents:
if dependent.endswith(".dep"):
_load_metaconfigs_from_one_dependency(dependent)
else:
if dependent in _WATCHED_METACONFIGS:
continue
# Set watch on the MetaConfig znode and load the content from S3
metaconfig_zk_path = METACONFIG_ZK_PATH_FORMAT.format(dependent)
if not _kazoo_client(ZK_HOSTS).exists(metaconfig_zk_path):
log.error("The metaconfig %s does not exist" % dependent)
continue
log.info("Watching Metaconfig %s" % dependent)
watcher = DataWatcher(metaconfig_zk_path, ZK_HOSTS)
watcher.watch(functools.partial(update_metaconfig, dependent))
_WATCHED_METACONFIGS.add(dependent)
def update_dependency(dependents):
# For all children under dependency, set datawatch or children watches
# If this function is triggered not in the placing watchers time,
# restart ZUM to refresh to dependency list.
if _INITIALIZATION_COMPLETED:
_kill("Watched dependency changed, restart ZUM to catch the change")
for dependent in dependents:
if dependent.endswith(".dep"):
_load_metaconfigs_from_one_dependency(dependent)
else:
if dependent in _WATCHED_METACONFIGS:
continue
# Set watch on the MetaConfig znode and load the content from S3
metaconfig_zk_path = METACONFIG_ZK_PATH_FORMAT.format(dependent)
if not _kazoo_client(ZK_HOSTS).exists(metaconfig_zk_path):
log.error("The metaconfig %s does not exist" % dependent)
continue
log.info("Watching Metaconfig %s" % dependent)
watcher = DataWatcher(metaconfig_zk_path, ZK_HOSTS)
watcher.watch(functools.partial(update_metaconfig, dependent))
_WATCHED_METACONFIGS.add(dependent)
