def gevent_load_metaconfigs(dependency): try: log.info("Sleep 3 seconds for FLASK to start before actually loading") gevent.sleep(3) # We prioritize the bootstraping of configs from ConfigV3 _load_metaconfigs_from_one_dependency(dependency) global _INITIALIZATION_COMPLETED _INITIALIZATION_COMPLETED = True except BaseException: _kill("Restart because initialization error")
def gevent_load_metaconfigs(dependency): try: log.info("Sleep 3 seconds for FLASK to start before actually loading") gevent.sleep(3) _load_metaconfigs_from_one_dependency(_GLOBAL_SHARED_DEPENDENCY) if dependency != _GLOBAL_SHARED_DEPENDENCY: _load_metaconfigs_from_one_dependency(dependency) global _INITIALIZATION_COMPLETED _INITIALIZATION_COMPLETED = True except BaseException: _kill("Restart because initialization error")
def _check_local_session_state(): global _ZK_SESSION_ID while True: client = _kazoo_client() log.info("Current zk session id %s", client._session_id) if _ZK_SESSION_ID is None: _ZK_SESSION_ID = client._session_id elif _ZK_SESSION_ID != client._session_id: log.warning("Zookeeper session changes from %s to %s", _ZK_SESSION_ID,client._session_id) since_start = datetime.datetime.utcnow() - _START_TIME if since_start.total_seconds()>180: _kill("Restart since ZK session changes") gevent.sleep(60)
def _periodic_checking(): while True: try: log.info('sleeping... for %s seconds' % (str(WAKEUP_FREQUENCY_IN_SECS))) gevent.sleep(float(WAKEUP_FREQUENCY_IN_SECS)) # Wait up to MAX_JITTER_TIME_IN_SECS seconds before invoking to lower # the likelihood that many machines invoke do this at the same time gevent.sleep(random.randint(0, MAX_JITTER_TIME_IN_SECS)) log.info('Executing all associated commands...') if random.uniform(0, 1) < RESTART_PROBABILITY: _kill("Restart from periodic_checking") _refresh_all_commands() _retry_nonexistent_zk_paths(list(_CONFIGS_WITH_NONEXISTENT_PATH)) except Exception: log.exception("Periodic checking error")
def _notification_processor(): while True: (zk_path, command, value, version, max_wait_in_secs, watch_type, notification_timestamp) = _NOTIFICATION_EVENT_QUEUE.get() if zk_path == "kill": _kill("Restart via kill api") # ignore all notifications with an older version if _is_older_version(zk_path, version, notification_timestamp): continue # TODO: we need to deal with it if the number of spawned greenlets # becomes an issue. gevent.spawn(_process_notification, command, value, version, max_wait_in_secs, watch_type, zk_path, notification_timestamp)
def _periodic_checking(): while True: global _LAST_SUCCESS_PERIODIC_CHECK try: log.info('sleeping... for %s seconds' % (str(WAKEUP_FREQUENCY_IN_SECS))) gevent.sleep(float(WAKEUP_FREQUENCY_IN_SECS)) # Wait up to MAX_JITTER_TIME_IN_SECS seconds before invoking to lower # the likelihood that many machines invoke do this at the same time gevent.sleep(random.randint(0, MAX_JITTER_TIME_IN_SECS)) log.info('Executing all associated commands...') if random.uniform(0, 1) < RESTART_PROBABILITY: _kill("Restart from periodic_checking") _refresh_all_commands() _retry_nonexistent_zk_paths(list(_CONFIGS_WITH_NONEXISTENT_PATH)) _LAST_SUCCESS_PERIODIC_CHECK = datetime.datetime.now() except Exception: log.exception("Periodic checking error")
def update_dependency(dependents): # For all children under dependency, set datawatch or children watches # If this function is triggered not in the placing watchers time, # restart ZUM to refresh to dependency list. if _INITIALIZATION_COMPLETED: _kill("Watched dependency changed, restart ZUM to catch the change") for dependent in dependents: if dependent.endswith(".dep"): _load_metaconfigs_from_one_dependency(dependent) else: if dependent in _WATCHED_METACONFIGS: continue # Set watch on the MetaConfig znode and load the content from S3 metaconfig_zk_path = METACONFIG_ZK_PATH_FORMAT.format(dependent) if not _kazoo_client(ZK_HOSTS).exists(metaconfig_zk_path): log.error("The metaconfig %s does not exist" % dependent) continue log.info("Watching Metaconfig %s" % dependent) watcher = DataWatcher(metaconfig_zk_path, ZK_HOSTS) watcher.watch(functools.partial(update_metaconfig, dependent)) _WATCHED_METACONFIGS.add(dependent)