def _create_dep_message(self, *args): """ :rtype: ApplicationDependenciesMessage """ mes = ApplicationDependenciesMessage() for path in args: data = { path: {"configuration_path": path, "dependencies": list()} } mes.update(data) return mes
def __init__(self, configuration, zoo_keeper, web_socket_clients, time_estimate_cache): """ :type configuration: zoom.config.configuration.Configuration :type zoo_keeper: kazoo.client.KazooClient :type web_socket_clients: list """ self._cache = ApplicationDependenciesMessage() self._configuration = configuration self._zoo_keeper = zoo_keeper self._web_socket_clients = web_socket_clients self._time_estimate_cache = time_estimate_cache self._message_throttle = MessageThrottle(configuration, web_socket_clients)
def __init__(self, configuration, zoo_keeper, web_socket_clients, time_estimate_cache): """ :type configuration: zoom.config.configuration.Configuration :type zoo_keeper: zoom.www.zoo_keeper.ZooKeeper :type web_socket_clients: list """ self._cache = ApplicationDependenciesMessage() self._configuration = configuration self._zoo_keeper = zoo_keeper self._web_socket_clients = web_socket_clients self._time_estimate_cache = time_estimate_cache self._message_throttle = MessageThrottle(configuration, web_socket_clients)
def _on_update_path(self, path): try: message = ApplicationDependenciesMessage() self._walk(path, message) self._cache.update(message.application_dependencies) self._recalc_downstream_dependencies() self._message_throttle.add_message(message) self._time_estimate_cache.update_dependencies( self._cache.application_dependencies) except Exception: logging.exception('An unhandled Exception has occurred for path: ' '{0}'.format(path))
def test_update(self): path = '/foo' mes = ApplicationDependenciesMessage() d1 = {"configuration_path": path, "dependencies": list()} # test dict gets update data = {path: d1} mes.update(data) self.assertEqual(mes.application_dependencies, data) # test data change for same key d2 = d1.copy() d2['dependencies'] = [1, 2, 3] data = {path: d2} mes.update(data) self.assertEqual(mes.application_dependencies.get(path), d2)
class ApplicationDependencyCache(object): def __init__(self, configuration, zoo_keeper, web_socket_clients, time_estimate_cache): """ :type configuration: zoom.config.configuration.Configuration :type zoo_keeper: kazoo.client.KazooClient :type web_socket_clients: list """ self._cache = ApplicationDependenciesMessage() self._configuration = configuration self._zoo_keeper = zoo_keeper self._web_socket_clients = web_socket_clients self._time_estimate_cache = time_estimate_cache self._message_throttle = MessageThrottle(configuration, web_socket_clients) def start(self): self._message_throttle.start() def stop(self): self._message_throttle.stop() def load(self): """ :rtype: ApplicationDependenciesMessage """ try: if not self._cache.application_dependencies: self._load() return self._cache except Exception: logging.exception('An unhandled Exception has occurred') def reload(self): """ Clear cache, and re-walk agent config path. """ self._cache.clear() logging.info("Application dependency cache cleared") self._on_update_path(self._configuration.agent_configuration_path) @TimeThis(__file__) def _load(self): """ Walk full agent config path to get data. Load self._cache object """ self._cache.clear() self._walk(self._configuration.agent_configuration_path, self._cache) logging.info( "Application dependency cache loaded from ZooKeeper {0}".format( self._configuration.agent_configuration_path)) self._recalc_downstream_dependencies() self._time_estimate_cache.update_dependencies( self._cache.application_dependencies) @connected_with_return(None) def _walk(self, path, result): """ :type path: str :type result: ApplicationDependenciesMessage """ try: children = self._zoo_keeper.get_children(path, watch=self._on_update) if children: for child in children: self._walk(zk_path_join(path, child), result) else: self._get_application_dependency(path, result) except NoNodeError: logging.debug('Node at {0} no longer exists.'.format(path)) def _get_application_dependency(self, path, result): """ Load result object with application dependencies :type path: str :type result: ApplicationDependenciesMessage """ if self._zoo_keeper.exists(path): data, stat = self._zoo_keeper.get(path, watch=self._on_update) if not data: return try: root = ElementTree.fromstring(data) for node in root.findall('Automation/Component'): app_id = node.attrib.get('id') registrationpath = node.attrib.get('registrationpath', None) if registrationpath is None: registrationpath = zk_path_join( self._configuration.application_state_path, app_id) start_action = node.find('Actions/Action[@id="start"]') if start_action is None: logging.warn("No Start Action Found for {0}".format( registrationpath)) dependencies = list() else: dependencies = self._parse_dependencies(start_action) data = { "configuration_path": registrationpath, "dependencies": dependencies, "downstream": list() } result.update({registrationpath: data}) except Exception: logging.exception('An unhandled exception occurred') else: logging.warn("config path does not exist: {0}".format(path)) def _parse_dependencies(self, action): """ Parse dependencies out of XML :type action: xml.etree.ElementTree.Element :rtype: list """ # TODO: rename 'path' when it really isn't a path. this is a hack... # prev_was_not keeps track of whether the outer class was 'not' dependencies = [] prev_was_not = False for predicate in action.iter('Predicate'): pred_type = predicate.get('type').lower() pred_path = predicate.get('path', None) # pred_oper = predicate.get('operational', False) pred_oper = bool( verify_attribute(predicate, 'operational', none_allowed=True)) if pred_type == PredicateType.ZOOKEEPERHASCHILDREN: dependencies.append({ 'type': pred_type, 'path': pred_path, 'operational': pred_oper }) prev_was_not = False elif pred_type == PredicateType.ZOOKEEPERHASGRANDCHILDREN: dependencies.append({ 'type': pred_type, 'path': pred_path, 'operational': pred_oper }) prev_was_not = False elif pred_type == PredicateType.ZOOKEEPERGOODUNTILTIME: if len(pred_path.split('gut/')) > 1: dependencies.append({ 'type': pred_type, 'path': ("I should be up between: {0}".format( pred_path.split("gut/")[1])), 'operational': pred_oper }) else: logging.debug('Invalid GUT path: {0}'.format(pred_path)) prev_was_not = False elif pred_type == PredicateType.HOLIDAY: dependencies.append({ 'type': pred_type, 'path': ("Does NOT run on holidays" if prev_was_not else "Runs on holidays"), 'operational': pred_oper }) prev_was_not = False elif pred_type == PredicateType.WEEKEND: dependencies.append({ 'type': pred_type, 'path': ("Does NOT run on weekends" if prev_was_not else "Runs on weekends"), 'operational': pred_oper }) prev_was_not = False elif pred_type == PredicateType.TIMEWINDOW: begin = predicate.get('begin', None) end = predicate.get('end', None) weekdays = predicate.get('weekdays', None) msg = 'I should be up ' if begin is not None: msg += 'after: {0} '.format(begin) if end is not None: msg += 'until: {0}'.format(end) # only send dependency if there is something to send if begin is not None or end is not None: dependencies.append({ 'type': pred_type, 'path': msg, 'operational': pred_oper }) # pretend this is a weekend predicate for convenience if weekdays is not None: day_range = TimeWindow.parse_range(weekdays) if Weekdays.SATURDAY in day_range or Weekdays.SUNDAY in day_range: wk_msg = 'Runs on weekends' else: wk_msg = 'Does NOT run on weekends' dependencies.append({ 'type': PredicateType.WEEKEND, 'path': wk_msg, 'operational': pred_oper }) elif pred_type == PredicateType.NOT: prev_was_not = True return dependencies @TimeThis(__file__) def _recalc_downstream_dependencies(self, tries=0): """ Loop over existing cache and link upstream with downstream elements """ # clear existing downstream try: for data in self._cache.application_dependencies.itervalues(): downstream = data.get('downstream') del downstream[:] dep_copy = self._cache.application_dependencies.copy() for path, data in dep_copy.iteritems(): for key, value in self._cache.application_dependencies.iteritems( ): for dep in data.get('dependencies'): if dep['type'] == PredicateType.ZOOKEEPERHASGRANDCHILDREN: if key.startswith(dep['path']): value.get('downstream').append(path) elif dep['type'] == PredicateType.ZOOKEEPERHASCHILDREN: if dep['path'] == key: value.get('downstream').append(path) except RuntimeError: time.sleep(1) tries += 1 if tries < 3: self._recalc_downstream_dependencies(tries=tries) def _on_update(self, event): """ Callback to send updates via websocket on application state changes. :type event: kazoo.protocol.states.WatchedEvent """ self._on_update_path(event.path) def _on_update_path(self, path): try: message = ApplicationDependenciesMessage() self._walk(path, message) self._cache.update(message.application_dependencies) self._recalc_downstream_dependencies() self._message_throttle.add_message(message) self._time_estimate_cache.update_dependencies( self._cache.application_dependencies) except Exception: logging.exception('An unhandled Exception has occurred for path: ' '{0}'.format(path))
class DataStore(object): def __init__(self, configuration, zoo_keeper, task_server): """ :type configuration: zoom.config.configuration.Configuration :type zoo_keeper: kazoo.client.KazooClient :type task_server: zoom.www.entities.task_server.TaskServer """ self._configuration = configuration self._zoo_keeper = zoo_keeper self._task_server = task_server self._alert_exceptions = list() self._pd = \ PagerDuty(self._configuration.pagerduty_subdomain, self._configuration.pagerduty_api_token, self._configuration.pagerduty_default_svc_key, alert_footer=self._configuration.pagerduty_alert_footer) self._alert_manager = AlertManager( configuration.alert_path, configuration.override_node, configuration.application_state_path, zoo_keeper, self._pd, self._alert_exceptions) self._web_socket_clients = list() self._time_estimate_cache = TimeEstimateCache(self._configuration, self._web_socket_clients) self._application_dependency_cache = \ ApplicationDependencyCache(self._configuration, self._zoo_keeper, self._web_socket_clients, self._time_estimate_cache) self._application_state_cache = \ ApplicationStateCache(self._configuration, self._zoo_keeper, self._web_socket_clients, self._time_estimate_cache) self._global_cache = GlobalCache(self._configuration, self._zoo_keeper, self._web_socket_clients) self._pd_svc_list_cache = {} def start(self): logging.info('Starting data store.') self._global_cache.start() self._application_state_cache.start() self._application_dependency_cache.start() self._time_estimate_cache.start() self._alert_manager.start() def stop(self): logging.info('Stopping data store.') self._global_cache.stop() self._application_state_cache.stop() self._application_dependency_cache.stop() self._time_estimate_cache.stop() self._alert_manager.stop() @connected_with_return(ApplicationStatesMessage()) def load_application_state_cache(self): """ :rtype: zoom.messages.application_states.ApplicationStatesMessage """ logging.info('Loading application states.') return self._application_state_cache.load() @connected_with_return(ApplicationDependenciesMessage()) def load_application_dependency_cache(self): """ :rtype: zoom.messages.application_dependencies.ApplicationDependenciesMessage """ logging.info('Loading application dependencies.') return self._application_dependency_cache.load() @connected_with_return(TimeEstimateMessage()) def load_time_estimate_cache(self): """ :rtype: zoom.messages.timing_estimate.TimeEstimateMessage """ return self._time_estimate_cache.load() def get_start_time(self, path): """ :rtype: dict """ return self._time_estimate_cache.get_graphite_data(path) @connected_with_return(GlobalModeMessage('{"mode":"Unknown"}')) def get_global_mode(self): """ :rtype: zoom.messages.global_mode_message.GlobalModeMessage """ logging.info('Loading global mode.') return self._global_cache.get_mode() def reload(self): """ Clear all cache objects and send reloaded data as updates. """ # restart client to destroy any existing watches # self._zoo_keeper.restart() logging.info('Reloading all cache types.') self._task_server.clear_all_tasks() self._global_cache.on_update() self._application_state_cache.reload() self._application_dependency_cache.reload() self._time_estimate_cache.reload() self._alert_manager.start() self._pd_svc_list_cache = self._pd.get_service_dict() return {'cache_clear': 'okay'} def load(self): """ Clear all cache objects and send reloaded data as updates. """ logging.info('Loading all cache types.') self._global_cache.on_update() self._application_state_cache.load() self._application_dependency_cache.load() self._time_estimate_cache.load() self._pd_svc_list_cache = self._pd.get_service_dict() return {'cache_load': 'okay'} @property def pd_client(self): """ :rtype: zoom.common.pagerduty.PagerDuty """ return self._pd @property def web_socket_clients(self): """ :rtype: list """ return self._web_socket_clients @property def application_state_cache(self): """ :rtype: zoom.www.cache.application_state_cache.ApplicationStateCache """ return self._application_state_cache @property def alert_exceptions(self): """ :rtype: list """ return self._alert_exceptions @property def pagerduty_services(self): """ :rtype: dict """ return self._pd_svc_list_cache
class ApplicationDependencyCache(object): def __init__(self, configuration, zoo_keeper, web_socket_clients, time_estimate_cache): """ :type configuration: zoom.config.configuration.Configuration :type zoo_keeper: zoom.www.zoo_keeper.ZooKeeper :type web_socket_clients: list """ self._cache = ApplicationDependenciesMessage() self._configuration = configuration self._zoo_keeper = zoo_keeper self._web_socket_clients = web_socket_clients self._time_estimate_cache = time_estimate_cache self._message_throttle = MessageThrottle(configuration, web_socket_clients) def start(self): self._message_throttle.start() def stop(self): self._message_throttle.stop() def load(self): """ :rtype: ApplicationDependenciesMessage """ try: if not self._cache.application_dependencies: self._load() return self._cache except Exception: logging.exception('An unhandled Exception has occurred') def reload(self): """ Clear cache, and re-walk agent config path. """ self._cache.clear() logging.info("Application dependency cache cleared") self._on_update_path(self._configuration.agent_configuration_path) def _load(self): """ Walk full agent config path to get data. Load self._cache object """ self._cache.clear() self._walk(self._configuration.agent_configuration_path, self._cache) logging.info("Application dependency cache loaded from ZooKeeper {0}" .format(self._configuration.agent_configuration_path)) self._recalc_downstream_dependencies() self._time_estimate_cache.update_dependencies( self._cache.application_dependencies) @connected_with_return(None) def _walk(self, path, result): """ :type path: str :type result: ApplicationDependenciesMessage """ try: children = self._zoo_keeper.get_children(path, watch=self._on_update) if children: for child in children: self._walk(os.path.join(path, child), result) else: self._get_application_dependency(path, result) except NoNodeError: logging.debug('Node at {0} no longer exists.'.format(path)) def _get_application_dependency(self, path, result): """ Load result object with application dependencies :type path: str :type result: ApplicationDependenciesMessage """ if self._zoo_keeper.exists(path): data, stat = self._zoo_keeper.get(path, watch=self._on_update) if not data: return try: root = ElementTree.fromstring(data) for node in root.findall('Automation/Component'): app_id = node.attrib.get('id') registrationpath = node.attrib.get('registrationpath', None) if registrationpath is None: registrationpath = os.path.join( self._configuration.application_state_path, app_id) start_action = node.find('Actions/Action[@id="start"]') if start_action is None: logging.warn("No Start Action Found for {0}" .format(registrationpath)) continue dependencies = self._parse_dependencies(start_action) data = { "configuration_path": registrationpath, "dependencies": dependencies, "downstream": list() } result.update({registrationpath: data}) except Exception: logging.exception('An unhandled exception occurred') else: logging.warn("config path does not exist: {0}".format(path)) def _parse_dependencies(self, action): """ Parse dependencies out of XML :type action: xml.etree.ElementTree.Element :rtype: list """ # TODO: rename 'path' when it really isn't a path. this is a hack... # prev_was_not keeps track of whether the outer class was 'not' dependencies = [] prev_was_not = False for predicate in action.iter('Predicate'): pred_type = predicate.get('type').lower() pred_path = predicate.get('path', None) if pred_type == PredicateType.ZOOKEEPERHASCHILDREN: dependencies.append({'type': pred_type, 'path': pred_path}) prev_was_not = False if pred_type == PredicateType.ZOOKEEPERHASGRANDCHILDREN: dependencies.append({'type': pred_type, 'path': pred_path}) prev_was_not = False if pred_type == PredicateType.ZOOKEEPERGOODUNTILTIME: if len(pred_path.split('gut/')) > 1: dependencies.append( {'type': pred_type, 'path': ("I should be up between: {0}" .format(pred_path.split("gut/")[1]))}) else: logging.debug('Invalid GUT path: {0}'.format(pred_path)) prev_was_not = False if pred_type == PredicateType.HOLIDAY: dependencies.append( {'type': pred_type, 'path': ("Does NOT run on holidays" if prev_was_not else "Runs on holidays")}) prev_was_not = False if pred_type == PredicateType.WEEKEND: dependencies.append( {'type': pred_type, 'path': ("Does NOT run on weekends" if prev_was_not else "Runs on weekends")}) prev_was_not = False if pred_type == PredicateType.TIME: start = predicate.get('start', None) stop = predicate.get('stop', None) msg = 'I should be up ' if start is not None: msg += 'after: {0} '.format(start) if stop is not None: msg += 'until: {0}'.format(stop) if not start and not stop: msg += '?' dependencies.append({'type': pred_type, 'path': msg}) if pred_type == PredicateType.NOT: prev_was_not = True return dependencies def _recalc_downstream_dependencies(self): """ Loop over existing cache and link upstream with downstream elements """ # clear existing downstream for data in self._cache.application_dependencies.itervalues(): downstream = data.get('downstream') del downstream[:] dep_copy = self._cache.application_dependencies.copy() for path, data in dep_copy.iteritems(): for key, value in self._cache.application_dependencies.iteritems(): for dep in data.get('dependencies'): if dep['type'] == PredicateType.ZOOKEEPERHASGRANDCHILDREN: if key.startswith(dep['path']): value.get('downstream').append(path) elif dep['type'] == PredicateType.ZOOKEEPERHASCHILDREN: if dep['path'] == key: value.get('downstream').append(path) def _on_update(self, event): """ Callback to send updates via websocket on application state changes. :type event: kazoo.protocol.states.WatchedEvent """ self._on_update_path(event.path) def _on_update_path(self, path): try: message = ApplicationDependenciesMessage() self._walk(path, message) self._recalc_downstream_dependencies() self._cache.update(message.application_dependencies) self._message_throttle.add_message(message) self._time_estimate_cache.update_dependencies( self._cache.application_dependencies) except Exception: logging.exception('An unhandled Exception has occurred')