def _create_dep_message(self, *args): """ :rtype: ApplicationDependenciesMessage """ mes = ApplicationDependenciesMessage() for path in args: data = { path: {"configuration_path": path, "dependencies": list()} } mes.update(data) return mes
def test_update(self): path = '/foo' mes = ApplicationDependenciesMessage() d1 = {"configuration_path": path, "dependencies": list()} # test dict gets update data = {path: d1} mes.update(data) self.assertEqual(mes.application_dependencies, data) # test data change for same key d2 = d1.copy() d2['dependencies'] = [1, 2, 3] data = {path: d2} mes.update(data) self.assertEqual(mes.application_dependencies.get(path), d2)
class ApplicationDependencyCache(object): def __init__(self, configuration, zoo_keeper, web_socket_clients, time_estimate_cache): """ :type configuration: zoom.config.configuration.Configuration :type zoo_keeper: kazoo.client.KazooClient :type web_socket_clients: list """ self._cache = ApplicationDependenciesMessage() self._configuration = configuration self._zoo_keeper = zoo_keeper self._web_socket_clients = web_socket_clients self._time_estimate_cache = time_estimate_cache self._message_throttle = MessageThrottle(configuration, web_socket_clients) def start(self): self._message_throttle.start() def stop(self): self._message_throttle.stop() def load(self): """ :rtype: ApplicationDependenciesMessage """ try: if not self._cache.application_dependencies: self._load() return self._cache except Exception: logging.exception('An unhandled Exception has occurred') def reload(self): """ Clear cache, and re-walk agent config path. """ self._cache.clear() logging.info("Application dependency cache cleared") self._on_update_path(self._configuration.agent_configuration_path) @TimeThis(__file__) def _load(self): """ Walk full agent config path to get data. Load self._cache object """ self._cache.clear() self._walk(self._configuration.agent_configuration_path, self._cache) logging.info( "Application dependency cache loaded from ZooKeeper {0}".format( self._configuration.agent_configuration_path)) self._recalc_downstream_dependencies() self._time_estimate_cache.update_dependencies( self._cache.application_dependencies) @connected_with_return(None) def _walk(self, path, result): """ :type path: str :type result: ApplicationDependenciesMessage """ try: children = self._zoo_keeper.get_children(path, watch=self._on_update) if children: for child in children: self._walk(zk_path_join(path, child), result) else: self._get_application_dependency(path, result) except NoNodeError: logging.debug('Node at {0} no longer exists.'.format(path)) def _get_application_dependency(self, path, result): """ Load result object with application dependencies :type path: str :type result: ApplicationDependenciesMessage """ if self._zoo_keeper.exists(path): data, stat = self._zoo_keeper.get(path, watch=self._on_update) if not data: return try: root = ElementTree.fromstring(data) for node in root.findall('Automation/Component'): app_id = node.attrib.get('id') registrationpath = node.attrib.get('registrationpath', None) if registrationpath is None: registrationpath = zk_path_join( self._configuration.application_state_path, app_id) start_action = node.find('Actions/Action[@id="start"]') if start_action is None: logging.warn("No Start Action Found for {0}".format( registrationpath)) dependencies = list() else: dependencies = self._parse_dependencies(start_action) data = { "configuration_path": registrationpath, "dependencies": dependencies, "downstream": list() } result.update({registrationpath: data}) except Exception: logging.exception('An unhandled exception occurred') else: logging.warn("config path does not exist: {0}".format(path)) def _parse_dependencies(self, action): """ Parse dependencies out of XML :type action: xml.etree.ElementTree.Element :rtype: list """ # TODO: rename 'path' when it really isn't a path. this is a hack... # prev_was_not keeps track of whether the outer class was 'not' dependencies = [] prev_was_not = False for predicate in action.iter('Predicate'): pred_type = predicate.get('type').lower() pred_path = predicate.get('path', None) # pred_oper = predicate.get('operational', False) pred_oper = bool( verify_attribute(predicate, 'operational', none_allowed=True)) if pred_type == PredicateType.ZOOKEEPERHASCHILDREN: dependencies.append({ 'type': pred_type, 'path': pred_path, 'operational': pred_oper }) prev_was_not = False elif pred_type == PredicateType.ZOOKEEPERHASGRANDCHILDREN: dependencies.append({ 'type': pred_type, 'path': pred_path, 'operational': pred_oper }) prev_was_not = False elif pred_type == PredicateType.ZOOKEEPERGOODUNTILTIME: if len(pred_path.split('gut/')) > 1: dependencies.append({ 'type': pred_type, 'path': ("I should be up between: {0}".format( pred_path.split("gut/")[1])), 'operational': pred_oper }) else: logging.debug('Invalid GUT path: {0}'.format(pred_path)) prev_was_not = False elif pred_type == PredicateType.HOLIDAY: dependencies.append({ 'type': pred_type, 'path': ("Does NOT run on holidays" if prev_was_not else "Runs on holidays"), 'operational': pred_oper }) prev_was_not = False elif pred_type == PredicateType.WEEKEND: dependencies.append({ 'type': pred_type, 'path': ("Does NOT run on weekends" if prev_was_not else "Runs on weekends"), 'operational': pred_oper }) prev_was_not = False elif pred_type == PredicateType.TIMEWINDOW: begin = predicate.get('begin', None) end = predicate.get('end', None) weekdays = predicate.get('weekdays', None) msg = 'I should be up ' if begin is not None: msg += 'after: {0} '.format(begin) if end is not None: msg += 'until: {0}'.format(end) # only send dependency if there is something to send if begin is not None or end is not None: dependencies.append({ 'type': pred_type, 'path': msg, 'operational': pred_oper }) # pretend this is a weekend predicate for convenience if weekdays is not None: day_range = TimeWindow.parse_range(weekdays) if Weekdays.SATURDAY in day_range or Weekdays.SUNDAY in day_range: wk_msg = 'Runs on weekends' else: wk_msg = 'Does NOT run on weekends' dependencies.append({ 'type': PredicateType.WEEKEND, 'path': wk_msg, 'operational': pred_oper }) elif pred_type == PredicateType.NOT: prev_was_not = True return dependencies @TimeThis(__file__) def _recalc_downstream_dependencies(self, tries=0): """ Loop over existing cache and link upstream with downstream elements """ # clear existing downstream try: for data in self._cache.application_dependencies.itervalues(): downstream = data.get('downstream') del downstream[:] dep_copy = self._cache.application_dependencies.copy() for path, data in dep_copy.iteritems(): for key, value in self._cache.application_dependencies.iteritems( ): for dep in data.get('dependencies'): if dep['type'] == PredicateType.ZOOKEEPERHASGRANDCHILDREN: if key.startswith(dep['path']): value.get('downstream').append(path) elif dep['type'] == PredicateType.ZOOKEEPERHASCHILDREN: if dep['path'] == key: value.get('downstream').append(path) except RuntimeError: time.sleep(1) tries += 1 if tries < 3: self._recalc_downstream_dependencies(tries=tries) def _on_update(self, event): """ Callback to send updates via websocket on application state changes. :type event: kazoo.protocol.states.WatchedEvent """ self._on_update_path(event.path) def _on_update_path(self, path): try: message = ApplicationDependenciesMessage() self._walk(path, message) self._cache.update(message.application_dependencies) self._recalc_downstream_dependencies() self._message_throttle.add_message(message) self._time_estimate_cache.update_dependencies( self._cache.application_dependencies) except Exception: logging.exception('An unhandled Exception has occurred for path: ' '{0}'.format(path))
class ApplicationDependencyCache(object): def __init__(self, configuration, zoo_keeper, web_socket_clients, time_estimate_cache): """ :type configuration: zoom.config.configuration.Configuration :type zoo_keeper: zoom.www.zoo_keeper.ZooKeeper :type web_socket_clients: list """ self._cache = ApplicationDependenciesMessage() self._configuration = configuration self._zoo_keeper = zoo_keeper self._web_socket_clients = web_socket_clients self._time_estimate_cache = time_estimate_cache self._message_throttle = MessageThrottle(configuration, web_socket_clients) def start(self): self._message_throttle.start() def stop(self): self._message_throttle.stop() def load(self): """ :rtype: ApplicationDependenciesMessage """ try: if not self._cache.application_dependencies: self._load() return self._cache except Exception: logging.exception('An unhandled Exception has occurred') def reload(self): """ Clear cache, and re-walk agent config path. """ self._cache.clear() logging.info("Application dependency cache cleared") self._on_update_path(self._configuration.agent_configuration_path) def _load(self): """ Walk full agent config path to get data. Load self._cache object """ self._cache.clear() self._walk(self._configuration.agent_configuration_path, self._cache) logging.info("Application dependency cache loaded from ZooKeeper {0}" .format(self._configuration.agent_configuration_path)) self._recalc_downstream_dependencies() self._time_estimate_cache.update_dependencies( self._cache.application_dependencies) @connected_with_return(None) def _walk(self, path, result): """ :type path: str :type result: ApplicationDependenciesMessage """ try: children = self._zoo_keeper.get_children(path, watch=self._on_update) if children: for child in children: self._walk(os.path.join(path, child), result) else: self._get_application_dependency(path, result) except NoNodeError: logging.debug('Node at {0} no longer exists.'.format(path)) def _get_application_dependency(self, path, result): """ Load result object with application dependencies :type path: str :type result: ApplicationDependenciesMessage """ if self._zoo_keeper.exists(path): data, stat = self._zoo_keeper.get(path, watch=self._on_update) if not data: return try: root = ElementTree.fromstring(data) for node in root.findall('Automation/Component'): app_id = node.attrib.get('id') registrationpath = node.attrib.get('registrationpath', None) if registrationpath is None: registrationpath = os.path.join( self._configuration.application_state_path, app_id) start_action = node.find('Actions/Action[@id="start"]') if start_action is None: logging.warn("No Start Action Found for {0}" .format(registrationpath)) continue dependencies = self._parse_dependencies(start_action) data = { "configuration_path": registrationpath, "dependencies": dependencies, "downstream": list() } result.update({registrationpath: data}) except Exception: logging.exception('An unhandled exception occurred') else: logging.warn("config path does not exist: {0}".format(path)) def _parse_dependencies(self, action): """ Parse dependencies out of XML :type action: xml.etree.ElementTree.Element :rtype: list """ # TODO: rename 'path' when it really isn't a path. this is a hack... # prev_was_not keeps track of whether the outer class was 'not' dependencies = [] prev_was_not = False for predicate in action.iter('Predicate'): pred_type = predicate.get('type').lower() pred_path = predicate.get('path', None) if pred_type == PredicateType.ZOOKEEPERHASCHILDREN: dependencies.append({'type': pred_type, 'path': pred_path}) prev_was_not = False if pred_type == PredicateType.ZOOKEEPERHASGRANDCHILDREN: dependencies.append({'type': pred_type, 'path': pred_path}) prev_was_not = False if pred_type == PredicateType.ZOOKEEPERGOODUNTILTIME: if len(pred_path.split('gut/')) > 1: dependencies.append( {'type': pred_type, 'path': ("I should be up between: {0}" .format(pred_path.split("gut/")[1]))}) else: logging.debug('Invalid GUT path: {0}'.format(pred_path)) prev_was_not = False if pred_type == PredicateType.HOLIDAY: dependencies.append( {'type': pred_type, 'path': ("Does NOT run on holidays" if prev_was_not else "Runs on holidays")}) prev_was_not = False if pred_type == PredicateType.WEEKEND: dependencies.append( {'type': pred_type, 'path': ("Does NOT run on weekends" if prev_was_not else "Runs on weekends")}) prev_was_not = False if pred_type == PredicateType.TIME: start = predicate.get('start', None) stop = predicate.get('stop', None) msg = 'I should be up ' if start is not None: msg += 'after: {0} '.format(start) if stop is not None: msg += 'until: {0}'.format(stop) if not start and not stop: msg += '?' dependencies.append({'type': pred_type, 'path': msg}) if pred_type == PredicateType.NOT: prev_was_not = True return dependencies def _recalc_downstream_dependencies(self): """ Loop over existing cache and link upstream with downstream elements """ # clear existing downstream for data in self._cache.application_dependencies.itervalues(): downstream = data.get('downstream') del downstream[:] dep_copy = self._cache.application_dependencies.copy() for path, data in dep_copy.iteritems(): for key, value in self._cache.application_dependencies.iteritems(): for dep in data.get('dependencies'): if dep['type'] == PredicateType.ZOOKEEPERHASGRANDCHILDREN: if key.startswith(dep['path']): value.get('downstream').append(path) elif dep['type'] == PredicateType.ZOOKEEPERHASCHILDREN: if dep['path'] == key: value.get('downstream').append(path) def _on_update(self, event): """ Callback to send updates via websocket on application state changes. :type event: kazoo.protocol.states.WatchedEvent """ self._on_update_path(event.path) def _on_update_path(self, path): try: message = ApplicationDependenciesMessage() self._walk(path, message) self._recalc_downstream_dependencies() self._cache.update(message.application_dependencies) self._message_throttle.add_message(message) self._time_estimate_cache.update_dependencies( self._cache.application_dependencies) except Exception: logging.exception('An unhandled Exception has occurred')